def get_screenshot(interp, finfo, argstr):
try:
interpret = u.interpret_method(interp)
argstr = u.debracket(argstr, interpret, finfo=finfo)
args = u.parse_arg_string(argstr)
Config.log(argstr, tag='GET_SCREENSHOT')
if not u.have_required(args, 'url', 'dest', 'height', 'width'):
raise Exception("get_screenshot incomplete args '%s'" % argstr)
dest = u.debracket(args['dest'], interpret, finfo=finfo)
(dest_dir, dest_file) = os.path.split(dest)
u.ensure_path(dest_dir)
puppeteer_templates = Config.main.template_root + '/puppeteer'
template_file = puppeteer_templates + '/get_screenshot.js'
try:
template = open(template_file).read()
except Exception as exc:
Config.log("'%s' opening template file '%s" %
(str(exc), template_file), tag='GET_SCREENSHOT_ERROR')
raise
symbols = {
'url': args['url'],
'width': args['width'],
'height': args['height'],
'dest': dest
}
script = u.debracket(template, interpret, symbols=symbols)
script_path = Config.main.output + '/tmp'
u.ensure_path(script_path)
script_file = script_path + '/get_screenshot.js'
open(script_file, 'w').write(script)
working_dir = Config.main.get('tools', 'node_workdir')
if not os.path.isdir(working_dir):
err = "get_screenshot: invalid node_workdir '%s'" % working_dir
logging.error(err)
raise Exception(err)
call_args = ['node', script_file]
(returncode, stdout, stderr) = u.run_command(
call_args, working_dir)
# logging.debug("returncode: %s\nstdout: %s\nstderr: %s" % (returncode, stdout, stderr))
if returncode == 0:
# we know the file that was created, so make its metadata now
newfi = {}
newfi['parent_full'] = finfo['full'] # provenance
newfi['name'] = dest_file
newfi['path'] = dest_dir
newfi['full'] = dest
newfi['rules_run'] = False
newfi.pop('groups', None)
return {'new_finfo': newfi}
else:
Config.log("rc %i, stderr = '%s'" % (returncode, stderr),
tag='GET_SCREENSHOT_ERROR')
except Exception as exc:
Config.log(str(exc), tag='GET_SCREENSHOT_ERROR')
def _get_cur_url_and_file(self, settings):
if settings['current_daynum'] > self._daynum_today:
return None, None
yyyy, mm, dd = u.daynum_to_ymd(settings['current_daynum'])
symbols = {'YYYY': yyyy, 'MM': mm, 'DD': dd}
options = {'allow_verbatim': False}
url = u.debracket(settings['pattern'], self.interpret, symbols=symbols, options=options)
if settings['dest']:
fname = u.debracket(settings['dest'], self.interpret, symbols=symbols, options=options)
else:
parsed = urlparse(url)
fname = os.path.basename(parsed.path)
return url, fname
def smoke_test(interp, finfo, argstr):
try:
logger.info('I do not think this will work')
# REVIEW: faulthandler not safe in apache, assumes stdout available
#faulthandler.enable()
interpret = u.interpret_method(interp)
argstr = u.debracket(argstr, interpret, finfo=finfo)
args = u.parse_arg_string(argstr)
#Config.log(argstr, tag='NETCDF_SMOKE')
#if not u.have_required(args, 'dest'):
# raise Exception("get_screenshot incomplete args '%s'" % argstr)
#dest = u.debracket(args['dest'], interpret, finfo=finfo)
#(dest_dir, dest_file) = os.path.split(dest)
#u.ensure_path(dest_dir)
full = finfo['full']
if not os.path.exists(full):
# TODO config logging not working here?
Config.log(full + " not found", tag='NETCDF_BAD_ARG')
logger.info(full + ' not found')
return 'todo file not found ' + full
Config.log("here goes nc.Dataset with " + full)
rootgrp = nc.Dataset(finfo['full'], 'r', format='NETCDF4')
ret = 'variables:\n' + str(rootgrp.variables)
rootgrp.close()
return ret
except Exception as exc:
Config.log(str(exc), tag='NETCDF_SMOKE_ERROR')
return str(exc)
def render_part_args(self, args):
if not 'name' in args:
raise Exception("name not specified in args '%s'" % str(args))
part_file = self.config.template_root + '/parts/' + args[
'name'] + '.html'
part = open(part_file).read()
return u.debracket(part, self.interpret, symbols=args)
def captionize(interp, finfo, argstr):
try:
interpret = u.interpret_method(interp)
argstr = u.debracket(argstr, interpret, finfo=finfo)
args = u.parse_arg_string(argstr)
src = finfo['full']
Config.log("src '%s', argstr '%s'" % (src, argstr), tag='CAPTIONIZE')
if not u.have_required(args, 'dest', 'where', 'font_size', 'bar_size',
'pad_x', 'pad_y', 'text'):
raise Exception("captionize incomplete args '%s'" % argstr)
dest = args['dest']
(dest_dir, dest_file) = os.path.split(dest)
u.ensure_path(dest_dir)
params = []
if args['where'] == 'top':
params.extend(['-gravity', 'northwest'])
elif args['where'] == 'bottom':
params.extend(['-gravity', 'southwest'])
else:
raise Exception("captionize invalid 'where' arg in " + argstr)
# TODO: validate colors. See https://www.imagemagick.org/script/color.php
if 'background_color' in args:
params.extend(['-background', args['background_color']])
else:
params.extend(['-background', 'white'])
if 'text_color' in args:
params.extend(['-fill', args['text_color']])
else:
params.extend(['-fill', 'black'])
# TODO: validate font name. "convert -list font" will list them. system dependent.
if 'font' in args:
params.extend(['-font', args['font']])
else:
params.extend(['-font', 'Helvetica'])
params.extend(['-pointsize', args['font_size']])
params.extend(['-splice', '0x' + args['bar_size']])
x = u.force_sign(args['pad_x'])
y = u.force_sign(args['pad_y'])
params.extend(['-annotate', x + y])
fixed_text = args['text'].replace("'", "\\'")
params.append('"' + fixed_text + '"')
call_args = ['convert', src] + params + [dest]
(returncode, stdout, stderr) = u.run_command(call_args)
# logging.debug("returncode: %s\nstdout: %s\nstderr: %s" % (returncode, stdout, stderr))
if returncode == 0:
# we know the file that was created, so make its metadata now
newfi = {}
newfi['parent_full'] = finfo['full'] # provenance
newfi['name'] = dest_file
newfi['path'] = dest_dir
newfi['full'] = dest
newfi['rules_run'] = False
newfi.pop('groups', None)
return {'new_finfo': newfi}
else:
logging.error("captionize failed with rc %i, stderr = '%s'" %
(returncode, stderr))
except Exception as exc:
logging.error("captionize exception '%s'" % str(exc))
def _get_rules(self):
try:
rules = []
cur_rule = {'condition': {}, 'actions': []}
rule_file = self.config.get('process', 'rule_file', return_none=True)
if rule_file:
tmp_fh = u.process_includes(rule_file)
rule_lines = tmp_fh.read().splitlines()
tmp_fh.close()
logging.info("using rules from file %s" % rule_file)
else:
raise Exception("required config value 'rule_file' is missing")
# TODO add support for reformatting split lines
for line in rule_lines:
match = re.search(self._re['define'], line)
if match:
key = match.group(1)
val = u.debracket(match.group(2), self.interpret)
Config.log("'%s' = '%s'" % (key, val), tag='RULE_DEFINE')
self.symbols[key] = val
continue
if re.search(self._re['comment'], line):
continue # allow commented or blank lines
match = self._re['header'].search(line)
if match:
if cur_rule['condition']:
rules.append(cur_rule)
cur_rule = {'condition': {}, 'actions': []}
cur_rule['props'] = u.comma_split(match.group(2))
continue
match = self._re['cond'].search(line)
if match:
if cur_rule['condition']:
rules.append(cur_rule)
cur_rule = {'condition': {}, 'actions': []}
indent = match.group(1) # not used at present
cur_rule['condition']['text'] = match.group(2)
continue
match = self._re['action'].search(line)
if match:
if not cur_rule['condition']:
raise Exception("Logic error 1 in rule line '%s'" % line)
cur_rule['actions'].append({'text': match.group(2)})
continue
raise Exception("Logic error 2 in rule line '%s'" % line)
if cur_rule['condition']:
if not cur_rule['actions']:
err = "Rule has condition '%s' but no actions" % cur_rule['condition']['text']
raise Exception(err)
rules.append(cur_rule)
return rules
except Exception as exc:
Config.log(str(exc), tag='TP_RULES')
raise
def apply_copy(finfo):
if 'stop' in finfo:
del finfo['stop']
return False
# source is assumed to be [full]
# more is assumed for now to only contain dest expr
dest = u.debracket(more, tp, finfo=finfo)
newfi = self.copy_with_metadata(finfo, dest)
if newfi:
# note that full path is the key here, not bare filename
self.file_info[dest] = newfi
return True
def extract_frames(interp, finfo, argstr):
try:
interpret = u.interpret_method(interp)
argstr = u.debracket(argstr, interpret, finfo=finfo)
args = u.parse_arg_string(argstr)
src = finfo['full']
Config.log(src, tag='EXTRACT_FRAMES_CALLED')
src_name = finfo['name']
frame_ct = get_frame_count(src)
if frame_ct < 2:
msg = "extract_frames: '%s' not multi-frame, taking no action" % src
logging.warning(msg)
return frame_ct
if not 'out_dir' in args:
raise Exception("out_dir not specified in args '%s'" % argstr)
dest = args['out_dir']
logging.debug("extract_frames dest is %s" % dest)
if _extract_frames(src, dest):
raw_files = u.plain_files(dest, src_name + '_frame_(\d+)\.gif')
if 'frames_wanted' in args:
wanted = args['frames_wanted']
# TODO support more syntax, 1..max, 1..4, 0, max-10..max, etc.
# for now only support '0'
if wanted != '0':
raise Exception("unsupported 'frames_wanted' spec '%s'" %
wanted)
# todo - crude version for frame 0 support
tag = '00000'
the_files = []
for file in raw_files:
if tag in file:
the_files.append(file)
else:
os.remove(dest + '/' + file)
else:
the_files = raw_files
# save some metadata
finfo['frames_path'] = dest
finfo['frames_files'] = the_files
finfo['frames_count'] = len(the_files)
msg = "src '%s', dest '%s', count '%s'" % (src, dest,
finfo['frames_count'])
Config.log(msg, tag='EXTRACT_FRAMES_OK')
return finfo['frames_count']
except Exception as exc:
logging.error("extract_frames exception '%s'" % str(exc))
def scale_and_copy(interp, finfo, argstr):
try:
interpret = u.interpret_method(interp)
argstr = u.debracket(argstr, interpret, finfo=finfo)
args = u.parse_arg_string(argstr)
src = finfo['full']
Config.log("src '%s', argstr '%s'" % (src, argstr),
tag='SCALE_AND_COPY')
# image magick infers format from extension
if not u.have_required(args, 'dest', 'size'):
raise Exception("scale_and_copy incomplete args '%s'" % argstr)
if 'larger_dim' in args:
# TODO: this alternative to 'size' requires getting dims of original
raise Exception("scale_and_copy: 'larger_dim' not yet supported")
dest = args['dest']
size = int(args['size'])
size_str = "%ix%i" % (size, size)
# to convert only first frame of animated gif, specify 'file[0]'
if 'single_frame' in args and args['single_frame']:
src += '[0]'
### TODO more copied stuff from copy_with_metadata!
(dest_dir, dest_file) = os.path.split(dest)
u.ensure_path(dest_dir)
call_args = ['convert', src, '-resize', size_str, dest]
(returncode, stdout, stderr) = u.run_command(call_args)
# logging.debug("returncode: %s\nstdout: %s\nstderr: %s" % (returncode, stdout, stderr))
if returncode == 0:
# we know the file that was created, so make its metadata now
newfi = {}
newfi['parent_full'] = finfo['full'] # provenance
newfi['name'] = dest_file
newfi['path'] = dest_dir
newfi['full'] = dest
newfi['rules_run'] = False
newfi.pop('groups', None)
# add thumb dimensions to metadata
newfi['width'], newfi['height'] = get_image_size(dest)
return {'new_finfo': newfi}
else:
logging.error("scale_and_copy failed with rc %i, stderr = '%s'" %
(returncode, stderr))
return {}
except Exception as exc:
logging.error("scale_and_copy exception '%s'" % str(exc))
return {}
def default_template_file_action(self,
dir_name,
file_name,
dest_rel_path=None,
dest_name=None):
template_full = dir_name + '/' + file_name
Config.log("default_template_file_action '%s'" % template_full,
tag='DEFAULT_TEMPLATE_FILE_ACTION')
if dest_name:
rel_path = dest_rel_path
dest_path = u.pathify(self.output_root, dest_rel_path)
else:
rel_path = u.make_rel_path(self.site_root, dir_name)
dest_path = u.pathify(self.output_root, rel_path)
dest_name = file_name
u.ensure_path(dest_path)
dest_full = u.pathify(dest_path, dest_name)
info = {
'name': dest_name,
'path': dest_path,
'rel_path': rel_path,
'full': dest_full,
'key': u.make_key(rel_path, dest_name)
}
if self.config.is_template_type(file_name):
template = open(template_full).read()
output = u.debracket(template, self.interpret)
if not self.config.is_special_file(info['key']):
open(dest_full, 'w').write(output)
local = u.local_metadata(dest_path, dest_name)
info['size'] = local['size']
info['modified'] = local['modified']
info['md5'] = u.md5(dest_full)
self.track_file(info)
else:
shutil.copyfile(template_full, dest_full)
local = u.local_metadata(dest_path, dest_name)
info['size'] = local['size']
info['modified'] = local['modified']
info['md5'] = u.md5(dest_full)
self.track_file(info)
def web_handle(tp, finfo, argstr):
# bracket notation allowed in args
argstr = u.debracket(argstr, tp.interpret, finfo=finfo)
# logging.info("web_handle called, args = '%s'" % argstr)
Config.log("args = '%s'" % argstr, tag='WEB_HANDLE_CALLED')
tp.web_maker.add_file(finfo, argstr)
def render_mode_generic_nav_by_date(webmaker, dest_key, work_item, list_args):
try:
if dest_key == 'if_empty':
# no items to render. make 1 page with emptiness message, and
# a link to it.
# passing the URL of the apology page this way
page_url = list_args['if_empty']
links_wl = webmaker.ensure_worklist(list_args['links_wl_name'])
links_wl[dest_key] = {
'index': 0,
'roles': {
'default': []
},
'page_url': page_url,
'item_args': {}
}
# single-item wl for the page
per_page_worklist_name = page_url + Config.main.default_page_wl_name(
)
per_page_wl = webmaker.ensure_worklist(per_page_worklist_name)
per_page_wl[dest_key] = {
'index': 0,
'roles': {
'default': []
},
'item_args': {
'list_header': 'No items are available.',
'sort_key': ''
}
}
# wl to render page:
gen_pages_wl = webmaker.ensure_worklist(list_args['gen_pages'])
# TODO REVIEW think this is right...NOTE not valid if !django!!! TODO FIX
if 'short' in list_args:
static_page_url = page_url + '/' + list_args['url_suffix']
else:
static_page_url = page_url
gen_pages_wl['if_empty'] = {
'index': 0,
'item_args': {
'page_context': page_url,
'static_page_url': static_page_url
},
'list_args': list_args
}
# single-entry wl to point to latest and only page:
entry_wl = webmaker.ensure_worklist(
list_args['entry_worklist_name'])
django_page_url = '/' + list_args['url_prefix'] + '/' + '0000-00-00'
entry_wl['if_empty'] = {
'date_string': '0000-00-00',
'value': page_url,
'django_value': django_page_url,
'index': 0,
'item_args': {}
}
return ''
finfo = work_item['roles']['default'][0]
# ensure it's on the worklist of dates to link from left nav
### TODO - generated file name should not be created ad-hoc in two places,
### it should be stored as an arg where both the link and the saving of the
### page can use it
if not u.have_required(list_args, 'url_prefix', 'url_suffix',
'links_wl_name', 'display_name_spec',
'list_header_spec', 'gen_pages',
'entry_worklist_name'):
raise Exception('required arguments are missing from list_args')
# NOTE! the link dest is the page containing this image, not the image itself
# TODO do this further upstream, this needlessly overwrites the worklist item N times
page_url = '/' + list_args['url_prefix'] + '/' + work_item[
'item_args']['date_string']
if 'short' not in list_args:
page_url = page_url + '/' + list_args['url_suffix']
links_wl = webmaker.ensure_worklist(list_args['links_wl_name'])
links_wl[page_url] = {
'index': len(links_wl),
'roles': {
'default': [finfo]
},
'page_url': page_url,
'item_args': copy.deepcopy(work_item['item_args'])
}
# add to custom worklist for its page
# TODO fix hardcoded default name
per_page_worklist_name = page_url + '.page_wl'
per_page_wl = webmaker.ensure_worklist(per_page_worklist_name)
per_page_wl[dest_key] = {
'index': len(per_page_wl),
'roles': {
'default': [finfo]
},
'item_args': copy.deepcopy(work_item['item_args'])
}
# TODO find a less wonky solution to bracket in bracket (and colon!) issue
display_name_spec = u.fix_alt_arg(list_args['display_name_spec'])
# todo can we combine this with u.get_display_name somehow??
display_name = u.parse_subargs(display_name_spec, webmaker.interpret,
webmaker.config, finfo)
per_page_wl[dest_key]['item_args']['display_name'] = display_name
# TODO see above. Also, no good reason this is different from display name!!!
list_header_spec = u.fix_alt_arg(list_args['list_header_spec'])
list_header = u.debracket(list_header_spec,
webmaker.interpret,
finfo=finfo,
symbols=work_item['item_args'])
# todo: this is ad-hoc and inscrutable. need page context for debracketing.
per_page_wl[dest_key]['item_args']['list_header'] = list_header
per_page_wl[dest_key]['item_args']['image_url'] = dest_key
per_page_wl[dest_key]['item_args']['symbol_key'] = finfo['name']
# ensure it's on the worklist of pages to generate
gen_pages_wl = webmaker.ensure_worklist(list_args['gen_pages'])
# REVIEW why isn't the key page_url here?
gen_pages_wl[dest_key] = {
'index': len(gen_pages_wl),
'item_args': copy.deepcopy(work_item['item_args']),
'list_args': list_args
}
gen_pages_wl[dest_key]['item_args']['page_context'] = page_url
# ensure single-item worklist that links to most recent (used to select entry page)
entry_wl = webmaker.ensure_worklist(list_args['entry_worklist_name'])
setit = True
cur_date = work_item['item_args']['date_string']
if 'only' in entry_wl:
latest_date = entry_wl['only']['date_string']
# format YYYY-MM-DD so we can use string compare
setit = cur_date > latest_date
if setit:
django_page_url = '/' + list_args['url_prefix'] + '/' + work_item[
'item_args']['date_string']
entry_wl['only'] = {
'date_string': cur_date,
'value': page_url,
'django_value': django_page_url,
'index': 0,
'item_args': copy.deepcopy(work_item['item_args'])
}
return ''
except Exception as exc:
Config.log(str(exc), tag='NAV_BY_DATE')
raise
def panoply(interp, finfo, argstr):
Config.log("file '%s' argstr '%s'" % (finfo['full'], argstr),
tag='PANOPLY')
try:
interpret = u.interpret_method(interp)
argstr = u.debracket(argstr, interpret, finfo=finfo)
args = u.parse_arg_string(argstr)
if not 'action' in args:
raise Exception("panoply: 'action arg is required")
if not 'dest' in args:
raise Exception("panoply: dest arg is required")
action = args['action']
dest = u.debracket(args['dest'], interpret, finfo=finfo)
panoply_templates = Config.main.template_root + '/panoply'
src = finfo['full']
if not src.endswith('.nc'):
logging.error("panoply command: '%s' is not a dataset file" % src)
return
### TODO more copied stuff from copy_with_metadata!
(dest_dir, dest_file) = os.path.split(dest)
u.ensure_path(dest_dir)
jar = 'PanoplyCL.jar'
size = None
if 'size' in args:
size = int(args['size'])
size_factor = _panoply_size_to_size_factor(size)
template_file = panoply_templates + '/' + action + '.pclt'
try:
template = open(template_file).read()
except Exception as exc:
logging.error(
"panoply command: error '%s' opening template file '%s" %
(str(exc), template_file))
raise
symbols = {
'dataset': src,
'output_file': dest,
'size_factor': size_factor
}
script = u.debracket(template, interpret, symbols=symbols)
script_path = Config.main.output + '/tmp'
u.ensure_path(script_path)
script_file = script_path + '/' + action + '.pcl'
open(script_file, 'w').write(script)
working_dir = Config.main.get('tools', 'panoply_workdir')
if not os.path.isdir(working_dir):
err = "panoply: invalid panoply_workdir '%s'" % working_dir
logging.error(err)
raise Exception(err)
call_args = ['java', '-jar', jar, script_file]
(returncode, stdout, stderr) = u.run_command(call_args, working_dir)
logging.debug("returncode: %s\nstdout: %s\nstderr: %s" %
(returncode, stdout, stderr))
if returncode == 0:
# we know the file that was created, so make its metadata now
newfi = {}
newfi['name'] = dest_file
newfi['path'] = dest_dir
newfi['full'] = dest
newfi['rules_run'] = False
tmp = u.local_metadata(newfi['path'], newfi['name'])
newfi['size'] = tmp['size']
newfi['modified'] = tmp['modified']
return {'new_finfo': newfi}
else:
logging.error("panoply failed with rc '%i', stderr = '%s'" %
(returncode, stderr))
except Exception as exc:
# py2 logging.error("panoply exception '%s'" % exc.message)
logging.error("panoply exception '%s'" % str(exc))
