def smoke_test(interp, finfo, argstr):
try:
logger.info('I do not think this will work')
# REVIEW: faulthandler not safe in apache, assumes stdout available
#faulthandler.enable()
interpret = u.interpret_method(interp)
argstr = u.debracket(argstr, interpret, finfo=finfo)
args = u.parse_arg_string(argstr)
#Config.log(argstr, tag='NETCDF_SMOKE')
#if not u.have_required(args, 'dest'):
# raise Exception("get_screenshot incomplete args '%s'" % argstr)
#dest = u.debracket(args['dest'], interpret, finfo=finfo)
#(dest_dir, dest_file) = os.path.split(dest)
#u.ensure_path(dest_dir)
full = finfo['full']
if not os.path.exists(full):
# TODO config logging not working here?
Config.log(full + " not found", tag='NETCDF_BAD_ARG')
logger.info(full + ' not found')
return 'todo file not found ' + full
Config.log("here goes nc.Dataset with " + full)
rootgrp = nc.Dataset(finfo['full'], 'r', format='NETCDF4')
ret = 'variables:\n' + str(rootgrp.variables)
rootgrp.close()
return ret
except Exception as exc:
Config.log(str(exc), tag='NETCDF_SMOKE_ERROR')
return str(exc)
def captionize(interp, finfo, argstr):
try:
interpret = u.interpret_method(interp)
argstr = u.debracket(argstr, interpret, finfo=finfo)
args = u.parse_arg_string(argstr)
src = finfo['full']
Config.log("src '%s', argstr '%s'" % (src, argstr), tag='CAPTIONIZE')
if not u.have_required(args, 'dest', 'where', 'font_size', 'bar_size',
'pad_x', 'pad_y', 'text'):
raise Exception("captionize incomplete args '%s'" % argstr)
dest = args['dest']
(dest_dir, dest_file) = os.path.split(dest)
u.ensure_path(dest_dir)
params = []
if args['where'] == 'top':
params.extend(['-gravity', 'northwest'])
elif args['where'] == 'bottom':
params.extend(['-gravity', 'southwest'])
else:
raise Exception("captionize invalid 'where' arg in " + argstr)
# TODO: validate colors. See https://www.imagemagick.org/script/color.php
if 'background_color' in args:
params.extend(['-background', args['background_color']])
else:
params.extend(['-background', 'white'])
if 'text_color' in args:
params.extend(['-fill', args['text_color']])
else:
params.extend(['-fill', 'black'])
# TODO: validate font name. "convert -list font" will list them. system dependent.
if 'font' in args:
params.extend(['-font', args['font']])
else:
params.extend(['-font', 'Helvetica'])
params.extend(['-pointsize', args['font_size']])
params.extend(['-splice', '0x' + args['bar_size']])
x = u.force_sign(args['pad_x'])
y = u.force_sign(args['pad_y'])
params.extend(['-annotate', x + y])
fixed_text = args['text'].replace("'", "\\'")
params.append('"' + fixed_text + '"')
call_args = ['convert', src] + params + [dest]
(returncode, stdout, stderr) = u.run_command(call_args)
# logging.debug("returncode: %s\nstdout: %s\nstderr: %s" % (returncode, stdout, stderr))
if returncode == 0:
# we know the file that was created, so make its metadata now
newfi = {}
newfi['parent_full'] = finfo['full'] # provenance
newfi['name'] = dest_file
newfi['path'] = dest_dir
newfi['full'] = dest
newfi['rules_run'] = False
newfi.pop('groups', None)
return {'new_finfo': newfi}
else:
logging.error("captionize failed with rc %i, stderr = '%s'" %
(returncode, stderr))
except Exception as exc:
logging.error("captionize exception '%s'" % str(exc))
def get_screenshot(interp, finfo, argstr):
try:
interpret = u.interpret_method(interp)
argstr = u.debracket(argstr, interpret, finfo=finfo)
args = u.parse_arg_string(argstr)
Config.log(argstr, tag='GET_SCREENSHOT')
if not u.have_required(args, 'url', 'dest', 'height', 'width'):
raise Exception("get_screenshot incomplete args '%s'" % argstr)
dest = u.debracket(args['dest'], interpret, finfo=finfo)
(dest_dir, dest_file) = os.path.split(dest)
u.ensure_path(dest_dir)
puppeteer_templates = Config.main.template_root + '/puppeteer'
template_file = puppeteer_templates + '/get_screenshot.js'
try:
template = open(template_file).read()
except Exception as exc:
Config.log("'%s' opening template file '%s" %
(str(exc), template_file), tag='GET_SCREENSHOT_ERROR')
raise
symbols = {
'url': args['url'],
'width': args['width'],
'height': args['height'],
'dest': dest
}
script = u.debracket(template, interpret, symbols=symbols)
script_path = Config.main.output + '/tmp'
u.ensure_path(script_path)
script_file = script_path + '/get_screenshot.js'
open(script_file, 'w').write(script)
working_dir = Config.main.get('tools', 'node_workdir')
if not os.path.isdir(working_dir):
err = "get_screenshot: invalid node_workdir '%s'" % working_dir
logging.error(err)
raise Exception(err)
call_args = ['node', script_file]
(returncode, stdout, stderr) = u.run_command(
call_args, working_dir)
# logging.debug("returncode: %s\nstdout: %s\nstderr: %s" % (returncode, stdout, stderr))
if returncode == 0:
# we know the file that was created, so make its metadata now
newfi = {}
newfi['parent_full'] = finfo['full'] # provenance
newfi['name'] = dest_file
newfi['path'] = dest_dir
newfi['full'] = dest
newfi['rules_run'] = False
newfi.pop('groups', None)
return {'new_finfo': newfi}
else:
Config.log("rc %i, stderr = '%s'" % (returncode, stderr),
tag='GET_SCREENSHOT_ERROR')
except Exception as exc:
Config.log(str(exc), tag='GET_SCREENSHOT_ERROR')
def extract_frames(interp, finfo, argstr):
try:
interpret = u.interpret_method(interp)
argstr = u.debracket(argstr, interpret, finfo=finfo)
args = u.parse_arg_string(argstr)
src = finfo['full']
Config.log(src, tag='EXTRACT_FRAMES_CALLED')
src_name = finfo['name']
frame_ct = get_frame_count(src)
if frame_ct < 2:
msg = "extract_frames: '%s' not multi-frame, taking no action" % src
logging.warning(msg)
return frame_ct
if not 'out_dir' in args:
raise Exception("out_dir not specified in args '%s'" % argstr)
dest = args['out_dir']
logging.debug("extract_frames dest is %s" % dest)
if _extract_frames(src, dest):
raw_files = u.plain_files(dest, src_name + '_frame_(\d+)\.gif')
if 'frames_wanted' in args:
wanted = args['frames_wanted']
# TODO support more syntax, 1..max, 1..4, 0, max-10..max, etc.
# for now only support '0'
if wanted != '0':
raise Exception("unsupported 'frames_wanted' spec '%s'" %
wanted)
# todo - crude version for frame 0 support
tag = '00000'
the_files = []
for file in raw_files:
if tag in file:
the_files.append(file)
else:
os.remove(dest + '/' + file)
else:
the_files = raw_files
# save some metadata
finfo['frames_path'] = dest
finfo['frames_files'] = the_files
finfo['frames_count'] = len(the_files)
msg = "src '%s', dest '%s', count '%s'" % (src, dest,
finfo['frames_count'])
Config.log(msg, tag='EXTRACT_FRAMES_OK')
return finfo['frames_count']
except Exception as exc:
logging.error("extract_frames exception '%s'" % str(exc))
def scale_and_copy(interp, finfo, argstr):
try:
interpret = u.interpret_method(interp)
argstr = u.debracket(argstr, interpret, finfo=finfo)
args = u.parse_arg_string(argstr)
src = finfo['full']
Config.log("src '%s', argstr '%s'" % (src, argstr),
tag='SCALE_AND_COPY')
# image magick infers format from extension
if not u.have_required(args, 'dest', 'size'):
raise Exception("scale_and_copy incomplete args '%s'" % argstr)
if 'larger_dim' in args:
# TODO: this alternative to 'size' requires getting dims of original
raise Exception("scale_and_copy: 'larger_dim' not yet supported")
dest = args['dest']
size = int(args['size'])
size_str = "%ix%i" % (size, size)
# to convert only first frame of animated gif, specify 'file[0]'
if 'single_frame' in args and args['single_frame']:
src += '[0]'
### TODO more copied stuff from copy_with_metadata!
(dest_dir, dest_file) = os.path.split(dest)
u.ensure_path(dest_dir)
call_args = ['convert', src, '-resize', size_str, dest]
(returncode, stdout, stderr) = u.run_command(call_args)
# logging.debug("returncode: %s\nstdout: %s\nstderr: %s" % (returncode, stdout, stderr))
if returncode == 0:
# we know the file that was created, so make its metadata now
newfi = {}
newfi['parent_full'] = finfo['full'] # provenance
newfi['name'] = dest_file
newfi['path'] = dest_dir
newfi['full'] = dest
newfi['rules_run'] = False
newfi.pop('groups', None)
# add thumb dimensions to metadata
newfi['width'], newfi['height'] = get_image_size(dest)
return {'new_finfo': newfi}
else:
logging.error("scale_and_copy failed with rc %i, stderr = '%s'" %
(returncode, stderr))
return {}
except Exception as exc:
logging.error("scale_and_copy exception '%s'" % str(exc))
return {}
def set_file_info(tp, finfo, argstr):
try:
logging.info("set_file_info called, args = '%s'" % argstr)
# this has its own peculiar syntax, so don't blindly debracket
# argstr = u.debracket(argstr, tp.interpret, finfo=finfo)
args = u.parse_arg_string(argstr)
for key, value in args.items():
if is_protected_metadata(key) or not len(key):
logging.warning("metadata key '%s' is protected or invalid" %
key)
continue
if value.startswith('"'):
value = value.strip('"')
if value.startswith('['):
finfo[key] = u.parse_subargs(value,
tp.interpret,
tp.config,
finfo=finfo)
else:
finfo[key] = value
except Exception as exc:
logging.error("set_file_info exception '%s'" % str(exc))
def add_file(self, finfo, argstr):
args = u.parse_arg_string(argstr)
if not 'worklist' in args:
raise Exception("worklist_name argument is required")
worklist_name = args['worklist']
if 'role' in args:
role = args['role']
else:
role = 'default'
the_list = self.ensure_worklist(worklist_name)
if 'dest_key' in args:
key = args['dest_key']
finfo['dest_key'] = key
else:
if 'key' in finfo:
key = finfo['key']
else:
key = finfo['full']
if not key:
raise Exception("no dest_key available")
Config.log("key '%s' worklist '%s'" % (key, worklist_name),
tag='WEBMAKER_WL_ADD')
if not key in the_list:
the_list[key] = {'roles': {'default': []}, 'index': len(the_list)}
# save any extra args passed
# todo review - what if this overwrites old args?
the_list[key]['item_args'] = args
if role in the_list[key]['roles']:
if role == 'default':
the_list[key]['roles'][role].append(finfo)
else:
msg = "replacing existing '%s' role in worklist item (key = '%s', worklist = '%s')" \
% (role, key, worklist_name)
logging.warning(msg)
the_list[key]['roles'][role] = finfo
def render_part(self, argstr):
# NOTE: could do bracket sub here if needed
args = u.parse_arg_string(argstr, options={'merge': True})
return self.render_part_args(args)
def _render_worklist(self, argstr=''):
ret = ''
list_name = 'UNDEFINED'
try:
argstr = argstr.replace('\n', ' ').replace('\t', ' ')
list_args = u.parse_arg_string(argstr)
if not 'name' in list_args or not list_args['name']:
raise Exception("list name not specified in list_args '%s'" %
argstr)
list_name = list_args['name']
if self.page_context:
list_key = self.page_context + '.' + list_name
if not list_key in self._worklists:
# logging.info("no worklist '%s', falling back to '%s'." % (list_key, list_name))
list_key = list_name
else:
list_key = list_name
Config.log(list_key, tag='RENDER_WORKLIST_' + list_key)
if not list_key in self._worklists or not len(
self._worklists[list_key]):
msg = "worklist " + list_key + " is empty."
Config.log(msg, tag='RENDER_WORKLIST_EMPTY_' + list_key)
if 'if_empty' in list_args:
# put special work_item on list to render in empty case
the_list = self.ensure_worklist(list_key)
the_list['if_empty'] = {
'index': 0,
'roles': {
'default': []
},
'item_args': {}
}
Config.log(msg,
tag='RENDER_WORKLIST_HANDLE_IF_EMPTY_' +
list_key)
else:
return "<p>%s</p>" % msg
if 'mode' in list_args and list_args['mode']:
mode = list_args['mode']
else:
msg = "worklist '%s' has no mode specified. using default." % list_key
logging.debug(msg)
mode = 'default'
keyname = 'value'
if mode == 'literal' and 'keyname' in list_args:
keyname = list_args['keyname']
max_items = 1000000
if 'max_items' in list_args:
max_items = int(list_args['max_items'])
rel_path = ''
work = None
wl = self._worklists[list_key]
# attach symbols
if 'symbols' in list_args:
skip_if_no_symbols = self.config.is_true('symbols',
'skip_if_no_symbols',
absent_means_yes=True)
symbol_set_name = list_args['symbols']
for key, work_item in wl.items():
if key != 'if_empty':
# TODO how to handle lookup key?
# TODO want option to add directly rather than to item_args??
symbol_key = work_item['item_args']['symbol_key']
found = self.config.attach_symbols(
symbol_set_name, symbol_key,
work_item['item_args'])
if not found and skip_if_no_symbols:
work_item['skip'] = True
wl = {
k: wl[k]
for k in wl if 'skip' not in wl[k] or not wl[k]['skip']
}
if len(wl) < 2:
# sorting meaningless and might not work, so bypass it
work = collections.OrderedDict(wl)
else:
reverse = 'reverse' in list_args
if 'order' in list_args:
order = list_args['order']
if order == 'key':
# case-insensitive alpha sort of key
work = collections.OrderedDict(
sorted(wl.items(),
key=lambda item: item[0],
reverse=reverse))
else:
work = collections.OrderedDict(
sorted(
wl.items(),
key=lambda item: item[1]['item_args'][order],
reverse=reverse))
else:
# default ordering is as they were added
# TODO not needed if used OrderedDict in the first place
work = collections.OrderedDict(
sorted(wl.items(),
key=lambda item: item[1]['index'],
reverse=reverse))
#--------------- render it -----------------------------
msg = "list has %i items, rendering %i." % (
len(work), min(len(work), max_items))
Config.log(msg, tag='RENDER_WORKLIST_' + list_key)
rendered = 0
for key, work_item in work.items():
if 'skip' in work_item and work_item['skip']:
Config.log(key, tag='RENDER_WORKLIST_SKIP_KEY')
continue
if mode == 'literal':
ret += work_item[keyname]
continue
# check for plugin matching mode name
f = self._get_mode_plugin('render_mode_' + mode)
ret += f(self, key, work_item, list_args)
rendered += 1
if rendered >= max_items:
break
continue
return ret
except Exception as exc:
msg = "exception '%s' rendering worklist '%s' (context '%s')" % (
str(exc), list_name, self.page_context)
Config.log(msg, tag='RENDER_WORKLIST_ERR')
ret += 'An error has occurred.'
return ret
def panoply(interp, finfo, argstr):
Config.log("file '%s' argstr '%s'" % (finfo['full'], argstr),
tag='PANOPLY')
try:
interpret = u.interpret_method(interp)
argstr = u.debracket(argstr, interpret, finfo=finfo)
args = u.parse_arg_string(argstr)
if not 'action' in args:
raise Exception("panoply: 'action arg is required")
if not 'dest' in args:
raise Exception("panoply: dest arg is required")
action = args['action']
dest = u.debracket(args['dest'], interpret, finfo=finfo)
panoply_templates = Config.main.template_root + '/panoply'
src = finfo['full']
if not src.endswith('.nc'):
logging.error("panoply command: '%s' is not a dataset file" % src)
return
### TODO more copied stuff from copy_with_metadata!
(dest_dir, dest_file) = os.path.split(dest)
u.ensure_path(dest_dir)
jar = 'PanoplyCL.jar'
size = None
if 'size' in args:
size = int(args['size'])
size_factor = _panoply_size_to_size_factor(size)
template_file = panoply_templates + '/' + action + '.pclt'
try:
template = open(template_file).read()
except Exception as exc:
logging.error(
"panoply command: error '%s' opening template file '%s" %
(str(exc), template_file))
raise
symbols = {
'dataset': src,
'output_file': dest,
'size_factor': size_factor
}
script = u.debracket(template, interpret, symbols=symbols)
script_path = Config.main.output + '/tmp'
u.ensure_path(script_path)
script_file = script_path + '/' + action + '.pcl'
open(script_file, 'w').write(script)
working_dir = Config.main.get('tools', 'panoply_workdir')
if not os.path.isdir(working_dir):
err = "panoply: invalid panoply_workdir '%s'" % working_dir
logging.error(err)
raise Exception(err)
call_args = ['java', '-jar', jar, script_file]
(returncode, stdout, stderr) = u.run_command(call_args, working_dir)
logging.debug("returncode: %s\nstdout: %s\nstderr: %s" %
(returncode, stdout, stderr))
if returncode == 0:
# we know the file that was created, so make its metadata now
newfi = {}
newfi['name'] = dest_file
newfi['path'] = dest_dir
newfi['full'] = dest
newfi['rules_run'] = False
tmp = u.local_metadata(newfi['path'], newfi['name'])
newfi['size'] = tmp['size']
newfi['modified'] = tmp['modified']
return {'new_finfo': newfi}
else:
logging.error("panoply failed with rc '%i', stderr = '%s'" %
(returncode, stderr))
except Exception as exc:
# py2 logging.error("panoply exception '%s'" % exc.message)
logging.error("panoply exception '%s'" % str(exc))
