def smoke_test(interp, finfo, argstr): try: logger.info('I do not think this will work') # REVIEW: faulthandler not safe in apache, assumes stdout available #faulthandler.enable() interpret = u.interpret_method(interp) argstr = u.debracket(argstr, interpret, finfo=finfo) args = u.parse_arg_string(argstr) #Config.log(argstr, tag='NETCDF_SMOKE') #if not u.have_required(args, 'dest'): # raise Exception("get_screenshot incomplete args '%s'" % argstr) #dest = u.debracket(args['dest'], interpret, finfo=finfo) #(dest_dir, dest_file) = os.path.split(dest) #u.ensure_path(dest_dir) full = finfo['full'] if not os.path.exists(full): # TODO config logging not working here? Config.log(full + " not found", tag='NETCDF_BAD_ARG') logger.info(full + ' not found') return 'todo file not found ' + full Config.log("here goes nc.Dataset with " + full) rootgrp = nc.Dataset(finfo['full'], 'r', format='NETCDF4') ret = 'variables:\n' + str(rootgrp.variables) rootgrp.close() return ret except Exception as exc: Config.log(str(exc), tag='NETCDF_SMOKE_ERROR') return str(exc)
def captionize(interp, finfo, argstr): try: interpret = u.interpret_method(interp) argstr = u.debracket(argstr, interpret, finfo=finfo) args = u.parse_arg_string(argstr) src = finfo['full'] Config.log("src '%s', argstr '%s'" % (src, argstr), tag='CAPTIONIZE') if not u.have_required(args, 'dest', 'where', 'font_size', 'bar_size', 'pad_x', 'pad_y', 'text'): raise Exception("captionize incomplete args '%s'" % argstr) dest = args['dest'] (dest_dir, dest_file) = os.path.split(dest) u.ensure_path(dest_dir) params = [] if args['where'] == 'top': params.extend(['-gravity', 'northwest']) elif args['where'] == 'bottom': params.extend(['-gravity', 'southwest']) else: raise Exception("captionize invalid 'where' arg in " + argstr) # TODO: validate colors. See https://www.imagemagick.org/script/color.php if 'background_color' in args: params.extend(['-background', args['background_color']]) else: params.extend(['-background', 'white']) if 'text_color' in args: params.extend(['-fill', args['text_color']]) else: params.extend(['-fill', 'black']) # TODO: validate font name. "convert -list font" will list them. system dependent. if 'font' in args: params.extend(['-font', args['font']]) else: params.extend(['-font', 'Helvetica']) params.extend(['-pointsize', args['font_size']]) params.extend(['-splice', '0x' + args['bar_size']]) x = u.force_sign(args['pad_x']) y = u.force_sign(args['pad_y']) params.extend(['-annotate', x + y]) fixed_text = args['text'].replace("'", "\\'") params.append('"' + fixed_text + '"') call_args = ['convert', src] + params + [dest] (returncode, stdout, stderr) = u.run_command(call_args) # logging.debug("returncode: %s\nstdout: %s\nstderr: %s" % (returncode, stdout, stderr)) if returncode == 0: # we know the file that was created, so make its metadata now newfi = {} newfi['parent_full'] = finfo['full'] # provenance newfi['name'] = dest_file newfi['path'] = dest_dir newfi['full'] = dest newfi['rules_run'] = False newfi.pop('groups', None) return {'new_finfo': newfi} else: logging.error("captionize failed with rc %i, stderr = '%s'" % (returncode, stderr)) except Exception as exc: logging.error("captionize exception '%s'" % str(exc))
def get_screenshot(interp, finfo, argstr): try: interpret = u.interpret_method(interp) argstr = u.debracket(argstr, interpret, finfo=finfo) args = u.parse_arg_string(argstr) Config.log(argstr, tag='GET_SCREENSHOT') if not u.have_required(args, 'url', 'dest', 'height', 'width'): raise Exception("get_screenshot incomplete args '%s'" % argstr) dest = u.debracket(args['dest'], interpret, finfo=finfo) (dest_dir, dest_file) = os.path.split(dest) u.ensure_path(dest_dir) puppeteer_templates = Config.main.template_root + '/puppeteer' template_file = puppeteer_templates + '/get_screenshot.js' try: template = open(template_file).read() except Exception as exc: Config.log("'%s' opening template file '%s" % (str(exc), template_file), tag='GET_SCREENSHOT_ERROR') raise symbols = { 'url': args['url'], 'width': args['width'], 'height': args['height'], 'dest': dest } script = u.debracket(template, interpret, symbols=symbols) script_path = Config.main.output + '/tmp' u.ensure_path(script_path) script_file = script_path + '/get_screenshot.js' open(script_file, 'w').write(script) working_dir = Config.main.get('tools', 'node_workdir') if not os.path.isdir(working_dir): err = "get_screenshot: invalid node_workdir '%s'" % working_dir logging.error(err) raise Exception(err) call_args = ['node', script_file] (returncode, stdout, stderr) = u.run_command( call_args, working_dir) # logging.debug("returncode: %s\nstdout: %s\nstderr: %s" % (returncode, stdout, stderr)) if returncode == 0: # we know the file that was created, so make its metadata now newfi = {} newfi['parent_full'] = finfo['full'] # provenance newfi['name'] = dest_file newfi['path'] = dest_dir newfi['full'] = dest newfi['rules_run'] = False newfi.pop('groups', None) return {'new_finfo': newfi} else: Config.log("rc %i, stderr = '%s'" % (returncode, stderr), tag='GET_SCREENSHOT_ERROR') except Exception as exc: Config.log(str(exc), tag='GET_SCREENSHOT_ERROR')
def extract_frames(interp, finfo, argstr): try: interpret = u.interpret_method(interp) argstr = u.debracket(argstr, interpret, finfo=finfo) args = u.parse_arg_string(argstr) src = finfo['full'] Config.log(src, tag='EXTRACT_FRAMES_CALLED') src_name = finfo['name'] frame_ct = get_frame_count(src) if frame_ct < 2: msg = "extract_frames: '%s' not multi-frame, taking no action" % src logging.warning(msg) return frame_ct if not 'out_dir' in args: raise Exception("out_dir not specified in args '%s'" % argstr) dest = args['out_dir'] logging.debug("extract_frames dest is %s" % dest) if _extract_frames(src, dest): raw_files = u.plain_files(dest, src_name + '_frame_(\d+)\.gif') if 'frames_wanted' in args: wanted = args['frames_wanted'] # TODO support more syntax, 1..max, 1..4, 0, max-10..max, etc. # for now only support '0' if wanted != '0': raise Exception("unsupported 'frames_wanted' spec '%s'" % wanted) # todo - crude version for frame 0 support tag = '00000' the_files = [] for file in raw_files: if tag in file: the_files.append(file) else: os.remove(dest + '/' + file) else: the_files = raw_files # save some metadata finfo['frames_path'] = dest finfo['frames_files'] = the_files finfo['frames_count'] = len(the_files) msg = "src '%s', dest '%s', count '%s'" % (src, dest, finfo['frames_count']) Config.log(msg, tag='EXTRACT_FRAMES_OK') return finfo['frames_count'] except Exception as exc: logging.error("extract_frames exception '%s'" % str(exc))
def scale_and_copy(interp, finfo, argstr): try: interpret = u.interpret_method(interp) argstr = u.debracket(argstr, interpret, finfo=finfo) args = u.parse_arg_string(argstr) src = finfo['full'] Config.log("src '%s', argstr '%s'" % (src, argstr), tag='SCALE_AND_COPY') # image magick infers format from extension if not u.have_required(args, 'dest', 'size'): raise Exception("scale_and_copy incomplete args '%s'" % argstr) if 'larger_dim' in args: # TODO: this alternative to 'size' requires getting dims of original raise Exception("scale_and_copy: 'larger_dim' not yet supported") dest = args['dest'] size = int(args['size']) size_str = "%ix%i" % (size, size) # to convert only first frame of animated gif, specify 'file[0]' if 'single_frame' in args and args['single_frame']: src += '[0]' ### TODO more copied stuff from copy_with_metadata! (dest_dir, dest_file) = os.path.split(dest) u.ensure_path(dest_dir) call_args = ['convert', src, '-resize', size_str, dest] (returncode, stdout, stderr) = u.run_command(call_args) # logging.debug("returncode: %s\nstdout: %s\nstderr: %s" % (returncode, stdout, stderr)) if returncode == 0: # we know the file that was created, so make its metadata now newfi = {} newfi['parent_full'] = finfo['full'] # provenance newfi['name'] = dest_file newfi['path'] = dest_dir newfi['full'] = dest newfi['rules_run'] = False newfi.pop('groups', None) # add thumb dimensions to metadata newfi['width'], newfi['height'] = get_image_size(dest) return {'new_finfo': newfi} else: logging.error("scale_and_copy failed with rc %i, stderr = '%s'" % (returncode, stderr)) return {} except Exception as exc: logging.error("scale_and_copy exception '%s'" % str(exc)) return {}
def set_file_info(tp, finfo, argstr): try: logging.info("set_file_info called, args = '%s'" % argstr) # this has its own peculiar syntax, so don't blindly debracket # argstr = u.debracket(argstr, tp.interpret, finfo=finfo) args = u.parse_arg_string(argstr) for key, value in args.items(): if is_protected_metadata(key) or not len(key): logging.warning("metadata key '%s' is protected or invalid" % key) continue if value.startswith('"'): value = value.strip('"') if value.startswith('['): finfo[key] = u.parse_subargs(value, tp.interpret, tp.config, finfo=finfo) else: finfo[key] = value except Exception as exc: logging.error("set_file_info exception '%s'" % str(exc))
def add_file(self, finfo, argstr): args = u.parse_arg_string(argstr) if not 'worklist' in args: raise Exception("worklist_name argument is required") worklist_name = args['worklist'] if 'role' in args: role = args['role'] else: role = 'default' the_list = self.ensure_worklist(worklist_name) if 'dest_key' in args: key = args['dest_key'] finfo['dest_key'] = key else: if 'key' in finfo: key = finfo['key'] else: key = finfo['full'] if not key: raise Exception("no dest_key available") Config.log("key '%s' worklist '%s'" % (key, worklist_name), tag='WEBMAKER_WL_ADD') if not key in the_list: the_list[key] = {'roles': {'default': []}, 'index': len(the_list)} # save any extra args passed # todo review - what if this overwrites old args? the_list[key]['item_args'] = args if role in the_list[key]['roles']: if role == 'default': the_list[key]['roles'][role].append(finfo) else: msg = "replacing existing '%s' role in worklist item (key = '%s', worklist = '%s')" \ % (role, key, worklist_name) logging.warning(msg) the_list[key]['roles'][role] = finfo
def render_part(self, argstr): # NOTE: could do bracket sub here if needed args = u.parse_arg_string(argstr, options={'merge': True}) return self.render_part_args(args)
def _render_worklist(self, argstr=''): ret = '' list_name = 'UNDEFINED' try: argstr = argstr.replace('\n', ' ').replace('\t', ' ') list_args = u.parse_arg_string(argstr) if not 'name' in list_args or not list_args['name']: raise Exception("list name not specified in list_args '%s'" % argstr) list_name = list_args['name'] if self.page_context: list_key = self.page_context + '.' + list_name if not list_key in self._worklists: # logging.info("no worklist '%s', falling back to '%s'." % (list_key, list_name)) list_key = list_name else: list_key = list_name Config.log(list_key, tag='RENDER_WORKLIST_' + list_key) if not list_key in self._worklists or not len( self._worklists[list_key]): msg = "worklist " + list_key + " is empty." Config.log(msg, tag='RENDER_WORKLIST_EMPTY_' + list_key) if 'if_empty' in list_args: # put special work_item on list to render in empty case the_list = self.ensure_worklist(list_key) the_list['if_empty'] = { 'index': 0, 'roles': { 'default': [] }, 'item_args': {} } Config.log(msg, tag='RENDER_WORKLIST_HANDLE_IF_EMPTY_' + list_key) else: return "<p>%s</p>" % msg if 'mode' in list_args and list_args['mode']: mode = list_args['mode'] else: msg = "worklist '%s' has no mode specified. using default." % list_key logging.debug(msg) mode = 'default' keyname = 'value' if mode == 'literal' and 'keyname' in list_args: keyname = list_args['keyname'] max_items = 1000000 if 'max_items' in list_args: max_items = int(list_args['max_items']) rel_path = '' work = None wl = self._worklists[list_key] # attach symbols if 'symbols' in list_args: skip_if_no_symbols = self.config.is_true('symbols', 'skip_if_no_symbols', absent_means_yes=True) symbol_set_name = list_args['symbols'] for key, work_item in wl.items(): if key != 'if_empty': # TODO how to handle lookup key? # TODO want option to add directly rather than to item_args?? symbol_key = work_item['item_args']['symbol_key'] found = self.config.attach_symbols( symbol_set_name, symbol_key, work_item['item_args']) if not found and skip_if_no_symbols: work_item['skip'] = True wl = { k: wl[k] for k in wl if 'skip' not in wl[k] or not wl[k]['skip'] } if len(wl) < 2: # sorting meaningless and might not work, so bypass it work = collections.OrderedDict(wl) else: reverse = 'reverse' in list_args if 'order' in list_args: order = list_args['order'] if order == 'key': # case-insensitive alpha sort of key work = collections.OrderedDict( sorted(wl.items(), key=lambda item: item[0], reverse=reverse)) else: work = collections.OrderedDict( sorted( wl.items(), key=lambda item: item[1]['item_args'][order], reverse=reverse)) else: # default ordering is as they were added # TODO not needed if used OrderedDict in the first place work = collections.OrderedDict( sorted(wl.items(), key=lambda item: item[1]['index'], reverse=reverse)) #--------------- render it ----------------------------- msg = "list has %i items, rendering %i." % ( len(work), min(len(work), max_items)) Config.log(msg, tag='RENDER_WORKLIST_' + list_key) rendered = 0 for key, work_item in work.items(): if 'skip' in work_item and work_item['skip']: Config.log(key, tag='RENDER_WORKLIST_SKIP_KEY') continue if mode == 'literal': ret += work_item[keyname] continue # check for plugin matching mode name f = self._get_mode_plugin('render_mode_' + mode) ret += f(self, key, work_item, list_args) rendered += 1 if rendered >= max_items: break continue return ret except Exception as exc: msg = "exception '%s' rendering worklist '%s' (context '%s')" % ( str(exc), list_name, self.page_context) Config.log(msg, tag='RENDER_WORKLIST_ERR') ret += 'An error has occurred.' return ret
def panoply(interp, finfo, argstr): Config.log("file '%s' argstr '%s'" % (finfo['full'], argstr), tag='PANOPLY') try: interpret = u.interpret_method(interp) argstr = u.debracket(argstr, interpret, finfo=finfo) args = u.parse_arg_string(argstr) if not 'action' in args: raise Exception("panoply: 'action arg is required") if not 'dest' in args: raise Exception("panoply: dest arg is required") action = args['action'] dest = u.debracket(args['dest'], interpret, finfo=finfo) panoply_templates = Config.main.template_root + '/panoply' src = finfo['full'] if not src.endswith('.nc'): logging.error("panoply command: '%s' is not a dataset file" % src) return ### TODO more copied stuff from copy_with_metadata! (dest_dir, dest_file) = os.path.split(dest) u.ensure_path(dest_dir) jar = 'PanoplyCL.jar' size = None if 'size' in args: size = int(args['size']) size_factor = _panoply_size_to_size_factor(size) template_file = panoply_templates + '/' + action + '.pclt' try: template = open(template_file).read() except Exception as exc: logging.error( "panoply command: error '%s' opening template file '%s" % (str(exc), template_file)) raise symbols = { 'dataset': src, 'output_file': dest, 'size_factor': size_factor } script = u.debracket(template, interpret, symbols=symbols) script_path = Config.main.output + '/tmp' u.ensure_path(script_path) script_file = script_path + '/' + action + '.pcl' open(script_file, 'w').write(script) working_dir = Config.main.get('tools', 'panoply_workdir') if not os.path.isdir(working_dir): err = "panoply: invalid panoply_workdir '%s'" % working_dir logging.error(err) raise Exception(err) call_args = ['java', '-jar', jar, script_file] (returncode, stdout, stderr) = u.run_command(call_args, working_dir) logging.debug("returncode: %s\nstdout: %s\nstderr: %s" % (returncode, stdout, stderr)) if returncode == 0: # we know the file that was created, so make its metadata now newfi = {} newfi['name'] = dest_file newfi['path'] = dest_dir newfi['full'] = dest newfi['rules_run'] = False tmp = u.local_metadata(newfi['path'], newfi['name']) newfi['size'] = tmp['size'] newfi['modified'] = tmp['modified'] return {'new_finfo': newfi} else: logging.error("panoply failed with rc '%i', stderr = '%s'" % (returncode, stderr)) except Exception as exc: # py2 logging.error("panoply exception '%s'" % exc.message) logging.error("panoply exception '%s'" % str(exc))