def message_to_jobs(msg, product_list): """Convert a posttroll message *msg* to a list of jobs given a *product_list*.""" formats = product_list['product_list'].get('formats', None) for _product, pconfig in plist_iter(product_list['product_list'], level='product'): if 'formats' not in pconfig and formats is not None: pconfig['formats'] = copy.deepcopy(formats) jobs = OrderedDict() priorities = get_area_priorities(product_list) # TODO: check the uri is accessible from the current host. input_filenames = [urlparse(uri).path for uri in gen_dict_extract(msg.data, 'uri')] for prio, areas in priorities.items(): jobs[prio] = OrderedDict() jobs[prio]['input_filenames'] = input_filenames.copy() jobs[prio]['input_mda'] = msg.data.copy() jobs[prio]['product_list'] = {} for section in product_list: if section == 'product_list': if section not in jobs[prio]['product_list']: jobs[prio]['product_list'][section] = OrderedDict(product_list[section].copy()) del jobs[prio]['product_list'][section]['areas'] jobs[prio]['product_list'][section]['areas'] = OrderedDict() for area in areas: jobs[prio]['product_list'][section]['areas'][area] = product_list[section]['areas'][area] else: jobs[prio]['product_list'][section] = product_list[section] return jobs
def _extract_filenames(msg): """Extract the filenames from *msg*. If the message contains a `filesystem` item, use fsspec to decode it. """ filenames = [ urlparse(uri).path for uri in gen_dict_extract(msg.data, 'uri') ] filenames = _create_fs_file_instances(filenames, msg) return filenames
def _create_fs_file_instances(filenames, msg): """Create FSFile instances when filesystem is provided.""" filesystems = list(gen_dict_extract(msg.data, 'filesystem')) if filesystems: from satpy.readers import FSFile from fsspec.spec import AbstractFileSystem import json filenames = [ FSFile(filename, AbstractFileSystem.from_json(json.dumps(filesystem))) for filename, filesystem in zip(filenames, filesystems) ] return filenames