def message_to_jobs(msg, product_list):
"""Convert a posttroll message *msg* to a list of jobs given a *product_list*."""
formats = product_list['product_list'].get('formats', None)
for _product, pconfig in plist_iter(product_list['product_list'], level='product'):
if 'formats' not in pconfig and formats is not None:
pconfig['formats'] = copy.deepcopy(formats)
jobs = OrderedDict()
priorities = get_area_priorities(product_list)
# TODO: check the uri is accessible from the current host.
input_filenames = [urlparse(uri).path for uri in gen_dict_extract(msg.data, 'uri')]
for prio, areas in priorities.items():
jobs[prio] = OrderedDict()
jobs[prio]['input_filenames'] = input_filenames.copy()
jobs[prio]['input_mda'] = msg.data.copy()
jobs[prio]['product_list'] = {}
for section in product_list:
if section == 'product_list':
if section not in jobs[prio]['product_list']:
jobs[prio]['product_list'][section] = OrderedDict(product_list[section].copy())
del jobs[prio]['product_list'][section]['areas']
jobs[prio]['product_list'][section]['areas'] = OrderedDict()
for area in areas:
jobs[prio]['product_list'][section]['areas'][area] = product_list[section]['areas'][area]
else:
jobs[prio]['product_list'][section] = product_list[section]
return jobs
def _extract_filenames(msg):
"""Extract the filenames from *msg*.
If the message contains a `filesystem` item, use fsspec to decode it.
"""
filenames = [
urlparse(uri).path for uri in gen_dict_extract(msg.data, 'uri')
]
filenames = _create_fs_file_instances(filenames, msg)
return filenames
def _create_fs_file_instances(filenames, msg):
"""Create FSFile instances when filesystem is provided."""
filesystems = list(gen_dict_extract(msg.data, 'filesystem'))
if filesystems:
from satpy.readers import FSFile
from fsspec.spec import AbstractFileSystem
import json
filenames = [
FSFile(filename,
AbstractFileSystem.from_json(json.dumps(filesystem)))
for filename, filesystem in zip(filenames, filesystems)
]
return filenames