def _get_parsing_plan_for_multifile_children(
self, obj_on_fs: PersistedObject, desired_type: Type[Any],
logger: Logger) -> Dict[str, Any]:
"""
Simply inspects the required type to find the names and types of its constructor arguments.
Then relies on the inner ParserFinder to parse each of them.
:param obj_on_fs:
:param desired_type:
:param logger:
:return:
"""
if is_collection(desired_type, strict=True):
# if the destination type is 'strictly a collection' (not a subclass of a collection) we know that we can't
# handle it here, the constructor is not pep484-typed
raise TypeError(
'Desired object type \'' + get_pretty_type_str(desired_type) +
'\' is a collection, '
'so it cannot be parsed with this default object parser')
else:
# First get the file children
children_on_fs = obj_on_fs.get_multifile_children()
# Try the type itself
# try:
return self.__get_parsing_plan_for_multifile_children(
obj_on_fs, desired_type, children_on_fs, logger=logger)
def _get_parsing_plan_for_multifile_children(self, obj_on_fs: PersistedObject, desired_type: Type[Any],
logger: Logger) -> Dict[str, Any]:
"""
Simply simply inspects the required type to find the names and types of its constructor arguments.
Then relies on the inner ParserFinder to parse each of them.
:param obj_on_fs:
:param desired_type:
:param logger:
:return:
"""
if is_collection(desired_type, strict=True):
raise TypeError('Desired object type \'' + get_pretty_type_str(desired_type)+ '\' is a collection, '
'so it cannot be parsed with this default object parser')
# First get the file children
children_on_fs = obj_on_fs.get_multifile_children()
# -- (a) extract the schema from the class constructor
constructor_args_types_and_opt = get_constructor_attributes_types(desired_type)
# -- (b) plan to parse each attribute required by the constructor
children_plan = dict() # results will be put in this object
# --use sorting in order to lead to reproducible results in case of multiple errors
for attribute_name, att_desc in sorted(constructor_args_types_and_opt.items()):
attribute_is_mandatory = att_desc[1]
attribute_type = att_desc[0]
# get the child
if attribute_name in children_on_fs.keys():
child_on_fs = children_on_fs[attribute_name]
# find a parser
parser_found = self.parser_finder.build_parser_for_fileobject_and_desiredtype(child_on_fs,
attribute_type,
logger=logger)
# create a parsing plan
children_plan[attribute_name] = parser_found.create_parsing_plan(attribute_type, child_on_fs,
logger=logger)
else:
if attribute_is_mandatory:
raise MissingMandatoryAttributeFiles.create(obj_on_fs, desired_type, attribute_name)
else:
# we don't care : optional attribute
# dont use warning since it does not show up nicely
#print('----- WARNING: Attribute ' + attribute_name + ' was not found on file system. However '
# 'it is not mandatory for the constructor of type ' + get_pretty_type_str(desired_type)
# + ', so we\'ll build the object without it...')
logger.warning('----- Attribute ' + attribute_name + ' was not found on file system. However '
'it is not mandatory for the constructor of type ' + get_pretty_type_str(desired_type)
+ ', so we\'ll build the object without it...')
pass
return children_plan
def _get_parsing_plan_for_multifile_children(
self, obj_on_fs: PersistedObject, desired_type: Type[Any],
logger: Logger) -> Dict[str, Any]:
"""
Simply inspects the required type to find the base type expected for items of the collection,
and relies on the ParserFinder to find the parsing plan
:param obj_on_fs:
:param desired_type:
:param logger:
:return:
"""
# nb of file children
n_children = len(obj_on_fs.get_multifile_children())
# first extract base collection type
subtypes, key_type = _extract_collection_base_type(desired_type)
if isinstance(subtypes, tuple):
# -- check the tuple length
if n_children != len(subtypes):
raise FolderAndFilesStructureError.create_for_multifile_tuple(
obj_on_fs, len(subtypes),
len(obj_on_fs.get_multifile_children()))
else:
# -- repeat the subtype n times
subtypes = [subtypes] * n_children
# -- for each child create a plan with the appropriate parser
children_plan = OrderedDict()
# use sorting for reproducible results in case of multiple errors
for (child_name, child_fileobject), child_typ in zip(
sorted(obj_on_fs.get_multifile_children().items()), subtypes):
# -- use the parserfinder to find the plan
t, child_parser = self.parser_finder.build_parser_for_fileobject_and_desiredtype(
child_fileobject, child_typ, logger)
children_plan[child_name] = child_parser.create_parsing_plan(
t, child_fileobject, logger, _main_call=False)
return children_plan
def create(parser: Parser, obj: PersistedObject = None):
"""
Helper method provided because we actually can't put that in the constructor, it creates a bug in Nose tests
https://github.com/nose-devs/nose/issues/725
:param parser:
:param obj:
:return:
"""
if obj is not None:
return _InvalidParserException('Error ' + str(obj) + ' cannot be parsed using ' + str(parser) + ' since '
+ ' this parser does not support ' + obj.get_pretty_file_mode())
else:
return _InvalidParserException('Error this parser is neither SingleFile nor MultiFile !')
def create_parsing_plan(self, desired_type: Type[T], filesystem_object: PersistedObject, logger: Logger,
_main_call: bool = True):
"""
Implements the abstract parent method by using the recursive parsing plan impl. Subclasses wishing to produce
their own parsing plans should rather override _create_parsing_plan in order to benefit from this same log msg.
:param desired_type:
:param filesystem_object:
:param logger:
:param _main_call: internal parameter for recursive calls. Should not be changed by the user.
:return:
"""
in_root_call = False
# -- log msg only for the root call, not for the children that will be created by the code below
if _main_call and (not hasattr(AnyParser.thrd_locals, 'flag_init') or AnyParser.thrd_locals.flag_init == 0):
# print('Building a parsing plan to parse ' + str(filesystem_object) + ' into a ' +
# get_pretty_type_str(desired_type))
logger.debug('Building a parsing plan to parse [{location}] into a {type}'
''.format(location=filesystem_object.get_pretty_location(append_file_ext=False),
type=get_pretty_type_str(desired_type)))
AnyParser.thrd_locals.flag_init = 1
in_root_call = True
# -- create the parsing plan
try:
pp = self._create_parsing_plan(desired_type, filesystem_object, logger, log_only_last=(not _main_call))
finally:
# remove threadlocal flag if needed
if in_root_call:
AnyParser.thrd_locals.flag_init = 0
# -- log success only if in root call
if in_root_call:
# print('Parsing Plan created successfully')
logger.debug('Parsing Plan created successfully')
# -- finally return
return pp
def __get_parsing_plan_for_multifile_children(self, obj_on_fs: PersistedObject, desired_type: Type[Any],
children_on_fs: Dict[str, PersistedObject], logger: Logger) \
-> Dict[str, Any]:
"""
Simply inspects the required type to find the names and types of its constructor arguments.
Then relies on the inner ParserFinder to parse each of them.
:param obj_on_fs:
:param desired_type:
:param children_on_fs:
:param logger:
:return:
"""
# -- (a) collect pep-484 information in the class constructor to be able to understand what is required
constructor_args_types_and_opt = get_constructor_attributes_types(
desired_type)
# -- (b) plan to parse each attribute required by the constructor
children_plan = dict() # results will be put in this object
# --use sorting in order to lead to reproducible results in case of multiple errors
for attribute_name, att_desc in sorted(
constructor_args_types_and_opt.items()):
attribute_is_mandatory = att_desc[1]
attribute_type = att_desc[0]
# get the child
if attribute_name in children_on_fs.keys():
child_on_fs = children_on_fs[attribute_name]
# find a parser
t, parser_found = self.parser_finder.build_parser_for_fileobject_and_desiredtype(
child_on_fs, attribute_type, logger=logger)
# create a parsing plan
children_plan[
attribute_name] = parser_found.create_parsing_plan(
t, child_on_fs, logger=logger, _main_call=False)
else:
if attribute_is_mandatory:
raise MissingMandatoryAttributeFiles.create(
obj_on_fs, desired_type, attribute_name)
else:
# we don't care : optional attribute
# dont use warning since it does not show up nicely
msg = 'NOT FOUND - This optional constructor attribute for type ' \
+ get_pretty_type_str(desired_type) + ' was not found on file system, but this may be normal'\
' - this message is displayed \'just in case\'.'
if logger.isEnabledFor(DEBUG):
logger.warning(
'(B) ' + obj_on_fs.get_pretty_child_location(
attribute_name, blank_parent_part=True) +
': ' + msg)
else:
logger.warning(
'WARNING parsing [{loc}] as a [{typ}]: optional constructor attribute [{att}] '
'not found on file system. This may be normal - this message is displayed \'just'
' in case\'.'.format(
loc=obj_on_fs.get_pretty_location(
blank_parent_part=False,
append_file_ext=False),
typ=get_pretty_type_str(desired_type),
att=attribute_name))
return children_plan
def _parse_multifile(self, desired_type: Type[Union[Dict, List, Set, Tuple]], obj: PersistedObject,
parsing_plan_for_children: Dict[str, ParsingPlan], logger: Logger,
options: Dict[str, Dict[str, Any]]) \
-> Union[Dict, List, Set, Tuple]:
"""
Options may contain a section with id 'MultifileCollectionParser' containing the following options:
* lazy_parsing: if True, the method will return immediately without parsing all the contents. Instead, the
returned collection will perform the parsing the first time an item is required.
* background_parsing: if True, the method will return immediately while a thread parses all the contents in
the background. Note that users cannot set both lazy_parsing and background_parsing to True at the same time
:param desired_type:
:param obj:
:param parsing_plan_for_children:
:param logger:
:param options:
:return:
"""
# first get the options and check them
lazy_parsing = False
background_parsing = False
opts = self._get_applicable_options(options)
for opt_key, opt_val in opts.items():
if opt_key is 'lazy_parsing':
lazy_parsing = opt_val
elif opt_key is 'background_parsing':
background_parsing = opt_val
else:
raise Exception(
'Invalid option in MultiFileCollectionParser : ' + opt_key)
check_var(lazy_parsing, var_types=bool, var_name='lazy_parsing')
check_var(background_parsing,
var_types=bool,
var_name='background_parsing')
if lazy_parsing and background_parsing:
raise ValueError(
'lazy_parsing and background_parsing cannot be set to true at the same time'
)
if lazy_parsing:
# build a lazy dictionary
results = LazyDictionary(
sorted(list(parsing_plan_for_children.keys())),
loading_method=lambda x: parsing_plan_for_children[x].execute(
logger, options))
# logger.debug('Assembling a ' + get_pretty_type_str(desired_type) + ' from all children of ' + str(obj)
# + ' (lazy parsing: children will be parsed when used) ')
logger.debug(
'(P) {loc} : lazy parsing ON, children will be parsed only if/when used'
.format(loc=obj.get_pretty_location(blank_parent_part=(
not GLOBAL_CONFIG.full_paths_in_logs),
compact_file_ext=True)))
elif background_parsing:
# -- TODO create a thread to perform the parsing in the background
raise ValueError('Background parsing is not yet supported')
else:
# Parse right now
results = OrderedDict()
# parse all children according to their plan
# -- use key-based sorting on children to lead to reproducible results
# (in case of multiple errors, the same error will show up first everytime)
for child_name, child_plan in sorted(
parsing_plan_for_children.items()):
results[child_name] = child_plan.execute(logger, options)
# logger.debug('Assembling a ' + get_pretty_type_str(desired_type) + ' from all parsed children of '
# + str(obj))
if issubclass(desired_type, list):
# return a list facade
return KeySortedListFacadeForDict(results)
elif issubclass(desired_type, tuple):
# return a tuple facade
return KeySortedTupleFacadeForDict(results)
elif issubclass(desired_type, set):
# return a set facade
return SetFacadeForDict(results)
elif issubclass(desired_type, dict):
# return the dict directly
return results
else:
raise TypeError(
'Cannot build the desired collection out of the multifile children: desired type is not '
'supported: ' + get_pretty_type_str(desired_type))