def get_default_yaml_parsers(
parser_finder: ParserFinder,
conversion_finder: ConversionFinder) -> List[AnyParser]:
"""
Utility method to return the default parsers able to parse an object from a file.
Note that MultifileObjectParser is not provided in this list, as it is already added in a hardcoded way in
RootParser
:return:
"""
return [ # yaml for any object
SingleFileParserFunction(
parser_function=read_object_from_yaml,
streaming_mode=True,
supported_exts={'.yaml', '.yml'},
supported_types={AnyObject},
),
# yaml for collection objects
SingleFileParserFunction(
parser_function=read_object_from_yaml,
custom_name='read_collection_from_yaml',
streaming_mode=True,
supported_exts={'.yaml', '.yml'},
supported_types={Tuple, Dict, List, Set},
)
]
def get_default_primitive_parsers():
return [
SingleFileParserFunction(parser_function=read_str_from_txt,
streaming_mode=True,
supported_exts={'.txt'},
supported_types={str})
]
def test_parse_with_custom_parser(self):
"""
Parses a subclass of DataFrame with a custom converter.
:return:
"""
from typing import Type
from parsyfiles.converting_core import T
from logging import Logger
from xml.etree.ElementTree import ElementTree, parse, tostring
def read_xml(desired_type: Type[T], file_path: str, encoding: str,
logger: Logger, **kwargs):
"""
Opens an XML file and returns the tree parsed from it as an ElementTree.
:param desired_type:
:param file_path:
:param encoding:
:param logger:
:param kwargs:
:return:
"""
return parse(file_path)
my_parser = SingleFileParserFunction(parser_function=read_xml,
streaming_mode=False,
supported_exts={'.xml'},
supported_types={ElementTree})
parser = RootParser('parsyfiles with timeseries')
parser.register_parser(my_parser)
xmls = parser.parse_collection(fix_path('./xml_collection'),
ElementTree)
pprint({name: tostring(x.getroot()) for name, x in xmls.items()})
def get_default_pandas_parsers() -> List[AnyParser]:
"""
Utility method to return the default parsers able to parse a dictionary from a file.
:return:
"""
return [
SingleFileParserFunction(parser_function=read_dataframe_from_xls,
streaming_mode=False,
supported_exts={'.xls', '.xlsx', '.xlsm'},
supported_types={pd.DataFrame},
option_hints=pandas_parsers_option_hints_xls),
SingleFileParserFunction(parser_function=read_df_or_series_from_csv,
streaming_mode=False,
supported_exts={'.csv', '.txt'},
supported_types={pd.DataFrame, pd.Series},
option_hints=pandas_parsers_option_hints_csv),
]
def get_default_config_parsers() -> List[AnyParser]:
"""
Utility method to return the default parsers able to parse a dictionary from a file.
:return:
"""
return [SingleFileParserFunction(parser_function=read_config,
streaming_mode=True,
supported_exts={'.cfg', '.ini'},
supported_types={ConfigParser}),
]
def get_default_object_parsers(parser_finder: ParserFinder, conversion_finder: ConversionFinder) -> List[AnyParser]:
"""
Utility method to return the default parsers able to parse an object from a file.
Note that MultifileObjectParser is not provided in this list, as it is already added in a hardcoded way in
RootParser
:return:
"""
return [SingleFileParserFunction(parser_function=read_object_from_pickle,
streaming_mode=False,
supported_exts={'.pyc'},
supported_types={AnyObject}),
MultifileObjectParser(parser_finder, conversion_finder)
]
def get_default_jprops_parsers(parser_finder: ParserFinder, conversion_finder: ConversionFinder) -> List[AnyParser]:
"""
Utility method to return the default parsers able to parse a dictionary from a properties file.
:return:
"""
return [SingleFileParserFunction(parser_function=read_dict_from_properties,
streaming_mode=True, custom_name='read_dict_from_properties',
supported_exts={'.properties', '.txt'},
supported_types={dict},
function_args={'conversion_finder': conversion_finder}),
# SingleFileParserFunction(parser_function=read_list_from_properties,
# streaming_mode=True,
# supported_exts={'.properties', '.txt'},
# supported_types={list}),
]
def get_default_collection_parsers(
parser_finder: ParserFinder,
conversion_finder: ConversionFinder) -> List[AnyParser]:
"""
Utility method to return the default parsers able to parse a dictionary from a file.
:return:
"""
return [
SingleFileParserFunction(
parser_function=read_dict_or_list_from_json,
streaming_mode=True,
custom_name='read_dict_or_list_from_json',
supported_exts={'.json'},
supported_types={dict, list},
function_args={'conversion_finder': conversion_finder}),
MultifileCollectionParser(parser_finder)
]
def test_custom_parser_ok_for_subclasses():
"""
Tests that if you register a custom parser for a subclass of A, it gets correctly used to parse A (in non-strict
mode, which is the default)
:return:
"""
root_parser = RootParser()
class A:
def __init__(self, txt):
self.txt = txt
class B(A):
""" a subclass of A """
pass
def read_B_from_txt(desired_type: Type[dict], file_object: TextIOBase,
logger: Logger, *args, **kwargs) -> str:
# read the entire stream into a string
str_io = StringIO()
shutil.copyfileobj(file_object, str_io)
# only return the first character
return B(str_io.getvalue()[0])
# before registering a parser for B, only generic parsers are able to parse a A
before_capa = root_parser.get_capabilities_for_type(A)['.txt']
assert list(before_capa.keys()) == ['3_generic']
# register a parser for B
root_parser.register_parser(
SingleFileParserFunction(parser_function=read_B_from_txt,
streaming_mode=True,
supported_exts={'.txt'},
supported_types={B}))
# after registering the new parser appears in the list able to parse A
after_capa = root_parser.get_capabilities_for_type(A)['.txt']
assert str(after_capa['2_approx_match'][0]) == '<read_B_from_txt>'
a = root_parser.parse_item(get_path('b64pickle-float-1.0=True'), A)
# check that the custom parser was used, not the generic 'construct from string'
assert len(a.txt) == 1
assert a.txt == 'g'
def setUp(self):
# 2 simple classes with inheritance
class A(object):
pass
class B(A):
pass
# 2 generic classes with inheritance
Y = TypeVar('Y')
Z = TypeVar('Z')
class C(Generic[Y]):
pass
class D(Generic[Z, Y], C[Y]):
pass
self.A = A
self.B = B
self.C = C
self.D = D
self.all_types = {A, B, C, D, AnyObject}
self.all_parsers_for_a = set()
self.all_parsers_for_b = set()
self.all_parsers_for_c = set()
self.all_parsers_for_d = set()
self.all_parsers_generic = set()
# create all combinations of file extensions
self.all_a_extensions = []
all_b_extensions = []
all_c_extensions = []
all_d_extensions = []
self.all_extensions = {MULTIFILE_EXT}
for a in [False, True]:
for b in [False, True]:
for c in [False, True]:
for d in [False, True]:
ext = '.'
if a:
ext += 'a'
if b:
ext += 'b'
if c:
ext += 'c'
if d:
ext += 'd'
print(ext)
if a or b or c or d:
self.all_extensions.add(ext)
if a:
self.all_a_extensions.append(ext)
if b:
all_b_extensions.append(ext)
if c:
all_c_extensions.append(ext)
if d:
all_d_extensions.append(ext)
# parsers
# *********** Specific SingleFile ************
# -- one type
def parse_a():
pass
def parse_b():
pass
def parse_c():
pass
def parse_d():
pass
a_parser_single = SingleFileParserFunction(parse_a,
supported_types={A},
supported_exts=set(
self.all_a_extensions))
b_parser_single = SingleFileParserFunction(
parse_b, supported_types={B}, supported_exts=set(all_b_extensions))
c_parser_single = SingleFileParserFunction(
parse_c, supported_types={C}, supported_exts=set(all_c_extensions))
d_parser_single = SingleFileParserFunction(
parse_d, supported_types={D}, supported_exts=set(all_d_extensions))
parsers_specific_singlefile_onetype = [
a_parser_single, b_parser_single, c_parser_single, d_parser_single
]
self.all_parsers_for_a.add(a_parser_single)
self.all_parsers_for_b.add(b_parser_single)
self.all_parsers_for_c.add(c_parser_single)
self.all_parsers_for_d.add(d_parser_single)
# -- several types
def parse_a_or_d():
pass
def parse_b_or_c():
pass
ad_parser_single = SingleFileParserFunction(
parse_a_or_d,
supported_types={A, D},
supported_exts=set(self.all_a_extensions).union(all_d_extensions))
bc_parser_single = SingleFileParserFunction(
parse_b_or_c,
supported_types={B, C},
supported_exts=set(all_b_extensions).union(all_c_extensions))
parsers_specific_singlefile_severaltypes = [
ad_parser_single, bc_parser_single
]
self.all_parsers_for_a.add(ad_parser_single)
self.all_parsers_for_b.add(bc_parser_single)
self.all_parsers_for_c.add(bc_parser_single)
self.all_parsers_for_d.add(ad_parser_single)
# ******** Specific Multifile ************
class DummyMultifileParser(MultiFileParser):
def __init__(self, supported_types):
super(DummyMultifileParser, self).__init__(supported_types)
def __str__(self):
return 'MultifileParser for ' + str([get_pretty_type_str(typ) for typ in self.supported_types]) \
+ 'for ' + str(self.supported_exts)
def _get_parsing_plan_for_multifile_children(
self, obj_on_fs: PersistedObject, desired_type: Type[T],
logger: Logger) -> Dict[str, _BaseParsingPlan[Any]]:
pass
def _parse_multifile(self, desired_type: Type[T],
obj: PersistedObject,
parsing_plan_for_children: Dict[
str, AnyParser._RecursiveParsingPlan],
logger: Logger,
options: Dict[str, Dict[str, Any]]) -> T:
pass
# -- one type
a_parser_multi = DummyMultifileParser(supported_types={A})
b_parser_multi = DummyMultifileParser(supported_types={B})
c_parser_multi = DummyMultifileParser(supported_types={C})
d_parser_multi = DummyMultifileParser(supported_types={D})
parsers_specific_multifile_onetype = [
a_parser_multi, b_parser_multi, c_parser_multi, d_parser_multi
]
self.all_parsers_for_a.add(a_parser_multi)
self.all_parsers_for_b.add(b_parser_multi)
self.all_parsers_for_c.add(c_parser_multi)
self.all_parsers_for_d.add(d_parser_multi)
# -- several types
bd_parser_multi = DummyMultifileParser(supported_types={B, D})
ac_parser_multi = DummyMultifileParser(supported_types={A, C})
parsers_specific_multifile_severaltypes = [
bd_parser_multi, ac_parser_multi
]
self.all_parsers_for_a.add(ac_parser_multi)
self.all_parsers_for_b.add(bd_parser_multi)
self.all_parsers_for_c.add(ac_parser_multi)
self.all_parsers_for_d.add(bd_parser_multi)
# ******** Specific BOTH **************
class DummyParser(AnyParser):
def __init__(self, supported_types, supported_exts):
super(DummyParser,
self).__init__(supported_types=supported_types,
supported_exts=supported_exts)
def _get_parsing_plan_for_multifile_children(
self, obj_on_fs: PersistedObject, desired_type: Type[T],
logger: Logger) -> Dict[str, _BaseParsingPlan[Any]]:
pass
def _parse_multifile(self, desired_type: Type[T],
obj: PersistedObject,
parsing_plan_for_children: Dict[
str, AnyParser._RecursiveParsingPlan],
logger: Logger,
options: Dict[str, Dict[str, Any]]) -> T:
pass
def _parse_singlefile(self, desired_type: Type[T], file_path: str,
encoding: str, logger: Logger,
options: Dict[str, Dict[str, Any]]) -> T:
pass
# -- one type
a_parser_both = DummyParser(supported_types={A},
supported_exts=set(self.all_a_extensions))
b_parser_both = DummyParser(supported_types={B},
supported_exts=set(all_b_extensions))
c_parser_both = DummyParser(supported_types={C},
supported_exts=set(all_c_extensions))
d_parser_both = DummyParser(supported_types={D},
supported_exts=set(all_d_extensions))
parsers_specific_bothfile_onetype = [
a_parser_both, b_parser_both, c_parser_both, d_parser_both
]
self.all_parsers_for_a.add(a_parser_both)
self.all_parsers_for_b.add(b_parser_both)
self.all_parsers_for_c.add(c_parser_both)
self.all_parsers_for_d.add(d_parser_both)
# -- several types
abc_parser_both = DummyParser(
supported_types={A, B, C},
supported_exts=set(self.all_a_extensions).union(
all_b_extensions).union(all_c_extensions))
acd_parser_both = DummyParser(
supported_types={A, C, D},
supported_exts=set(self.all_a_extensions).union(
all_c_extensions).union(all_d_extensions))
parsers_specific_bothfile_severaltypes = [
abc_parser_both, acd_parser_both
]
self.all_parsers_for_a = self.all_parsers_for_a.union(
{abc_parser_both, acd_parser_both})
self.all_parsers_for_b.add(abc_parser_both)
self.all_parsers_for_c = self.all_parsers_for_c.union(
{abc_parser_both, acd_parser_both})
self.all_parsers_for_d.add(acd_parser_both)
# ******** GENERIC *******
def parse_any():
pass
any_parser_singlefile = SingleFileParserFunction(
parse_any,
supported_types={AnyObject},
supported_exts=set(all_d_extensions))
parsers_generic_singlefile = [any_parser_singlefile]
any_parser_multifile = DummyMultifileParser(
supported_types={AnyObject})
parsers_generic_multifile = [any_parser_multifile]
any_parser_bothfile = DummyParser(supported_types={AnyObject},
supported_exts=set(
self.all_a_extensions))
parsers_generic_bothfile = [any_parser_bothfile]
self.all_parsers_generic = self.all_parsers_generic.union(set(parsers_generic_singlefile)) \
.union(set(parsers_generic_multifile)) \
.union(set(parsers_generic_bothfile))
self.all_parsers_lists = [
parsers_specific_singlefile_onetype,
parsers_specific_singlefile_severaltypes,
parsers_specific_multifile_onetype,
parsers_specific_multifile_severaltypes,
parsers_specific_bothfile_onetype,
parsers_specific_bothfile_severaltypes, parsers_generic_singlefile,
parsers_generic_multifile, parsers_generic_bothfile
]
self.all_parsers = {
A: self.all_parsers_for_a,
B: self.all_parsers_for_b,
C: self.all_parsers_for_c,
D: self.all_parsers_for_d,
AnyObject: self.all_parsers_generic
}
def setUp(self):
"""
This setup function defines
* the classes that will be used in the whole test class.
* file extensions associated to one or several classes
* parsers
:return:
"""
# defines the classes
A, B, C, D = self.define_types()
self.A, self.B, self.C, self.D, = A, B, C, D
self.all_types = {A, B, C, D, AnyObject}
# create all combinations of file extensions
all_a_exts, all_b_exts, all_c_exts, all_d_exts, all_extensions = self.define_file_extensions(
)
self.all_extensions = all_extensions
# defines the parsers
self.all_parsers_for_a = set()
self.all_parsers_for_b = set()
self.all_parsers_for_c = set()
self.all_parsers_for_d = set()
self.all_parsers_generic = set()
# *********** Specific SingleFile ************
# -- one type
def parse_a():
pass
def parse_b():
pass
def parse_c():
pass
def parse_d():
pass
a_parser_single = SingleFileParserFunction(
parse_a, supported_types={A}, supported_exts=set(all_a_exts))
b_parser_single = SingleFileParserFunction(
parse_b, supported_types={B}, supported_exts=set(all_b_exts))
c_parser_single = SingleFileParserFunction(
parse_c, supported_types={C}, supported_exts=set(all_c_exts))
d_parser_single = SingleFileParserFunction(
parse_d, supported_types={D}, supported_exts=set(all_d_exts))
parsers_specific_singlefile_onetype = [
a_parser_single, b_parser_single, c_parser_single, d_parser_single
]
self.all_parsers_for_a.add(a_parser_single)
self.all_parsers_for_b.add(b_parser_single)
self.all_parsers_for_c.add(c_parser_single)
self.all_parsers_for_d.add(d_parser_single)
# -- several types
def parse_a_or_d():
pass
def parse_b_or_c():
pass
ad_parser_single = SingleFileParserFunction(
parse_a_or_d,
supported_types={A, D},
supported_exts=set(all_a_exts).union(all_d_exts))
bc_parser_single = SingleFileParserFunction(
parse_b_or_c,
supported_types={B, C},
supported_exts=set(all_b_exts).union(all_c_exts))
parsers_specific_singlefile_severaltypes = [
ad_parser_single, bc_parser_single
]
self.all_parsers_for_a.add(ad_parser_single)
self.all_parsers_for_b.add(bc_parser_single)
self.all_parsers_for_c.add(bc_parser_single)
self.all_parsers_for_d.add(ad_parser_single)
# ******** Specific Multifile ************
class DummyMultifileParser(MultiFileParser):
def __init__(self, supported_types):
super(DummyMultifileParser, self).__init__(supported_types)
def __str__(self):
return 'MultifileParser for ' + str([get_pretty_type_str(typ) for typ in self.supported_types]) \
+ 'for ' + str(self.supported_exts)
def _get_parsing_plan_for_multifile_children(
self, obj_on_fs: PersistedObject, desired_type: Type[T],
logger: Logger) -> Dict[str, _BaseParsingPlan[Any]]:
pass
def _parse_multifile(self, desired_type: Type[T],
obj: PersistedObject,
parsing_plan_for_children: Dict[
str, AnyParser._RecursiveParsingPlan],
logger: Logger,
options: Dict[str, Dict[str, Any]]) -> T:
pass
# -- one type
a_parser_multi = DummyMultifileParser(supported_types={A})
b_parser_multi = DummyMultifileParser(supported_types={B})
c_parser_multi = DummyMultifileParser(supported_types={C})
d_parser_multi = DummyMultifileParser(supported_types={D})
parsers_specific_multifile_onetype = [
a_parser_multi, b_parser_multi, c_parser_multi, d_parser_multi
]
self.all_parsers_for_a.add(a_parser_multi)
self.all_parsers_for_b.add(b_parser_multi)
self.all_parsers_for_c.add(c_parser_multi)
self.all_parsers_for_d.add(d_parser_multi)
# -- several types
bd_parser_multi = DummyMultifileParser(supported_types={B, D})
ac_parser_multi = DummyMultifileParser(supported_types={A, C})
parsers_specific_multifile_severaltypes = [
bd_parser_multi, ac_parser_multi
]
self.all_parsers_for_a.add(ac_parser_multi)
self.all_parsers_for_b.add(bd_parser_multi)
self.all_parsers_for_c.add(ac_parser_multi)
self.all_parsers_for_d.add(bd_parser_multi)
# ******** Specific BOTH **************
class DummyParser(AnyParser):
def __init__(self, supported_types, supported_exts):
super(DummyParser,
self).__init__(supported_types=supported_types,
supported_exts=supported_exts)
def _get_parsing_plan_for_multifile_children(
self, obj_on_fs: PersistedObject, desired_type: Type[T],
logger: Logger) -> Dict[str, _BaseParsingPlan[Any]]:
pass
def _parse_multifile(self, desired_type: Type[T],
obj: PersistedObject,
parsing_plan_for_children: Dict[
str, AnyParser._RecursiveParsingPlan],
logger: Logger,
options: Dict[str, Dict[str, Any]]) -> T:
pass
def _parse_singlefile(self, desired_type: Type[T], file_path: str,
encoding: str, logger: Logger,
options: Dict[str, Dict[str, Any]]) -> T:
pass
# -- one type
a_parser_both = DummyParser(supported_types={A},
supported_exts=set(all_a_exts))
b_parser_both = DummyParser(supported_types={B},
supported_exts=set(all_b_exts))
c_parser_both = DummyParser(supported_types={C},
supported_exts=set(all_c_exts))
d_parser_both = DummyParser(supported_types={D},
supported_exts=set(all_d_exts))
parsers_specific_bothfile_onetype = [
a_parser_both, b_parser_both, c_parser_both, d_parser_both
]
self.all_parsers_for_a.add(a_parser_both)
self.all_parsers_for_b.add(b_parser_both)
self.all_parsers_for_c.add(c_parser_both)
self.all_parsers_for_d.add(d_parser_both)
# -- several types
abc_parser_both = DummyParser(
supported_types={A, B, C},
supported_exts=set(all_a_exts).union(all_b_exts).union(all_c_exts))
acd_parser_both = DummyParser(
supported_types={A, C, D},
supported_exts=set(all_a_exts).union(all_c_exts).union(all_d_exts))
parsers_specific_bothfile_severaltypes = [
abc_parser_both, acd_parser_both
]
self.all_parsers_for_a = self.all_parsers_for_a.union(
{abc_parser_both, acd_parser_both})
self.all_parsers_for_b.add(abc_parser_both)
self.all_parsers_for_c = self.all_parsers_for_c.union(
{abc_parser_both, acd_parser_both})
self.all_parsers_for_d.add(acd_parser_both)
# ******** GENERIC *******
def parse_any():
pass
any_parser_singlefile = SingleFileParserFunction(
parse_any,
supported_types={AnyObject},
supported_exts=set(all_d_exts))
parsers_generic_singlefile = [any_parser_singlefile]
any_parser_multifile = DummyMultifileParser(
supported_types={AnyObject})
parsers_generic_multifile = [any_parser_multifile]
any_parser_bothfile = DummyParser(supported_types={AnyObject},
supported_exts=set(all_a_exts))
parsers_generic_bothfile = [any_parser_bothfile]
self.all_parsers_generic = self.all_parsers_generic.union(set(parsers_generic_singlefile)) \
.union(set(parsers_generic_multifile)) \
.union(set(parsers_generic_bothfile))
self.all_parsers_lists = [
parsers_specific_singlefile_onetype,
parsers_specific_singlefile_severaltypes,
parsers_specific_multifile_onetype,
parsers_specific_multifile_severaltypes,
parsers_specific_bothfile_onetype,
parsers_specific_bothfile_severaltypes, parsers_generic_singlefile,
parsers_generic_multifile, parsers_generic_bothfile
]
self.all_parsers = {
A: self.all_parsers_for_a,
B: self.all_parsers_for_b,
C: self.all_parsers_for_c,
D: self.all_parsers_for_d,
AnyObject: self.all_parsers_generic
}
def test_old_demo(self):
# In this demonstrative example, we will parse 'test cases' for an imaginary function that performs operations :
# op_function(a:int, b:int, operation:str = '+') -> int
#
# Each of our 'test case' items will be made of several things:
# * mandatory input data (here, a and b)
# * optional configuration (here, operation)
# * mandatory expected result (here, the output)
# We would like these things stored in four separate files. Typically the reason is that you will want to
# separate the various formats that you wish to use: csv, xml, json...
# So our data folder structure looks like this:
# test_cases
# ├── case1
# │ ├── input_a.txt
# │ ├── input_b.txt
# │ └── output.txt
# ├── case2
# │ ├── input_a.txt
# │ ├── input_b.txt
# │ ├── options.txt
# │ └── output.txt
# └── case3
# ├── input_a.txt
# ├── input_b.txt
# ├── options.cfg
# └── output.txt
# Note that the configuration file is optional. Here, only `case2` and `case3` will have a non-default configuration.
# You may also have noticed that the configuration file is present with two different extensions :
# `.txt` (in case2) and `.cfg` (in case3). This framework allows to register several file extensions for the
# same type of object to parse. Each extension may have its own parser function.
# First import the package and create a root parser.
# for test : we dont register the default parsers, just to check
root_parser = RootParser('parsyfiles defaults+test custom functions',
register_default_parsers=False)
# install multifile support at least
root_parser.install_basic_multifile_support()
# Then register a parser function for all items that will be represented as **single** files.
# * In this example, all inputs and outputs are `int` so we create a first function to parse an int from a text file:
#from io import TextIOBase
def test_parse_int_file(desired_type: Type[int],
file_object: TextIOBase,
logger: logging.Logger, *args,
**kwargs) -> int:
integer_str = file_object.readline()
return int(integer_str)
root_parser.register_parser(
SingleFileParserFunction(test_parse_int_file,
streaming_mode=True,
supported_types={int},
supported_exts={'.txt'}))
# Note that the parsing framework automatically opens and closes the file for you, even in case of exception.
# We also need to be able to read an `configuration`, that is a `Dict[str, str]` in our case. We propose two formats:
# * one `.txt` format where the first row will directly contain the value for the operation, and
# * one `.cfg` format where the configuration will be available in the configparser format.
#from typing import Dict
class OpConfig(dict):
"""
An OpConfig object is a Dict[str, str] object
"""
def __init__(self, config: Dict[str, str]):
check_var(config, var_types=dict, var_name='config')
super(OpConfig, self).__init__()
self.__wrapped_impl = config
# here you may wish to perform additional checks on the wrapped object
unrecognized = set(config.keys()) - {'operation'}
if len(unrecognized) > 0:
raise ValueError('Unrecognized options : ' +
str(unrecognized))
def __getitem__(self, item):
return self.__wrapped_impl.__getitem__(item)
def __setitem__(self, key, value):
return self.__wrapped_impl.__setitem__(key, value)
# Delegate all calls to the implementation:
def __getattr__(self, name):
return getattr(self.__wrapped_impl, name)
def test_parse_configuration_txt_file(desired_type: Type[Dict],
file_object: TextIOBase,
logger: logging.Logger, *args,
**kwargs) -> Dict[str, str]:
return OpConfig({'operation': file_object.readline()})
def test_parse_configuration_cfg_file(desired_type: Type[Dict],
file_object: TextIOBase,
logger: logging.Logger, *args,
**kwargs) -> Dict[str, str]:
import configparser
config = configparser.ConfigParser()
config.read_file(file_object)
return OpConfig(dict(config['main'].items()))
root_parser.register_parser(
SingleFileParserFunction(test_parse_configuration_txt_file,
streaming_mode=True,
supported_types={OpConfig},
supported_exts={'.txt'}))
root_parser.register_parser(
SingleFileParserFunction(test_parse_configuration_cfg_file,
streaming_mode=True,
supported_types={OpConfig},
supported_exts={'.cfg'}))
# Finally we define the 'test case' objects
class OpTestCase(object):
def __init__(self,
input_a: int,
input_b: int,
output: int,
options: OpConfig = None):
self.input_a, self.input_b, self.output = input_a, input_b, output
if options is None:
self.op = '+'
else:
self.op = options['operation']
def __str__(self):
return self.__repr__()
def __repr__(self):
return str(self.input_a) + ' ' + self.op + ' ' + str(
self.input_b) + ' =? ' + str(self.output)
# And we parse a collection of these
results = root_parser.parse_collection(
fix_path('./test_data/custom_old_demo'), OpTestCase)
pprint(results)
conf = FlatFileMappingConfiguration(separator='--')
results = root_parser.parse_collection(
fix_path('./test_data/custom_old_demo_flat'),
OpTestCase,
file_mapping_conf=conf)
pprint(results)
class OpTestCaseColl(object):
def __init__(self,
input_a: int,
input_b: int,
output: int,
input_c: Dict[str, List[int]] = None,
options: OpConfig = None):
self.input_a, self.input_b, self.output = input_a, input_b, output
if options is None:
self.op = '+'
else:
self.op = options['operation']
self.input_c = input_c or None
def __str__(self):
return self.__repr__()
def __repr__(self):
return str(self.input_a) + ' ' + self.op + ' ' + str(
self.input_b) + ' =? ' + str(self.output) + ' ' + str(
self.input_c)
results = root_parser.parse_collection(
fix_path('./test_data/custom_old_demo_flat_coll'),
OpTestCaseColl,
file_mapping_conf=conf)
pprint(results['case3'].input_c)
results = root_parser.parse_collection(
fix_path('./test_data/custom_old_demo_coll'), OpTestCaseColl)
pprint(results['case3'].input_c)
def _create_root_parser(self):
# This framework allows users to parse (collections of) objects from files. Each object is a dictionary with
# user-defined fields. Each field is mapped to a file, or a collection of files in a folder.
# Step 1: define the applicative data model.
# ------------------------------------------
# a) define the *basic* types of the objects you want to read from files.
# Each of these object types should be readable from *one* (and only one) file.
#
# -- Sometimes you will feel the need to use a class of your own, that probably already exists in your program:
class OneLiner(object):
"""
A very simple type wrapping some text contents.
"""
def __init__(self, contents: str):
self.contents = contents
def __str__(self):
""" readable representations """
return self.contents
def __repr__(self):
""" unambiguous representation """
return self.contents
# -- In many cases, you will feel the need to wrap an existing object, that a parsing library already returns
from configparser import ConfigParser
class Config(ConfigParser):
"""
A Config object is a ConfigParser object with only one 'main' section
"""
def __init__(self, config: ConfigParser):
super(Config, self).__init__()
self.__wrapped_impl = config
# here you may wish to perform additional checks on the wrapped object,
# for example here we check that the config only has one section named 'main'
if 'main' not in config.sections():
raise Exception(
'Wrong Config configuration : mandatory main section is missing'
)
if len(config.sections()) != 1:
raise Exception(
'Wrong Config configuration : there should only be one main section'
)
# Delegate all calls to the implementation:
def __getattr__(self, name):
return getattr(self.__wrapped_impl, name)
# OneLiner = type('OneLiner', (), {}) # NewType('ParamDict', Dict[str, str])
# b) Define the main schema of the dictionary object that you want to parse.
#
# , that spans across several files. This is the object you
# *really* want as the outcome of the parsing step. For example it might represent a test case.
class MainFooBarItem(object):
"""
The main class of objects that we want to parse. The signature of its constructor will be used to infer
the parsers to use and to check which attributes are optional or not.
"""
def __init__(self,
input_simple: OneLiner,
expected_out: Config,
expected_perf: Config = None):
self.input_simple = input_simple
self.expected_out = expected_out
self.expected_perf = expected_perf
def get_as_dict(self):
return {
'input_simple': self.input_simple,
'expected_out': self.expected_out,
'expected_perf': self.expected_perf
}
def __str__(self):
return self.get_as_dict().__str__()
def __repr__(self):
return self.get_as_dict().__repr__()
# Step 2: for each basic type, define at least one parsing function, that has one mandatory input
# (file_object: TextIOBase, stream opened from the file by the framework) and that returns an object of the
# expected type. Note that the stream will be closed automatically by the framework
#
def test_custom_read_oneliner_from_txt_file_stream(
desired_type: Type[Config], file_object: TextIOBase,
logger: logging.Logger, *args, **kwargs) -> OneLiner:
"""
Helper method to read a txt file and return its content as a OneLiner object.
:param file_object: stream opened from the file by the framework, and closed automatically after parsing
:return: a new object created from the file.
"""
# read the file - or at least the first row
first_row = file_object.readline()
# no need to close, that will be done by caller
return OneLiner(first_row)
def test_custom_read_config_from_config_file_stream(
desired_type: Type[Config], file_object: TextIOBase,
logger: logging.Logger, *args, **kwargs) -> Config:
"""
Helper method to read a txt file and return its content as a Config object.
:param file_object: stream opened from the file by the framework, and closed automatically after parsing
:return: a new object created from the file.
"""
import configparser
config = configparser.ConfigParser()
config.read_file(file_object)
return Config(config)
# Step 3: register all basic types, along with at least one parsing function. The framework allows to register
# several file extensions for each basic type - each file extension being associated with a unique parsing
# function. For example an object might be available as a .cfg, a .txt or a .csv three parsing functions would
# then be registered. Note that you may register the same function twice if it is able to handle several file
# extensions.
#
# parsers = {
# OneLiner: {
# '.txt': read_oneliner_from_txt_file_stream
# },
# Config: {
# '.cfg': read_config_from_config_file_stream,
# '.txt': read_config_from_config_file_stream
# }
# }
parsers = [
SingleFileParserFunction(
parser_function=test_custom_read_oneliner_from_txt_file_stream,
supported_exts={'.txt'},
supported_types={OneLiner}),
SingleFileParserFunction(
parser_function=test_custom_read_config_from_config_file_stream,
supported_exts={'.cfg', '.txt'},
supported_types={Config}),
]
# create root parser
root_parser = RootParser('parsyfiles defaults+test custom functions')
root_parser.register_parsers(parsers)
return root_parser, MainFooBarItem