def __lazyloadclass__(self):
"""
defers evaluation of the attribute class. If it occured during init there would be unresolved dependencies
:return: object
"""
_class = self.__init__['__class__']
if _class in globals():
return globals()[_class]
elif _class.upper() in namespace():
return namespace()[_class.upper()]
elif _class.upper() in pluginnamespace():
return pluginnamespace()[_class.upper()]
def loadmethod(self, key, val, force=False):
#print('initializing method: %s' % key)
modulename = val['module']
_method = pluginnamespace()[modulename.upper()](self,
config=val['config'])
"""
We use an orderedDict to ensure that dependencies in methods that take other methods as input aren't broken.
A normal dict get's read randomly and will result in attribute errors when you call.
If the user declared an input on a method that had not been processed yet, an AttributeError would be thrown.
see classprocessor.parseconfig and processClass.generateMethods
also see classgen.factory.methodbinder
"""
_result = None
try:
_result = _method.result(force=force)
#self.__method_results__[key] = _result.result()
except NotComputed:
if force:
print('reraising not computed')
raise
else:
self.__method_deferred__.append(key)
else:
self.__method_results__[key] = _result
def listmethods():
# TODO add errors
_base = inspect.getmro(coretypes.FunctionType)[0]
for k, v in pluginnamespace().items():
if _base in inspect.getmro(v):
print(k.title())
return True
def listaggregates():
# TODO add errors
_base = inspect.getmro(coretypes.Aggregator)[0]
for k, v in pluginnamespace().items():
if _base in inspect.getmro(v):
print(k.title())
return True
def listoutputplugins():
#TODO add errors
_base = inspect.getmro(Output)[0]
for k, v in pluginnamespace().items():
if _base in inspect.getmro(v):
print(k)
return True
def __validationrequired__(self):
"""
Checks if validation is required during data load.
Should always be true except for classes that handle their own validation
:return: True if attribute validation routines should be called when data is loaded.
"""
_class = self.__init__['__class__']
if _class in globals():
# user created class
return True
elif _class.upper() in namespace():
# an attribute that is a user created class, it will handle it's own validation
return False
elif _class.upper() in pluginnamespace():
# an attribute that relies on a plugin
return True
else:
raise TypeError(
'I cannot find %s in the globals, plugins or class namespaces'
% _class)
def checkalignment(project):
if not isinstance(project, StructureFactory.Container):
raise TypeError(
'function checkalignment expected an arg of type StructureFactory.Container but received an object of type %s'
% type(project))
uniquebranches = set()
_tree = project.treestructure()
for i in structureflattentolist(_tree):
uniquebranches.add('%s' % i) #cheap hack to make list hashable
_longest = findlongest(uniquebranches) #(s)
if len(uniquebranches) == 1:
return True
for i in uniquebranches: #s:
if i[1:-1] not in _longest[1:-1]:
_nonmatching = '\n '.join(
reversematch(project=project, matchstring=i))
_outputplugin = partialnamespace('output')[
runtimenamespace()['outputscheme']]['plugin']
# have to compare strings instead of using isinstance to avoid circular imports
_treeoutputplugins = '* ' + '\n *'.join([
'%s' % k for k, v in pluginnamespace().items()
if 'TreeOutput' in str(v.__bases__)
])
_exception = 'Found an object structure {0} ' \
'that does not align with the longest object structure {1}. ' \
'The non-aligned objects can be found at:\n{2}. ' \
'\n\nThis error occured because the output plugin, {3}, requires aligned output. ' \
'If you do not want to align your data structure, please use an output plugin ' \
'that does not require aligned data. Available plugins that do not require ' \
'aligned data include:\n\n{4}'.format(i, _longest, _nonmatching, _outputplugin, _treeoutputplugins)
raise ValueError(_exception)
return True
def aggregates(self):
_retdict = defaultdict(list)
_aggreagatorlist = self.__aggregates__()
if _aggreagatorlist is None or len(_aggreagatorlist) == 0:
return None
else:
for aggregator in _aggreagatorlist:
_aggregatorconfig = aggregatornamespace().get(
aggregator.upper(), None)
_aggregatorfunctionname = _aggregatorconfig.get(
'function', None)
_aggregatortargetattribute = _aggregatorconfig.get(
'select', None).get('attribute', None)
if _aggregatorfunctionname is not None:
_aggregatorfunction = pluginnamespace().get(
_aggregatorfunctionname.upper(), None)
if _aggregatorfunction is not None:
aggregator = _aggregatorfunction(
config={aggregator: _aggregatorconfig})
_retdict[_aggregatortargetattribute].append(
aggregator.compute())
return _retdict
def aggregates(self):
_retlist = []
_aggreagatorlist = self.__aggregates__()
if _aggreagatorlist is None or len(_aggreagatorlist) == 0:
return None
else:
for aggregator in _aggreagatorlist:
_aggregatorconfig = aggregatornamespace().get(
aggregator.upper(), None)
_aggregatorfunctionname = _aggregatorconfig.get(
'function', None)
if _aggregatorfunctionname is not None:
_aggregatorfunction = pluginnamespace().get(
_aggregatorfunctionname.upper(), None)
if _aggregatorfunction is not None:
aggregator = _aggregatorfunction(
config={aggregator: _aggregatorconfig})
_resultdict = aggregator.compute()
_retlist.append({
_resultdict.get('name', None):
_resultdict.get('result', None)
})
return _retlist
def listplugins():
# TODO add errors
for k, v in pluginnamespace().items():
print(k)
return True
def processproject(path=None,
scheme=None,
output=None,
verbose=False,
silent=False,
debug=False):
if verbose and silent:
click.echo(
'cannot use both the --verbose and --silent options together.')
sys.exit(1)
if verbose:
click.echo('[VERBOSE] Loading project from %s...' % path)
elif not silent:
click.echo('Loading project from %s...' % path)
dbg = debug
projectconfig = {}
try:
if verbose:
click.echo('[VERBOSE] Loading bootstrap config...')
projectconfig = loadconfig(path)
except Exception as err:
click.echo('While processing bootstrap configuration directives '
'an error occurred. The following message was received '
'during the error: %s' % err)
sys.exit(status=1)
try:
if verbose:
click.echo(
'[VERBOSE] Loading plugins from code base and %s\\plugins...' %
projectconfig['root'])
__loadplugins(projectconfig['root'], verbose=verbose, silent=silent)
except Exception as err:
if dbg:
raise
click.echo('While loading plugins '
'an error occurred. The following message was received '
'during the error: %s' % err)
sys.exit(1)
try:
if verbose:
click.echo('[VERBOSE] Loading project config from %s...' %
projectconfig['configpath'])
__configloader(projectconfig['configpath'])
except Exception as err:
if dbg:
raise
click.echo('While processing bootstrap configuration directives '
'an error occurred. The following message was received '
'during the error: %s' % err)
sys.exit(1)
try:
if verbose:
click.echo('[VERBOSE] Loading user defined classes from %s...' %
projectconfig['classes'])
__loadclasses(projectconfig['classes'],
verbose=verbose,
silent=silent,
dbg=False) #hard coded dbg, see TODO
# TODO fix dbg routines in class loading
except Exception as err:
if dbg:
raise
click.echo('While loading user configured classes '
'an error occurred. The following message was received '
'during the error: %s' % err)
sys.exit(1)
try:
if verbose:
click.echo(
'[VERBOSE] Loading user defined aggregators from %s...' %
projectconfig['aggregators'])
__loadaggregators(projectconfig['aggregators'],
verbose=verbose,
silent=silent)
except Exception as err:
if dbg:
raise
click.echo('While loading user configured aggregates '
'an error occurred. The following message was received '
'during the error: %s' % err)
sys.exit(1)
if verbose:
click.echo('[VERBOSE] Registering run time options...')
try:
__registeroption('outputscheme', scheme)
except Exception as err:
if dbg:
raise
click.echo(
'While registering the output scheme specified via the command line '
'switch an error occurred. '
'Please make sure that the section [output.%s] exists in gtool.cfg. '
'The following message was received during the error: %s' %
(scheme, err))
sys.exit(1)
try:
__registeroption('debug', dbg)
except Exception as err:
if dbg:
raise
click.echo(
'While registering a runtime debug option an error occurred. '
'This error is internal and you need to file a bug report. '
'The following message was received during the error: %s' % err)
sys.exit(1)
try:
__outputparser(outputscheme=scheme)
except Exception as err:
if dbg:
raise
click.echo(
'While registering output schemes for user configured classes '
'an error occurred. The following message was received '
'during the error: %s' % err)
sys.exit(1)
dataobject = None
try:
if verbose:
click.echo('[VERBOSE] Loading data from %s...' %
projectconfig['dataroot'])
dataobject = process(projectconfig['dataroot'])
except Exception as err:
if dbg:
raise
click.echo('While processing the data structure an error occurred. '
'The following message was received '
'during the error: %s' % err)
sys.exit(1)
outputprocessor = None
if verbose:
click.echo('[VERBOSE] Using output scheme %s...' % scheme)
outputschemeconfig = partialnamespace('output').get(scheme, None)
if outputschemeconfig is None:
click.echo('Could not find [output.%s] in gtool.cfg.')
sys.exit(1)
outputschemeplugin = outputschemeconfig.get('plugin', None)
if outputschemeplugin is None:
click.echo(
'an output plugin is not specified in [output.%s] in gtool.cfg.')
sys.exit(1)
try:
if verbose:
click.echo('[VERBOSE] Preparing output processor %s...' %
outputschemeplugin.upper())
outputprocessor = pluginnamespace()[outputschemeplugin.upper()](
) #TODO read cfg variables and pass in via init
except Exception as err:
if dbg:
raise
click.echo(
'While loading the output processor, %s, specified in gtool.cfg '
'for output scheme [output.%s] an error occurred. The '
'following message was received during the error: %s' %
(outputschemeplugin.upper(), scheme, err))
sys.exit(1)
result = None
try:
if verbose:
click.echo('[VERBOSE] Processing the data...')
result = outputprocessor.output(dataobject, output=output)
except Exception as err:
if dbg:
raise
click.echo('While processing the data into output an error occurred. '
'The following message was received '
'during the error: %s' % err)
sys.exit(1)
if verbose:
click.echo('[VERBOSE] Rendering output to %s...' %
output if output is not None else "standard out")
elif not silent and output is not None:
click.echo('Output written to %s' % output)
if not isinstance(result, bool):
if not isinstance(result, str) and hasattr(result, '__iter__'):
for row in result:
print(row)
else:
print(result)
if not silent:
click.echo('Done')
