def named_tuple_to_workflow_closure(add_date=general_utils.get_timestamp()):
"""Takes a named tuple workflow object and transforms it into the dictionary
version, removing extraneous identifiers and including metadata.
"""
def named_tuple_to_workflow(workflow_tuple):
workflow = workflow_tuple._asdict()
add_dates(workflow)
add_empty_storage(workflow)
add_empty_function_collection(workflow)
add_empty_parameter_collection(workflow)
remove_identifiers(workflow)
del workflow_tuple
return workflow
def add_dates(workflow):
workflow['add_date'] = add_date
workflow['remove_date'] = None
def add_empty_storage(workflow):
workflow['structure'] = None
def add_empty_function_collection(workflow):
workflow['functions'] = []
def add_empty_parameter_collection(workflow):
workflow['parameters'] = {}
workflow['parameters']['inputs'] = []
workflow['parameters']['outputs'] = []
def remove_identifiers(workflow):
identifiers = ['project']
general_utils.remove_dictionary_keys(workflow, identifiers)
return named_tuple_to_workflow
def write_records(records, properties=None):
"""This method takes a list of named tuple records and a properties file and
produces an output file containing the records. The properties that are necessary
are: field_names, and either field_start_positions or a field_separator.
Optionally, an output file should be specified.
"""
try:
file_name = properties['output_file'][0]
except:
file_name = get_name(records[0]) + '_' + get_timestamp()
try:
fields = properties['field_names']
except:
fields = records[0]._fields
try:
fields = list(
zip(fields, properties['field_start_positions'],
properties['field_end_positions'],
properties['field_justify']))
write_fixed_length_strings(file_name, records, fields)
except:
print('Trying to write records as delimited strings.')
delimiter = translate_delimiter(properties['field_separator'][0])
print(delimiter, ' (delimiter)')
write_delimited_strings(file_name, delimiter, records, fields)
return True
def create_new_template(database,
structure,
template,
order=None,
setup=False,
add_date=general_utils.get_timestamp()):
"""Creates the passed template using the given structure in the given database.
Returns the newly created template. If setup is specified as True, loads from
a filesystem type setup.
"""
if type(database) in [dict, OrderedDict]:
database = database['database']
if type(structure) not in [dict, OrderedDict]:
structure = structure_processor.get_structure_by_name(
database, structure)
template_maker = template_model.make_template_closure(structure,
setup=setup,
add_date=add_date)
new_template = template_maker(template)
new_template = template_dao.add_template(database, template)
structure_processor.add_template_to_structure(database,
structure,
template,
order=order)
return new_template
def make_functions_from_tuples(function_tuples):
"""Takes a list of function named tuples and transforms them into a list of
function dictionary objects.
"""
add_date = general_utils.get_timestamp()
function_maker = function_model.named_tuple_to_function_closure(add_date)
functions = list(map(function_maker, function_tuples))
return functions
def make_workflows_from_tuples(workflow_tuples):
"""Takes workflow named tuples and returns them as a list of dictionaries.
Additionally, uses an add date to indicate when the component was added.
"""
add_date = general_utils.get_timestamp()
workflow_maker = workflow_model.named_tuple_to_workflow_closure(add_date)
workflows = list(map(workflow_maker, workflow_tuples))
return workflows
def make_projects_from_tuples(project_tuples):
"""Makes project records from an array of named tuples. Returns the new list
of project dictionary objects.
"""
add_date = general_utils.get_timestamp()
project_maker = project_model.named_tuple_to_project_closure(add_date)
projects = list(map(project_maker, project_tuples))
return projects
def reset_component_dates(component):
"""Resets the dates attached to the component to default dates. Returns the updated
component object.
"""
new_component = copy.deepcopy(component)
new_component['add_date'] = general_utils.get_timestamp()
new_component['remove_date'] = None
return new_component
def make_components_from_tuples(component_tuples):
"""Takes component named tuples and returns them as a list of dictionaries.
Additionally, uses an add date to indicate when the component was added.
"""
add_date = general_utils.get_timestamp()
component_maker = component_model.named_tuple_to_component_closure(
add_date)
components = list(map(component_maker, component_tuples))
return components
def make_expressions_from_tuples(expression_tuples):
"""Takes a list of expression named tuples and transforms them into
a list of expression dictionary objects.
"""
add_date = general_utils.get_timestamp()
expression_maker = expression_model.named_tuple_to_expression_closure(
add_date)
expressions = list(map(expression_maker, expression_tuples))
return expressions
def add_keys_to_existing_profile(profile_dictionary, default=True):
""" Adds database keys to already existing profile.
"""
profile_dictionary['name'] = 'default'
profile_dictionary['type'] = 'path'
profile_dictionary['add_date'] = general_utils.get_timestamp()
profile_dictionary['remove_date'] = None
profile_dictionary['default'] = default
return profile_dictionary
def make_translators_from_filesystem(template):
"""Retrieves all the translators for a template from a filesystem
and adds them to the database.
"""
template = processor_utils.get_fully_qualified_paths(database, template, profile=profile)
translators = get_translators_from_filesystem(template)
translator_maker = translator_model.make_translator_closure(template, general_utils.get_timestamp())
translators = list(map(translator_maker, translators))
return translators
def make_group(group_name, add_date=general_utils.get_timestamp(), order=None):
group = {}
group['name'] = group_name
group['type'] = 'group'
group['structures'] = []
group['order'] = order
group['remove_date'] = None
group['add_date'] = add_date
return group
def remove_component_by_id(component_id):
"""Uses the given id to remove the component. Returns the removed component.
"""
timestamp = general_utils.get_timestamp()
component = mongo_dao.remove_component_by_id(database, component_id,
timestamp)
if remove_storage:
storage_location = component['location']
remove_component_storage(storage_location)
return component
def make_evaluation_closure(workflow, template, add_date=general_utils.get_timestamp()):
"""Creates an evaluation record for database entry.
"""
def make_evaluation(evaluation=None):
if not evaluation:
evaluation = {}
add_blank_name(evaluation)
add_blank_description(evaluation)
add_dates(evaluation)
add_workflow(evaluation)
add_structure(evaluation)
add_collection(evaluation)
add_fields(evaluation)
add_functions(evaluation)
add_info(evaluation)
add_record(evaluation)
return evaluation
def add_dates(evaluation):
evaluation['add_date'] = add_date
def add_blank_name(evaluation):
evaluation['name'] = 'None'
def add_blank_description(evaluation):
evaluation['description'] = 'None'
def add_workflow(evaluation):
evaluation['workflow'] = workflow['name']
def add_structure(evaluation):
evaluation['structure'] = workflow['structure']
def add_collection(evaluation):
evaluation['template'] = template['name']
evaluation['collection'] = template['collection']
def add_fields(evaluation):
if 'fields' in list(template.keys()):
evaluation['fields'] = get_evaluation_fields(template)
else:
evaluation['fields'] = []
def add_info(evaluation):
evaluation['info'] = {}
evaluation['info']['success'] = False
evaluation['info']['time'] = None
def add_functions(evaluation):
evaluation['functions'] = []
def add_record(evaluation):
evaluation['record'] = None
return make_evaluation
def make_templates_from_filesystem(structure):
"""Retrieves all the templates for a structure from a filesystem
and adds them to the database.
"""
structure = processor_utils.get_fully_qualified_paths(database,
structure,
profile=profile)
templates = get_templates_from_filesystem(structure)
template_maker = template_model.make_template_closure(
structure, general_utils.get_timestamp())
templates = list(map(template_maker, templates))
return templates
def add_structures_from_filesystem(database, profile=None):
"""Adds all structures in a filesystem. Uses the default profile or gets the
profile that is specified by the user.
"""
profile = profile_processor.get_fully_qualified_profile(database, profile)
structure_directory = profile['structures']
structures = get_structures_from_filesystem(structure_directory)
structure_tool = structure_model.make_structure_closure(general_utils.get_timestamp())
structures = list(map(structure_tool, structures))
create_structure_collection(database, structures)
structures = get_current_structures(database)
return structures
def make_records_from_filesystem(translator):
"""This method gathers and adds filesystem records to the structure for the
translator. It adds relevant information to each record that it returns, based
on association and ability information from the template and structure.
It returns the added records.
"""
translator = processor_utils.get_fully_qualified_paths(database, translator, profile=profile)
records = get_records_from_filesystem(translator)
template = template_processor.get_template_by_name(translator['structure'],
translator['template'])
record_maker = record_model.make_record_closure(template, translator, general_utils.get_timestamp())
records = list(map(record_maker, records))
return records
def make_profile(profile, profile_dictionary, profile_name, full_path):
"""Adds information to a passed profile dictionary and returns it.
"""
profile_dictionary = {
key: profile_dictionary[key][0]
for key in list(profile_dictionary.keys())
}
if full_path:
profile_dictionary['name'] = profile_name
else:
profile_dictionary['name'] = profile
profile_dictionary['type'] = 'path'
profile_dictionary['default'] = False
profile_dictionary['add_date'] = general_utils.get_timestamp()
profile_dictionary['remove_date'] = None
return profile_dictionary
def update_workflow_template(project, workflow, function, parameter, target,
scope):
"""Creates a new template for the given workflow with the given fields.
Processes the fields by updating or adding. Returns the newly formed template.
"""
database = project['database']
structure = workflow_model.get_workflow_structure(workflow)
templates = mars.template.get_current_templates_by_structure_name(
database, structure)
current_template = templates[0]
modify_date = general_utils.get_timestamp()
current_fields = mars.template.copy_template_fields(current_template)
new_field = make_storage_field(parameter, target, scope)
update_storage_fields(current_fields, new_field)
mars.template.set_template_removal_date(database, current_template,
modify_date)
new_template = {}
new_template['fields'] = current_fields
new_template = mars.template.create_new_template(database,
workflow['structure'],
new_template,
add_date=modify_date)
return new_template
def make_structure_closure(add_date=general_utils.get_timestamp(), setup=True):
def get_setup_keys():
return ['template_directory']
def make_structure(structure):
"""Prepares a structure dictionary for addition to the structure database.
"""
add_dates(structure)
add_type(structure)
adjust_templates(structure)
if setup:
move_setup_properties(structure)
return structure
def add_dates(structure):
structure['add_date'] = add_date
structure['remove_date'] = None
return structure
def add_type(structure):
structure['type'] = 'structure'
return structure
def adjust_templates(structure):
if 'templates' not in list(
structure.keys()) or not structure['templates']:
structure['templates'] = []
if type(structure['templates']) in [str, str]:
structure['templates'] = [structure['templates']]
return structure
def move_setup_properties(structure):
structure['setup'] = {key: structure[key] for key in get_setup_keys()}
general_utils.remove_dictionary_keys(structure, get_setup_keys())
return structure
return make_structure
def assemble_groups_from_structures(structures, complete=True):
"""Returns the unique set of groups for the array of dictionaries that is passed.
If keyword parameter 'complete' is True, adds complete structures to each group.
If keyword parameter 'complete' is False, only adds structure names to the group.
"""
add_date = general_utils.get_timestamp()
group_names = []
groups = []
counter = 0
for structure in structures:
if structure['group'] not in group_names:
groups.append(
parameter_model.make_group(structure['group'],
add_date,
order=counter))
group_names.append(structure['group'])
counter += 1
if complete:
get_group_by_name(
groups, structure['group'])['structures'].append(structure)
else:
get_group_by_name(groups, structure['group'])['structures'].append(
structure['name'])
return groups
def update_remove_date(template, remove_date=general_utils.get_timestamp()):
"""Sets the remove date for a given template, returning the updated template.
"""
template['remove_date'] = str(remove_date)
return template
def make_template_closure(structure,
add_date=general_utils.get_timestamp(),
setup=True):
"""Closure function returning the function that creates a structure template.
"""
field_translator_dictionary = {}
field_translator_dictionary['field_names'] = 'name'
field_translator_dictionary['field_types'] = 'type'
field_translator_dictionary['field_required'] = 'required'
field_translator_dictionary['field_defaults'] = 'default'
index_translator_dictionary = {}
index_translator_dictionary['index_fields'] = 'field'
index_translator_dictionary['index_types'] = 'type'
def get_empties():
return [None, False, 'None', 'False']
def get_setup_keys():
return ['translator_directory']
def is_field_key(key):
if key in list(field_translator_dictionary.keys()):
return True
return False
def is_index_key(key):
if key in list(index_translator_dictionary.keys()):
return True
return False
def parse_template_file_name(file_name):
return file_name.rsplit('/', 1)[-1].rsplit('.')[0]
def make_template(template):
add_dates(template)
add_type(template)
adjust_name(template)
adjust_description(template)
adjust_collection(template)
adjust_structure(template)
adjust_fields(template)
adjust_indeces(template)
adjust_translators(template)
if setup:
move_setup_properties(template)
return template
def add_dates(template):
template['add_date'] = add_date
template['remove_date'] = None
return template
def add_type(template):
template['type'] = 'template'
return template
def adjust_name(template):
if 'name' not in list(template.keys()):
if 'template_file' in list(template.keys()):
template['name'] = parse_template_file_name(
template['template_file'])
else:
template['name'] = 'auto_' + general_utils.get_random_string(
length=10)
general_utils.remove_dictionary_keys(template, ['template_file'])
return template
def adjust_description(template):
if 'description' not in list(template.keys()):
template['description'] = ''
return template
def adjust_collection(template):
if 'collection' not in list(template.keys()):
unique_collection = 'mars_' + general_utils.get_random_string(
numbers=True, length=10)
template['collection'] = unique_collection
return template
def adjust_structure(template):
if type(structure) in [dict, OrderedDict]:
template['structure'] = structure['name']
else:
template['structure'] = structure
return template
def adjust_fields(template):
if setup:
field_properties = {
key: template[key]
for key in list(template.keys()) if is_field_key(key)
}
field_tool = generic_model.field_converter_closure(
field_properties, field_translator_dictionary)
field_properties = list(map(field_tool, field_properties))
template['fields'] = list(
map(generic_model.combine_fields,
list(zip(*field_properties))))
general_utils.remove_dictionary_keys(
template, list(field_translator_dictionary.keys()))
elif 'fields' not in list(template.keys()):
template['fields'] = []
return template
def adjust_indeces(template):
index_properties = {
key: template[key]
for key in list(template.keys()) if is_index_key(key)
}
field_tool = generic_model.field_converter_closure(
index_properties, index_translator_dictionary)
index_properties = list(map(field_tool, index_properties))
template['indeces'] = list(
map(generic_model.combine_fields, list(zip(*index_properties))))
general_utils.remove_dictionary_keys(
template, list(index_translator_dictionary.keys()))
return template
def adjust_translators(template):
if 'translators' not in list(
template.keys()) or not template['translators']:
template['translators'] = []
if type(template['translators']) in [str, str]:
template['translators'] = [template['translators']]
return template
def move_setup_properties(template):
template['setup'] = {key: template[key] for key in get_setup_keys()}
general_utils.remove_dictionary_keys(template, get_setup_keys())
return template
return make_template