def make_stations_from_metadata(profile=None):
print "Making stations from filesystem"
station_type_dict = generic_processor.get_dictionary_from_filesystem("station",
"properties",
profile=profile)
data_type_dict = generic_processor.get_dictionary_from_filesystem("datatype",
"properties",
profile=profile)
stations = make_all_station_types(station_type_dict, data_type_dict, profile=profile)
print len(stations), " (number of stations created)"
print "Clearing mongo stations collection"
remove_status = mongo_dao.remove_station_collection()
print remove_status, " (None means station collection removal success)"
station_collection = create_station_collection(stations)
print station_collection, " (station collection pointer)"
all_returned_stations = get_all_stations()
print len(all_returned_stations), " (all stations in mongo)"
ghcnm_returned_stations = get_all_stations_by_program("ghcnm")
print len(ghcnm_returned_stations), " (ghcnm stations in mongo)"
ushcn_returned_stations = get_all_stations_by_program("ushcn")
print len(ushcn_returned_stations), " (ushcn stations in mongo)"
northam_returned_stations = get_all_stations_by_program("northam")
print len(northam_returned_stations), " (northam stations in mongo)"
print "Testing adding geospatial index"
result = add_station_geospatial_index()
print result, " (result of adding station geospatial index)"
print "Testing geoindex search"
program = "ghcnm" # program for testing
test1_stations = get_random_program_stations(program)
print len(test1_stations), ' (length of test stations)'
print test1_stations[0]
status = write_station_records(test1_stations, program, utils.get_timestamp(), profile=profile)
print status, '(status of effort)'
def make_random_station_file_by_program(program):
profile = "berkheimer"
stations = get_random_program_stations(program)
if program in ['ushcn']:
ushcn_stations = filter(lambda station: station['id'].startswith('USH'), stations)
northam_stations = filter(lambda station: not station['id'].startswith('USH'), stations)
stations = ushcn_stations + northam_stations
status = write_station_records(stations, program, utils.get_timestamp(), profile=profile)
return status
def make_group(group_name, add_date=utils.get_timestamp(), order=None):
group = {}
group['name'] = group_name
group['type'] = 'group'
group['structures'] = []
group['order'] = order
group['remove_date'] = None
group['add_date'] = add_date
return group
def make_templates_from_filesystem(structure):
"""Retrieves all the templates for a structure from a filesystem
and adds them to the database.
"""
structure = processor_utils.get_fully_qualified_paths(database, structure, profile=profile)
templates = get_templates_from_filesystem(structure)
template_maker = template_model.make_template_closure(structure, utils.get_timestamp())
templates = map(template_maker, templates)
return templates
def add_keys_to_existing_profile(profile_dictionary, default=True):
""" Adds database keys to already existing profile.
"""
profile_dictionary['name'] = 'default'
profile_dictionary['type'] = 'path'
profile_dictionary['add_date'] = utils.get_timestamp()
profile_dictionary['remove_date'] = None
profile_dictionary['default'] = default
return profile_dictionary
def make_translators_from_filesystem(template):
"""Retrieves all the translators for a template from a filesystem
and adds them to the database.
"""
template = processor_utils.get_fully_qualified_paths(database, template, profile=profile)
translators = get_translators_from_filesystem(template)
translator_maker = translator_model.make_translator_closure(template, utils.get_timestamp())
translators = map(translator_maker, translators)
return translators
def add_structures_from_filesystem(database, profile=None):
"""Adds all structures in a filesystem. Uses the default profile or gets the
profile that is specified by the user.
"""
profile = profile_processor.get_fully_qualified_profile(database, profile)
structure_directory = profile['structures']
structures = get_structures_from_filesystem(structure_directory)
structure_tool = structure_model.make_structure_closure(utils.get_timestamp())
structures = map(structure_tool, structures)
create_structure_collection(database, structures)
structures = get_current_structures(database)
return structures
def make_records_from_filesystem(translator):
"""This method gathers and adds filesystem records to the structure for the
translator. It adds relevant information to each record that it returns, based
on association and ability information from the template and structure.
It returns the added records.
"""
translator = processor_utils.get_fully_qualified_paths(database, translator, profile=profile)
records = get_records_from_filesystem(translator)
template = template_processor.get_template_by_name(translator['structure'],
translator['template'])
record_maker = record_model.make_record_closure(template, translator, utils.get_timestamp())
records = map(record_maker, records)
return records
def make_profile(profile, profile_dictionary, profile_name, full_path):
"""Adds information to a passed profile dictionary and returns it.
"""
profile_dictionary = {key: profile_dictionary[key][0] for key in profile_dictionary.keys()}
if full_path:
profile_dictionary['name'] = profile_name
else:
profile_dictionary['name'] = profile
profile_dictionary['type'] = 'path'
profile_dictionary['default'] = False
profile_dictionary['add_date'] = utils.get_timestamp()
profile_dictionary['remove_date'] = None
return profile_dictionary
def create_new_template(database, structure, template, setup=False, add_date=utils.get_timestamp()):
"""Creates the passed template using the given structure in the given database.
Returns the newly created template. If setup is specified as True, loads from
a filesystem type setup.
"""
if type(database) in [dict, OrderedDict]:
database = database['database']
if type(structure) not in [dict, OrderedDict]:
structure = structure_processor.get_structure_by_name(database, structure)
template_maker = template_model.make_template_closure(structure, setup=setup, add_date=add_date)
new_template = template_maker(template)
new_template = template_dao.add_template(database, template)
structure_processor.add_template_to_structure(database, structure, template)
return new_template
def make_structure_closure(add_date=utils.get_timestamp(), setup=True):
def get_setup_keys():
return ['template_directory']
def make_structure(structure):
"""Prepares a structure dictionary for addition to the structure database.
"""
add_dates(structure)
add_type(structure)
adjust_templates(structure)
if setup:
move_setup_properties(structure)
return structure
def add_dates(structure):
structure['add_date'] = add_date
structure['remove_date'] = None
return structure
def add_type(structure):
structure['type'] = 'structure'
return structure
def adjust_templates(structure):
if 'templates' not in structure.keys() or not structure['templates']:
structure['templates'] = []
if type(structure['templates']) in [str, unicode]:
structure['templates'] = [structure['templates']]
return structure
def move_setup_properties(structure):
structure['setup'] = {key: structure[key] for key in get_setup_keys()}
utils.remove_dictionary_keys(structure, get_setup_keys())
return structure
return make_structure
def make_template_closure(structure, add_date=utils.get_timestamp(), setup=True):
"""Closure function returning the function that creates a structure template.
"""
field_translator_dictionary = {}
field_translator_dictionary['field_names'] = 'name'
field_translator_dictionary['field_types'] = 'type'
field_translator_dictionary['field_required'] = 'required'
field_translator_dictionary['field_defaults'] = 'default'
index_translator_dictionary = {}
index_translator_dictionary['index_fields'] = 'field'
index_translator_dictionary['index_types'] = 'type'
def get_empties():
return [None, False, 'None', 'False']
def get_setup_keys():
return ['translator_directory']
def is_field_key(key):
if key in field_translator_dictionary.keys():
return True
return False
def is_index_key(key):
if key in index_translator_dictionary.keys():
return True
return False
def parse_template_file_name(file_name):
return file_name.rsplit('/', 1)[-1].rsplit('.')[0]
def make_template(template):
add_dates(template)
add_type(template)
adjust_name(template)
adjust_description(template)
adjust_collection(template)
adjust_structure(template)
adjust_fields(template)
adjust_indeces(template)
adjust_translators(template)
if setup:
move_setup_properties(template)
return template
def add_dates(template):
template['add_date'] = add_date
template['remove_date'] = None
return template
def add_type(template):
template['type'] = 'template'
return template
def adjust_name(template):
if 'name' not in template.keys():
if 'template_file' in template.keys():
template['name'] = parse_template_file_name(template['template_file'])
else:
template['name'] = 'auto_' + utils.get_random_string(length=10)
utils.remove_dictionary_keys(template, ['template_file'])
return template
def adjust_description(template):
if 'description' not in template.keys():
template['description'] = ''
return template
def adjust_collection(template):
if 'collection' not in template.keys():
unique_collection = 'mars_' + utils.get_random_string(numbers=True, length=10)
template['collection'] = unique_collection
return template
def adjust_structure(template):
template['structure'] = structure['name']
return template
def adjust_fields(template):
if setup:
field_properties = {key: template[key] for key in template.keys() if is_field_key(key)}
field_tool = generic_model.field_converter_closure(field_properties, field_translator_dictionary)
field_properties = map(field_tool, field_properties)
template['fields'] = map(generic_model.combine_fields, zip(*field_properties))
utils.remove_dictionary_keys(template, field_translator_dictionary.keys())
elif 'fields' not in template.keys():
template['fields'] = []
return template
def adjust_indeces(template):
index_properties = {key: template[key] for key in template.keys() if is_index_key(key)}
field_tool = generic_model.field_converter_closure(index_properties, index_translator_dictionary)
index_properties = map(field_tool, index_properties)
template['indeces'] = map(generic_model.combine_fields, zip(*index_properties))
utils.remove_dictionary_keys(template, index_translator_dictionary.keys())
return template
def adjust_translators(template):
if 'translators' not in template.keys() or not template['translators']:
template['translators'] = []
if type(template['translators']) in [str, unicode]:
template['translators'] = [template['translators']]
return template
def move_setup_properties(template):
template['setup'] = {key: template[key] for key in get_setup_keys()}
utils.remove_dictionary_keys(template, get_setup_keys())
return template
return make_template
def update_remove_date(template, remove_date=utils.get_timestamp()):
"""Sets the remove date for a given template, returning the updated template.
"""
template['remove_date'] = str(remove_date)
return template