def make_stations_from_metadata(profile=None): print "Making stations from filesystem" station_type_dict = generic_processor.get_dictionary_from_filesystem("station", "properties", profile=profile) data_type_dict = generic_processor.get_dictionary_from_filesystem("datatype", "properties", profile=profile) stations = make_all_station_types(station_type_dict, data_type_dict, profile=profile) print len(stations), " (number of stations created)" print "Clearing mongo stations collection" remove_status = mongo_dao.remove_station_collection() print remove_status, " (None means station collection removal success)" station_collection = create_station_collection(stations) print station_collection, " (station collection pointer)" all_returned_stations = get_all_stations() print len(all_returned_stations), " (all stations in mongo)" ghcnm_returned_stations = get_all_stations_by_program("ghcnm") print len(ghcnm_returned_stations), " (ghcnm stations in mongo)" ushcn_returned_stations = get_all_stations_by_program("ushcn") print len(ushcn_returned_stations), " (ushcn stations in mongo)" northam_returned_stations = get_all_stations_by_program("northam") print len(northam_returned_stations), " (northam stations in mongo)" print "Testing adding geospatial index" result = add_station_geospatial_index() print result, " (result of adding station geospatial index)" print "Testing geoindex search" program = "ghcnm" # program for testing test1_stations = get_random_program_stations(program) print len(test1_stations), ' (length of test stations)' print test1_stations[0] status = write_station_records(test1_stations, program, utils.get_timestamp(), profile=profile) print status, '(status of effort)'
def make_stations_from_filesystem(profile=None): """Creates (recreates if preexisting) a station collection in mongo. """ remove_station_collection() station_type_dict = generic_processor.get_dictionary_from_filesystem("station", "properties", profile=profile) data_type_dict = generic_processor.get_dictionary_from_filesystem("datatype", "properties", profile=profile) stations = make_all_station_types(station_type_dict, data_type_dict, profile=profile) if stations: create_station_collection(stations) add_station_geospatial_index() return "Successfully created station collection." return "Failed to create station collection."
def make_all_station_types(station_types_dict, data_types_dict, profile=None): all_stations = [] for station_type in station_types_dict['station_types']: station_props = generic_processor.get_dictionary_from_filesystem(station_type, "metadata", profile=profile) station_props['keyword_converter'] = None data_types = data_types_dict[station_type] current_stations = make_stations(station_props, station_type, data_types) current_stations = modify_stations(station_type, current_stations) all_stations += current_stations return all_stations
def write_station_records(records, program, timestamp, profile=None): """Writes station records to a file, returning False if failure, full filepath if it succeeds. """ properties = generic_processor.get_dictionary_from_filesystem(program, "metadata", profile=profile) properties['output_file'] = [properties['output_directory'][0] + program + '.' + timestamp.replace('-', '.').replace(':', '.') + '.metadata'] station_translator = station_model.station_tuple_translator_closure(program, properties['type_name'][0], properties['field_names'], properties['field_precision']) stations = map(station_translator, records) write_success = file_dao.write_records(stations, properties) if write_success: return properties['output_file'][0] return False