Exemplo n.º 1
0
def make_stations_from_metadata(profile=None):
    print "Making stations from filesystem"
    station_type_dict = generic_processor.get_dictionary_from_filesystem("station",
                                                                         "properties",
                                                                         profile=profile)
    data_type_dict = generic_processor.get_dictionary_from_filesystem("datatype",
                                                                      "properties",
                                                                      profile=profile)
    stations = make_all_station_types(station_type_dict, data_type_dict, profile=profile)
    print len(stations), " (number of stations created)"
    print "Clearing mongo stations collection"
    remove_status = mongo_dao.remove_station_collection()
    print remove_status, " (None means station collection removal success)"
    station_collection = create_station_collection(stations)
    print station_collection, " (station collection pointer)"
    all_returned_stations = get_all_stations()
    print len(all_returned_stations), " (all stations in mongo)"
    ghcnm_returned_stations = get_all_stations_by_program("ghcnm")
    print len(ghcnm_returned_stations), " (ghcnm stations in mongo)"
    ushcn_returned_stations = get_all_stations_by_program("ushcn")
    print len(ushcn_returned_stations), " (ushcn stations in mongo)"
    northam_returned_stations = get_all_stations_by_program("northam")
    print len(northam_returned_stations), " (northam stations in mongo)"
    print "Testing adding geospatial index"
    result = add_station_geospatial_index()
    print result, " (result of adding station geospatial index)"
    print "Testing geoindex search"
    program = "ghcnm"  # program for testing
    test1_stations = get_random_program_stations(program)
    print len(test1_stations), ' (length of test stations)'
    print test1_stations[0]
    status = write_station_records(test1_stations, program, utils.get_timestamp(), profile=profile)
    print status, '(status of effort)'
Exemplo n.º 2
0
def make_stations_from_filesystem(profile=None):
    """Creates (recreates if preexisting) a station collection in mongo.
    """
    remove_station_collection()
    station_type_dict = generic_processor.get_dictionary_from_filesystem("station",
                                                                         "properties",
                                                                         profile=profile)
    data_type_dict = generic_processor.get_dictionary_from_filesystem("datatype",
                                                                      "properties",
                                                                      profile=profile)
    stations = make_all_station_types(station_type_dict, data_type_dict, profile=profile)
    if stations:
        create_station_collection(stations)
        add_station_geospatial_index()
        return "Successfully created station collection."
    return "Failed to create station collection."
Exemplo n.º 3
0
def make_all_station_types(station_types_dict, data_types_dict, profile=None):
    all_stations = []
    for station_type in station_types_dict['station_types']:
        station_props = generic_processor.get_dictionary_from_filesystem(station_type,
                                                                         "metadata",
                                                                         profile=profile)
        station_props['keyword_converter'] = None
        data_types = data_types_dict[station_type]
        current_stations = make_stations(station_props, station_type, data_types)
        current_stations = modify_stations(station_type, current_stations)
        all_stations += current_stations
    return all_stations
Exemplo n.º 4
0
def write_station_records(records, program, timestamp, profile=None):
    """Writes station records to a file, returning False if failure, full filepath
    if it succeeds.
    """
    properties = generic_processor.get_dictionary_from_filesystem(program, "metadata",
                                                                  profile=profile)
    properties['output_file'] = [properties['output_directory'][0] +
                                 program + '.' +
                                 timestamp.replace('-', '.').replace(':', '.') +
                                 '.metadata']
    station_translator = station_model.station_tuple_translator_closure(program,
                                                                        properties['type_name'][0],
                                                                        properties['field_names'],
                                                                        properties['field_precision'])
    stations = map(station_translator, records)
    write_success = file_dao.write_records(stations, properties)
    if write_success:
        return properties['output_file'][0]
    return False