options.max_buddies, options.max_buddy_distance) # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - exceptions_encountered = [] station_num = 0 for station in stations: ucanid = station['ucanid'] station_id = station['sid'] station_name = station['name'] if options.datasets == 'all': datasets = station['datasets'] station_num += 1 # hourly data file must already exist filepath = factory.getFilepathForUcanid(ucanid, 'hours') if not os.path.exists(filepath): print skipmsg % (station_num, total_stations, ucanid, station_id, station_name) errmsg = 'Hourly data file for station %d does not exist : %s' print errmsg % (ucanid, filepath) continue # gotta quit if there is no hourly data else: filepath = factory.getFilepathForUcanid(ucanid, 'statistics') if os.path.exists(filepath): if not replace_existing: print skipmsg % (station_num, total_stations, ucanid, station_id, station_name) errmsg = 'Statistics database for station %d already exists : %s' print errmsg % (ucanid, filepath) continue
factory = ObsnetDataFactory(options) stations = factory.argsToStationData(args, options, 'all') total_stations = len(stations) station_num = 0 for station in stations: if 'id' in station: station['sid'] = station['id'] del station['id'] ucanid = station['ucanid'] station_id = station['sid'] station_name = station['name'] station_num += 1 # hourly data file must already exist filepath = factory.getFilepathForUcanid(ucanid, 'hours') if not os.path.exists(filepath): print skipmsg % (station_num, total_stations, ucanid, station_id, station_name) errmsg = 'Hourly data file for station %d does not exist : %s' print errmsg % (ucanid, filepath) continue if replace_existing: stats_filepath = factory.getFilepathForUcanid(ucanid, 'statistics') if os.path.exists(stats_filepath): os.remove(stats_filepath) # open spike log file if not debug: log_filepath = factory.getFilepathForUcanid(ucanid, 'spikes') log_file = open(log_filepath, 'wt')
max_sample_size = durtions_days * durations_hours min_sample_size = int(max_sample_size * (1.0 - (percent_missing / 100.))) rel_hours_cushion = relativedelta(hours=hours_cushion) # create a factory, then use it to get the list of stations factory = ObsnetDataFactory(options) if len(args) > 0: ucanids = [int(arg) for arg in args] else: criteria = factory._validCriteria(options, SEARCH_KEYS) ucanids = factory.getStationIdsFromArgs(args, criteria) # apply rule to replace existing data files if replace_existing: for ucanid in ucanids: filepath = factory.getFilepathForUcanid(ucanid,'hour-stats') if os.path.exists(filepath): os.remove(filepath) else: ucanids = [uid for uid in ucanids if not os.path.exists(factory.getFilepathForUcanid(uid,'hour-stats'))] # process each station in the list for ucanid in ucanids: hours_manager = factory.getStationFileManager((ucanid,'hours'), 'r') if options.elements is None: elements = hours_manager.hdf5_file.keys() else: elements = stringToTuple(options.elements) stats_manager = factory.getStationFileManager((ucanid,'hour-stats'), 'w')
factory = ObsnetDataFactory(options) stations = factory.argsToStationData(args, options, 'all') total_stations = len(stations) station_num = 0 for station in stations: station_num += 1 ucanid = station['ucanid'] if 'id' in station: station['sid'] = station['id'] del station['id'] # hourly data file must already exist filepath = factory.getFilepathForUcanid(ucanid, 'hours') if not os.path.exists(filepath): print skipmsg % (station_num, total_stations, ucanid, station['sid'], station['name']) errmsg = 'Hourly data file for station %d does not exist : %s' print errmsg % (station['ucanid'], filepath) # open sequence log file if not debug: seq_log_filename = '%d_sequences.log' % ucanid seq_log_filepath = os.path.join(factory.config.working_dir, seq_log_filename) seq_log_file = open(seq_log_filepath, 'wt') else: seq_flog_file = None # we're going to process this station
procmsg = '\nprocessing station %d of %d : %d : %s (%s)' skipmsg = 'skipping station %d of %d : %d : %s (%s)' days_per_request = options.days_per_request replace_existing = options.replace_existing test_run = options.test if test_run: debug = True else: debug = options.debug station_num = 0 for station in stations: station_num += 1 ucan = UcanConnection(None, days_per_request) filepath = factory.getFilepathForUcanid(station['ucanid'], 'hours') if os.path.exists(filepath): if replace_existing: os.remove(filepath) else: print skipmsg % (station_num, total_stations, station['ucanid'], station['sid'], station['name']) continue # we're going to process this station print procmsg % (station_num, total_stations, station['ucanid'], station['sid'], station['name']) # get a manager for the new file manager = factory.getFileManager(filepath, 'w') manager.setFileAttribute('created', manager._timestamp()) manager.setFileAttributes(**station)