options.max_buddies, options.max_buddy_distance)
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
exceptions_encountered = []
station_num = 0
for station in stations:
ucanid = station['ucanid']
station_id = station['sid']
station_name = station['name']
if options.datasets == 'all': datasets = station['datasets']
station_num += 1
# hourly data file must already exist
filepath = factory.getFilepathForUcanid(ucanid, 'hours')
if not os.path.exists(filepath):
print skipmsg % (station_num, total_stations, ucanid, station_id,
station_name)
errmsg = 'Hourly data file for station %d does not exist : %s'
print errmsg % (ucanid, filepath)
continue # gotta quit if there is no hourly data
else:
filepath = factory.getFilepathForUcanid(ucanid, 'statistics')
if os.path.exists(filepath):
if not replace_existing:
print skipmsg % (station_num, total_stations, ucanid,
station_id, station_name)
errmsg = 'Statistics database for station %d already exists : %s'
print errmsg % (ucanid, filepath)
continue
factory = ObsnetDataFactory(options)
stations = factory.argsToStationData(args, options, 'all')
total_stations = len(stations)
station_num = 0
for station in stations:
if 'id' in station:
station['sid'] = station['id']
del station['id']
ucanid = station['ucanid']
station_id = station['sid']
station_name = station['name']
station_num += 1
# hourly data file must already exist
filepath = factory.getFilepathForUcanid(ucanid, 'hours')
if not os.path.exists(filepath):
print skipmsg % (station_num, total_stations, ucanid, station_id,
station_name)
errmsg = 'Hourly data file for station %d does not exist : %s'
print errmsg % (ucanid, filepath)
continue
if replace_existing:
stats_filepath = factory.getFilepathForUcanid(ucanid, 'statistics')
if os.path.exists(stats_filepath): os.remove(stats_filepath)
# open spike log file
if not debug:
log_filepath = factory.getFilepathForUcanid(ucanid, 'spikes')
log_file = open(log_filepath, 'wt')
max_sample_size = durtions_days * durations_hours
min_sample_size = int(max_sample_size * (1.0 - (percent_missing / 100.)))
rel_hours_cushion = relativedelta(hours=hours_cushion)
# create a factory, then use it to get the list of stations
factory = ObsnetDataFactory(options)
if len(args) > 0:
ucanids = [int(arg) for arg in args]
else:
criteria = factory._validCriteria(options, SEARCH_KEYS)
ucanids = factory.getStationIdsFromArgs(args, criteria)
# apply rule to replace existing data files
if replace_existing:
for ucanid in ucanids:
filepath = factory.getFilepathForUcanid(ucanid,'hour-stats')
if os.path.exists(filepath): os.remove(filepath)
else:
ucanids = [uid for uid in ucanids
if not os.path.exists(factory.getFilepathForUcanid(uid,'hour-stats'))]
# process each station in the list
for ucanid in ucanids:
hours_manager = factory.getStationFileManager((ucanid,'hours'), 'r')
if options.elements is None:
elements = hours_manager.hdf5_file.keys()
else:
elements = stringToTuple(options.elements)
stats_manager = factory.getStationFileManager((ucanid,'hour-stats'), 'w')
factory = ObsnetDataFactory(options)
stations = factory.argsToStationData(args, options, 'all')
total_stations = len(stations)
station_num = 0
for station in stations:
station_num += 1
ucanid = station['ucanid']
if 'id' in station:
station['sid'] = station['id']
del station['id']
# hourly data file must already exist
filepath = factory.getFilepathForUcanid(ucanid, 'hours')
if not os.path.exists(filepath):
print skipmsg % (station_num, total_stations, ucanid,
station['sid'], station['name'])
errmsg = 'Hourly data file for station %d does not exist : %s'
print errmsg % (station['ucanid'], filepath)
# open sequence log file
if not debug:
seq_log_filename = '%d_sequences.log' % ucanid
seq_log_filepath = os.path.join(factory.config.working_dir,
seq_log_filename)
seq_log_file = open(seq_log_filepath, 'wt')
else: seq_flog_file = None
# we're going to process this station
procmsg = '\nprocessing station %d of %d : %d : %s (%s)'
skipmsg = 'skipping station %d of %d : %d : %s (%s)'
days_per_request = options.days_per_request
replace_existing = options.replace_existing
test_run = options.test
if test_run: debug = True
else: debug = options.debug
station_num = 0
for station in stations:
station_num += 1
ucan = UcanConnection(None, days_per_request)
filepath = factory.getFilepathForUcanid(station['ucanid'], 'hours')
if os.path.exists(filepath):
if replace_existing: os.remove(filepath)
else:
print skipmsg % (station_num, total_stations, station['ucanid'],
station['sid'], station['name'])
continue
# we're going to process this station
print procmsg % (station_num, total_stations, station['ucanid'],
station['sid'], station['name'])
# get a manager for the new file
manager = factory.getFileManager(filepath, 'w')
manager.setFileAttribute('created', manager._timestamp())
manager.setFileAttributes(**station)
