def updateWithUcanMetadata(station,
ucan_connection=None,
update_datasets=True):
""" Update a station dictionary with metadata from Ucan/Tsvar
"""
if ucan_connection is not None:
ucan = ucan_connection
else:
ucan = UcanConnection()
ucanid = ucan.ucanid(station)
station['ucanid'] = ucanid
station_keys = station.keys()
meta = ucan.getMetadata(station)
for key in ('county', 'elev', 'lat', 'lon'):
if key not in station_keys: station[key] = meta[key]
if 'state' not in station_keys: station['state'] = meta['postal']
if 'gmt' not in station_keys:
gmt = int(meta['gmt_offset'])
if gmt > 0: gmt = -gmt
station['gmt'] = gmt
datasets = []
first_hour = 9999999999
last_report = 0
for dataset_name in CONFIG.raw_datasets:
try:
ts_var = ucan.getTsVar(station, dataset_name)
except:
pass
else:
datasets.append(dataset_name)
start_hour, end_hour = ts_var.getValidDateRange()
first_hour = min(first_hour, dateAsInt(start_hour, True))
last_report = max(last_report, dateAsInt(end_hour, True))
del ts_var
if ucan_connection is not None: del ucan
if update_datasets:
station['datasets'] = ','.join(datasets)
if first_hour == 9999999999: station['first_hour'] = -32768
else: station['first_hour'] = first_hour
if last_report == 0: station['last_report'] = -32768
else: station['last_report'] = last_report
return station
def updateLastReport(self, station):
valid_hours = station['valid_hours']
if len(valid_hours) > 0:
last_hour = max(valid_hours)
last_hour = self.start_time + relativedelta(hours=last_hour)
last_report = dateAsInt(last_hour, True)
self.last_report_column[station['index']] = last_report
station['last_report'] = last_report
return station
def getStationPrecip(factory, station, _date_, cushion=0, test_run=False):
_datetime_ = asDatetime(_date_)
start_time = _datetime_ - ONE_DAY + relativedelta(hours=7)
end_time = _datetime_ + relativedelta(hours=7)
if cushion != 0:
start_time -= relativedelta(hours=cushion)
end_time += relativedelta(hours=cushion)
if test_run:
if station['sid'] == STATION_CACHE['sid']:
data = STATION_CACHE['data']
dates = STATION_CACHE['dates']
else:
# hourly data file must already exist
filepath = factory.getFilepathForUcanid(station['ucanid'], 'hours')
if not os.path.exists(filepath):
print SKIP_MSG % station
print 'Hourly data file does not exist : %s' % filepath
return None
manager = HDF5DataFileManager(filepath, 'r')
data = manager.getData('pcpn.value')
STATION_CACHE['data'] = data
dates = manager.getData('pcpn.date')
STATION_CACHE['dates'] = dates
STATION_CACHE['sid'] = station['sid']
_start_time = dateAsInt(start_time, True)
_end_time = dateAsInt(end_time, True)
indexes = N.where((dates > (_start_time - 1)) & (dates < _end_time))
if len(indexes[0]) > 0: return data[indexes]
return None
else:
start_time = dateAsTuple(start_time, True)
end_time = dateAsTuple(end_time, True)
ucan = HourlyDataConnection(days_per_request=1)
_start_date_, _end_date_, precip =\
ucan.getData(station, 'pcpn', start_time, end_time)
precip = N.array(precip)
precip[N.where(N.isinf(precip))] = N.nan
return precip
(where, exception_type, formatted, details))
else:
if log_file:
log_file.close()
msg = 'Created spike log file for %s at %s'
print msg % (station_name, log_filepath)
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
if debug: print 'locate buddies for %d' % ucanid
try:
num_buddies =\
buddy_locator.findAndSave(station['ucanid'], station['lon'],
station['lat'],
dateAsInt(station['first_hour']),
dateAsInt(station['last_report']))
except:
where = 'buddy locator for station %d' % ucanid
exception_type, formatted, details = captureLastException()
if True: #debug:
print where
print exception_type
if details is not None: print details
print ''.join(formatted)
os._exit(99)
else:
exceptions_encountered.append(
(where, exception_type, formatted, details))
else:
print ' %d buddies found for %d' % (num_buddies, ucanid)
# make connection to UCAN server
connection = HourlyDataConnection(2, first_hour_in_day=1)
end_time = base_time
start_time = end_time - relativedelta(hours=23)
invalid = True
while invalid:
missing = [ ]
for element in raw_elements:
#if debug: print '\nelement', element
first_hour, last_hour, data = \
connection.getData(station,element,start_time,end_time,debug)
num_hours = len(data)
if num_hours == 24 and len(N.where(N.isfinite(data))[0]) > 0:
station['last_report'] = dateAsInt(start_time)
invalid = False
break
end_time -= ONE_DAY
start_time -= ONE_DAY
dump_file = open(dump_filepath, 'w')
dump_file.write(repr(stations[0]))
for station in stations[1:]:
dump_file.write('\n')
dump_file.write(repr(station))
dump_file.close()
today = datetime.now()
today = datetime(today.year, today.month, today.day, 23)
icao_base_time = today - ONE_DAY
# date is input, use it as the latest possible report date
if len(args) > 0:
year = int(args[0])
month = int(args[1])
day = int(args[2])
if len(args) > 3: newa_base_time = datetime(year, month, day, int(args[3]))
else: newa_base_time = datetime(year, month, day, 23)
# otherwise, latest_possible report date is yesterday
else:
newa_base_time = today - ONE_DAY
test_last_day = dateAsInt(newa_base_time)
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
criteria = ('bbox', 'county', 'name', 'network', 'sid', 'state')
factory = ObsnetDataFactory(options)
criteria = factory._validCriteria(options, criteria)
metadata = list(factory._parseMetadata(options.metadata))
if 'datasets' not in metadata: metadata.append('datasets')
if 'first_hour' not in metadata: metadata.append('first_hour')
if 'last_report' not in metadata: metadata.append('last_report')
if 'name' not in metadata: metadata.append('name')
if 'sid' not in metadata: metadata.append('sid')
if 'state' not in metadata: metadata.append('state')
if 'ucanid' not in metadata: metadata.append('ucanid')
stats_manager = factory.getStationFileManager((ucanid,'hour-stats'), 'w')
stats_manager.setFileAttributes(**hours_manager.getFileAttributes())
stats_manager.setFileAttribute('created', stats_manager._timestamp())
earliest_hour = 9999999999
latest_hour = -32768
for element in elements:
if debug:
print ' '
print ' '
# get start/end time
hours_attrs = hours_manager.getDataAttributes('%s.date' % element)
first_hour = tuple(hours_attrs['first_hour'])
earliest_hour = min(earliest_hour, dateAsInt(first_hour))
base_hour = datetime(*first_hour)
last_hour = tuple(hours_attrs['last_hour'])
latest_hour = max(latest_hour, dateAsInt(last_hour))
last_hour = datetime(*last_hour)
num_years = (last_hour.year - first_hour.year) + 1
if num_years < min_year_span:
print SKIPPING_MSG % (element, ucanid)
print TOO_SHORT_MSG % (min_year_span, num_years)
continue
else:
print PROCESSING_MSG % (element, ucanid)
raw_data, raw_attrs = hours_manager.getData('%s.value' % element, True)
def latestReportDate(station, newa_base_time, icao_base_time, debug=False):
""" Returns latest date that any sensor reported
"""
last_report = 0
if debug:
print '\nprocessing station %(sid)s : %(name)s' % station
# get list of datasets to check
sensor_types = [sensor_type.strip()
for sensor_type in station['datasets'].split(',')]
sensor_types.sort()
# make connection to UCAN server
connection = HourlyDataConnection()
# reset start/end times
if station['network'] == 'icao':
requested_end_time = icao_base_time
else: requested_end_time = newa_base_time
# looking for latest date that any data sensor reported
for sensor_type in sensor_types:
# find out the range of dates available for this dataset
data_start_time, data_end_time =\
connection.getValidDatetimeRange(station, sensor_type, False)
if debug:
msg = ' %s : ucan advertised time span = %s thru %s'
print msg % (sensor_type, data_start_time.strftime('%Y%m%d:%H'),
data_end_time.strftime('%Y%m%d:%H'))
end_time = min(requested_end_time, data_end_time)
start_time = end_time - relativedelta(hours=23)
if debug:
msg = ' %s : interval start time = %s, interval end time = %s'
print msg % (sensor_type, start_time.strftime('%Y%m%d%H'),
end_time.strftime('%Y%m%d%H'))
# get data for first interval
data_from, data_to, data = \
connection.getData(station, sensor_type, start_time, end_time, False)
data_from = dateAsInt(data_from,True)
data_to = dateAsInt(data_to,True)
# work backwards to latest finite value returned, if any
_continue_ = True
while _continue_:
# must contain 24 hours to be valid data
if len(data) == 24:
if debug:
msg = ' %s : data returned for period %d to %d'
print msg % (sensor_type, data_from, data_to)
# look for latest hour that a finite data value was returned
for hour in range(23,-1,-1):
if N.isfinite(data[hour]):
valid_time = start_time + relativedelta(hours=hour)
valid_time = dateAsInt(valid_time, True)
last_report = max(valid_time, last_report)
if debug:
msg = ' %s : finite value (%-.2f) found at data[%d] : %d'
print msg % (sensor_type, data[hour], hour, valid_time)
_continue_ = False
break
else:
if debug:
msg = ' %s : ALL data missing in period %d to %d'
print msg % (sensor_type, data_from, data_to)
# go back one day and check again
start_time -= ONE_DAY
if (int(start_time.strftime('%Y%m%d%H')) > last_report
and start_time >= data_start_time):
end_time -= ONE_DAY
data_from, data_to, data = \
connection.getData(station,sensor_type,start_time,end_time,False)
data_from = dateAsInt(data_from,True)
data_to = dateAsInt(data_to,True)
else: _continue_ = False
# delete the UCAN connection for this station
del connection
return last_report
options, args = parser.parse_args()
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
if len(args) > 0:
year = int(args[0])
month = int(args[1])
day = int(args[2])
if len(args) > 3: first_hour_in_day = int(args[3])
end_time = datetime(year, month, day, 23)
start_time = datetime(year, month, day, 0)
else:
date = datetime.now() - ONE_DAY
end_time = datetime(date.year, date.month, date.day, 23)
start_time = datetime(date.year, date.month, date.day, 0)
end_time_as_int = dateAsInt(end_time, True)
date_str = end_time.strftime('%B %d, %Y')
prev_date = dateAsInt(end_time - ONE_DAY, True)
debug = options.debug
verbose = options.verbose
if verbose: print 'verbose debug output requested'
networks = stringToTuple(options.networks)
test_run = options.test_run
update_index = options.update_index
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# create factory and get required data from index file
factory = ObsnetDataFactory(options)
index_manager = factory.getFileManager('index', 'r')
parser.add_option('-w', action='store', type='string', dest='working_dir',
default=None)
parser.add_option('-y', action='store_true', dest='test', default=False)
parser.add_option('-z', action='store_true', dest='debug', default=False)
options, args = parser.parse_args()
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
year = int(args[0])
month = int(args[1])
day = int(args[2])
if len(args) > 3: hour = int(args[3])
else: hour = 23
max_report_date = dateAsInt((year, month, day))
max_report_time = (max_report_date * 100) + hour
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
factory = ObsnetDataFactory(options)
index_manager = factory.getFileManager('index', mode='r')
last_reports = index_manager.getData('last_report')
index_manager.closeFile()
for indx in range(len(last_reports)):
last_report = last_reports[indx]
if last_report == max_report_date:
last_reports[indx] = max_report_time
elif last_report > max_report_time:
