def updateWithUcanMetadata(station, ucan_connection=None, update_datasets=True): """ Update a station dictionary with metadata from Ucan/Tsvar """ if ucan_connection is not None: ucan = ucan_connection else: ucan = UcanConnection() ucanid = ucan.ucanid(station) station['ucanid'] = ucanid station_keys = station.keys() meta = ucan.getMetadata(station) for key in ('county', 'elev', 'lat', 'lon'): if key not in station_keys: station[key] = meta[key] if 'state' not in station_keys: station['state'] = meta['postal'] if 'gmt' not in station_keys: gmt = int(meta['gmt_offset']) if gmt > 0: gmt = -gmt station['gmt'] = gmt datasets = [] first_hour = 9999999999 last_report = 0 for dataset_name in CONFIG.raw_datasets: try: ts_var = ucan.getTsVar(station, dataset_name) except: pass else: datasets.append(dataset_name) start_hour, end_hour = ts_var.getValidDateRange() first_hour = min(first_hour, dateAsInt(start_hour, True)) last_report = max(last_report, dateAsInt(end_hour, True)) del ts_var if ucan_connection is not None: del ucan if update_datasets: station['datasets'] = ','.join(datasets) if first_hour == 9999999999: station['first_hour'] = -32768 else: station['first_hour'] = first_hour if last_report == 0: station['last_report'] = -32768 else: station['last_report'] = last_report return station
def updateLastReport(self, station): valid_hours = station['valid_hours'] if len(valid_hours) > 0: last_hour = max(valid_hours) last_hour = self.start_time + relativedelta(hours=last_hour) last_report = dateAsInt(last_hour, True) self.last_report_column[station['index']] = last_report station['last_report'] = last_report return station
def getStationPrecip(factory, station, _date_, cushion=0, test_run=False): _datetime_ = asDatetime(_date_) start_time = _datetime_ - ONE_DAY + relativedelta(hours=7) end_time = _datetime_ + relativedelta(hours=7) if cushion != 0: start_time -= relativedelta(hours=cushion) end_time += relativedelta(hours=cushion) if test_run: if station['sid'] == STATION_CACHE['sid']: data = STATION_CACHE['data'] dates = STATION_CACHE['dates'] else: # hourly data file must already exist filepath = factory.getFilepathForUcanid(station['ucanid'], 'hours') if not os.path.exists(filepath): print SKIP_MSG % station print 'Hourly data file does not exist : %s' % filepath return None manager = HDF5DataFileManager(filepath, 'r') data = manager.getData('pcpn.value') STATION_CACHE['data'] = data dates = manager.getData('pcpn.date') STATION_CACHE['dates'] = dates STATION_CACHE['sid'] = station['sid'] _start_time = dateAsInt(start_time, True) _end_time = dateAsInt(end_time, True) indexes = N.where((dates > (_start_time - 1)) & (dates < _end_time)) if len(indexes[0]) > 0: return data[indexes] return None else: start_time = dateAsTuple(start_time, True) end_time = dateAsTuple(end_time, True) ucan = HourlyDataConnection(days_per_request=1) _start_date_, _end_date_, precip =\ ucan.getData(station, 'pcpn', start_time, end_time) precip = N.array(precip) precip[N.where(N.isinf(precip))] = N.nan return precip
(where, exception_type, formatted, details)) else: if log_file: log_file.close() msg = 'Created spike log file for %s at %s' print msg % (station_name, log_filepath) # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - if debug: print 'locate buddies for %d' % ucanid try: num_buddies =\ buddy_locator.findAndSave(station['ucanid'], station['lon'], station['lat'], dateAsInt(station['first_hour']), dateAsInt(station['last_report'])) except: where = 'buddy locator for station %d' % ucanid exception_type, formatted, details = captureLastException() if True: #debug: print where print exception_type if details is not None: print details print ''.join(formatted) os._exit(99) else: exceptions_encountered.append( (where, exception_type, formatted, details)) else: print ' %d buddies found for %d' % (num_buddies, ucanid)
# make connection to UCAN server connection = HourlyDataConnection(2, first_hour_in_day=1) end_time = base_time start_time = end_time - relativedelta(hours=23) invalid = True while invalid: missing = [ ] for element in raw_elements: #if debug: print '\nelement', element first_hour, last_hour, data = \ connection.getData(station,element,start_time,end_time,debug) num_hours = len(data) if num_hours == 24 and len(N.where(N.isfinite(data))[0]) > 0: station['last_report'] = dateAsInt(start_time) invalid = False break end_time -= ONE_DAY start_time -= ONE_DAY dump_file = open(dump_filepath, 'w') dump_file.write(repr(stations[0])) for station in stations[1:]: dump_file.write('\n') dump_file.write(repr(station)) dump_file.close()
today = datetime.now() today = datetime(today.year, today.month, today.day, 23) icao_base_time = today - ONE_DAY # date is input, use it as the latest possible report date if len(args) > 0: year = int(args[0]) month = int(args[1]) day = int(args[2]) if len(args) > 3: newa_base_time = datetime(year, month, day, int(args[3])) else: newa_base_time = datetime(year, month, day, 23) # otherwise, latest_possible report date is yesterday else: newa_base_time = today - ONE_DAY test_last_day = dateAsInt(newa_base_time) # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - criteria = ('bbox', 'county', 'name', 'network', 'sid', 'state') factory = ObsnetDataFactory(options) criteria = factory._validCriteria(options, criteria) metadata = list(factory._parseMetadata(options.metadata)) if 'datasets' not in metadata: metadata.append('datasets') if 'first_hour' not in metadata: metadata.append('first_hour') if 'last_report' not in metadata: metadata.append('last_report') if 'name' not in metadata: metadata.append('name') if 'sid' not in metadata: metadata.append('sid') if 'state' not in metadata: metadata.append('state') if 'ucanid' not in metadata: metadata.append('ucanid')
stats_manager = factory.getStationFileManager((ucanid,'hour-stats'), 'w') stats_manager.setFileAttributes(**hours_manager.getFileAttributes()) stats_manager.setFileAttribute('created', stats_manager._timestamp()) earliest_hour = 9999999999 latest_hour = -32768 for element in elements: if debug: print ' ' print ' ' # get start/end time hours_attrs = hours_manager.getDataAttributes('%s.date' % element) first_hour = tuple(hours_attrs['first_hour']) earliest_hour = min(earliest_hour, dateAsInt(first_hour)) base_hour = datetime(*first_hour) last_hour = tuple(hours_attrs['last_hour']) latest_hour = max(latest_hour, dateAsInt(last_hour)) last_hour = datetime(*last_hour) num_years = (last_hour.year - first_hour.year) + 1 if num_years < min_year_span: print SKIPPING_MSG % (element, ucanid) print TOO_SHORT_MSG % (min_year_span, num_years) continue else: print PROCESSING_MSG % (element, ucanid) raw_data, raw_attrs = hours_manager.getData('%s.value' % element, True)
def latestReportDate(station, newa_base_time, icao_base_time, debug=False): """ Returns latest date that any sensor reported """ last_report = 0 if debug: print '\nprocessing station %(sid)s : %(name)s' % station # get list of datasets to check sensor_types = [sensor_type.strip() for sensor_type in station['datasets'].split(',')] sensor_types.sort() # make connection to UCAN server connection = HourlyDataConnection() # reset start/end times if station['network'] == 'icao': requested_end_time = icao_base_time else: requested_end_time = newa_base_time # looking for latest date that any data sensor reported for sensor_type in sensor_types: # find out the range of dates available for this dataset data_start_time, data_end_time =\ connection.getValidDatetimeRange(station, sensor_type, False) if debug: msg = ' %s : ucan advertised time span = %s thru %s' print msg % (sensor_type, data_start_time.strftime('%Y%m%d:%H'), data_end_time.strftime('%Y%m%d:%H')) end_time = min(requested_end_time, data_end_time) start_time = end_time - relativedelta(hours=23) if debug: msg = ' %s : interval start time = %s, interval end time = %s' print msg % (sensor_type, start_time.strftime('%Y%m%d%H'), end_time.strftime('%Y%m%d%H')) # get data for first interval data_from, data_to, data = \ connection.getData(station, sensor_type, start_time, end_time, False) data_from = dateAsInt(data_from,True) data_to = dateAsInt(data_to,True) # work backwards to latest finite value returned, if any _continue_ = True while _continue_: # must contain 24 hours to be valid data if len(data) == 24: if debug: msg = ' %s : data returned for period %d to %d' print msg % (sensor_type, data_from, data_to) # look for latest hour that a finite data value was returned for hour in range(23,-1,-1): if N.isfinite(data[hour]): valid_time = start_time + relativedelta(hours=hour) valid_time = dateAsInt(valid_time, True) last_report = max(valid_time, last_report) if debug: msg = ' %s : finite value (%-.2f) found at data[%d] : %d' print msg % (sensor_type, data[hour], hour, valid_time) _continue_ = False break else: if debug: msg = ' %s : ALL data missing in period %d to %d' print msg % (sensor_type, data_from, data_to) # go back one day and check again start_time -= ONE_DAY if (int(start_time.strftime('%Y%m%d%H')) > last_report and start_time >= data_start_time): end_time -= ONE_DAY data_from, data_to, data = \ connection.getData(station,sensor_type,start_time,end_time,False) data_from = dateAsInt(data_from,True) data_to = dateAsInt(data_to,True) else: _continue_ = False # delete the UCAN connection for this station del connection return last_report
options, args = parser.parse_args() # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # if len(args) > 0: year = int(args[0]) month = int(args[1]) day = int(args[2]) if len(args) > 3: first_hour_in_day = int(args[3]) end_time = datetime(year, month, day, 23) start_time = datetime(year, month, day, 0) else: date = datetime.now() - ONE_DAY end_time = datetime(date.year, date.month, date.day, 23) start_time = datetime(date.year, date.month, date.day, 0) end_time_as_int = dateAsInt(end_time, True) date_str = end_time.strftime('%B %d, %Y') prev_date = dateAsInt(end_time - ONE_DAY, True) debug = options.debug verbose = options.verbose if verbose: print 'verbose debug output requested' networks = stringToTuple(options.networks) test_run = options.test_run update_index = options.update_index # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - # create factory and get required data from index file factory = ObsnetDataFactory(options) index_manager = factory.getFileManager('index', 'r')
parser.add_option('-w', action='store', type='string', dest='working_dir', default=None) parser.add_option('-y', action='store_true', dest='test', default=False) parser.add_option('-z', action='store_true', dest='debug', default=False) options, args = parser.parse_args() # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # year = int(args[0]) month = int(args[1]) day = int(args[2]) if len(args) > 3: hour = int(args[3]) else: hour = 23 max_report_date = dateAsInt((year, month, day)) max_report_time = (max_report_date * 100) + hour # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - factory = ObsnetDataFactory(options) index_manager = factory.getFileManager('index', mode='r') last_reports = index_manager.getData('last_report') index_manager.closeFile() for indx in range(len(last_reports)): last_report = last_reports[indx] if last_report == max_report_date: last_reports[indx] = max_report_time elif last_report > max_report_time: