def __init__(self, options, **kwargs):
self.config = ConfigObject('config', None)
if isinstance(options, dict): opt_dict = options
else: opt_dict = optionsAsDict(options)
for key, value in opt_dict.items():
if not key.startswith('_') and value is not None:
self.config[key] = opt_dict[key]
for key, value in kwargs.items():
if not key.startswith('_') and value is not None:
self.config[key] = kwargs[key]
if 'pkg_path' not in self.config: self.config['pkg_path'] = PKG_DIRPATH
config_keys = self.config.keys()
if 'working_dir' not in config_keys:
self.config['working_dir'] = WORKING_DIR
if 'ManagerClass' not in config_keys:
self.config.newChild('ManagerClass')
self.config.ManagerClass.hdf5 = HDF5DataFileManager
self.config.ManagerClass.index = HDF5DataFileManager
self.config.ManagerClass.stats = HDF5DataFileManager
self.config.ManagerClass.statistics = HDF5DataFileManager
self.config.ManagerClass.hour = ObsnetDataFileManager
self.config.ManagerClass.hours = ObsnetDataFileManager
self.config.ManagerClass.day = ObsnetDataFileManager
self.config.ManagerClass.days = ObsnetDataFileManager
if 'station_filename_tmpl' not in config_keys:
self.config['station_filename_tmpl'] = '%d_%s.h5'
if 'station_index_file' not in config_keys:
index_filepath = os.path.join(self.config.working_dir,
'station_index.h5')
self.config['station_index'] = index_filepath
if 'debug' not in config_keys:
self.config.debug = False
self.debug = self.config.debug
import numpy as N
from rccpy.utils.config import ConfigObject
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
PRECIP_ELEMENTS = ('pcpn', )
TEMPERATURE_ELEMENTS = ('dewpt', 'dewpt_depr', 'srad', 'st4i', 'st8i', 'temp')
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
elements = ConfigObject('elements', None)
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
#
# elements contains configuration parameters that MUST be set for each element
#
# description : detailed description of element
# raw_type : used to specify how the dat is stored in the data files - expressed
# as a tuple (dtype, data units, missing value, frequency, interval)
# frequency : observation frequency (time span e.g.'hour','day',etc.)
# interval : number of time spans per data value
# serial_type : used to specify output data transforms for extracting data from
# hdf5 files via getSerialData() of HDF5DataFileManager an it's
# subclasses : expressed as tuple (dtype, units, missing values)
# tsvar_type : used for extracting data using UCAN interface:
# expressed as tuple (dtype, missing, units, tsv name, tsv units)
# generated data elements do not have tsvar_type
# units : used expressed as tuple (hdf5 type, hdf5 units, script type, script units)
# value_type : used to convey character and limits of observed data values
# expressed as tuple (data character, min reasonable value,
# validation.email.header.missing
# validation.email.header.missing[bcc, cc, mail_to, sender, signature, subject]
# validation.email.header.summary
# validation.email.header.summary[bcc, cc, mail_to, sender, signature, subject]
# validation.email.header.test
# validation.email.header.test[bcc, cc, mail_to, sender, signature, subject]
#
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
#
# initialize validation configuration
#
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
validation = ConfigObject('validation', None)
validation.constraints = (('network', '!=', 'icao'), ('active', '=', 'Y'))
validation.search_criteria = (('network', '!=', 'icao'), )
# station.(activate or deactivate or missing).(html or text)
activate = ConfigObject('activate', None, 'html', 'text')
deactivate = ConfigObject('deactivate', None, 'html', 'text')
missing = ConfigObject('missing', None, 'html', 'text')
station = ConfigObject('station', None, activate, deactivate, missing)
del activate, deactivate, missing
# summary.(activate or deactivate or missing)
summary = ConfigObject('summary', None, 'activate', 'deactivate', 'missing')
# content.(station or summary)
content = ConfigObject('content', None, station, summary)
# sensors.email.header.error[bcc, cc, mail_to, sender, signature, subject]
# sensors.email.header.summary
# sensors.email.header.summary[bcc, cc, mail_to, sender, signature, subject]
# sensors.email.header.test
# sensors.email.header.test[bcc, cc, mail_to, sender, signature, subject]
#
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
#
# initialize sensor error detection configuration
#
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
sensors = ConfigObject('sensors', None, 'email.header', 'email.content',
'email.content.error.html', 'email.content.error.text',
'email.content.summary.detail', 'pcpn')
# some attributes coomon to all sensors
sensors.constraints = (('network', '!=', 'icao'), ('active', '=', 'Y'))
sensors.end_hour = 7
sensors.grid_offset = 0.0625 # 5 x 5 grid, 20 km**2
sensors.search_criteria = (('network', '!=', 'icao'), )
sensors.start_hour = 8
sensors.station_metadata =\
'sid,name,ucanid,active,network,lat,lon,first_hour,last_report'
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
#
# precip specific attributes
#
class ObsnetDataFactory(object):
def __init__(self, options, **kwargs):
self.config = ConfigObject('config', None)
if isinstance(options, dict): opt_dict = options
else: opt_dict = optionsAsDict(options)
for key, value in opt_dict.items():
if not key.startswith('_') and value is not None:
self.config[key] = opt_dict[key]
for key, value in kwargs.items():
if not key.startswith('_') and value is not None:
self.config[key] = kwargs[key]
if 'pkg_path' not in self.config: self.config['pkg_path'] = PKG_DIRPATH
config_keys = self.config.keys()
if 'working_dir' not in config_keys:
self.config['working_dir'] = WORKING_DIR
if 'ManagerClass' not in config_keys:
self.config.newChild('ManagerClass')
self.config.ManagerClass.hdf5 = HDF5DataFileManager
self.config.ManagerClass.index = HDF5DataFileManager
self.config.ManagerClass.stats = HDF5DataFileManager
self.config.ManagerClass.statistics = HDF5DataFileManager
self.config.ManagerClass.hour = ObsnetDataFileManager
self.config.ManagerClass.hours = ObsnetDataFileManager
self.config.ManagerClass.day = ObsnetDataFileManager
self.config.ManagerClass.days = ObsnetDataFileManager
if 'station_filename_tmpl' not in config_keys:
self.config['station_filename_tmpl'] = '%d_%s.h5'
if 'station_index_file' not in config_keys:
index_filepath = os.path.join(self.config.working_dir,
'station_index.h5')
self.config['station_index'] = index_filepath
if 'debug' not in config_keys:
self.config.debug = False
self.debug = self.config.debug
# * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
def argsToStationData(self,
args,
options,
metadata='all',
filepath=None,
search_keys=SEARCH_KEYS,
sort_by=None):
criteria = self._validCriteria(options, search_keys)
num_args = len(args)
if num_args > 0:
# request for list of stations
if args[0].isdigit():
if metadata == 'ucanid': return [int(arg) for arg in args]
else:
stations = [
self.getStations(metadata, {
'ucanid': int(arg),
})[0] for arg in args
]
# request for range of stations from index file
elif ':' in args[0]:
stations = self.getStations(metadata, criteria)
# request for range of stations
stations = self._rangeOfStations(stations, args)
# request for stations from source other than index file
else:
filepath = args[0]
if filepath != 'newa' and filepath not in DIRECTORY_KEYS:
filepath = os.path.normpath(filepath)
stations = self.getStations(metadata, criteria, filepath)
# request was for all stations
if num_args > 1:
if ':' not in args[1]:
errmsg = 'Invalid argument list : "%s:'
raise ValueError, errmsg % ' '.join(args)
# request for range of stations
stations = self._rangeOfStations(stations, args[1:])
else:
stations = self.getStations(metadata, criteria, filepath)
if sort_by is not None and len(stations) > 1:
stations = self._sortStations(stations, sort_by)
if metadata == 'ucanid':
return [station['ucanid'] for station in stations]
else:
return stations
# * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
def backupIndexFile(self, keep_original=True):
index_filepath = self.getFilepath('index')
if not os.path.exists(index_filepath):
errmsg = 'Station index file not accessable : %s' % index_filepath
raise IOError, errmsg
time_str = datetime.now().strftime(CONFIG.backup_date_format)
index_rootpath, ext = os.path.splitext(index_filepath)
backup_filepath = self._backupFilePath(index_filepath)
try:
if keep_original:
os.system('cp %s %s' % (index_filepath, backup_filepath))
else:
os.system('mv %s %s' % (index_filepath, backup_filepath))
except:
e, param, ttback = sys.exc_info()
print e, param
print tback
errmsg = 'Unable to create backup file :', backup_filepath
raise IOError, errmsg
return index_filepath, backup_filepath
# * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
def fileExists(self, file_key_or_path):
filepath = self.getFilepath(file_key_or_path)
return os.path.isfile(filepath)
# * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
def getDirectoryPath(self, directory_key):
if directory_key in ('day', 'days'):
return os.path.join(self.config.working_dir, 'days')
elif directory_key in ('hour', 'hours'):
return os.path.join(self.config.working_dir, 'hours')
elif directory_key in ('index', 'work', 'working'):
return self.config.working_dir
elif directory_key in ('log', 'logs'):
return os.path.join(self.config.working_dir, 'logs')
elif directory_key in ('stats', 'statistics'):
return os.path.join(self.config.working_dir, 'statistics')
elif directory_key in ('test', 'tests'):
return os.path.join(self.config.working_dir, 'tests')
elif directory_key in ('scripts', 'validation'):
if directory_key in self.config:
return self.config[directory_key]
else:
return os.path.join(self.config['pkg_path'], directory_key)
else:
return os.path.join(self.config.working_dir, directory_key)
def getFileManager(self, file_key_or_path, mode='r'):
filepath = self.getFilepath(file_key_or_path)
if filepath is None:
raise ValueError, errmsg % file_key_or_path
if os.path.exists(filepath):
if mode == 'w':
return self.getManagerClass(file_key_or_path)(filepath, 'a')
else:
return self.getManagerClass(file_key_or_path)(filepath, mode)
else:
if mode == 'r': raise IOError, 'File not found : %s' % filepath
dirpath, filename = os.path.split(filepath)
if not os.path.exists(dirpath): os.makedirs(dirpath)
manager = self.getManagerClass(file_key_or_path)(filepath, 'w')
manager.setFileAttribute('created', manager._timestamp())
return manager
def getFilepath(self, file_key_or_path):
if isinstance(file_key_or_path, int):
return self.getFilepathForUcanid(file_key_or_path)
elif isinstance(file_key_or_path, (tuple, list)):
return self.getFilepathForUcanid(*file_key_or_path)
elif isinstance(file_key_or_path, basestring):
if file_key_or_path.endswith('.h5'):
return file_key_or_path
elif file_key_or_path == 'index':
return self.config.station_index
return None
def getFilepathForUcanid(self, ucanid, file_type='hours'):
if not isinstance(ucanid, int): ucaind = int(ucanid)
filename_tmpl = FILEPATH_TEMPLATES.get(file_type, None)
if filename_tmpl is None:
filepath = DEFAULT_FILEPATH_TEMPLATE % (ucanid, file_type)
else:
filepath = filename_tmpl % ucanid
filepath = os.path.join(self.config.working_dir, filepath)
dirpath = os.path.split(filepath)[0]
if not os.path.exists(dirpath): os.makedirs(dirpath)
return filepath
def getManagerClass(self, file_key_or_path):
if isinstance(file_key_or_path, (tuple, list)):
filetype = file_key_or_path[1]
elif file_key_or_path.endswith('.h5'):
filetype = 'hdf5'
else:
filetype = file_key_or_path
if filetype in self.config.ManagerClass.keys():
return self.config.ManagerClass[filetype]
errmsg = 'Unable to detect file type for %s' % str(file_key_or_path)
raise ValueError, errmsg
# * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
def getStations(self,
metadata='ucanid',
criteria=None,
filepath=None,
sort_by=None):
if filepath is None or filepath == 'index':
return self.getIndexedStations(metadata, criteria, sort_by)
elif filepath == 'newa':
return self.getNewaStations(metadata, criteria, sort_by)
elif filepath == 'ucan':
return self.getUcanStations(metadata, criteria, sort_by)
elif isinstance(filepath, (list, tuple)):
return self.getStationsInDirectory(filepath[0], filepath[1],
metadata, criteria, sort_by)
try:
return self.getStationsInDirectory(filepath, metadata, criteria,
sort_by)
except ValueError:
return self.readStationListFile(filepath, metadata, criteria,
sort_by)
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
def getIndexedStations(self, metadata='all', criteria=None, sort_by=None):
manager = self.getFileManager('index', mode='r')
dataset_names = self._parseMetadata(metadata, manager)
_criteria = self._validCriteria(criteria, dataset_names)
datasets = manager.getDataWhere(dataset_names, _criteria)
manager.closeFile()
stations = []
for stn_indx in range(len(datasets[0])):
station = {}
for indx in range(len(dataset_names)):
station[safeDataKey(dataset_names[indx])] =\
safevalue(datasets[indx][stn_indx])
stations.append(station)
if sort_by is not None:
stations = self._sortStations(stations, sort_by)
return stations
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
def getNewaStations(self, metadata=None, criteria=None, sort_by=None):
from newa.services import NEWA_METADATA, NewaWebServicesClient
client = NewaWebServicesClient()
stations = []
for station in client.request('stationList', 'all'):
station = safedict(station, True)
station['sid'] = str(station['id']) # str() converts numeric ids
del station['id']
stations.append(station)
del client
if metadata is None:
stations = self._constrain(stations, NEWA_METADATA, criteria)
else:
NEWA_METADATA
stations = self._constrain(stations, metadata, criteria)
if sort_by is not None:
stations = self._sortStations(stations, sort_by)
return stations
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
def getUcanStations(self, metadata='all', criteria=None, sort_by=None):
from rccpy.stations.ucan import UcanConnection, UnknownUcanId
dataset_names = self._parseMetadata(metadata)
missing_county = INDEX.county.missing
missing_gmt = INDEX.gmt.missing
stations = []
ucan = UcanConnection()
for station in self.getNewaStations(dataset_names, criteria):
if 'sid' in dataset_names: station['sid'] = station['id']
del station['id']
try:
ucanid = ucan.ucanid(station)
except KeyError as e:
if self.debug:
msg = 'Unable to acquire UCAN ID for %s : %s'
print msg % (station['sid'], station['name'])
continue
else:
if 'ucanid' in dataset_names:
station['ucanid'] = ucanid
try:
meta = ucan.getMetadata(station)
except UnknownUcanId:
if self.debug:
print 'No metadata avaiable for UCAN ID :', ucanid
if 'county' in dataset_names:
station['county'] = missing_county
if 'gmt' in dataset_names:
station['gmt'] = missing_gmt
else:
if 'county' in dataset_names:
station['county'] = meta['county']
if 'gmt' in dataset_names:
gmt = meta['gmt_offset']
if gmt > 0: station['gmt'] = -gmt
else: station['gmt'] = gmt
del ucan
if sort_by is not None:
stations = self._sortStations(stations, sort_by)
return stations
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
def getStationsInDirectory(self,
directory_key_or_path,
metadata='all',
criteria=None,
file_type=None,
sort_by=None):
dataset_names = self._parseMetadata(metadata)
_criteria = self._validCriteria(criteria, station.keys())
constraints = dictToConstraints(_criteria, ') and (')
if os.path.exists(directory_key_or_path):
directory_path = directory_key_or_path
else:
directory_path = self.getDirectoryPath(directory_key_or_path)
if file_type is None:
file_suffix = '_%s.h5' % directory_key_or_path
else:
file_suffix = '_%s.h5' % file_type
has_elements = 'hour' in file_suffix
filepaths = []
for name in os.listdir(directory_path):
if name.endswith(file_suffix):
filepaths.append(os.path.join(directory_path, name))
if len(filepaths) < 1: return ()
stations = []
ManagerClass = self.getManagerClass(directory_key_or_path)
for filepath in filepaths:
manager = ManagerClass(filepaths[0], 'r')
attrs = manager.getFileAttributes()
manager.closeFile()
if constraints is None or eval(constraints, globals(), attrs):
station = {}
for key, value in attrs:
if key in dataset_names: station[key] = value
if has_elements and 'elements' in dataset_names:
station['elements'] = ','.join(manager.listGroups())
stations.append(station)
if sort_by is not None:
stations = self._sortStations(stations, sort_by)
return stations
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
def readStationListFile(self,
filepath,
metadata='all',
criteria=None,
sort_by=None):
dataset_names = self._parseMetadata(metadata)
dataset_names.sort()
_criteria = self._validCriteria(criteria, station.keys())
constraints = dictToConstraints(_criteria, ') and (')
stations = []
filename, file_ext = os.path.splitext(filepath)
stn_file = open(filepath, 'r')
if file_ext == '.json':
for station in json.loads(stn_file.read())['stations']:
station = safedict(station)
if 'id' in station:
station['sid'] = station['id']
del station['id']
if constraints is not None or eval(constraints, globals(),
station):
stations.append(station)
else:
for station in eval(stn_file.read().replace('\n',
'').replace(' ', '')):
station = safedict(station)
if 'id' in station:
station['sid'] = station['id']
del station['id']
if constraints is not None or eval(constraints, globals(),
station):
stations.append(station)
stn_file.close()
station_datasets = stations[0].keys()
station_datasets.sort()
if station_datasets != dataset_names:
for station in stations:
for key in station:
if key not in dataset_names: del statin[key]
if sort_by is not None:
stations = self._sortStations(stations, sort_by)
return stations
# * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
def createHourlyDataGroup(self, manager, element, data, start_hour,
end_hour, **kwargs):
# create element group
attrs = {}
attrs['created'] = manager._timestamp()
attrs['description'] = kwargs.get('description', DESCRIPTIONS[element])
attrs['first_hour'] = start_hour.timetuple()[:4]
attrs['frequency'] = 'hour'
attrs['interval'] = 1
attrs['last_hour'] = end_hour.timetuple()[:4]
manager.createGroup(element, attrs)
# generate the dates array and use it to create a dataset
dataset_name = '%s.date' % element
hours = generateHoursArray(start_hour,
end_hour,
date_format=int,
interval=1,
as_numpy=True)
manager.createDataset(dataset_name, hours, attrs)
manager.setDatasetAttribute(dataset_name, 'description',
'Year,Month,Day,Hour')
manager.setDatasetAttribute(dataset_name, 'date_formula',
DATE_FORMULAE['hour'])
# cretae value dataset
dataset_name = '%s.value' % element
for attr_name, attr_value in kwargs.items():
if attr_name not in attrs: attrs[attr_name] = attr_value
if 'value_type' not in attrs:
attrs['value_type'] = VALUE_TYPES[element]
if 'missing' not in attrs: attrs['missing'] = -32768
if 'units' not in attrs: attrs['units'] = HOURLY_DATA_TYPES[element][2]
manager.createDataset(dataset_name, self.transformData(element, data),
attrs)
# * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
def getConfigParameter(self, parameter_name):
return self.config.get(parameter_name, None)
def transformData(self, element, data, to_dataset=True):
if to_dataset:
if element in PRECIP_ELEMENTS: data *= 100.
elif element in TEMPERATURE_ELEMENTS: data *= 10.
data[N.where(N.isnan(data))] = -32768
data[N.where(N.isinf(data))] = -32768
return N.array(data, dtype='i2')
else:
data = N.array(data, dtype=float)
data[data == -32768] = N.nan
if element in PRECIP_ELEMENTS: data /= 100.
elif element in TEMPERATURE_ELEMENTS: data /= 10.
return data
# * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
def _backupFilePath(self, filepath):
time_str = datetime.now().strftime(CONFIG.backup_date_format)
rootpath, ext = os.path.splitext(filepath)
backup_filepath = '%s_%s%s' % (rootpath, time_str, ext)
return backup_filepath
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
def _constrain(self, stations, metadata, criteria, manager=None):
if criteria is not None:
_criteria = self._validCriteria(criteria, metadata)
constraints = dictToConstraints(_criteria, ' & ')
stations = [
station for station in stations
if eval(constraints, globals(), station)
]
if metadata != 'all':
dataset_names = self._parseMetadata(metadata, manager)
for station in stations:
for key in station:
if key not in dataset_names:
del station[key]
return stations
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
def _getFrequencyFromFilename(self, filename):
for frequency in VALID_FREQUENCIES:
if frequency in filename: return frequency
if 'dbm' in filename: return 'hour'
raise KeyError, 'Cannot determine frequency from file name'
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
def _parseMetadata(self, metadata, manager=None):
if isinstance(metadata, basestring):
if metadata == 'all':
if manager is not None:
return tuple(manager.listDatasets())
else:
return RAW_DATA_ELEMENTS
else:
return stringToTuple(metadata)
elif isinstance(metadata, (list, tuple)):
return tuple(metadata)
else:
errmsg = "'metadata' argument is an invalid type: %s"
raise TypeError, errmsg % type(metadata)
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
def _rangeOfStations(self, stations, *args):
""" return range of stations as sepcified in args)
"""
gop = 'gt'
lop = 'lt'
ucanid_low = 0
ucanid_high = 9999999999
if ':' in args[0]:
op, limit = args[arg_indx].split(':')
if limit.isdigit():
if op in ('ge', 'gt'):
gop = op
ucanid_low = int(limit)
elif op in ('le', 'lt'):
lop = op
ucanid_high = int(limit)
else:
raise ValueError, 'Invalid argument : %s' % args[0]
if len(args) > 1:
op, limit = args[1].split(':')
if limit.isdigit():
if gop and op in ('le', 'lt'):
lop = op
ucanid_high = int(limit)
elif lop and op in ('ge', 'gt'):
lop = op
ucanid_low = int(limit)
else:
raise ValueError, 'Invalid argument : %s' % args[arg_indx]
else:
raise ValueError, 'Invalid argument list : "%s:' % ' '.join(args)
if gop == 'ge':
if lop == 'le':
return [
station for station in stations
if station['ucanid'] >= ucanid_low
and station['ucanid'] <= ucanid_high
]
else:
return [
station for station in stations
if station['ucanid'] >= ucanid_low
and station['ucanid'] < ucanid_high
]
else:
if lop == 'le':
return [
station for station in stations
if station['ucanid'] > ucanid_low
and station['ucanid'] <= ucanid_high
]
else:
return [
station for station in stations
if station['ucanid'] > ucanid_low
and station['ucanid'] < ucanid_high
]
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
def _sortStations(self, stations, sort_by):
if isinstance(sort_by, basestring):
sort_keys = [key.strip() for key in sort_by.split(',')]
elif isinstance(sort_by, (list, tuple)):
sort_keys = sort_by
else:
errmsg = "'sort_by' argument is an invalid type: %s"
raise TypeError, errmsg % type(sort_by)
keys = ["station['%s']" % key for key in sort_keys]
template = '"(%s,)"' % ','.join(keys)
def sortBy(station):
return eval(template, globals(), station)
return sorted(stations, key=sortBy)
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
def _validCriteria(self, criteria, dataset_names):
if criteria is not None:
valid_criteria = {}
if isinstance(criteria, optparse.Values):
for key, value in vars(criteria).items():
if value is None: continue
if (key == 'bbox' or key in dataset_names):
valid_criteria[key] = value
elif isinstance(criteria, dict):
for key, value in criteria.items():
if (key == 'bbox' or key in dataset_names):
valid_criteria[key] = value
elif isinstance(criteria, (list, tuple)):
valid_criteria = []
for rule in criteria:
if rule[0] in dataset_names:
valid_criteria.append(rule)
if valid_criteria:
valid_criteria = tuple(valid_criteria)
if valid_criteria: return valid_criteria
return None
from rccpy.utils.config import ConfigObject
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
#
# initialize validation configuration
#
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
validation = ConfigObject('validation',None)
validation.constraints = (('network','!=','icao'),('active','=','Y'))
validation.search_criteria = (('network','!=','icao'),)
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
#
# initialize limits for data extremes
#
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
extremes = ConfigObject('extremes',None)
extremes.stddevs = {
'invalid' : {'daily_pcpn':10,'dewpt':7,'dewpt_depr':7,'lwet':7,'pcpn':10,
'rhum':7,'srad':7,'st4i':7,'st8i':7,'temp':7,'wdir':7,'wspd':7},
'suspect' : {'daily_pcpn':7,'dewpt':4,'dewpt_depr':4,'lwet':4,'pcpn':7,
'rhum':4,'srad':4,'st4i':4,'st8i':4,'temp':4,'wdir':4,'wspd':4},
}
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
#
# initialize sequence filters and limits
#
import numpy as N
from rccpy.utils.config import ConfigObject
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
PRECIP_ELEMENTS = ('pcpn', )
TEMPERATURE_ELEMENTS = ('dewpt', 'dewpt_depr', 'srad', 'st4i', 'st8i', 'temp')
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
datasets = ConfigObject('datasets', None)
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
#
# datasets contains configuration parameters that MUST be set for each dataset
#
# description : detailed description of dataset
# raw_type : used to specify how the dat is stored in the data files - expressed
# as a tuple (dtype, data units, missing value, frequency, interval)
# frequency : observation frequency (time span e.g.'hour','day',etc.)
# interval : number of time spans per data value
# serial_type : used to specify output data transforms for extracting data from
# hdf5 files via getSerialData() of HDF5DataFileManager an it's
# subclasses : expressed as tuple (dtype, units, missing values)
# tsvar_type : used for extracting data using UCAN interface:
# expressed as tuple (dtype, missing, units, tsv name, tsv units)
# generated data datasets do not have tsvar_type
# units : used expressed as tuple (hdf5 type, hdf5 units, script type, script units)
# value_type : used to convey character and limits of observed data values
# expressed as tuple (data character, min reasonable value,
from rccpy.utils.config import ConfigObject
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
DEFAULT_DATASETS = ('lwet','pcpn','rhum','srad','st4i','st8i','temp','wdir','wspd')
DEFAULT_METADATA = ('network','state','id','name','lon','lat','elev')
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
networks = ConfigObject('networks',None)
networks.cu_log = { 'metadata' : DEFAULT_METADATA,
'elements' : DEFAULT_DATASETS,
}
networks.icao = { 'metadata' : DEFAULT_METADATA,
'elements' : ('pcpn','rhum','temp','wdir','wspd'),
}
networks.newa = { 'metadata' : DEFAULT_METADATA,
'elements' : DEFAULT_DATASETS,
}
networks.newa.coordinators = { 'CT' : { 'contact' : 'Mary Concklin',
'email' : '*****@*****.**' },
'MA' : { 'contact' : 'Jon Clements ',
'email' : '*****@*****.**' },
'PA' : { 'contact' : 'Rob Crassweller ',
'email' : '*****@*****.**' },
'NY' : { 'contact' : 'Juliet Carroll ',
'email' : '*****@*****.**' },
'VT' : { 'contact' : 'Terry Bradshaw ',
from rccpy.utils.config import ConfigObject
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
#
# initialize configuration of services
#
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
services = ConfigObject('services', None, 'web', 'email')
services.web.acis = 'http://data.rcc-acis.org/'
services.web.newa = 'http://newa.nrcc.cornell.edu/newaUtil/'
#services.email.smtp_host = 'virga.nrcc.cornell.edu'
services.email.smtp_host = 'appsmtp.mail.cornell.edu'