#! /Users/rem63/venvs/nrcc_prod/bin/python
import os, sys
from newa.manager import ObsnetDataFileManager
from newa.factory import ObsnetDataFactory
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
from optparse import OptionParser
parser = OptionParser()
parser.add_option('-z', action='store_true', dest='debug', default=False,
help='show all available debug output')
options, args = parser.parse_args()
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
factory = ObsnetDataFactory(options)
dirpath = factory.getDirectoryPath('hours')
for filename in os.listdir(dirpath):
if filename.endswith('.h5'):
print 'fixing', filename
manager = ObsnetDataFileManager(os.path.join(dirpath,filename),mode='a')
if 'daily_pcpn' in manager.listGroups():
manager.setDatasetAttribute('daily_pcpn.value','missing',-32768)
manager.closeFile()
default=None)
parser.add_option('-w', action='store', type='string', dest='working_dir',
default=None)
parser.add_option('-y', action='store_true', dest='test', default=False)
parser.add_option('-z', action='store_true', dest='debug', default=False)
options, args = parser.parse_args()
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
if os.path.isfile(os.path.normpath(args[0])):
input_filepath = os.path.normpath(args[0])
dirpath, filename = os.path.split(input_filepath)
else:
factory = ObsnetDataFactory(options)
dirpath = factory.getDirectoryPath('updates')
filename = args[0]
input_filepath = os.path.join(dirpath, filename)
if options.output_filepath is None:
output_filepath = '%s.dump' % os.path.splitext(input_filepath)[0]
else: output_filepath = os.path.abspath(option.output_filepath)
stations = [ ]
station_ids = [ ]
index_dict = { }
for station in readDumpFile(input_filepath):
# make sure not to add any new data columns
for key in station:
date_str = download_date.strftime('%Y%m%d')
time_str = datetime.now().strftime('%y%m%d.%H%M')
debug = options.debug
# get path to station index file
index_filepath = factory.getFilepath('index')
if not os.path.exists(index_filepath):
raise IOError, 'Station index file not accessable : %s' % index_filepath
# make a backup copy of index file
if create_backup: factory.backupIndexFile()
# get log directory path
log_dirpath = os.path.join(factory.getDirectoryPath('working'), 'updates')
if not os.path.exists(log_dirpath):
os.makedirs(log_dirpath)
# get file path for change log
change_log_name = '%s_changes.log' % time_str
change_log_path = os.path.join(log_dirpath, change_log_name)
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
# get 'sid' dataset from index file
manager = factory.getFileManager('index', mode='r')
sids = manager.getData('sid')
all_datasets = manager.listDatasets()
index_map = dict([(enum[1],enum[0]) for enum in enumerate(sids)])
# list of states to update
states = options.states
if states is None:
states = CONFIG.metadata.states
else:
if ',' in states:
states = tuple([state.strip().upper() for state in states.split(',')])
elif len(states) == 2:
states = (states.upper(), )
else:
errmsg = 'Value of input option -s is invalid : %s' % states
raise ValueError, errmsg
# creat a factory instance and get directory/file paths
factory = ObsnetDataFactory(options)
index_dirpath = factory.getDirectoryPath('index')
download_dirpath = os.path.join(index_dirpath, 'downloads')
if not os.path.exists(download_dirpath): os.makedirs(download_dirpath)
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
date_str = datetime.now().strftime('%Y%m%d')
for state in states:
filepath = downloadMetadata(root_url,
state,
date_str,
file_format,
download_dirpath,
debug=debug)
if filepath is not None: