#! /Users/rem63/venvs/nrcc_prod/bin/python import os, sys from newa.manager import ObsnetDataFileManager from newa.factory import ObsnetDataFactory # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # from optparse import OptionParser parser = OptionParser() parser.add_option('-z', action='store_true', dest='debug', default=False, help='show all available debug output') options, args = parser.parse_args() # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # factory = ObsnetDataFactory(options) dirpath = factory.getDirectoryPath('hours') for filename in os.listdir(dirpath): if filename.endswith('.h5'): print 'fixing', filename manager = ObsnetDataFileManager(os.path.join(dirpath,filename),mode='a') if 'daily_pcpn' in manager.listGroups(): manager.setDatasetAttribute('daily_pcpn.value','missing',-32768) manager.closeFile()
default=None) parser.add_option('-w', action='store', type='string', dest='working_dir', default=None) parser.add_option('-y', action='store_true', dest='test', default=False) parser.add_option('-z', action='store_true', dest='debug', default=False) options, args = parser.parse_args() # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # if os.path.isfile(os.path.normpath(args[0])): input_filepath = os.path.normpath(args[0]) dirpath, filename = os.path.split(input_filepath) else: factory = ObsnetDataFactory(options) dirpath = factory.getDirectoryPath('updates') filename = args[0] input_filepath = os.path.join(dirpath, filename) if options.output_filepath is None: output_filepath = '%s.dump' % os.path.splitext(input_filepath)[0] else: output_filepath = os.path.abspath(option.output_filepath) stations = [ ] station_ids = [ ] index_dict = { } for station in readDumpFile(input_filepath): # make sure not to add any new data columns for key in station:
date_str = download_date.strftime('%Y%m%d') time_str = datetime.now().strftime('%y%m%d.%H%M') debug = options.debug # get path to station index file index_filepath = factory.getFilepath('index') if not os.path.exists(index_filepath): raise IOError, 'Station index file not accessable : %s' % index_filepath # make a backup copy of index file if create_backup: factory.backupIndexFile() # get log directory path log_dirpath = os.path.join(factory.getDirectoryPath('working'), 'updates') if not os.path.exists(log_dirpath): os.makedirs(log_dirpath) # get file path for change log change_log_name = '%s_changes.log' % time_str change_log_path = os.path.join(log_dirpath, change_log_name) # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # get 'sid' dataset from index file manager = factory.getFileManager('index', mode='r') sids = manager.getData('sid') all_datasets = manager.listDatasets() index_map = dict([(enum[1],enum[0]) for enum in enumerate(sids)])
# list of states to update states = options.states if states is None: states = CONFIG.metadata.states else: if ',' in states: states = tuple([state.strip().upper() for state in states.split(',')]) elif len(states) == 2: states = (states.upper(), ) else: errmsg = 'Value of input option -s is invalid : %s' % states raise ValueError, errmsg # creat a factory instance and get directory/file paths factory = ObsnetDataFactory(options) index_dirpath = factory.getDirectoryPath('index') download_dirpath = os.path.join(index_dirpath, 'downloads') if not os.path.exists(download_dirpath): os.makedirs(download_dirpath) # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # date_str = datetime.now().strftime('%Y%m%d') for state in states: filepath = downloadMetadata(root_url, state, date_str, file_format, download_dirpath, debug=debug) if filepath is not None: