print msg % ('new', elem_string)
index_datasets[station_index] = elem_string
num_changed += 1
ucanid_list = tuple(do_over)
do_over = [ ]
attempt += 1
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
num_left = len(ucanid_list)
attempt -= 1
if num_left == 0:
print '\nCompleted discovery after %d attempts.' % attempt
if num_changed > 0:
factory.backupIndexFile()
msg = 'Differences were discovered in %d of %d stations.'
print msg % (num_changed, num_stations)
if update:
index_manager = factory.getFileManager('index', mode='a')
index_manager.replaceDataset('datasets', index_datasets,
datasets_attrs)
index_manager.closeFile()
print 'The database has been updated.'
else:
print 'The database was not updated during this test run.'
else:
print 'No changes were discovered in the element list of any station.'
print 'The database was not updated.'
else:
print '\n******************************************************************'
'ucanid': ucanids[station_index],
}
is_missing = last_reports[station_index] < 0
if missing_only and not is_missing: continue
print 'updating last report date for :\n', station
last_report = latestReportDate(station, newa_base_time, icao_base_time,
debug)
# set last_reports to latest date found
if last_report != last_reports[station_index]:
last_reports[station_index] = last_report
num_changed += 1
station['last'] = last_report
print message % station
sys.stdout.flush()
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
if num_changed > 0 and not test_run:
backup_filepath = factory.backupIndexFile()
print 'Index file backed up to', backup_filepath
print 'Last report date will be updated for %d stations.' % num_changed
manager = factory.getFileManager('index', mode='a')
manager.replaceDataset('last_report', last_reports, last_report_attrs)
manager.closeFile()
print 'Station index file has been updated.'
print factory.getFilepath('index')
# create the index array dictionary
arrays = { }
for key in INDEX_KEYS:
arrays[key] = [ ]
# populate the index arrays with station data
for station in stations:
for key in INDEX_KEYS:
if key in station:
arrays[key].append(station[key])
else:
arrays[key].append(MISSING[key])
# save the index arrays to the index file
factory = ObsnetDataFactory(options)
if factory.fileExists('index'): factory.backupIndexFile()
index_manager = factory.getFileManager('index', mode='w')
for key, dataset in INDEX.items():
print 'creating array for', key
data = N.array(arrays[key], dtype=dataset.data_type)
attrs = { 'missing' : dataset.missing,
'description' : dataset.description,
}
if key in ('lon','lat','elev'):
valid = data[N.where(N.isfinite(data))]
attrs['min'] = N.min(valid)
attrs['max'] = N.max(valid)
if dataset.units is not None: attrs['units'] = dataset.units
factory = ObsnetDataFactory(options)
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
date_str = download_date.strftime('%Y%m%d')
time_str = datetime.now().strftime('%y%m%d.%H%M')
debug = options.debug
# get path to station index file
index_filepath = factory.getFilepath('index')
if not os.path.exists(index_filepath):
raise IOError, 'Station index file not accessable : %s' % index_filepath
# make a backup copy of index file
if create_backup: factory.backupIndexFile()
# get log directory path
log_dirpath = os.path.join(factory.getDirectoryPath('working'), 'updates')
if not os.path.exists(log_dirpath):
os.makedirs(log_dirpath)
# get file path for change log
change_log_name = '%s_changes.log' % time_str
change_log_path = os.path.join(log_dirpath, change_log_name)
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
# get 'sid' dataset from index file
manager = factory.getFileManager('index', mode='r')
sids = manager.getData('sid')
print 'new index will contain %d stations' % num_stations
# fill the column list with data from each station
columns = dict([(name, []) for name in column_names])
for station in stations:
for name in column_names:
columns[name].append(station[name])
# make sure that all of the new columns are the correct length
for name, data in columns.items():
if len(data) != num_stations:
errmsg = '%s column has %d entries, but there are %d stations'
raise RuntimeError, errmsg % (name, len(data), num_stations)
# backup the existing index file
index_filepath, backup_filepath = factory.backupIndexFile(keep_original=False)
time_str = datetime.now().strftime('%y%m%d.%H%M')
# create a new version of the index file that contains valid stations
# from the log file
manager = factory.getFileManager('index', 'w')
update_time = manager._timestamp()
for name, data in columns.items():
attrs = {
'created': column_created[name],
'updated': update_time,
'description': INDEX[name].description,
'missing': MISSING[name],
}
units = INDEX[name].units
if units: attrs['units'] = units