def updateTurfModelFiles(factory, threat_key, replace_prev, verbose, debug): threat_fullname = factory.threatName(threat_key) if threat_key == 'hstress': periods = ('daily', ) else: periods = ('daily', 'average') num_periods = len(periods) MODEL_START_TIME = datetime.datetime.now() common_obs_end = factory.commonObsEnd(threat_key, TODAY) max_obs_end = maxReanalysisEndTime(factory, threat_key) while max_obs_end > common_obs_end: # most common case max_obs_end -= HOURS_IN_DAY max_obs_start = max_obs_end - HOURS_TO_START max_obs_date = max_obs_start.date() if debug: print '\nupdateTurfModelFiles debug info >>' print ' commonObsEnd :', common_obs_end print ' max_obs_end :', max_obs_end print ' max_obs_start :', max_obs_start print ' max_obs_date :', max_obs_date max_fcast_time = maxForecastEndTime(factory, threat_key) common_fcast_end = factory.commonFcastEnd(threat_key, max_fcast_time) if debug: print ' max_fcast_time :', max_fcast_time print ' commonFcastEnd :', common_fcast_end print '\nProcessing %s risk :' % threat_fullname period_count = 0 for period_key in periods: PERIOD_START_TIME = datetime.datetime.now() key_dates = updateTurfModelPeriod(threat_key, period_key, max_obs_date, common_fcast_end, max_fcast_time, verbose, debug) # period_start, fcast_start, period_end = key_dates elapsed_time = elapsedTime(PERIOD_START_TIME, True) print ' Completed %s %s grid file update in %s' % ( threat_fullname, period_key.title(), elapsed_time) if verbose: print ' data start = %s : fcast start = %s : data end = %s' % key_dates period_count += 1 if period_count < num_periods: print ' ' elapsed_time = elapsedTime(MODEL_START_TIME, True) print '\nCompleted %s grid updates in %s' % (threat_fullname, elapsed_time)
def completeProcess(self, retcode): elapsed_time = elapsedTime(self.process_start_time, True) if retcode > 0: print FAILURE_MSG % (elapsed_time, self.active_script, retcode) else: print COMPLETE_MSG % (elapsed_time, self.active_script) sys.stdout.flush() self.active_script = None
def __call__(self, request): start_response = datetime.datetime.now() # decode request variables into a dictionary request_dict = self.requestAsDict(request) print '\n\nGrapeHardinessTempextHandler.__call__' # extract location coordinates location = self.extractLocationParameters(request_dict) lat, lon = location['coords'] print ' location :\n', location target_year = request_dict.get('season', None) print '\n target_year :', target_year # get the configured season limits dates = self.extractSeasonDates(request_dict) print ' dates :\n', dates season_start = asDatetimeDate(dates['season_start']) season_end = asDatetimeDate(dates['season_end']) target_year = dates['season'] del dates['season'] # initialize response string with season dates response = \ '{"tempexts":{%s,"data":{' % self.tightJsonString(dates)[1:-1] reader = \ self.tempextFileReader(target_year, self.source, self.region) if self.mode in ('dev', 'test'): print 'tempexts file :', reader.filepath # add recent averages data = \ reader.getSliceAtNode('mint', season_start, season_end, lon, lat) response = \ '%s"mint":%s' % (response, self.serializeData(data, '%.1f')) # add climate normal averages data = \ reader.getSliceAtNode('maxt', season_start, season_end, lon, lat) response = \ '%s,"maxt":%s' % (response, self.serializeData(data, '%.1f')) reader.close() del data if self.mode in ('dev', 'test'): print 'tempexts data retrieved in', elapsedTime( start_response, True) print '\n\nTEMPEXTS RESPONSE\n', response, '\n\n' self.respondWithJSON(request, '%s}}}' % response)
date_args = tuple([int(n) for n in args[0].split('.')]) if utc_date: # input date is already UTC corrected end_hour = datetime.datetime(*date_args) else: hour = tz.localize(datetime.datetime(*date_args)) end_hour = hour.astimezone(pytz.utc) else: hour = tz.localize(datetime.datetime.now()) end_hour = hour.astimezone(pytz.utc) if debug: print 'num_hours', num_hours print 'end_hour', end_hour utc_hour = end_hour - datetime.timedelta(hours=num_hours - 1) while utc_hour <= end_hour: if debug: print '\nprocessing download for', utc_hour files = downloadRTMA(utc_hour, server, verbose, debug) file_count += len(files) utc_hour += ONE_HOUR elapsed_time = elapsedTime(download_start, True) print '\ncompleted downloaded %d in %s' % (file_count, elapsed_time) transport_dirpath = \ '/Volumes/Transport/data/app_data/shared/reanalysis/conus/rtma' if file_count > 0 and os.path.exists(transport_dirpath): command = \ '/usr/bin/rsync -cgloprtuD %s %s' % (RTMA_DIRPATH, transport_dirpath) print '\n', command os.system(command)
default=False) parser.add_option('-n', action='store_true', dest='use_ndfd_cache', default=False) parser.add_option('-v', action='store_true', dest='verbose', default=False) parser.add_option('-z', action='store_true', dest='debug', default=False) options, args = parser.parse_args() # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # debug = options.debug use_dev_paths = options.use_dev_paths use_ndfd_cache = options.use_ndfd_cache verbose = options.verbose or debug factory = NDFDProjectFactory() if use_dev_paths: factory.config.dirpaths.update(factory.config.dev_dirpaths) if use_ndfd_cache: factory.setServerUrl(factory.ndfd_config.cache_server) target_date, filepaths = factory.downloadLatestForecast(True) if verbose: print 'NDFD grib files downloaded for %s :', str(target_date) for filepath in filepaths: print ' ', filepath elapsed_time = elapsedTime(DOWNLOAD_START_TIME, True) fmt = 'completed download for %s in %s' print fmt % (target_date.isoformat(), elapsed_time)
#factory = TurfThreatGridFileFactory(source, region, sub_region) factory = TurfThreatJsonGeneratorFactory() if dev_mode: factory.useDirpathsForMode('dev') grand_total = 0 # generate JSON files for threats that have both daily and average risk data generator = factory.riskFileGenerator('avg,daily', source, region, sub_region) for threat in ('anthrac', 'bpatch', 'dspot', 'pblight'): threat_fullname = factory.threatName(threat) JSON_START_TIME = datetime.datetime.now() count = generator(threat, target_year, debug) if count > 0: elapsed_time = elapsedTime(JSON_START_TIME, True) info = (threat_fullname, count, elapsed_time) print 'Generated JSON files for %s @ %d grid nodes in %s' % info grand_total += count else: print 'No valid data for %s yet.' % threat_fullname # generate JSON files for threats that have only daily risk data threat_fullname = factory.threatName('hstress') generator = factory.riskFileGenerator('daily', source, region, sub_region) JSON_START_TIME = datetime.datetime.now() count = generator('hstress', 'daily', target_year, debug) if count > 0:
if debug: print ' building hardtemp dataset' builder.config.datasets.hardtemp.chunks = chunks data = reader.dateSlice('hardiness.temp', season_start, season_end, dtype=float, missing=N.nan, units='F') builder.open('a') builder.buildDataset('hardtemp', data=data, start_date=season_start, end_date=season_end) builder.close() attrs = reader.dateAttributes('hardiness.temp') attrs['start_date'] = season_start_str attrs['end_date'] = season_end_str attrs['units'] = 'F' if debug: print ' setting hardtemp attributes :\n ', attrs builder.open('a') builder.setDatasetAttributes('hardtemp', **attrs) builder.close() # turn annoying numpy warnings back on warnings.resetwarnings() msg = '... finished building files for temp extremes and %d grape varieties in %s' print msg % (len(project.varieties), elapsedTime(build_start, True))
if len(N.where(N.isnan(daily))[0]) == 0: lat = N.round(lats[y, x], 3) lon = N.round(lons[y, x], 3) params = {'lat': lat, 'lon': lon} params['daily'] = ','.join(['%d' % value for value in daily]) filename = json_filename % { 'node': factory.gridNodeToFilename((lon, lat)), } filepath = os.path.join(json_dirpath, filename) with open(filepath, 'w') as writer: writer.write(json_template % params) num_files += 1 if verbose: print num_files, filepath else: num_nans = len(N.where(N.isnan(daily))[0]) location = (y, x, lons[y, x], lats[y, x]) if num_nans == daily.size: print 'All NAN daily @ node[%s,%d] (%.5f, %.5f)' % location else: print '%d NANs in daily @ node[%s,%d] (%.5f, %.5f)' % ( num_nans, ) + location threat_fullname = factory.threatName(threat) elapsed_time = elapsedTime(JSON_START_TIME, True) info = (threat_fullname, num_files, elapsed_time) print '\nGenerated %s JSON files for %d grid nodes in %s' % info
from optparse import OptionParser parser = OptionParser() parser.add_option('-v', action='store_true', dest='verbose', default=False) parser.add_option('-z', action='store_true', dest='debug', default=False) options, args = parser.parse_args() # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # debug = options.debug verbose = options.verbose region_key = args[0] factory = AcisProjectFactory() bbox = factory.config.regions[region_key].data source = factory.getSourceConfig('ndfd') builder = factory.getStaticFileBuilder('ndfd', region_key) builder.initFileAttributes() data = factory.getAcisGridData(source_key, 'mint', date, None, False, meta=('ll','elev'), bbox=bbox, debug=debug) print builder.filepath builder.build(True, True, data['lon'], data['lat'], elev_data=data['elev'], bbox=bbox) elapsed_time = elapsedTime(BUILD_START_TIME, True) print 'completed build of %s static file in' % source.tag, elapsed_time
drawScatterMap(risk, lats, lons, **map_options) mapped = True else: index = reader._dateToIndex('risk', date) print 'No data for %s (index=%d)' % (date.strftime('%Y-%m-%d'), index) mapped = False if mapped: finishMap(fig, axes, fig1, **options) if verbose: print ' ', map_filepath else: print ' ', map_filename thumb_filename = thumbfile_template.replace('||DATE||', file_date_str) thumb_filepath = os.path.join(thumb_dirpath, thumb_filename) image = PIL.Image.open(map_filepath) image.thumbnail(thumbnail_shape, PIL.Image.ANTIALIAS) image.save(thumb_filepath, 'PNG', quality=100, optimize=True) if debug: print ' ', thumb_filepath date += ONE_DAY days += 1 # turn annoying numpy warnings back on warnings.resetwarnings() info = (days, start_date, end_date) print '\nProcessed maps for %d days : %s thru %s' % info elapsed_time = elapsedTime(MAPS_START_TIME, True) info = (threat_fullname, elapsed_time) print 'Completed %s risk maps in %s' % info
def __call__(self, request): start_response = datetime.datetime.now() # decode request variables into a dictionary request_dict = self.requestAsDict(request) # extract location coordinates location = request_dict['location'] coords = location.get('coords', None) if coords is not None: lat = float(coords[0]) lon = float(coords[1]) else: lat = float(location['lat']) lon = float(location['lon']) # GDD threshold and target_year gdd_threshold = str(request_dict['gdd_threshold']) target_year = request_dict.get('season', None) #if target_year is None: target_year = datetime.date.today().year if target_year is None: target_year = self.maxAvailableYear() # get the configured season limits dates = self.extractSeasonDates(request_dict, target_year) # initialize the response response_json = \ '{"season":{"gdd_threshold":"%s"' % gdd_threshold response_json = \ '%s,"location":%s' % (response_json,self.tightJsonString(location)) # create a POR file reader reader = \ self.getTargetYearFileReader(target_year, self.source, self.region) if self.mode in ('dev', 'test'): print 'season data file :', reader.filepath # create path to GDD dataset dataset_path = reader.gddDatasetPath(gdd_threshold) # capture the significant dates for the dataset if 'dates' in self.mode_config \ and target_year == self.mode_config.season: dates.update(self.mode_config.dates.attrs) else: dates.update(reader.getSignificantDates(dataset_path)) season_end = dates['season_end'] end_date = asDatetimeDate(season_end) start_date = asDatetimeDate(dates['season_start']) if 'fcast_start' in dates: fcast_start = asDatetimeDate(dates['fcast_start']) if fcast_start > end_date: del dates['fcast_start'] if 'fcast_end' in dates: fcast_end = asDatetimeDate(dates['fcast_end']) if fcast_end > end_date: if 'fcast_start' in dates: dates['fcast_end'] = season_end else: del dates['fcast_end'] last_obs = asDatetimeDate(dates['last_obs']) if last_obs > end_date: dates['last_obs'] = season_end last_valid = asDatetimeDate(dates['last_valid']) if last_valid > end_date: dates['last_valid'] = season_end last_valid = asDatetimeDate(season_end) # temporarily free up the POR file reader.close() # add season dates to response response_json = \ '%s,"dates":%s' % (response_json, self.tightJsonString(dates)) del dates # get the accumulated GDD for Y axis of data plots reader.open() response_json = '%s,"data":{"season":%s}}}' % ( response_json, self.serializeData( reader.getDataAtNode(dataset_path, lon, lat, start_date, last_valid))) reader.close() if self.mode in ('dev', 'test'): print 'season data retrieved in ', elapsedTime( start_response, True) # send the response self.respond(request, response_json)
print "\nstatic file for %s region already exists, do you want to replace it ?" % region response = raw_input("Enter 'yes' or 'no' : ") if response in ('y', 'yes'): os.remove(ndfd_filepath) print ndfd_filepath, 'exists :', os.path.exists(ndfd_filepath) break elif response in ('n', 'no'): print 'Execution ending, will not replace', ndfd_filepath exit() builder = NdfdStaticGridFileBuilder(conus_filepath, ndfd_filepath, CONFIG, region, tolerance=tolerance, debug=debug) with warnings.catch_warnings(): print '\nBuilding Acis datasets' builder.buildAcisDatasets(debug=debug) print '\nBuilding NDFD group datasets datasets' builder.buildNdfdGroup(debug=debug) builder.close() elapsed_time = elapsedTime(START_TIME, True) fmt = 'completed creation of NDFD static file for %s in %s' print fmt % (region, elapsed_time) print 'filepath :', builder.filepath
bbox=bbox, debug=debug) builder.open('a') builder.updateTempGroup(span_start, data['mint'], data['maxt'], source.tag) builder.close() # incrment to next sequence of dates if INCREMENT is not None: span_start = span_end + ONE_DAY span_end = span_start + INCREMENT else: span_start = span_start + ONE_DAY del data elapsed_time = elapsedTime(year_start_time, True) print PERFORMANCE_MSG % (target_year, elapsed_time) if wait_time > 0 and num_years > 1: time.sleep(wait_time) total_wait_time += wait_time # turn annoying numpy warnings back on warnings.resetwarnings() elapsed_time = elapsedTime(BUILD_START_TIME, True) print 'completed build for %d years in %s' % (len(target_years), elapsed_time) if total_wait_time > 0: msg = 'total wait time between %d years = %s seconds' print msg % (num_years, total_wait_time)
def __call__(self, request): print '\n\GrapeHardinessTempDataHandler.__call__' start_response = datetime.datetime.now() # decode request variables into a dictionary request_dict = self.requestAsDict(request) varieties = self.extractVarietyParameters(request_dict) variety = varieties['variety'] # extract location coordinates location = self.extractLocationParameters(request_dict) if 'coords' in location: lat, lon = location['coords'] else: lat = location['lat'] lon = location['lon'] # extract season date limits from request dates = self.extractSeasonDates(request_dict) target_year = dates['season'] print ' target_year :', target_year print ' dates :\n', dates # create a variety file reader reader = self.varietyFileReader(variety, target_year, self.source, self.region, 'season') if self.mode in ('dev', 'test'): print 'hardtemp data file :', reader.filepath # path to the hardiness temp dataset dataset_path = 'hardtemp' # capture the significant dates for the dataset if 'dates' in self.mode_config \ and target_year == self.mode_config.season: dates.update(self.mode_config.dates.attrs) else: dates.update(reader.significantDates(dataset_path)) # temporarily free up the file reader.close() season_end = dates['season_end'] end_date = asDatetimeDate(season_end) start_date = asDatetimeDate(dates['season_start']) if 'fcast_start' in dates: fcast_start = asDatetimeDate(dates['fcast_start']) if fcast_start > end_date: del dates['fcast_start'] if 'fcast_end' in dates: fcast_end = asDatetimeDate(dates['fcast_end']) if fcast_end > end_date: if 'fcast_start' in dates: dates['fcast_end'] = season_end else: del dates['fcast_end'] last_obs = asDatetimeDate(dates['last_obs']) if last_obs > end_date: dates['last_obs'] = season_end last_valid = asDatetimeDate(dates['last_valid']) if last_valid > end_date: dates['last_valid'] = season_end last_valid = asDatetimeDate(season_end) #dates = self.tightJsonString(dates) # get the accumulated GDD for Y axis of data plots reader.open() data = \ reader.dataAtNode(dataset_path, lon, lat, start_date, last_valid) reader.close() if self.mode in ('dev', 'test'): print 'hardtemp data retrieved in ', elapsedTime( start_response, True) # initialize the response response_dict = { "hardtemp": { "variety": variety, "location": location, "dates": dates, "data": "data_array" } } response = self.tightJsonString(response_dict).replace('\\"', '"') response = \ response.replace('"data_array"', self.serializeData(data, '%.1f')) # send the response self.respondWithJSON(request, response)
ndfd_lats[i, j] = grib_lats[y, x] ndfd_lons[i, j] = neg_lons[y, x] if extreme_debug: print '\n\nmin_diff :', min_diff print 'nodes :', nodes print 'x_indexes :', x_indexes[i, j] print 'y_indexes :', y_indexes[i, j] print ' distance :', distance[i, j] print ' ndfd_lat :', ndfd_lats[i, j] print ' acis_lat :', acis_lats[i, j] print ' ndfd_lon :', ndfd_lons[i, j] print ' acis_lon :', acis_lons[i, j] if j > 5: exit() elapsed_time = elapsedTime(COORD_MAPPING_START, True) fmt = 'finished mapping NDFD grid nodes to ACIS grid in %s' print fmt % elapsed_time print '\nbuilding static file for %s region' % region static_reader.open() elevation = static_reader.get2DSlice('elev', bbox[0], bbox[2], bbox[1], bbox[3]) static_reader.close() #filepath = static_factory.staticGridFilepath('ndfd', region) #print "building", filepath builder = ndfd_static_factory.ndfdStaticFileBuilder(region, dataset_shape) with warnings.catch_warnings(): builder.build(False, False, None, None, bbox=bbox, debug=debug)
thumb_filename = thumbfile_template.replace('||DATE||', file_date_str) thumb_filepath = os.path.join(thumb_dirpath, thumb_filename) image = PIL.Image.open(map_filepath) image.thumbnail(thumbnail_shape, PIL.Image.ANTIALIAS) if debug: print ' ', thumb_filepath image.save(thumb_filepath, 'PNG', quality=100, optimize=True) date += ONE_DAY maps += 1 # turn annoying numpy warnings back on warnings.resetwarnings() info = (maps, start_date, end_date) print '\nProcessed maps for %d days : %s thru %s' % info elapsed_time = elapsedTime(MAPS_START_TIME, True) info = (treatment.description, threat.fullname, elapsed_time) print 'Completed %s %s control maps in %s' % info return maps # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # MAIN PROGRAM # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # factory for accessing data for maps data_factory = TurfControlsFactory() if dev_mode: data_factory.useDirpathsForMode('dev')
# turn annoying numpy warnings back on warnings.resetwarnings() if len(missing_times) > 0: for missing in missing_times: reason, hour, filepath = missing info = (timeString(hour), reason, filepath.split('conus/')[1]) print '\nDate missing for %s : %s\n %s' % info success, message = grid_factory.repairMissingReanalysis(hour, grid_variable, grid_region) print ' %s' % message if success: total_hours += 1 elapsed_time = elapsedTime(VARIABLE_START_TIME, True) msg = 'completed update of %d hours of "%s" data in %s' print msg % (total_hours, grid_variable, elapsed_time) if update_rhum: if update_start_time > update_end_time: print 'RHUM grid file is already at latest available analysis time : %s' % update_end_time.strftime('%Y-%m-%d:%H') else: # update RHUM from TMP and DPT # filter annoying numpy warnings warnings.filterwarnings('ignore',"All-NaN axis encountered") warnings.filterwarnings('ignore',"All-NaN slice encountered") warnings.filterwarnings('ignore',"invalid value encountered in greater") warnings.filterwarnings('ignore',"invalid value encountered in less")
def __call__(self, request): start_response = datetime.datetime.now() # decode request variables into a dictionary request_dict = self.requestAsDict(request) # extract location coordinates location = request_dict['location'] coords = location.get('coords', None) if coords is not None: lat = float(coords[0]) lon = float(coords[1]) else: lat = float(location['lat']) lon = float(location['lon']) # GDD threshold and target_year gdd_threshold = request_dict['gdd_threshold'] target_year = request_dict.get('season', None) # get the configured season limits dates = self.extractSeasonDates(request_dict, target_year) season_start = asDatetimeDate(dates['season_start']) season_end = asDatetimeDate(dates['season_end']) target_year = dates['season'] del dates['season'] # initialize response string with season dates response = \ '{"history":{%s,"data":{' % self.tightJsonString(dates)[1:-1] reader = self.getHistoryFileReader(target_year, self.source, self.region, gdd_threshold) if self.mode in ('dev', 'test'): print 'history file :', reader.filepath # add recent averages data = \ reader.getSliceAtNode('recent', season_start, season_end, lon, lat) response = '%s"recent":%s' % (response, self.serializeData(data)) # add climate normal averages data = \ reader.getSliceAtNode('normal', season_start, season_end, lon, lat) response = '%s,"normal":%s' % (response, self.serializeData(data)) # add period of record averages data = \ reader.getSliceAtNode('por.avg', season_start, season_end, lon, lat) response = '%s,"poravg":%s' % (response, self.serializeData(data, 4)) # add period of record - percent diffrence max GDD to average GDD data = \ reader.getSliceAtNode('por.max', season_start, season_end, lon, lat) response = '%s,"pormax":%s' % (response, self.serializeData(data, 4)) # add period of record - percent diffrence min GDD to average GDD data = \ reader.getSliceAtNode('por.min', season_start, season_end, lon, lat) response = '%s,"pormin":%s' % (response, self.serializeData(data, 4)) reader.close() del data if self.mode in ('dev', 'test'): print 'history data retrieved in', elapsedTime( start_response, True) self.respond(request, '%s}}}' % response)
row_y = [] for col in range(num_cols): lat = lats[row, col] lon = lons[row, col] y, x, ndfdlon, ndfdlat, dist = \ nearestNode(lon, lat, ndfd_lons, ndfd_lats, search_radius, debug) row_dist.append(dist) row_lat.append(ndfdlat) row_lon.append(ndfdlon) row_x.append(x) row_y.append(y) if debug: print row, col, y, x, lat, ndfdlat, lon, ndfdlon, dist if csv_filepath: line = fmt % (row, col, y, x, lat, ndfdlat, lon, ndfdlon, dist) csv_file.write(line) if verbose: print status % (row, elapsedTime(index_start_time, True)) distance.append(row_dist) ndfd_x.append(row_x) ndfd_y.append(row_y) region_lats.append(row_lat) region_lons.append(row_lon) elapsed_time = elapsedTime(index_start_time, True) print 'completed indexing %d rows in %s' % (num_rows, elapsed_time) del lats, lons, ndfd_lats, ndfd_lons if csv_filepath: csv_file.close()
for utc_hour in download_hours: info = (reanalysis_name, utc_hour.strftime('%Y-%m-%d:%H')) print '\nProcessing %s DATA download for %s' % info grib_filepath = downloadData(factory, utc_hour, grib_server, verbose, debug) if grib_filepath is not None: data_count += 1 if debug: print ' saved DATA grib file to :', grib_filepath updateDataGrids(factory, utc_hour, grib_filepath, debug) # turn annoying numpy warnings back on warnings.resetwarnings() if data_count > 0: info = (data_count, reanalysis.upper(), elapsedTime(SECTION_START, True)) print '\nCompleted download of %d %s DATA grib files in %s' % info # download precip files pcpn_count = 0 pcpn_files = [] if download_pcpn: if start_hour is None: download_hours = determineTimespan(factory, end_hour, 'PCPN', grib_region, max_backward, verbose, debug) if len(download_hours) == 0: info = (end_hour.strftime('%Y=%m-%d:%H UTC'), max_backward, reanalysis.upper()) print '\nSearched from %s back %d hours. No previous %s DATA grib files were found.' % info
grid_variable).items(): if not time_key in skip: print ' %s = %s' % (time_key, hour.strftime('%Y-%m-%d:%H')) manager.close() print ' ' else: if slice_end > slice_start: print timespan_warning % (slice_start.strftime('%Y-%m-%d:%H'), slice_end.strftime('%Y-%m-%d:%H')) else: print hour_warning % slice_start.strftime('%Y-%m-%d:%H') # turn annoying numpy warnings back on warnings.resetwarnings() if len(missing_times) > 0: for missing in missing_times: reason, hour, filepath = missing info = (timeString(hour), reason, filepath.split('conus/')[1]) print '\nDate missing for %s : %s\n %s' % info success, message = \ grid_factory.repairMissingReanalysis(hour, grid_variable, grid_region) print ' %s' % message if success: total_hours += 1 elapsed_time = elapsedTime(UPDATE_START_TIME, True) msg = '\ncompleted update of %d hours of "%s" data in %s' print msg % (total_hours, grid_variable, elapsed_time)
reader.close() del reader # create the average GDD arrays for each time span for group in factory.config.filetypes.history50.groups: scope = group[1]['path'] first_year, last_year = scopes[scope] span_start = datetime.datetime.now() msg = '\ngenerating %d (%d days) history of %s GDD extremes for %s scope' print msg % (target_year, days_in_history, coverage, scope) gdd_dict = buildGddArrays(factory, days_in_history, first_year, last_year, source, region, gdd_threshold, coverage) # update the GDD datasets in the history file msg = ' updating %s dataset in %d %s GDD history file' manager = factory.getHistoryFileManager(target_year, source, region, gdd_threshold, coverage, 'a') for dataset_name, gdd_grid in gdd_dict.items(): dataset_path = '%s.%s' % (scope, dataset_name) print msg % (dataset_path, target_year, gdd_threshold) manager.open('a') manager.updateDataset(dataset_path, 1, gdd_grid) manager.close() msg = '....finished %s GDD history in %s' print msg % (scope, elapsedTime(span_start, True)) # turn annoying numpy warnings back on warnings.resetwarnings()
# reprocess range of dates in the current year elif num_args == 4: start_date = datetime.date(target_year, int(args[0]), int(args[1])) end_date = datetime.date(target_year, int(args[2]), int(args[3])) # reprocess range of dates in a specific year elif num_args == 5: start_date = datetime.date(target_year, int(args[1]), int(args[2])) end_date = datetime.date(target_year, int(args[3]), int(args[4])) if end_date == start_date: end_date = None else: end_date = min(end_date, last_valid_date) # get the temperature datasets and close thereader maxt = reader.getTimeSlice('temps.maxt', start_date, end_date) mint = reader.getTimeSlice('temps.mint', start_date, end_date) reader.close() del reader # get the appropriate GDD manager for the source manager = factory.getPORFileManager(target_year, source, region, mode='a') print 'recalculating gdd in', manager.filepath # recalculate GDD and update the file for threshold, th_string in gdd_thresholds: start_time = datetime.datetime.now() manager.open('a') manager.updateThresholdGroup(threshold, start_date, mint, maxt, source.tag, **kwargs) manager.close() elapsed_time = elapsedTime(start_time, True) print ' processed GDD %s in %s', (th_string, elapsed_time)
stages = N.zeros(num_days, dtype='<i2') for index, threshold in enumerate(treatment.thresholds[1:]): where = N.where(node_data >= threshold) if len(where[0] > 0): stages[where] = index + 1 control_json.append( json_fmt % (name, ','.join(['%d' % s for s in stages]))) params['data'] = '{%s}' % ','.join(control_json) with open(filepath, 'w') as writer: writer.write(json_template % params) num_files += 1 if verbose: print num_files, filepath else: print 'ISSUE : All NAN node found at :', y, x, lons[y, x], lats[y, x] # turn annoying numpy warnings back on warnings.resetwarnings() total_files += num_files elapsed_time = elapsedTime(JSON_START_TIME, True) msg = 'Generated %d json files for %d treatments of %s in %s' print msg % (total_files, len(treatments), control.fullname, elapsed_time) elapsed_time = elapsedTime(JOB_START_TIME, True) msg = 'Generated a total of %d json files for %d threats in %s' print msg % (total_files, num_threats, elapsedTime(JOB_START_TIME, True))