def create_tide_data_file(tide_station, date_list, output_file): logger = logging.getLogger(__name__) try: #Open the file and read any tide entries that exist. We'll then just try and fill in the missing #ones. #initial_tide_data = tide_data_file(logger=True) #initial_tide_data.open(output_file) with open(output_file, "w") as tide_csv_file: tide_csv_file.write("Station,Date,Range,Hi,Lo\n") for date_rec in date_list: tide = noaaTideDataExt(use_raw=True, logger=logger) #Date/Time format for the NOAA is YYYYMMDD, get previous 24 hours. wq_utc_date = date_rec.astimezone(timezone('UTC')) tide_start_time = (wq_utc_date - timedelta(hours=24)) tide_end_time = wq_utc_date """ date_key = date_data_utc.strftime('%Y-%m-%dT%H:%M:%S') if len(initial_tide_data): if date_key in initial_tide_data: if logger: logger.debug("Station: %s date: %s in history file, not retrieving." % (tide_station, wq_utc_date.strftime("%Y-%m-%dT%H:%M:%S"))) tide_csv_file.write("%s,%s,%f,%f,%f\n"\ % (tide_station, date_data_utc.strftime("%Y-%m-%dT%H:%M:%S"), initial_tide_data[date_key]['range'], initial_tide_data[date_key]['hi'], initial_tide_data[date_key]['lo'])) tide_start_time = None else: if logger: logger.debug("Station: %s date: %s not in history file, retrieving." % (tide_station, wq_utc_date.strftime("%Y-%m-%dT%H:%M:%S"))) """ if logger: logger.debug("Start retrieving tide data for station: %s date: %s-%s" % (tide_station, tide_start_time, tide_end_time)) if tide_start_time is not None: for x in range(0, 5): if logger: logger.debug("Attempt: %d retrieving tide data for station." % (x+1)) tide_data = tide.calcTideRangeExt(beginDate = tide_start_time, endDate = tide_end_time, station=tide_station, datum='MLLW', units='feet', timezone='GMT', smoothData=False) if tide_data and tide_data['HH'] is not None and tide_data['LL'] is not None: try: tide_range = tide_data['HH']['value'] - tide_data['LL']['value'] #Save tide station values. tide_hi = tide_data['HH']['value'] tide_lo = tide_data['LL']['value'] except TypeError, e: if logger: logger.exception(e) else: if logger: logger.error("Tide data for station: %s date: %s not available or only partial, using Peak data." % (tide_station, wq_utc_date.strftime("%Y-%m-%dT%H:%M:%S"))) try: tide_hi = tide_data['PeakValue']['value'] tide_lo = tide_data['ValleyValue']['value'] tide_range = tide_hi - tide_lo except TypeError, e: if logger: logger.exception(e) if tide_range is not None: tide_csv_file.write("%s,%s,%f,%f,%f\n"\ % (tide_station,wq_utc_date.strftime("%Y-%m-%dT%H:%M:%S"),tide_range,tide_hi,tide_lo)) break if logger: logger.debug("Finished retrieving tide data for station: %s date: %s" % (tide_station, wq_utc_date.strftime("%Y-%m-%dT%H:%M:%S")))
def get_tide_data(self, start_date, wq_tests_data): start_time = time.time() primary_tide_station = self.tide_station primary_station_range = 'tide_range_%s' % (primary_tide_station) #If we don't have data for this tide station, retrieve it. The self.tide_station_settings #list has the primary and subordinate stations, and the primary station can occur in multiple #subrdinates, so we only need to query the data once for the primary. #if wq_tests_data[primary_station_range] == wq_defines.NO_DATA: if self.logger: self.logger.debug( "Start retrieving tide data for station: %s date: %s" % (primary_tide_station, start_date)) tide = noaaTideDataExt(use_raw=True, logger=self.logger) tide_start_time = (start_date - timedelta(hours=24)) tide_end_time = start_date #Try and query the NOAA soap service. We give ourselves 5 tries. for x in range(0, 5): if self.logger: self.logger.debug( "Attempt: %d retrieving tide data for station." % (x + 1)) tide_data = tide.calcTideRangePeakDetect( beginDate=tide_start_time, endDate=tide_end_time, station=primary_tide_station, datum='MLLW', units='feet', timezone='GMT', smoothData=False, write_tide_data=False) if tide_data is not None: break if tide_data is not None: tide_range = tide_data['HH']['value'] - tide_data['LL']['value'] primary_tide = {} primary_tide['tide_range_%s' % (primary_tide_station)] = tide_range primary_tide['tide_hi_%s' % (primary_tide_station)] = float( tide_data['HH']['value']) primary_tide['tide_lo_%s' % (primary_tide_station)] = float( tide_data['LL']['value']) primary_tide['tide_stage_%s' % (primary_tide_station)] = tide_data['tide_stage'] ''' wq_tests_data['tide_range_%s' % (primary_tide_station)] = tide_range wq_tests_data['tide_hi_%s' % (primary_tide_station)] = float(tide_data['HH']['value']) wq_tests_data['tide_lo_%s' % (primary_tide_station)] = float(tide_data['LL']['value']) wq_tests_data['tide_stage_%s' % (primary_tide_station)] = tide_data['tide_stage'] ''' # Save subordinate station values subordinate_station_var_name = 'tide_range_%s' % ( self.tide_station_settings['offset_tide_station']) if wq_tests_data[subordinate_station_var_name] == wq_defines.NO_DATA: if primary_tide[primary_station_range] != wq_defines.NO_DATA: #The subordinate calcualtions are made from the primary station. NOAA provides #the offset constants to apply against the primary results. offset_hi = primary_tide[ 'tide_hi_%s' % (primary_tide_station )] * self.tide_station_settings['hi_tide_height_offset'] offset_lo = primary_tide[ 'tide_lo_%s' % (primary_tide_station )] * self.tide_station_settings['lo_tide_height_offset'] wq_tests_data[ subordinate_station_var_name] = offset_hi - offset_lo wq_tests_data[ 'tide_hi_%s' % (self.tide_station_settings['offset_tide_station'] )] = offset_hi wq_tests_data[ 'tide_lo_%s' % (self.tide_station_settings['offset_tide_station'] )] = offset_lo if self.logger: self.logger.debug( "Finished retrieving tide data for station: %s date: %s in %f seconds" % (self.tide_station, start_date, time.time() - start_time)) return
def get_tide_data(**kwargs): try: processing_start_time = time.time() logging.config.dictConfig(kwargs['worker_log_config']) logger = logging.getLogger(current_process().name) logger.info("%s starting get_tide_data." % (current_process().name)) inputQueue = kwargs['input_queue'] resultsQueue = kwargs['results_queue'] tide_station = kwargs['tide_station'] tide = noaaTideDataExt(use_raw=True, logger=logger) rec_cnt = 0 for date_rec in iter(inputQueue.get, 'STOP'): wq_utc_date = date_rec.astimezone(timezone('UTC')) tide_start_time = (wq_utc_date - timedelta(hours=24)) tide_end_time = wq_utc_date """ date_key = date_data_utc.strftime('%Y-%m-%dT%H:%M:%S') if len(initial_tide_data): if date_key in initial_tide_data: if logger: logger.debug("Station: %s date: %s in history file, not retrieving." % (tide_station, wq_utc_date.strftime("%Y-%m-%dT%H:%M:%S"))) tide_csv_file.write("%s,%s,%f,%f,%f\n"\ % (tide_station, date_data_utc.strftime("%Y-%m-%dT%H:%M:%S"), initial_tide_data[date_key]['range'], initial_tide_data[date_key]['hi'], initial_tide_data[date_key]['lo'])) tide_start_time = None else: if logger: logger.debug("Station: %s date: %s not in history file, retrieving." % (tide_station, wq_utc_date.strftime("%Y-%m-%dT%H:%M:%S"))) """ logger.debug("%s Start retrieving tide data for station: %s date: %s-%s" % (current_process().name, tide_station, tide_start_time, tide_end_time)) if tide_start_time is not None: successful = False for x in range(0, 5): if logger: logger.debug("%s Attempt: %d retrieving tide data for station." % (current_process().name, x+1)) pda_tide_data = tide.calcTideRangePeakDetect(beginDate = tide_start_time, endDate = tide_end_time, station=tide_station, datum='MLLW', units='feet', timezone='GMT', smoothData=False) ''' tide_stage = tide.get_tide_stage(begin_date = tide_start_time, end_date = tide_end_time, station=tide_station, datum='MLLW', units='feet', time_zone='GMT', write_tide_data=True) ''' if pda_tide_data is not None: pda_tide_data['date'] = wq_utc_date resultsQueue.put(pda_tide_data) successful = True break if not successful: if logger: logger.error("Unable to retrieve data for: %s" % (wq_utc_date)) #logger.error("Unable to retrieve data for: %s, putting back on queue" % (wq_utc_date)) #inputQueue.put(wq_utc_date) rec_cnt += 1 logger.info("%s finished get_tide_data. Processed: %d dates in %f seconds" % (current_process().name, rec_cnt, time.time()-processing_start_time)) except Exception as e: if logger: logger.exception(e) return