def create_tide_data_file(tide_station, date_list, output_file):
logger = logging.getLogger(__name__)
try:
#Open the file and read any tide entries that exist. We'll then just try and fill in the missing
#ones.
#initial_tide_data = tide_data_file(logger=True)
#initial_tide_data.open(output_file)
with open(output_file, "w") as tide_csv_file:
tide_csv_file.write("Station,Date,Range,Hi,Lo\n")
for date_rec in date_list:
tide = noaaTideDataExt(use_raw=True, logger=logger)
#Date/Time format for the NOAA is YYYYMMDD, get previous 24 hours.
wq_utc_date = date_rec.astimezone(timezone('UTC'))
tide_start_time = (wq_utc_date - timedelta(hours=24))
tide_end_time = wq_utc_date
"""
date_key = date_data_utc.strftime('%Y-%m-%dT%H:%M:%S')
if len(initial_tide_data):
if date_key in initial_tide_data:
if logger:
logger.debug("Station: %s date: %s in history file, not retrieving." % (tide_station, wq_utc_date.strftime("%Y-%m-%dT%H:%M:%S")))
tide_csv_file.write("%s,%s,%f,%f,%f\n"\
% (tide_station,
date_data_utc.strftime("%Y-%m-%dT%H:%M:%S"),
initial_tide_data[date_key]['range'],
initial_tide_data[date_key]['hi'],
initial_tide_data[date_key]['lo']))
tide_start_time = None
else:
if logger:
logger.debug("Station: %s date: %s not in history file, retrieving." % (tide_station, wq_utc_date.strftime("%Y-%m-%dT%H:%M:%S")))
"""
if logger:
logger.debug("Start retrieving tide data for station: %s date: %s-%s" % (tide_station, tide_start_time, tide_end_time))
if tide_start_time is not None:
for x in range(0, 5):
if logger:
logger.debug("Attempt: %d retrieving tide data for station." % (x+1))
tide_data = tide.calcTideRangeExt(beginDate = tide_start_time,
endDate = tide_end_time,
station=tide_station,
datum='MLLW',
units='feet',
timezone='GMT',
smoothData=False)
if tide_data and tide_data['HH'] is not None and tide_data['LL'] is not None:
try:
tide_range = tide_data['HH']['value'] - tide_data['LL']['value']
#Save tide station values.
tide_hi = tide_data['HH']['value']
tide_lo = tide_data['LL']['value']
except TypeError, e:
if logger:
logger.exception(e)
else:
if logger:
logger.error("Tide data for station: %s date: %s not available or only partial, using Peak data." % (tide_station, wq_utc_date.strftime("%Y-%m-%dT%H:%M:%S")))
try:
tide_hi = tide_data['PeakValue']['value']
tide_lo = tide_data['ValleyValue']['value']
tide_range = tide_hi - tide_lo
except TypeError, e:
if logger:
logger.exception(e)
if tide_range is not None:
tide_csv_file.write("%s,%s,%f,%f,%f\n"\
% (tide_station,wq_utc_date.strftime("%Y-%m-%dT%H:%M:%S"),tide_range,tide_hi,tide_lo))
break
if logger:
logger.debug("Finished retrieving tide data for station: %s date: %s" % (tide_station, wq_utc_date.strftime("%Y-%m-%dT%H:%M:%S")))
def get_tide_data(self, start_date, wq_tests_data):
start_time = time.time()
primary_tide_station = self.tide_station
primary_station_range = 'tide_range_%s' % (primary_tide_station)
#If we don't have data for this tide station, retrieve it. The self.tide_station_settings
#list has the primary and subordinate stations, and the primary station can occur in multiple
#subrdinates, so we only need to query the data once for the primary.
#if wq_tests_data[primary_station_range] == wq_defines.NO_DATA:
if self.logger:
self.logger.debug(
"Start retrieving tide data for station: %s date: %s" %
(primary_tide_station, start_date))
tide = noaaTideDataExt(use_raw=True, logger=self.logger)
tide_start_time = (start_date - timedelta(hours=24))
tide_end_time = start_date
#Try and query the NOAA soap service. We give ourselves 5 tries.
for x in range(0, 5):
if self.logger:
self.logger.debug(
"Attempt: %d retrieving tide data for station." % (x + 1))
tide_data = tide.calcTideRangePeakDetect(
beginDate=tide_start_time,
endDate=tide_end_time,
station=primary_tide_station,
datum='MLLW',
units='feet',
timezone='GMT',
smoothData=False,
write_tide_data=False)
if tide_data is not None:
break
if tide_data is not None:
tide_range = tide_data['HH']['value'] - tide_data['LL']['value']
primary_tide = {}
primary_tide['tide_range_%s' % (primary_tide_station)] = tide_range
primary_tide['tide_hi_%s' % (primary_tide_station)] = float(
tide_data['HH']['value'])
primary_tide['tide_lo_%s' % (primary_tide_station)] = float(
tide_data['LL']['value'])
primary_tide['tide_stage_%s' %
(primary_tide_station)] = tide_data['tide_stage']
'''
wq_tests_data['tide_range_%s' % (primary_tide_station)] = tide_range
wq_tests_data['tide_hi_%s' % (primary_tide_station)] = float(tide_data['HH']['value'])
wq_tests_data['tide_lo_%s' % (primary_tide_station)] = float(tide_data['LL']['value'])
wq_tests_data['tide_stage_%s' % (primary_tide_station)] = tide_data['tide_stage']
'''
# Save subordinate station values
subordinate_station_var_name = 'tide_range_%s' % (
self.tide_station_settings['offset_tide_station'])
if wq_tests_data[subordinate_station_var_name] == wq_defines.NO_DATA:
if primary_tide[primary_station_range] != wq_defines.NO_DATA:
#The subordinate calcualtions are made from the primary station. NOAA provides
#the offset constants to apply against the primary results.
offset_hi = primary_tide[
'tide_hi_%s' %
(primary_tide_station
)] * self.tide_station_settings['hi_tide_height_offset']
offset_lo = primary_tide[
'tide_lo_%s' %
(primary_tide_station
)] * self.tide_station_settings['lo_tide_height_offset']
wq_tests_data[
subordinate_station_var_name] = offset_hi - offset_lo
wq_tests_data[
'tide_hi_%s' %
(self.tide_station_settings['offset_tide_station']
)] = offset_hi
wq_tests_data[
'tide_lo_%s' %
(self.tide_station_settings['offset_tide_station']
)] = offset_lo
if self.logger:
self.logger.debug(
"Finished retrieving tide data for station: %s date: %s in %f seconds"
% (self.tide_station, start_date, time.time() - start_time))
return
def get_tide_data(**kwargs):
try:
processing_start_time = time.time()
logging.config.dictConfig(kwargs['worker_log_config'])
logger = logging.getLogger(current_process().name)
logger.info("%s starting get_tide_data." % (current_process().name))
inputQueue = kwargs['input_queue']
resultsQueue = kwargs['results_queue']
tide_station = kwargs['tide_station']
tide = noaaTideDataExt(use_raw=True, logger=logger)
rec_cnt = 0
for date_rec in iter(inputQueue.get, 'STOP'):
wq_utc_date = date_rec.astimezone(timezone('UTC'))
tide_start_time = (wq_utc_date - timedelta(hours=24))
tide_end_time = wq_utc_date
"""
date_key = date_data_utc.strftime('%Y-%m-%dT%H:%M:%S')
if len(initial_tide_data):
if date_key in initial_tide_data:
if logger:
logger.debug("Station: %s date: %s in history file, not retrieving." % (tide_station, wq_utc_date.strftime("%Y-%m-%dT%H:%M:%S")))
tide_csv_file.write("%s,%s,%f,%f,%f\n"\
% (tide_station,
date_data_utc.strftime("%Y-%m-%dT%H:%M:%S"),
initial_tide_data[date_key]['range'],
initial_tide_data[date_key]['hi'],
initial_tide_data[date_key]['lo']))
tide_start_time = None
else:
if logger:
logger.debug("Station: %s date: %s not in history file, retrieving." % (tide_station, wq_utc_date.strftime("%Y-%m-%dT%H:%M:%S")))
"""
logger.debug("%s Start retrieving tide data for station: %s date: %s-%s" % (current_process().name, tide_station, tide_start_time, tide_end_time))
if tide_start_time is not None:
successful = False
for x in range(0, 5):
if logger:
logger.debug("%s Attempt: %d retrieving tide data for station." % (current_process().name, x+1))
pda_tide_data = tide.calcTideRangePeakDetect(beginDate = tide_start_time,
endDate = tide_end_time,
station=tide_station,
datum='MLLW',
units='feet',
timezone='GMT',
smoothData=False)
'''
tide_stage = tide.get_tide_stage(begin_date = tide_start_time,
end_date = tide_end_time,
station=tide_station,
datum='MLLW',
units='feet',
time_zone='GMT',
write_tide_data=True)
'''
if pda_tide_data is not None:
pda_tide_data['date'] = wq_utc_date
resultsQueue.put(pda_tide_data)
successful = True
break
if not successful:
if logger:
logger.error("Unable to retrieve data for: %s" % (wq_utc_date))
#logger.error("Unable to retrieve data for: %s, putting back on queue" % (wq_utc_date))
#inputQueue.put(wq_utc_date)
rec_cnt += 1
logger.info("%s finished get_tide_data. Processed: %d dates in %f seconds" % (current_process().name, rec_cnt, time.time()-processing_start_time))
except Exception as e:
if logger:
logger.exception(e)
return
