def _download_files(self, datestamp: date, server_mode: bool = False):
"""Actually, just try to connect with the remote files
For a given queried date, we may have to use the forecast from the previous
day since the current nowcast doesn't hold data for today (solved?)
"""
progress = CliProgress()
if not isinstance(datestamp, date):
raise RuntimeError("invalid date passed: %s" % type(datestamp))
# check if the files are loaded and that the date matches
if self._has_data_loaded:
# logger.info("%s" % self.last_loaded_day)
if self._last_loaded_day == datestamp:
return True
else: # the data are old
logger.info("cleaning data: %s %s" %
(self._last_loaded_day, datestamp))
self.clear_data()
progress.start(text="Download GoMOFS", is_disabled=server_mode)
# check if the data are available on the RTOFS server
url_ck = self._build_check_url(datestamp)
if not self._check_url(url_ck):
datestamp -= timedelta(days=1)
url_ck = self._build_check_url(datestamp)
if not self._check_url(url_ck):
logger.warning(
'unable to retrieve data from GoMOFS server for date: %s and next day'
% datestamp)
self.clear_data()
progress.end()
return False
progress.update(30)
# Try to download the data grid grids
url = self._build_opendap_url(datestamp)
# logger.debug('downloading RTOFS data for %s' % datestamp)
try:
self._file = Dataset(url)
progress.update(70)
self._day_idx = 0
except (RuntimeError, IOError) as e:
logger.warning("unable to access data: %s -> %s" %
(datestamp.strftime("%Y%m%d"), e))
self.clear_data()
progress.end()
return False
# success!
self._has_data_loaded = True
self._last_loaded_day = datestamp
# logger.info("loaded data for %s" % datestamp)
progress.end()
return True
def test_run(self):
progress = CliProgress()
progress.start(title='Test Bar',
text='Doing stuff',
min_value=100,
max_value=300,
init_value=100)
time.sleep(.1)
progress.update(value=150, text='Updating')
time.sleep(.1)
progress.add(quantum=50, text='Updating')
time.sleep(.1)
self.assertFalse(progress.canceled)
progress.end()
def query(self, nc_path: str, lat: float,
lon: float) -> Optional[ProfileList]:
if not os.path.exists(nc_path):
raise RuntimeError('Unable to locate %s' % nc_path)
logger.debug('nc path: %s' % nc_path)
if (lat is None) or (lon is None):
logger.error("invalid location query: (%s, %s)" % (lon, lat))
return None
logger.debug('query location: %s, %s' % (lat, lon))
progress = CliProgress()
try:
self._file = Dataset(nc_path)
progress.update(20)
except (RuntimeError, IOError) as e:
logger.warning("unable to access data: %s" % e)
self.clear_data()
progress.end()
return None
try:
self.name = self._file.title
time = self._file.variables['time']
self._timestamp = num2date(time[0], units=time.units)
logger.debug("Retrieved time: %s" % self._timestamp.isoformat())
# Now get latitudes, longitudes and depths for x,y,z referencing
self._lats = self._file.variables['lat'][:]
self._lons = self._file.variables['lon'][:]
# logger.debug('lat:(%s)\n%s' % (self._lats.shape, self._lats))
# logger.debug('lon:(%s)\n%s' % (self._lons.shape, self._lons))
self._zeta = self._file.variables['zeta'][0, :]
self._siglay = self._file.variables['siglay'][:]
self._h = self._file.variables['h'][:]
# logger.debug('zeta:(%s)\n%s' % (self._zeta.shape, self._zeta))
# logger.debug('siglay:(%s)\n%s' % (self._siglay.shape, self._siglay[:, 0]))
# logger.debug('h:(%s)\n%s' % (self._h.shape, self._h))
self._temp = self._file.variables['temp'][:]
self._sal = self._file.variables['salinity'][:]
# logger.debug('temp:(%s)\n%s' % (self._temp.shape, self._temp[:, 0]))
# logger.debug('sal:(%s)\n%s' % (self._sal.shape, self._sal[:, 0]))
except Exception as e:
logger.error(
"troubles in variable lookup for lat/long grid and/or depth: %s"
% e)
self.clear_data()
progress.end()
return None
min_dist = 100000.0
min_idx = None
for idx, _ in enumerate(self._lats):
nc_lat = self._lats[idx]
nc_lon = self._lons[idx]
if nc_lon > 180.0:
nc_lon = nc_lon - 360.0
nc_dist = self.g.distance(nc_lon, nc_lat, lon, lat)
# logger.debug('loc: %.6f, %.6f -> %.6f' % (nc_lat, nc_lon, nc_dist))
if nc_dist < min_dist:
min_dist = nc_dist
min_idx = idx
if min_dist >= 10000.0:
logger.error("location too far from model nodes: %.f" % min_dist)
self.clear_data()
progress.end()
return None
self._loc_idx = min_idx
self._lon = self._lons[self._loc_idx]
if self._lon > 180.0:
self._lon = self._lon - 360.0
self._lat = self._lats[self._loc_idx]
logger.debug('closest node: %d [%s, %s] -> %s' %
(self._loc_idx, self._lat, self._lon, min_dist))
zeta = self._zeta[self._loc_idx]
h = self._h[self._loc_idx]
siglay = -self._siglay[:, self._loc_idx]
# logger.debug('zeta: %s, h: %s, siglay: %s' % (zeta, h, siglay))
self._d = siglay * (h + zeta)
# logger.debug('d:(%s)\n%s' % (self._h.shape, self._d))
# Make a new SV object to return our query in
ssp = Profile()
ssp.meta.sensor_type = Dicts.sensor_types['Synthetic']
ssp.meta.probe_type = Dicts.probe_types[self.name]
ssp.meta.latitude = self._lat
ssp.meta.longitude = self._lon
ssp.meta.utc_time = dt(year=self._timestamp.year,
month=self._timestamp.month,
day=self._timestamp.day,
hour=self._timestamp.hour,
minute=self._timestamp.minute,
second=self._timestamp.second)
ssp.meta.original_path = "%s_%s" % (
self.name, self._timestamp.strftime("%Y%m%d_%H%M%S"))
ssp.init_data(self._d.shape[0])
ssp.data.depth = self._d[:]
ssp.data.temp = self._temp[0, :, self._loc_idx]
ssp.data.sal = self._sal[0, :, self._loc_idx]
ssp.calc_data_speed()
ssp.clone_data_to_proc()
ssp.init_sis()
profiles = ProfileList()
profiles.append_profile(ssp)
progress.end()
return profiles
import time
import logging
from hyo2.abc.lib.logging import set_logging
from hyo2.abc.lib.progress.cli_progress import CliProgress
logger = logging.getLogger(__name__)
set_logging(ns_list=["hyo2.abc"])
progress = CliProgress()
progress.start(title='Test Bar',
text='Doing stuff',
min_value=100,
max_value=300,
init_value=100)
time.sleep(.1)
progress.update(value=150, text='Updating')
time.sleep(.1)
progress.add(quantum=50, text='Updating')
time.sleep(.1)
print("canceled? %s" % progress.canceled)
progress.end()
class OneDrive:
def __init__(self,
show_progress: bool = False,
debug_mode: bool = False,
progress: AbstractProgress = None):
if debug_mode:
self.debug_level = 2
else:
self.debug_level = 0
self.show_progress = show_progress
self.chunk_count = None
self.filesize = None
self.file_count = None
self.file_nr = None
if progress is None:
self.progress = CliProgress()
else:
self.progress = progress
def get_file(self, file_src: str, file_dst: str, unzip_it: bool = False):
""" Retrieve a file
Args:
file_src: File source
file_dst: File destination
unzip_it: Unzip the retrieved file
"""
file_dst = os.path.abspath(file_dst)
if os.path.exists(file_dst):
os.remove(file_dst)
response = requests.get(file_src, stream=True)
total_size_in_bytes = int(response.headers.get('content-length', 0))
logger.debug("size in bytes: %d" % total_size_in_bytes)
block_size = 1024 * 1024
blocks = total_size_in_bytes / block_size + 2.0
quantum = 100.0 / blocks
if self.show_progress:
self.progress.start(text="Downloading", has_abortion=True)
with open(file_dst, 'wb') as file:
for data in response.iter_content(chunk_size=block_size):
if self.show_progress:
if self.progress.canceled:
raise RuntimeError("download stopped by user")
self.progress.add(quantum=quantum)
file.write(data)
if self.show_progress:
self.progress.end()
if unzip_it:
import zipfile
try:
z = zipfile.ZipFile(file_dst, "r")
unzip_path = os.path.dirname(file_dst)
logger.debug("unzipping %s to %s" % (file_dst, unzip_path))
name_list = z.namelist()
self.file_nr = len(name_list)
if self.show_progress:
self.progress.start(text="Unzipping", has_abortion=True)
self.file_count = 0
for item in name_list:
# print(item)
z.extract(item, unzip_path)
self.file_count += 1
if self.show_progress:
if self.progress.canceled:
raise RuntimeError("unzip stopped by user")
pct = int((self.file_count / self.file_nr) * 100.0)
self.progress.update(pct)
z.close()
os.remove(file_dst)
if self.show_progress:
self.progress.end()
except Exception as e:
traceback.print_exc()
raise RuntimeError(
"unable to unzip the downloaded file: %s -> %s" %
(file_dst, e))