def _download_files(self, datestamp: date, server_mode: bool = False):
"""Actually, just try to connect with the remote files
For a given queried date, we may have to use the forecast from the previous
day since the current nowcast doesn't hold data for today (solved?)
"""
progress = CliProgress()
if not isinstance(datestamp, date):
raise RuntimeError("invalid date passed: %s" % type(datestamp))
# check if the files are loaded and that the date matches
if self._has_data_loaded:
# logger.info("%s" % self.last_loaded_day)
if self._last_loaded_day == datestamp:
return True
else: # the data are old
logger.info("cleaning data: %s %s" %
(self._last_loaded_day, datestamp))
self.clear_data()
progress.start(text="Download GoMOFS", is_disabled=server_mode)
# check if the data are available on the RTOFS server
url_ck = self._build_check_url(datestamp)
if not self._check_url(url_ck):
datestamp -= timedelta(days=1)
url_ck = self._build_check_url(datestamp)
if not self._check_url(url_ck):
logger.warning(
'unable to retrieve data from GoMOFS server for date: %s and next day'
% datestamp)
self.clear_data()
progress.end()
return False
progress.update(30)
# Try to download the data grid grids
url = self._build_opendap_url(datestamp)
# logger.debug('downloading RTOFS data for %s' % datestamp)
try:
self._file = Dataset(url)
progress.update(70)
self._day_idx = 0
except (RuntimeError, IOError) as e:
logger.warning("unable to access data: %s -> %s" %
(datestamp.strftime("%Y%m%d"), e))
self.clear_data()
progress.end()
return False
# success!
self._has_data_loaded = True
self._last_loaded_day = datestamp
# logger.info("loaded data for %s" % datestamp)
progress.end()
return True
def test_run(self):
progress = CliProgress()
progress.start(title='Test Bar',
text='Doing stuff',
min_value=100,
max_value=300,
init_value=100)
time.sleep(.1)
progress.update(value=150, text='Updating')
time.sleep(.1)
progress.add(quantum=50, text='Updating')
time.sleep(.1)
self.assertFalse(progress.canceled)
progress.end()
import time
import logging
from hyo2.abc.lib.logging import set_logging
from hyo2.abc.lib.progress.cli_progress import CliProgress
logger = logging.getLogger(__name__)
set_logging(ns_list=["hyo2.abc"])
progress = CliProgress()
progress.start(title='Test Bar',
text='Doing stuff',
min_value=100,
max_value=300,
init_value=100)
time.sleep(.1)
progress.update(value=150, text='Updating')
time.sleep(.1)
progress.add(quantum=50, text='Updating')
time.sleep(.1)
print("canceled? %s" % progress.canceled)
progress.end()
class OneDrive:
def __init__(self,
show_progress: bool = False,
debug_mode: bool = False,
progress: AbstractProgress = None):
if debug_mode:
self.debug_level = 2
else:
self.debug_level = 0
self.show_progress = show_progress
self.chunk_count = None
self.filesize = None
self.file_count = None
self.file_nr = None
if progress is None:
self.progress = CliProgress()
else:
self.progress = progress
def get_file(self, file_src: str, file_dst: str, unzip_it: bool = False):
""" Retrieve a file
Args:
file_src: File source
file_dst: File destination
unzip_it: Unzip the retrieved file
"""
file_dst = os.path.abspath(file_dst)
if os.path.exists(file_dst):
os.remove(file_dst)
response = requests.get(file_src, stream=True)
total_size_in_bytes = int(response.headers.get('content-length', 0))
logger.debug("size in bytes: %d" % total_size_in_bytes)
block_size = 1024 * 1024
blocks = total_size_in_bytes / block_size + 2.0
quantum = 100.0 / blocks
if self.show_progress:
self.progress.start(text="Downloading", has_abortion=True)
with open(file_dst, 'wb') as file:
for data in response.iter_content(chunk_size=block_size):
if self.show_progress:
if self.progress.canceled:
raise RuntimeError("download stopped by user")
self.progress.add(quantum=quantum)
file.write(data)
if self.show_progress:
self.progress.end()
if unzip_it:
import zipfile
try:
z = zipfile.ZipFile(file_dst, "r")
unzip_path = os.path.dirname(file_dst)
logger.debug("unzipping %s to %s" % (file_dst, unzip_path))
name_list = z.namelist()
self.file_nr = len(name_list)
if self.show_progress:
self.progress.start(text="Unzipping", has_abortion=True)
self.file_count = 0
for item in name_list:
# print(item)
z.extract(item, unzip_path)
self.file_count += 1
if self.show_progress:
if self.progress.canceled:
raise RuntimeError("unzip stopped by user")
pct = int((self.file_count / self.file_nr) * 100.0)
self.progress.update(pct)
z.close()
os.remove(file_dst)
if self.show_progress:
self.progress.end()
except Exception as e:
traceback.print_exc()
raise RuntimeError(
"unable to unzip the downloaded file: %s -> %s" %
(file_dst, e))
class TestABCLibCliProgress(unittest.TestCase):
def setUp(self):
self.progress = CliProgress()
def test_start_minimal(self):
try:
self.progress.start()
except Exception as e:
self.fail(e)
def test_start_custom_title_text(self):
try:
self.progress.start(title='Test Bar', text='Doing stuff')
except Exception as e:
self.fail(e)
def test_start_custom_min_max(self):
try:
self.progress.start(min_value=100, max_value=300, init_value=100)
except Exception as e:
self.fail(e)
def test_start_minimal_update(self):
try:
self.progress.start()
self.progress.update(50)
except Exception as e:
self.fail(e)
def test_start_minimal_update_raising(self):
with self.assertRaises(Exception) as context:
self.progress.start()
self.progress.update(1000)
def test_start_minimal_add(self):
try:
self.progress.start()
self.progress.add(50)
except Exception as e:
self.fail(e)
def test_start_minimal_add_raising(self):
with self.assertRaises(Exception) as context:
self.progress.start()
self.progress.add(1000)
def test_run(self):
progress = CliProgress()
progress.start(title='Test Bar',
text='Doing stuff',
min_value=100,
max_value=300,
init_value=100)
time.sleep(.1)
progress.update(value=150, text='Updating')
time.sleep(.1)
progress.add(quantum=50, text='Updating')
time.sleep(.1)
self.assertFalse(progress.canceled)
progress.end()