def __init__(self, projects_folder, profile=project_profiles['office'], progress=CliProgress()):
# output folder
self._output_folder = None # the project's output folder
if (projects_folder is None) or (not os.path.exists(projects_folder)):
projects_folder = self.default_output_folder()
logger.debug("using default output folder: %s" % projects_folder)
self.output_folder = projects_folder
# profile
self._active_profile = profile
# progress bar
if not isinstance(progress, AbstractProgress):
raise RuntimeError("invalid progress object")
self.progress = progress
# used to name the output folder
self._survey = str()
# grids
self._gr = GridsManager()
self._gr2 = GridsManager()
# features
self._ft = Features()
# outputs
self._output_shp = True
self._output_kml = True
self._output_subfolders = False
self._output_project_folder = True
# callback
self._cb = None
def __init__(self, lib_info: LibInfo, app_info: AppInfo, progress=CliProgress()):
self._li = lib_info
self._ai = app_info
self.progress = progress
self.local_zip_path = os.path.abspath(os.path.join(Helper(lib_info=self._li).package_folder(),
"Caris_Support_Files_%s.zip" % self.v_version()))
self.local_batch_file = None
def __init__(self,
prj_path: Path,
force_prj_creation: bool = False,
progress: AbstractProgress = CliProgress(use_logger=True)):
# check extension for passed project path
if prj_path.suffix != self.ext:
raise RuntimeError("invalid project extension: %s" % prj_path)
# delete project if force variable is true
if force_prj_creation is True:
if prj_path.exists() is True:
shutil.rmtree(str(prj_path))
prj_path.mkdir(parents=True, exist_ok=True)
self._path = prj_path
_ = self.raws_folder
_ = self.process_folder
_ = self.products_folder
self.progress = progress
self._i = ProjectInfo(prj_path=self._path)
self._r = Raws(raws_path=self.raws_folder)
self._p = Process(process_path=self.process_folder)
self._healthy = False
self.check_health()
def __init__(self,
show_progress: bool = False,
debug_mode: bool = False,
progress: AbstractProgress = None):
if debug_mode:
self.debug_level = 2
else:
self.debug_level = 0
self.show_progress = show_progress
self.chunk_count = None
self.filesize = None
self.file_count = None
self.file_nr = None
if progress is None:
self.progress = CliProgress()
else:
self.progress = progress
def __init__(self):
self._type = anomaly_algos["ANOMALY_DETECTOR_v1"]
self._version = 1
self._visual_debug = False
self._plot_settings = PlotSettings()
self._progress = CliProgress()
self._progress_span = 100
self._write_kml = False
self._write_shp = False
def test_run(self):
progress = CliProgress()
progress.start(title='Test Bar',
text='Doing stuff',
min_value=100,
max_value=300,
init_value=100)
time.sleep(.1)
progress.update(value=150, text='Updating')
time.sleep(.1)
progress.add(quantum=50, text='Updating')
time.sleep(.1)
self.assertFalse(progress.canceled)
progress.end()
def __init__(self,
grid_list: List[str],
output_folder: str,
output_project_folder: bool,
output_subfolders: bool,
use_nooa_nbs_profile: bool = False,
check_structure: bool = False,
check_metadata: bool = False,
check_elevation: bool = False,
check_uncertainty: bool = False,
check_tracking_list: bool = False,
progress: AbstractProgress = CliProgress(),
open_output_folder: bool = True):
self.grid_list = grid_list
self.output_folder = output_folder
self.output_project_folder = output_project_folder
self.output_subfolders = output_subfolders
self.open_output_folder = open_output_folder
self._noaa_nbs_profile = use_nooa_nbs_profile
self._structure = check_structure
self._metadata = check_metadata
self._elevation = check_elevation
self._uncertainty = check_uncertainty
self._tracking_list = check_tracking_list
self._msg = str()
self._bc_structure_errors = 0 # type: int
self._bc_structure_warnings = 0 # type: int
self._bc_metadata_errors = 0 # type: int
self._bc_metadata_warnings = 0 # type: int
self._bc_elevation_errors = 0 # type: int
self._bc_elevation_warnings = 0 # type: int
self._bc_uncertainty_errors = 0 # type: int
self._bc_uncertainty_warnings = 0 # type: int
self._bc_tracking_list_errors = 0 # type: int
self._bc_tracking_list_warnings = 0 # type: int
self._bc_report = None
self._bc_pdf = str()
self._cur_min_depth = None # type: Optional[float]
self._cur_max_depth = None # type: Optional[float]
self._cur_vr_min_depth = None # type: Optional[float]
self._cur_vr_max_depth = None # type: Optional[float]
self.progress = progress
self._is_vr = False
self._survey = str()
self._grid_basename = str()
def __init__(self):
self._progress = CliProgress()
self._profile = str()
self._flier_finder = False
self._holiday_finder = False
self._grid_qa = False
self._designated_scan = False
self._feature_scan = False
self._valsou_check = False
self._write_kml = False
self._write_shp = False
self._project_folder = False
self._subfolders = False
def __init__(
self,
prj_name: str = "default",
setup_name: str = "default",
force_new: bool = False,
progress: AbstractProgress = CliProgress(use_logger=True)
) -> None:
self.progress = progress
self._setup = Setup(name=setup_name,
prj_name=prj_name,
setups_folder=self.setups_folder())
cur_proj_path = self.projects_folder().joinpath(
self._setup.current_project + Project.ext)
self._prj = Project(prj_path=cur_proj_path,
force_prj_creation=force_new,
progress=self.progress)
def __init__(self,
output_folder=None,
profile=BaseProject.project_profiles['office'],
progress=CliProgress()):
super().__init__(projects_folder=output_folder,
profile=profile,
progress=progress)
# scan features
self._scan = None
self.file_scan_s57 = str()
self.file_scan_pdf = str()
self.scan_msg = str()
# triangle features
self._triangle = None
self.file_triangle_s57 = str()
self.triangle_msg = str()
def _download_files(self, datestamp: date, server_mode: bool = False):
"""Actually, just try to connect with the remote files
For a given queried date, we may have to use the forecast from the previous
day since the current nowcast doesn't hold data for today (solved?)
"""
progress = CliProgress()
if not isinstance(datestamp, date):
raise RuntimeError("invalid date passed: %s" % type(datestamp))
# check if the files are loaded and that the date matches
if self._has_data_loaded:
# logger.info("%s" % self.last_loaded_day)
if self._last_loaded_day == datestamp:
return True
else: # the data are old
logger.info("cleaning data: %s %s" %
(self._last_loaded_day, datestamp))
self.clear_data()
progress.start(text="Download GoMOFS", is_disabled=server_mode)
# check if the data are available on the RTOFS server
url_ck = self._build_check_url(datestamp)
if not self._check_url(url_ck):
datestamp -= timedelta(days=1)
url_ck = self._build_check_url(datestamp)
if not self._check_url(url_ck):
logger.warning(
'unable to retrieve data from GoMOFS server for date: %s and next day'
% datestamp)
self.clear_data()
progress.end()
return False
progress.update(30)
# Try to download the data grid grids
url = self._build_opendap_url(datestamp)
# logger.debug('downloading RTOFS data for %s' % datestamp)
try:
self._file = Dataset(url)
progress.update(70)
self._day_idx = 0
except (RuntimeError, IOError) as e:
logger.warning("unable to access data: %s -> %s" %
(datestamp.strftime("%Y%m%d"), e))
self.clear_data()
progress.end()
return False
# success!
self._has_data_loaded = True
self._last_loaded_day = datestamp
# logger.info("loaded data for %s" % datestamp)
progress.end()
return True
def query(self, nc_path: str, lat: float,
lon: float) -> Optional[ProfileList]:
if not os.path.exists(nc_path):
raise RuntimeError('Unable to locate %s' % nc_path)
logger.debug('nc path: %s' % nc_path)
if (lat is None) or (lon is None):
logger.error("invalid location query: (%s, %s)" % (lon, lat))
return None
logger.debug('query location: %s, %s' % (lat, lon))
progress = CliProgress()
try:
self._file = Dataset(nc_path)
progress.update(20)
except (RuntimeError, IOError) as e:
logger.warning("unable to access data: %s" % e)
self.clear_data()
progress.end()
return None
try:
self.name = self._file.title
time = self._file.variables['time']
self._timestamp = num2date(time[0], units=time.units)
logger.debug("Retrieved time: %s" % self._timestamp.isoformat())
# Now get latitudes, longitudes and depths for x,y,z referencing
self._lats = self._file.variables['lat'][:]
self._lons = self._file.variables['lon'][:]
# logger.debug('lat:(%s)\n%s' % (self._lats.shape, self._lats))
# logger.debug('lon:(%s)\n%s' % (self._lons.shape, self._lons))
self._zeta = self._file.variables['zeta'][0, :]
self._siglay = self._file.variables['siglay'][:]
self._h = self._file.variables['h'][:]
# logger.debug('zeta:(%s)\n%s' % (self._zeta.shape, self._zeta))
# logger.debug('siglay:(%s)\n%s' % (self._siglay.shape, self._siglay[:, 0]))
# logger.debug('h:(%s)\n%s' % (self._h.shape, self._h))
self._temp = self._file.variables['temp'][:]
self._sal = self._file.variables['salinity'][:]
# logger.debug('temp:(%s)\n%s' % (self._temp.shape, self._temp[:, 0]))
# logger.debug('sal:(%s)\n%s' % (self._sal.shape, self._sal[:, 0]))
except Exception as e:
logger.error(
"troubles in variable lookup for lat/long grid and/or depth: %s"
% e)
self.clear_data()
progress.end()
return None
min_dist = 100000.0
min_idx = None
for idx, _ in enumerate(self._lats):
nc_lat = self._lats[idx]
nc_lon = self._lons[idx]
if nc_lon > 180.0:
nc_lon = nc_lon - 360.0
nc_dist = self.g.distance(nc_lon, nc_lat, lon, lat)
# logger.debug('loc: %.6f, %.6f -> %.6f' % (nc_lat, nc_lon, nc_dist))
if nc_dist < min_dist:
min_dist = nc_dist
min_idx = idx
if min_dist >= 10000.0:
logger.error("location too far from model nodes: %.f" % min_dist)
self.clear_data()
progress.end()
return None
self._loc_idx = min_idx
self._lon = self._lons[self._loc_idx]
if self._lon > 180.0:
self._lon = self._lon - 360.0
self._lat = self._lats[self._loc_idx]
logger.debug('closest node: %d [%s, %s] -> %s' %
(self._loc_idx, self._lat, self._lon, min_dist))
zeta = self._zeta[self._loc_idx]
h = self._h[self._loc_idx]
siglay = -self._siglay[:, self._loc_idx]
# logger.debug('zeta: %s, h: %s, siglay: %s' % (zeta, h, siglay))
self._d = siglay * (h + zeta)
# logger.debug('d:(%s)\n%s' % (self._h.shape, self._d))
# Make a new SV object to return our query in
ssp = Profile()
ssp.meta.sensor_type = Dicts.sensor_types['Synthetic']
ssp.meta.probe_type = Dicts.probe_types[self.name]
ssp.meta.latitude = self._lat
ssp.meta.longitude = self._lon
ssp.meta.utc_time = dt(year=self._timestamp.year,
month=self._timestamp.month,
day=self._timestamp.day,
hour=self._timestamp.hour,
minute=self._timestamp.minute,
second=self._timestamp.second)
ssp.meta.original_path = "%s_%s" % (
self.name, self._timestamp.strftime("%Y%m%d_%H%M%S"))
ssp.init_data(self._d.shape[0])
ssp.data.depth = self._d[:]
ssp.data.temp = self._temp[0, :, self._loc_idx]
ssp.data.sal = self._sal[0, :, self._loc_idx]
ssp.calc_data_speed()
ssp.clone_data_to_proc()
ssp.init_sis()
profiles = ProfileList()
profiles.append_profile(ssp)
progress.end()
return profiles
import time
import logging
from hyo2.abc.lib.logging import set_logging
from hyo2.abc.lib.progress.cli_progress import CliProgress
logger = logging.getLogger(__name__)
set_logging(ns_list=["hyo2.abc"])
progress = CliProgress()
progress.start(title='Test Bar',
text='Doing stuff',
min_value=100,
max_value=300,
init_value=100)
time.sleep(.1)
progress.update(value=150, text='Updating')
time.sleep(.1)
progress.add(quantum=50, text='Updating')
time.sleep(.1)
print("canceled? %s" % progress.canceled)
progress.end()
class OneDrive:
def __init__(self,
show_progress: bool = False,
debug_mode: bool = False,
progress: AbstractProgress = None):
if debug_mode:
self.debug_level = 2
else:
self.debug_level = 0
self.show_progress = show_progress
self.chunk_count = None
self.filesize = None
self.file_count = None
self.file_nr = None
if progress is None:
self.progress = CliProgress()
else:
self.progress = progress
def get_file(self, file_src: str, file_dst: str, unzip_it: bool = False):
""" Retrieve a file
Args:
file_src: File source
file_dst: File destination
unzip_it: Unzip the retrieved file
"""
file_dst = os.path.abspath(file_dst)
if os.path.exists(file_dst):
os.remove(file_dst)
response = requests.get(file_src, stream=True)
total_size_in_bytes = int(response.headers.get('content-length', 0))
logger.debug("size in bytes: %d" % total_size_in_bytes)
block_size = 1024 * 1024
blocks = total_size_in_bytes / block_size + 2.0
quantum = 100.0 / blocks
if self.show_progress:
self.progress.start(text="Downloading", has_abortion=True)
with open(file_dst, 'wb') as file:
for data in response.iter_content(chunk_size=block_size):
if self.show_progress:
if self.progress.canceled:
raise RuntimeError("download stopped by user")
self.progress.add(quantum=quantum)
file.write(data)
if self.show_progress:
self.progress.end()
if unzip_it:
import zipfile
try:
z = zipfile.ZipFile(file_dst, "r")
unzip_path = os.path.dirname(file_dst)
logger.debug("unzipping %s to %s" % (file_dst, unzip_path))
name_list = z.namelist()
self.file_nr = len(name_list)
if self.show_progress:
self.progress.start(text="Unzipping", has_abortion=True)
self.file_count = 0
for item in name_list:
# print(item)
z.extract(item, unzip_path)
self.file_count += 1
if self.show_progress:
if self.progress.canceled:
raise RuntimeError("unzip stopped by user")
pct = int((self.file_count / self.file_nr) * 100.0)
self.progress.update(pct)
z.close()
os.remove(file_dst)
if self.show_progress:
self.progress.end()
except Exception as e:
traceback.print_exc()
raise RuntimeError(
"unable to unzip the downloaded file: %s -> %s" %
(file_dst, e))
def setUp(self):
self.progress = CliProgress()
class TestABCLibCliProgress(unittest.TestCase):
def setUp(self):
self.progress = CliProgress()
def test_start_minimal(self):
try:
self.progress.start()
except Exception as e:
self.fail(e)
def test_start_custom_title_text(self):
try:
self.progress.start(title='Test Bar', text='Doing stuff')
except Exception as e:
self.fail(e)
def test_start_custom_min_max(self):
try:
self.progress.start(min_value=100, max_value=300, init_value=100)
except Exception as e:
self.fail(e)
def test_start_minimal_update(self):
try:
self.progress.start()
self.progress.update(50)
except Exception as e:
self.fail(e)
def test_start_minimal_update_raising(self):
with self.assertRaises(Exception) as context:
self.progress.start()
self.progress.update(1000)
def test_start_minimal_add(self):
try:
self.progress.start()
self.progress.add(50)
except Exception as e:
self.fail(e)
def test_start_minimal_add_raising(self):
with self.assertRaises(Exception) as context:
self.progress.start()
self.progress.add(1000)
def test_run(self):
progress = CliProgress()
progress.start(title='Test Bar',
text='Doing stuff',
min_value=100,
max_value=300,
init_value=100)
time.sleep(.1)
progress.update(value=150, text='Updating')
time.sleep(.1)
progress.add(quantum=50, text='Updating')
time.sleep(.1)
self.assertFalse(progress.canceled)
progress.end()