def handle_single_dataset(self, context):
self.check_attr("single_dataset__datapath", (str, type(None)))
self.check_attr("single_dataset__title", (str, type(None)))
self.check_attr("single_dataset__about", (str, type(None)))
self.check_attr("single_dataset__obs_names", (str, type(None)))
self.check_attr("single_dataset__var_names", (str, type(None)))
if self.single_dataset__datapath is None:
if self.multi_dataset__dataroot is None:
# TODO: change the error message once dataroot is fully supported
raise ConfigurationError("missing datapath")
return
else:
if self.multi_dataset__dataroot is not None:
raise ConfigurationError(
"must supply only one of datapath or dataroot")
# create the matrix data cache manager:
if self.matrix_data_cache_manager is None:
self.matrix_data_cache_manager = MatrixDataCacheManager(
max_cached=1, timelimit_s=None)
# preload this data set
matrix_data_loader = MatrixDataLoader(self.single_dataset__datapath,
app_config=self.app_config)
try:
matrix_data_loader.pre_load_validation()
except DatasetAccessError as e:
raise ConfigurationError(str(e))
file_size = matrix_data_loader.file_size()
file_basename = basename(self.single_dataset__datapath)
if file_size > BIG_FILE_SIZE_THRESHOLD:
context["messagefn"](
f"Loading data from {file_basename}, this may take a while...")
else:
context["messagefn"](f"Loading data from {file_basename}.")
if self.single_dataset__about:
def url_check(url):
try:
result = urlparse(url)
if all([result.scheme, result.netloc]):
return True
else:
return False
except ValueError:
return False
if not url_check(self.single_dataset__about):
raise ConfigurationError(
"Must provide an absolute URL for --about. (Example format: http://example.com)"
)
def handle_single_dataset(self, context):
self.validate_correct_type_of_configuration_attribute(
"single_dataset__datapath", (str, type(None)))
self.validate_correct_type_of_configuration_attribute(
"single_dataset__title", (str, type(None)))
self.validate_correct_type_of_configuration_attribute(
"single_dataset__about", (str, type(None)))
self.validate_correct_type_of_configuration_attribute(
"single_dataset__obs_names", (str, type(None)))
self.validate_correct_type_of_configuration_attribute(
"single_dataset__var_names", (str, type(None)))
# preload this data set
matrix_data_loader = MatrixDataLoader(self.single_dataset__datapath,
app_config=self.app_config)
try:
matrix_data_loader.pre_load_validation()
except DatasetAccessError as e:
raise ConfigurationError(str(e))
file_size = matrix_data_loader.file_size()
file_basename = basename(self.single_dataset__datapath)
if file_size > BIG_FILE_SIZE_THRESHOLD:
context["messagefn"](
f"Loading data from {file_basename}, this may take a while...")
else:
context["messagefn"](f"Loading data from {file_basename}.")
if self.single_dataset__about:
def url_check(url):
try:
result = urlparse(url)
if all([result.scheme, result.netloc]):
return True
else:
return False
except ValueError:
return False
if not url_check(self.single_dataset__about):
raise ConfigurationError(
"Must provide an absolute URL for --about. (Example format: http://example.com)"
)