def test_up_and_download(self): with self.assertRaises(ConnectionError): # Should be empty! self.downloader.test_find() file_name = 'test.txt' self.assertFalse(self.downloader.md5_stored(file_name)) self.assertEqual(self.downloader.compute_md5(file_name), '') file_content = 'This is a test' with open(file_name, 'w') as f: f.write(file_content) self.assertTrue(os.path.exists(file_name)) self.uploader.upload_from_dict({file_name: os.path.abspath(file_name)}) self.assertTrue(self.uploader.md5_stored(file_name)) self.assertTrue(self.downloader.config_exists(file_name)) path = self.downloader.download_single(file_name) path_hr = self.downloader.download_single( file_name, human_readable_file_name=True) abs_path = self.downloader.get_abs_path(file_name) for p in [path, path_hr, abs_path]: self.assertTrue(os.path.exists(p)) read_file = straxen.get_resource(path) self.assertTrue(file_content == read_file) os.remove(file_name) self.assertFalse(os.path.exists(file_name)) self.downloader.test_find() self.downloader.download_all() # Now the test on init should work, let's double try straxen.MongoDownloader( collection=self.collection, file_database=None, _test_on_init=True, )
def get_resource(name: str, fmt: str = 'text', **kwargs): """ Fetch a straxen resource Allow a direct download using <fmt='abs_path'> otherwise kwargs are passed directly to straxen.get_resource. """ if fmt == 'abs_path': downloader = straxen.MongoDownloader() return downloader.download_single(name) return straxen.get_resource(name, fmt=fmt)
def test_invalid_methods(self): """ The following examples should NOT work, let's make sure the right errors are raised """ with self.assertRaises(ValueError): straxen.MongoDownloader( collection=self.collection, file_database='NOT NONE', ) with self.assertRaises(ValueError): straxen.MongoDownloader(collection='invalid type', ) with self.assertRaises(PermissionError): straxen.MongoUploader(readonly=True) with self.assertRaises(ValueError): self.uploader.upload_from_dict("A string is not a dict") with self.assertRaises(straxen.mongo_storage.CouldNotLoadError): self.uploader.upload_single('no_such_file', 'no_such_file') with self.assertWarns(UserWarning): self.uploader.upload_from_dict({'something': 'no_such_file'}) with self.assertRaises(ValueError): straxen.MongoDownloader( collection=self.collection, file_database=None, _test_on_init=False, store_files_at=False, ) with self.assertRaises(ValueError): self.downloader.download_single('no_existing_file') with self.assertRaises(ValueError): self.downloader._check_store_files_at('some_str') with self.assertRaises(PermissionError): self.downloader._check_store_files_at([])
def setup(self): self.model_file = self._get_model_file_name() if self.model_file is None: warn(f'No file provided for {self.algorithm}. Setting all values ' f'for {self.provides} to None.') # No further setup required return # Load the tensorflow model import tensorflow as tf if os.path.exists(self.model_file): print(f"Path is local. Loading {self.algorithm} TF model locally " f"from disk.") else: downloader = straxen.MongoDownloader() try: self.model_file = downloader.download_single(self.model_file) except straxen.mongo_storage.CouldNotLoadError as e: raise RuntimeError(f'Model files {self.model_file} is not found') from e with tempfile.TemporaryDirectory() as tmpdirname: tar = tarfile.open(self.model_file, mode="r:gz") tar.extractall(path=tmpdirname) self.model = tf.keras.models.load_model(tmpdirname)
def get_resource(x: str, fmt='text'): """ Get the resource from an online source to be opened here. We will sequentially try the following: 1. Load if from memory if we asked for it before; 2. load it from a file if the path exists; 3. (preferred option) Load it from our database 4. Load the file from some URL (e.g. raw github content) :param x: str, either it is : A.) a path to the file; B.) the identifier of the file as it's stored under in the database; C.) A URL to the file (e.g. raw github content). :param fmt: str, format of the resource x :return: the opened resource file x opened according to the specified format """ # 1. load from memory cached_name = _cache_name(x, fmt) if cached_name in _resource_cache: return _resource_cache[cached_name] # 2. load from file elif os.path.exists(x): return open_resource(x, fmt=fmt) # 3. load from database elif straxen.uconfig is not None: downloader = straxen.MongoDownloader() if x in downloader.list_files(): path = downloader.download_single(x) return open_resource(path, fmt=fmt) # 4. load from URL if '://' in x: return resource_from_url(x, fmt=fmt) raise FileNotFoundError( f'Cannot open {x} because it is either not stored or we ' f'cannot download it from anywhere.')
def setUp(self): # Just to make sure we are running some mongo server, see test-class docstring if 'TEST_MONGO_URI' not in os.environ: self._run_test = False return uri = os.environ.get('TEST_MONGO_URI') db_name = 'test_rundb' collection_name = 'fs.files' client = pymongo.MongoClient(uri) database = client[db_name] collection = database[collection_name] self.downloader = straxen.MongoDownloader( collection=collection, readonly=True, file_database=None, _test_on_init=False, ) self.uploader = straxen.MongoUploader( collection=collection, readonly=False, file_database=None, _test_on_init=False, ) self.collection = collection