def test_up_and_download(self):
with self.assertRaises(ConnectionError):
# Should be empty!
self.downloader.test_find()
file_name = 'test.txt'
self.assertFalse(self.downloader.md5_stored(file_name))
self.assertEqual(self.downloader.compute_md5(file_name), '')
file_content = 'This is a test'
with open(file_name, 'w') as f:
f.write(file_content)
self.assertTrue(os.path.exists(file_name))
self.uploader.upload_from_dict({file_name: os.path.abspath(file_name)})
self.assertTrue(self.uploader.md5_stored(file_name))
self.assertTrue(self.downloader.config_exists(file_name))
path = self.downloader.download_single(file_name)
path_hr = self.downloader.download_single(
file_name, human_readable_file_name=True)
abs_path = self.downloader.get_abs_path(file_name)
for p in [path, path_hr, abs_path]:
self.assertTrue(os.path.exists(p))
read_file = straxen.get_resource(path)
self.assertTrue(file_content == read_file)
os.remove(file_name)
self.assertFalse(os.path.exists(file_name))
self.downloader.test_find()
self.downloader.download_all()
# Now the test on init should work, let's double try
straxen.MongoDownloader(
collection=self.collection,
file_database=None,
_test_on_init=True,
)
def get_resource(name: str, fmt: str = 'text', **kwargs):
"""
Fetch a straxen resource
Allow a direct download using <fmt='abs_path'> otherwise kwargs are
passed directly to straxen.get_resource.
"""
if fmt == 'abs_path':
downloader = straxen.MongoDownloader()
return downloader.download_single(name)
return straxen.get_resource(name, fmt=fmt)
def test_invalid_methods(self):
"""
The following examples should NOT work, let's make sure the
right errors are raised
"""
with self.assertRaises(ValueError):
straxen.MongoDownloader(
collection=self.collection,
file_database='NOT NONE',
)
with self.assertRaises(ValueError):
straxen.MongoDownloader(collection='invalid type', )
with self.assertRaises(PermissionError):
straxen.MongoUploader(readonly=True)
with self.assertRaises(ValueError):
self.uploader.upload_from_dict("A string is not a dict")
with self.assertRaises(straxen.mongo_storage.CouldNotLoadError):
self.uploader.upload_single('no_such_file', 'no_such_file')
with self.assertWarns(UserWarning):
self.uploader.upload_from_dict({'something': 'no_such_file'})
with self.assertRaises(ValueError):
straxen.MongoDownloader(
collection=self.collection,
file_database=None,
_test_on_init=False,
store_files_at=False,
)
with self.assertRaises(ValueError):
self.downloader.download_single('no_existing_file')
with self.assertRaises(ValueError):
self.downloader._check_store_files_at('some_str')
with self.assertRaises(PermissionError):
self.downloader._check_store_files_at([])
def setup(self):
self.model_file = self._get_model_file_name()
if self.model_file is None:
warn(f'No file provided for {self.algorithm}. Setting all values '
f'for {self.provides} to None.')
# No further setup required
return
# Load the tensorflow model
import tensorflow as tf
if os.path.exists(self.model_file):
print(f"Path is local. Loading {self.algorithm} TF model locally "
f"from disk.")
else:
downloader = straxen.MongoDownloader()
try:
self.model_file = downloader.download_single(self.model_file)
except straxen.mongo_storage.CouldNotLoadError as e:
raise RuntimeError(f'Model files {self.model_file} is not found') from e
with tempfile.TemporaryDirectory() as tmpdirname:
tar = tarfile.open(self.model_file, mode="r:gz")
tar.extractall(path=tmpdirname)
self.model = tf.keras.models.load_model(tmpdirname)
def get_resource(x: str, fmt='text'):
"""
Get the resource from an online source to be opened here. We will
sequentially try the following:
1. Load if from memory if we asked for it before;
2. load it from a file if the path exists;
3. (preferred option) Load it from our database
4. Load the file from some URL (e.g. raw github content)
:param x: str, either it is :
A.) a path to the file;
B.) the identifier of the file as it's stored under in the database;
C.) A URL to the file (e.g. raw github content).
:param fmt: str, format of the resource x
:return: the opened resource file x opened according to the
specified format
"""
# 1. load from memory
cached_name = _cache_name(x, fmt)
if cached_name in _resource_cache:
return _resource_cache[cached_name]
# 2. load from file
elif os.path.exists(x):
return open_resource(x, fmt=fmt)
# 3. load from database
elif straxen.uconfig is not None:
downloader = straxen.MongoDownloader()
if x in downloader.list_files():
path = downloader.download_single(x)
return open_resource(path, fmt=fmt)
# 4. load from URL
if '://' in x:
return resource_from_url(x, fmt=fmt)
raise FileNotFoundError(
f'Cannot open {x} because it is either not stored or we '
f'cannot download it from anywhere.')
def setUp(self):
# Just to make sure we are running some mongo server, see test-class docstring
if 'TEST_MONGO_URI' not in os.environ:
self._run_test = False
return
uri = os.environ.get('TEST_MONGO_URI')
db_name = 'test_rundb'
collection_name = 'fs.files'
client = pymongo.MongoClient(uri)
database = client[db_name]
collection = database[collection_name]
self.downloader = straxen.MongoDownloader(
collection=collection,
readonly=True,
file_database=None,
_test_on_init=False,
)
self.uploader = straxen.MongoUploader(
collection=collection,
readonly=False,
file_database=None,
_test_on_init=False,
)
self.collection = collection
