def test_pyfolder_iterate(self):
"""
PyFolder can be iterated
:return:
"""
pyfolder = PyFolder(self.test_folders)
pyfolder["foo"] = b"bar"
pyfolder["foo2"] = b"bar2"
for element in pyfolder:
self.assertTrue(element in ["foo", "foo2"])
for element, content in pyfolder.items():
self.assertEqual(content, {
"foo": b"bar",
"foo2": b"bar2"
}[element])
pyfolder["foo3/foo"] = b"bar"
for element in pyfolder:
self.assertTrue(element in ["foo", "foo2", "foo3"])
for element, content in pyfolder.items():
if element == "foo3":
self.assertTrue(type(content) is PyFolder)
def test_pyfolder_deletes_elements(self):
"""
PyFolder accepts delete of elements
:return:
"""
pyfolder = PyFolder(self.test_folders)
pyfolder["prueba"] = b"hola"
pyfolder["test/example.txt"] = "jeje"
with self.assertRaises(Exception):
del pyfolder["."]
del pyfolder["prueba"]
del pyfolder["test/example.txt"]
del pyfolder["test"]
pyfolder = PyFolder(self.test_folders,
allow_remove_folders_with_content=True)
with self.assertRaises(Exception):
del pyfolder["."]
del pyfolder["test"]
del pyfolder["test/example.txt"]
pyfolder = PyFolder(self.test_folders, allow_override=True)
with self.assertRaises(OSError):
del pyfolder["test"]
del pyfolder["."]
pyfolder2 = PyFolder(os.path.join(self.test_folders, "example"),
allow_override=True)
self.assertTrue(
os.path.exists(os.path.join(self.test_folders, "example")))
del pyfolder["example"]
self.assertFalse(
os.path.exists(os.path.join(self.test_folders, "example")))
self.assertTrue(
os.path.exists(os.path.join(self.test_folders, "prueba")))
del pyfolder["prueba"]
self.assertFalse(
os.path.exists(os.path.join(self.test_folders, "prueba")))
pyfolder = PyFolder(self.test_folders,
allow_override=True,
allow_remove_folders_with_content=True)
self.assertTrue(os.path.exists(os.path.join(self.test_folders,
"test")))
del pyfolder["test"]
self.assertFalse(
os.path.exists(os.path.join(self.test_folders, "test")))
self.assertTrue(os.path.exists(self.test_folders))
del pyfolder["."]
self.assertFalse(os.path.exists(self.test_folders))
def test_pyfolder_values(self):
"""
PyFolder successfully returns the values o its keys
:return:
"""
pyfolder = PyFolder(self.test_folders)
pyfolder["foo"] = b"bar"
pyfolder["foo2"] = b"bar2"
self.assertTrue(all(x in pyfolder.values() for x in [b"bar", b"bar2"]))
self.assertEqual(len(pyfolder.values()), 2)
def test_pyfolder_keys(self):
"""
PyFolder is able to return its keys
:return:
"""
pyfolder = PyFolder(self.test_folders)
pyfolder["foo"] = b"bar"
pyfolder["foo2"] = b"bar2"
self.assertTrue(all(x in pyfolder.keys() for x in ["foo", "foo2"]))
self.assertEqual(len(pyfolder.keys()), 2)
def test_pyfolder_index(self):
"""
PyFolder is able to find elements
:return:
"""
pyfolder = PyFolder(self.test_folders)
pyfolder["foo"] = b"bar"
pyfolder["foo2"] = b"bar2"
pyfolder["foo3/foo"] = b"bar"
indexes = pyfolder.index("foo")
self.assertEqual(len(indexes), 2)
self.assertIn("foo", indexes)
self.assertIn("foo3/foo", indexes)
def test_pyfolder_edits_elements(self):
"""
PyFolder accepts edit of elements
:return:
"""
pyfolder = PyFolder(self.test_folders)
pyfolder["foo"] = b"bar"
with self.assertRaises(Exception):
pyfolder["foo"] = b"foo"
pyfolder = PyFolder(self.test_folders, allow_override=True)
pyfolder["foo"] = b"foo"
self.assertEqual(pyfolder["foo"], b"foo")
def test_pyfolder_iterate_only_folders(self):
"""
PyFolder is able to iterate only over folders.
:return:
"""
pyfolder = PyFolder(self.test_folders)
pyfolder["foo"] = b"bar"
pyfolder["foo2"] = b"bar2"
pyfolder["foo3/foo"] = b"bar"
for element in pyfolder.folders():
self.assertTrue(element in ["foo3"])
for element, content in pyfolder.folders_items():
self.assertTrue(type(content) is PyFolder)
self.assertIn("foo", content)
def load_from_folder(self, folder):
self.clear()
pyfolder = PyFolder(folder)
if "metadata.json" in pyfolder:
metadata = self.__load_json(pyfolder)
elif "metadata.csv" in pyfolder:
metadata = self.__load_csv(pyfolder)
else:
raise FileNotFoundError("No metadata.json or metadata.csv found.")
content_available = "content" in pyfolder
if not content_available:
content_folder = None
print("Warning: elements do not have content associated.")
else:
# We don't want PyFolder to interpret the elements of the dataset.
content_folder = PyFolder(pyfolder["content"].folder_root, interpret=False)
elements = []
batch_size = 0
for key, values in metadata.items():
element = {k: v for k, v in values.items() if k != "id"}
if content_available:
element['content'] = content_folder[key]
batch_size += len(element['content'])
elements.append(element)
if batch_size >= 1*1024*1024*1024: # each 1 GB, perform upload.
print("* Uploading elements...")
self.add_elements(elements)
batch_size = 0
elements = []
self.sync(update_size=False)
if len(elements) > 0:
self.add_elements(elements)
print("Uploading elements...")
self.sync()
print("\rFinished uploading.")
def model(prediction_strategy):
data = request.get_data()
PyZip(PyFolder(BASE_ROOT_FILE + '/' + prediction_strategy, interpret=False, allow_override=True,
allow_remove_folders_with_content=True, auto_create_folder=True)) \
.from_bytes(data)
load_files(BASE_ROOT_FILE, prediction_strategy)
return '', 204
def save_to_folder(self, folder, metadata_format="json", elements_extension=None, use_numbered_ids=False,
only_metadata=False):
try:
os.mkdir(folder)
except Exception as ex:
pass
format_saver = {
"csv": self.__save_csv,
"json": self.__save_json
}
if elements_extension is not None and elements_extension.startswith("."):
elements_extension = elements_extension[1:]
if metadata_format not in format_saver:
raise Exception("format {} for metadata not supported.".format(metadata_format))
pyfolder = PyFolder(folder)
print("Collecting elements...")
metadata = {}
id = -1
count = len(self)
it = -1
for element in self.filter_iter(cache_content=not only_metadata):
it += 1
print("\rProgress: {}%".format(round(it / (count + 0.0001) * 100, 2)), end="", flush=True)
if use_numbered_ids:
id += 1
else:
id = element.get_id()
if elements_extension is None:
element_id = id
else:
element_id = "{}.{}".format(id, elements_extension)
metadata[element_id] = {
'id': element.get_id(),
'title': element.get_title(),
'description': element.get_description(),
'http_ref': element.get_ref(),
'tags': element.get_tags(),
}
if not only_metadata:
pyfolder[os.path.join("content", element_id)] = element.get_content(interpret=False)
print("\rProgress: 100%", end="", flush=True)
format_saver[metadata_format](pyfolder, metadata)
print("\nSaved metadata in format {}".format(metadata_format))
print("Finished")
def test_pyfolder_creates_elements(self):
"""
PyFolder is able to create elements interpreted/not interpreted
:return:
"""
pyfolder = PyFolder(self.test_folders)
pyfolder["example"] = b"hi"
self.assertTrue(
os.path.exists(os.path.join(self.test_folders, "example")))
with open(os.path.join(self.test_folders, "example"), "rb") as f:
content = f.read()
self.assertEqual(content, b"hi")
pyfolder["example.txt"] = "hi"
self.assertTrue(
os.path.exists(os.path.join(self.test_folders, "example.txt")))
with open(os.path.join(self.test_folders, "example.txt"), "r") as f:
content = f.read()
self.assertEqual(content, "hi")
pyfolder["example.json"] = {"m": "hi"}
self.assertTrue(
os.path.exists(os.path.join(self.test_folders, "example.json")))
with open(os.path.join(self.test_folders, "example.json"), "r") as f:
content = json.load(f)
self.assertEqual(content, {"m": "hi"})
# Disable interpret
pyfolder = PyFolder(self.test_folders, interpret=False)
with self.assertRaises(Exception):
pyfolder["example2.txt"] = {"m": "hi"}
pyfolder = PyFolder(self.test_folders, interpret=True)
subdir = "subdir/subdir2/file.txt"
pyfolder[subdir] = "HELLO"
with open(os.path.join(self.test_folders, subdir), "r") as f:
content = f.read()
self.assertEqual(content, "HELLO")
def test_pyfolder_create_folders(self):
pyfolder = PyFolder(self.test_folders, auto_create_folder=False)
self.assertFalse(os.path.exists(self.test_folders))
# Can't create elements if flag is unset and folder doesn't exist
with self.assertRaises(FileNotFoundError):
pyfolder["example"] = "hi"
pyfolder = PyFolder(self.test_folders, auto_create_folder=True)
# It must have created the folder
self.assertTrue(os.path.exists(self.test_folders))
pyfolder = PyFolder(
os.path.join(self.test_folders, "example2", "example3"))
# It must have created the deep folder
self.assertTrue(
os.path.exists(
os.path.join(self.test_folders, "example2", "example3")))
def test_pyfolder_iterate_only_files(self):
"""
PyFolder is able to iterate only over files
:return:
"""
pyfolder = PyFolder(self.test_folders)
pyfolder["foo"] = b"bar"
pyfolder["foo2"] = b"bar2"
pyfolder["foo3/foo"] = b"bar"
for element in pyfolder.files():
self.assertTrue(element in ["foo", "foo2"])
for element, content in pyfolder.files_items():
self.assertEqual(content, {
"foo": b"bar",
"foo2": b"bar2"
}[element])
def updateFrameworkZipData(relativeFrameworkPath, zipDumpPath):
try:
frameworkPath = os.path.join(os.getcwd(), relativeFrameworkPath)
pyzip = PyZip(PyFolder(frameworkPath, interpret=False))
destinationPath = os.path.join(os.getcwd(), zipDumpPath)
f = open(destinationPath, "w")
f.write("frameworkZipData=" + str(pyzip.to_bytes()))
f.close()
except Exception as ex:
print(ex)
raise ex
def updateFrameworkZipData():
try:
frameworkPath = os.path.join(os.getcwd(),"framework.v2")
pyzip = PyZip(PyFolder(frameworkPath, interpret=False))
destinationPath = os.path.join(os.getcwd(),r"src\arc_reactor\cli\init\frameworkZip.py")
f = open(destinationPath, "w")
f.write("frameworkZipData="+str(pyzip.to_bytes()))
f.close()
except Exception as ex:
print(ex)
raise ex
def zipUnZipFrameWork():
try:
directory = (os.getcwd())
changePermission(directory)
removeTree(directory)
pyzip = PyZip().from_bytes(frameworkZipData)
pyzip.save("code.zip")
zip_path = os.path.join(directory,"code.zip")
pyzip = PyZip(PyFolder(directory, interpret=False)).from_file(zip_path, inflate=False)
os.remove("code.zip")
except Exception as ex:
print(ex)
def test_pyfolder_contains_elements(self):
"""
PyFolder accepts the "in" keyword to lookup for elements
:return:
"""
pyfolder = PyFolder(self.test_folders)
pyfolder["foo"] = b"bar"
pyfolder["foo2"] = b"bar2"
self.assertIn("foo", pyfolder)
self.assertIn("foo2", pyfolder)
self.assertNotIn("foo3", pyfolder)
def load(self, uri):
"""
Loads the model from the specified URI.
The model uses 4 files: one for the encoder, other for the decoder, other
for the autoencoder and one for the class options in JSON format.
:param uri: base filename
"""
self._encoder = load_model(uri + "_lstm_encoder.hdf5")
self._autoencoder = load_model(uri + "_lstm_autoencoder.hdf5")
pf = PyFolder(os.path.dirname(os.path.realpath(uri)))
dict_options = pf[os.path.basename(uri) + "_options.json"]
self._latent_space = dict_options['latent_space']
self._input_cells = dict_options['input_cells']
def save(self, uri):
"""
Saves the model into a given filename.
The model uses 4 files: one for the encoder, other for the decoder, other
for the autoencoder and one for the class options in JSON format.
:param uri: base filename.
"""
pf = PyFolder(os.path.dirname(os.path.realpath(uri)), allow_override=True)
pf[os.path.basename(uri)+"_options.json"] = {
'input_cells': self._input_cells,
'latent_space': self._latent_space,
}
save_model(self._autoencoder, uri+"_lstm_autoencoder.hdf5")
save_model(self._encoder, uri+"_lstm_encoder.hdf5")
def test_pyfolder_retrieves_elements(self):
"""
PyFolder accepts retrieval of elements
:return:
"""
pyfolder = PyFolder(self.test_folders)
with open(os.path.join(self.test_folders, "example"), "wb") as f:
f.write(b"hi")
self.assertEqual(pyfolder["example"], b"hi")
with open(os.path.join(self.test_folders, "example.txt"), "w") as f:
f.write("hi")
self.assertEqual(pyfolder["example.txt"], "hi")
with open(os.path.join(self.test_folders, "example.json"), "w") as f:
json.dump({"m": "h"}, f)
self.assertEqual(pyfolder["example.json"], {"m": "h"})
with self.assertRaises(KeyError):
a = pyfolder["UNKNOWN"]