def testMakeArchive(self):
# Part A creates a single file, archives it, then deletes it
# Part B extracts the file again from the archive and checks the content
base = self.tempdir
frel = "testfile"
fabs = opj(base,"testfile")
rar = opj(base,"testarchive1.rar")
# Part A:
file_helper.create_file(fabs,"content123test")
a = archive.Archive(rar,base)
a.add_file(frel)
self.assertEqual(a.run(),0)
self.assertEqual(file_helper.file_exists(rar),True)
file_helper.remove_file(fabs)
self.assertEqual(file_helper.file_exists(fabs),False)
# Part B:
b = archive.Archive(rar,base)
syscode = b.extract(target_path=base)
self.assertEqual(syscode,0)
self.assertEqual(file_helper.file_head(fabs),"content123test")
def testMakeArchiveSubdir1(self):
# test archiving a subdir with a file and extracting relative
base=self.tempdir
rar=opj(base,"testarchive1.rar")
a = archive.Archive(rar,base)
drel_top = "A"
drel=opj(drel_top,"1","a")
dabs=opj(base,drel)
frel=opj(drel,"testfile-1")
fabs=opj(base,frel)
file_helper.mkdir_p(dabs)
file_helper.create_file(fabs,"content-1")
a.add_dir(drel_top)
self.assertEqual(a.run(),0)
self.assertEqual(file_helper.file_exists(rar),True)
file_helper.remove_file(fabs)
self.assertEqual(file_helper.file_exists(fabs),False)
# test content of archive by extracting:
b = archive.Archive(rar,base)
syscode = b.extract(target_path=self.tempdir)
self.assertEqual(syscode,0)
self.assertEqual(file_helper.file_head(fabs),"content-1")
def testMakeArchive(self):
# Part A creates a single file, archives it, then deletes it
# Part B extracts the file again from the archive and checks the content
base = self.tempdir
frel = "testfile"
fabs = opj(base, "testfile")
rar = opj(base, "testarchive1.rar")
# Part A:
file_helper.create_file(fabs, "content123test")
a = archive.Archive(rar, base)
a.add_file(frel)
self.assertEqual(a.run(), 0)
self.assertEqual(file_helper.file_exists(rar), True)
file_helper.remove_file(fabs)
self.assertEqual(file_helper.file_exists(fabs), False)
# Part B:
b = archive.Archive(rar, base)
syscode = b.extract(target_path=base)
self.assertEqual(syscode, 0)
self.assertEqual(file_helper.file_head(fabs), "content123test")
def testMakeArchiveSubdir1(self):
# test archiving a subdir with a file and extracting relative
base = self.tempdir
rar = opj(base, "testarchive1.rar")
a = archive.Archive(rar, base)
drel_top = "A"
drel = opj(drel_top, "1", "a")
dabs = opj(base, drel)
frel = opj(drel, "testfile-1")
fabs = opj(base, frel)
file_helper.mkdir_p(dabs)
file_helper.create_file(fabs, "content-1")
a.add_dir(drel_top)
self.assertEqual(a.run(), 0)
self.assertEqual(file_helper.file_exists(rar), True)
file_helper.remove_file(fabs)
self.assertEqual(file_helper.file_exists(fabs), False)
# test content of archive by extracting:
b = archive.Archive(rar, base)
syscode = b.extract(target_path=self.tempdir)
self.assertEqual(syscode, 0)
self.assertEqual(file_helper.file_head(fabs), "content-1")
def testFileHelperFile(self):
# create file:
file_helper.create_file(opj(self.tempdir,"testfile"),"content123test")
# test if it exists:
self.assertEqual(file_helper.file_exists(opj(self.tempdir,"testfile")),True)
# test its content:
self.assertEqual(file_helper.file_head(opj(self.tempdir,"testfile")),"content123test")
# test if it can be removed:
file_helper.remove_file(opj(self.tempdir,"testfile"))
self.assertEqual(file_helper.file_exists(opj(self.tempdir,"testfile")),False)
def get_mnist_data(flattened=True):
if flattened:
if (file_exists(MNIST_FLATTENED_NORMALISED_PICKLE) == False):
setup_flattened_normalised_mnist_pickle()
dataset = read_from_pickle(MNIST_FLATTENED_NORMALISED_PICKLE)
return dataset['x_train'], dataset['y_train'], dataset[
'x_test'], dataset['y_test']
else:
if (file_exists(MNIST_PICKLE) == False):
setup_mnist_pickle()
dataset = read_from_pickle(MNIST_PICKLE)
return dataset['x_train'], dataset['y_train'], dataset[
'x_test'], dataset['y_test']
def setup_flattened_normalised_mnist_pickle():
print('Setting up normalised mnist pickle:')
destination_file = MNIST_FLATTENED_NORMALISED_PICKLE
if file_exists(destination_file) == True:
print('MNIST normalised file already exists.')
return
(x_train, y_train), (x_test, y_test) = mnist.load_data()
x_train = x_train.astype('float32') / 255.
x_test = x_test.astype('float32') / 255.
x_train = flatten_dataset(x_train)
x_test = flatten_dataset(x_test)
print('\t\tTraining data (x_train) : ', x_train.shape)
print('\t\tTraining labels (y_train) : ', y_train.shape)
print('\t\tTest data (y_train) : ', x_test.shape)
print('\t\tTest labels (y_test) : ', y_test.shape)
dataset = {
'x_train': x_train,
'y_train': y_train,
'x_test': x_test,
'y_test': y_test
}
save_as_pickle(dataset, destination_file)
def testFileHelperFile(self):
# create file:
file_helper.create_file(opj(self.tempdir, "testfile"),
"content123test")
# test if it exists:
self.assertEqual(
file_helper.file_exists(opj(self.tempdir, "testfile")), True)
# test its content:
self.assertEqual(file_helper.file_head(opj(self.tempdir, "testfile")),
"content123test")
# test if it can be removed:
file_helper.remove_file(opj(self.tempdir, "testfile"))
self.assertEqual(
file_helper.file_exists(opj(self.tempdir, "testfile")), False)
def testExclude(self):
# test archiving a subdir with a file and extracting relative
base=self.tempdir
rar=opj(base,"testarchive1.rar")
a = archive.Archive(rar,base)
drel_top = "A"
drel=opj(drel_top,"1","a")
dabs=opj(base,drel)
file_helper.mkdir_p(dabs)
frel=opj(drel,"testfile-1")
fabs=opj(base,frel)
file_helper.create_file(fabs,"content-2")
frel2=opj(drel,"testfile-2")
fabs2=opj(base,frel2)
file_helper.create_file(fabs2,"content-2")
a.add_dir(drel_top)
a.exclude(frel) # exclude the first file!
self.assertEqual(a.run(),0)
self.assertEqual(file_helper.file_exists(rar),True)
file_helper.remove_file(fabs)
self.assertEqual(file_helper.file_exists(fabs),False)
file_helper.remove_file(fabs2)
self.assertEqual(file_helper.file_exists(fabs2),False)
# test content of archive by extracting:
b = archive.Archive(rar,base)
syscode = b.extract(target_path=self.tempdir)
self.assertEqual(syscode,0)
# the first file was excluded, may not exist:
self.assertEqual(file_helper.file_exists(fabs),False)
# the second file must be there:
self.assertEqual(file_helper.file_head(fabs2),"content-2")
def testExclude(self):
# test archiving a subdir with a file and extracting relative
base = self.tempdir
rar = opj(base, "testarchive1.rar")
a = archive.Archive(rar, base)
drel_top = "A"
drel = opj(drel_top, "1", "a")
dabs = opj(base, drel)
file_helper.mkdir_p(dabs)
frel = opj(drel, "testfile-1")
fabs = opj(base, frel)
file_helper.create_file(fabs, "content-2")
frel2 = opj(drel, "testfile-2")
fabs2 = opj(base, frel2)
file_helper.create_file(fabs2, "content-2")
a.add_dir(drel_top)
a.exclude(frel) # exclude the first file!
self.assertEqual(a.run(), 0)
self.assertEqual(file_helper.file_exists(rar), True)
file_helper.remove_file(fabs)
self.assertEqual(file_helper.file_exists(fabs), False)
file_helper.remove_file(fabs2)
self.assertEqual(file_helper.file_exists(fabs2), False)
# test content of archive by extracting:
b = archive.Archive(rar, base)
syscode = b.extract(target_path=self.tempdir)
self.assertEqual(syscode, 0)
# the first file was excluded, may not exist:
self.assertEqual(file_helper.file_exists(fabs), False)
# the second file must be there:
self.assertEqual(file_helper.file_head(fabs2), "content-2")
def testMakeArchiveFileInSubdir(self):
# test archiving a file in a subdir and extracting relative
base = self.tempdir
rar = opj(base,"testarchive1.rar")
# Part A:
a = archive.Archive(rar,base)
drel = opj("A","1","a") # relative path to directory
dabs = opj(base,drel) # absolute
frel = opj("A","1","a","testfile-1")
fabs = opj(base,frel)
file_helper.mkdir_p(dabs)
file_helper.create_file(fabs,"content-1")
self.assertEqual(file_helper.file_exists(fabs),True)
a.add_file(frel) # must add relative paths
# run and check if rar archive exists:
self.assertEqual(a.run(),0)
self.assertEqual(file_helper.file_exists(rar),True)
# remove original file and check:
file_helper.remove_file(fabs)
self.assertEqual(file_helper.file_exists(fabs),False)
# Part B:
b = archive.Archive(rar,base)
syscode = b.extract(target_path=base)
self.assertEqual(syscode,0)
self.assertEqual(file_helper.file_exists(fabs),True)
# TODO: check content:
self.assertEqual(file_helper.file_head(fabs),"content-1")
def testMakeArchiveFileInSubdir(self):
# test archiving a file in a subdir and extracting relative
base = self.tempdir
rar = opj(base, "testarchive1.rar")
# Part A:
a = archive.Archive(rar, base)
drel = opj("A", "1", "a") # relative path to directory
dabs = opj(base, drel) # absolute
frel = opj("A", "1", "a", "testfile-1")
fabs = opj(base, frel)
file_helper.mkdir_p(dabs)
file_helper.create_file(fabs, "content-1")
self.assertEqual(file_helper.file_exists(fabs), True)
a.add_file(frel) # must add relative paths
# run and check if rar archive exists:
self.assertEqual(a.run(), 0)
self.assertEqual(file_helper.file_exists(rar), True)
# remove original file and check:
file_helper.remove_file(fabs)
self.assertEqual(file_helper.file_exists(fabs), False)
# Part B:
b = archive.Archive(rar, base)
syscode = b.extract(target_path=base)
self.assertEqual(syscode, 0)
self.assertEqual(file_helper.file_exists(fabs), True)
# TODO: check content:
self.assertEqual(file_helper.file_head(fabs), "content-1")
def setup_mnist_pickle():
print('Setting up raw mnist pickle:')
destination_file = MNIST_PICKLE
if file_exists(destination_file) == True:
print('MNIST file already exists.')
return
(x_train, y_train), (x_test, y_test) = mnist.load_data()
x_train = x_train.astype('float32')
x_test = x_test.astype('float32')
print('\t\tTraining data (x_train) : ', x_train.shape)
print('\t\tTraining labels (y_train) : ', y_train.shape)
print('\t\tTest data (y_train) : ', x_test.shape)
print('\t\tTest labels (y_test) : ', y_test.shape)
dataset = {
'x_train': x_train,
'y_train': y_train,
'x_test': x_test,
'y_test': y_test
}
save_as_pickle(dataset, destination_file)
def testMakeArchiveTreeAndFile(self):
# test archiving files in a tree and extracting relative
# method to communcate tree: add 1 file and 1 base dir of a tree
# Part A creates a single file, archives it, then deletes it
# Part B extracts the file again from the archive and checks the content
base = self.tempdir
rar = opj(base,"testarchive1.rar")
drel_top = "A"
drel=opj(drel_top,"1","a")
dabs = opj(base,drel)
frel=opj(drel_top,"1","a","testfile-1")
fabs=opj(base,frel)
drel_topB = "B"
drel2=opj(drel_topB,"1","a")
dabs2 = opj(base,drel2)
frel2=opj(drel_topB,"1","a","testfile-")
fabs2=opj(base,frel2)
drel3=opj(drel_topB,"2","a")
dabs3 = opj(base,drel3)
frel3=opj(drel_topB,"2","a","testfile-")
fabs3=opj(base,frel3)
# Part A:
a = archive.Archive(rar,base)
file_helper.mkdir_p(dabs)
file_helper.create_file(fabs,"content-1")
a.add_file(frel)
file_helper.mkdir_p(dabs2)
file_helper.create_file(fabs2,"content-2")
file_helper.mkdir_p(dabs3)
file_helper.create_file(fabs3,"content-3")
a.add_dir(drel_topB)
self.assertEqual(a.run(),0)
self.assertEqual(file_helper.file_exists(rar),True)
file_helper.remove_file(fabs)
file_helper.remove_file(fabs2)
file_helper.remove_file(fabs3)
self.assertEqual(file_helper.file_exists(fabs),False)
self.assertEqual(file_helper.file_exists(fabs2),False)
self.assertEqual(file_helper.file_exists(fabs3),False)
# Part B:
b = archive.Archive(rar,base)
syscode = b.extract(target_path=base)
self.assertEqual(syscode,0)
self.assertEqual(file_helper.file_head(fabs),"content-1")
self.assertEqual(file_helper.file_head(fabs2),"content-2")
self.assertEqual(file_helper.file_head(fabs3),"content-3")
def testMakeArchiveTreeAndFile(self):
# test archiving files in a tree and extracting relative
# method to communcate tree: add 1 file and 1 base dir of a tree
# Part A creates a single file, archives it, then deletes it
# Part B extracts the file again from the archive and checks the content
base = self.tempdir
rar = opj(base, "testarchive1.rar")
drel_top = "A"
drel = opj(drel_top, "1", "a")
dabs = opj(base, drel)
frel = opj(drel_top, "1", "a", "testfile-1")
fabs = opj(base, frel)
drel_topB = "B"
drel2 = opj(drel_topB, "1", "a")
dabs2 = opj(base, drel2)
frel2 = opj(drel_topB, "1", "a", "testfile-")
fabs2 = opj(base, frel2)
drel3 = opj(drel_topB, "2", "a")
dabs3 = opj(base, drel3)
frel3 = opj(drel_topB, "2", "a", "testfile-")
fabs3 = opj(base, frel3)
# Part A:
a = archive.Archive(rar, base)
file_helper.mkdir_p(dabs)
file_helper.create_file(fabs, "content-1")
a.add_file(frel)
file_helper.mkdir_p(dabs2)
file_helper.create_file(fabs2, "content-2")
file_helper.mkdir_p(dabs3)
file_helper.create_file(fabs3, "content-3")
a.add_dir(drel_topB)
self.assertEqual(a.run(), 0)
self.assertEqual(file_helper.file_exists(rar), True)
file_helper.remove_file(fabs)
file_helper.remove_file(fabs2)
file_helper.remove_file(fabs3)
self.assertEqual(file_helper.file_exists(fabs), False)
self.assertEqual(file_helper.file_exists(fabs2), False)
self.assertEqual(file_helper.file_exists(fabs3), False)
# Part B:
b = archive.Archive(rar, base)
syscode = b.extract(target_path=base)
self.assertEqual(syscode, 0)
self.assertEqual(file_helper.file_head(fabs), "content-1")
self.assertEqual(file_helper.file_head(fabs2), "content-2")
self.assertEqual(file_helper.file_head(fabs3), "content-3")
