def test_evaulate_write(self):
dir = TempDirectory()
d = TestContainer('parsed', FileBlock('foo', 'content', 'write'))
d.evaluate_with(Files('td'), globs={'td': dir})
compare([C(FileResult, passed=True, expected=None, actual=None)],
[r.evaluated for r in d])
dir.compare(['foo'])
compare(dir.read('foo', 'ascii'), 'content')
def test_evaulate_write(self):
dir = TempDirectory()
d = TestContainer('parsed',FileBlock('foo','content','write'))
d.evaluate_with(Files('td'),globs={'td':dir})
compare([C(FileResult,
passed=True,
expected=None,
actual=None)],
[r.evaluated for r in d])
dir.compare(['foo'])
compare(dir.read('foo', 'ascii'), 'content')
class TestPrepareTarget(TestCase):
def setUp(self):
self.dir = TempDirectory()
self.addCleanup(self.dir.cleanup)
replace = Replacer()
replace('workfront.generate.TARGET_ROOT', self.dir.path)
self.addCleanup(replace.restore)
self.session = Session('test')
def test_from_scratch(self):
path = prepare_target(self.session)
compare(path, expected=self.dir.getpath('unsupported.py'))
self.dir.compare(expected=[])
def test_everything(self):
self.dir.write('unsupported.py', b'yy')
path = prepare_target(self.session)
compare(path, expected=self.dir.getpath('unsupported.py'))
self.dir.compare(expected=['unsupported.py'])
compare(self.dir.read('unsupported.py'), b"yy")
def test_dots_in_version(self):
path = prepare_target(Session('test', api_version='v4.0'))
compare(path, expected=self.dir.getpath('v40.py'))
self.dir.compare(expected=[])
class TestPrepareTarget(TestCase):
def setUp(self):
self.dir = TempDirectory()
self.addCleanup(self.dir.cleanup)
replace = Replacer()
replace('workfront.generate.TARGET_ROOT', self.dir.path)
self.addCleanup(replace.restore)
self.session = Session('test')
def test_from_scratch(self):
path = prepare_target(self.session)
compare(path, expected=self.dir.getpath('unsupported.py'))
self.dir.compare(expected=[])
def test_everything(self):
self.dir.write('unsupported.py', b'yy')
path = prepare_target(self.session)
compare(path, expected=self.dir.getpath('unsupported.py'))
self.dir.compare(expected=['unsupported.py'])
compare(self.dir.read('unsupported.py'), b"yy")
def test_dots_in_version(self):
path = prepare_target(Session('test', api_version='v4.0'))
compare(path, expected=self.dir.getpath('v40.py'))
self.dir.compare(expected=[])
class TestFolderTestCase(unittest.TestCase):
def setUp(self):
self.tempdir = TempDirectory(encoding='utf-8')
def tearDown(self):
self.tempdir.cleanup()
def test_returns_false_if_parent_attributes_not_set(self):
d = Folder(self.tempdir.path)
with self.subTest(1):
compare(self.tempdir.path, d.path)
with self.subTest(2):
compare(os.path.basename(self.tempdir.path), d.name)
with self.subTest(3):
compare(os.path.dirname(self.tempdir.path), d.parent)
def test_returns_false_if_attributes_are_not_equal(self):
temp_path = self.tempdir.path
dir_1 = Folder(temp_path)
dir_2 = Folder(self.tempdir.makedir('abc/123/JPG'))
dir_3 = Folder(self.tempdir.makedir('abc/&%^/website'))
d4 = self.tempdir.makedir('abc/123/ghi')
d5 = Folder(os.path.join(temp_path, 'abc', 'hjig'))
sorter_identity = self.tempdir.write(
os.path.join(d4, SORTER_FOLDER_IDENTITY_FILENAME), '')
dir_4 = Folder(d4)
with self.subTest(1):
compare(True, dir_1.exists)
with self.subTest(2):
compare(False, dir_1.for_sorter)
with self.subTest(3):
compare('FOLDERS', dir_1.category_folder)
with self.subTest(4):
compare(True, dir_2.for_sorter)
with self.subTest(5):
compare('image', dir_2.category_folder)
with self.subTest(6):
compare(True, dir_3.for_sorter)
with self.subTest(7):
compare(None, dir_3.category_folder)
with self.subTest(8):
compare('FOLDERS', dir_4.category_folder)
with self.subTest(9):
compare(True, os.path.isfile(sorter_identity))
with self.subTest(10):
compare(True, dir_4.for_sorter)
with self.subTest(11):
compare(False, d5.for_sorter)
def test_returns_false_if_folder_not_exists(self):
temp_path = self.tempdir.path
path = os.path.join(temp_path, 'bdk/ksks/a94')
dir_1 = Folder(path)
with self.subTest(1):
compare(False, dir_1.exists)
with self.subTest(2):
self.assertRaises(FileNotFoundError, dir_1.create)
with self.subTest(3):
compare(False, dir_1.exists)
dir_1.create(parents=True)
with self.subTest(4):
compare(True, dir_1.exists)
def test_returns_false_if_grouping_failed(self):
dir_1 = self.tempdir.makedir('abc/for/document')
dir_2 = self.tempdir.makedir('abc/for/PDF')
dir_3 = self.tempdir.makedir('abc/for/last')
dir_ = self.tempdir.makedir('one/two')
d1 = Folder(dir_1)
d2 = Folder(dir_2)
d3 = Folder(dir_3)
with self.subTest(1):
compare(dir_1, d1.path)
d1.move_to(dir_)
with self.subTest(2):
compare(os.path.join(dir_, 'document'), d1.path)
with self.subTest(3):
compare(dir_2, d2.path)
d2.move_to(dir_)
with self.subTest(4):
compare(os.path.join(dir_, 'document', 'PDF'), d2.path)
with self.subTest(5):
compare(dir_3, d3.path)
d3.move_to(dir_)
with self.subTest(6):
compare(os.path.join(dir_, 'FOLDERS', 'last'), d3.path)
def test_folder_grouping_with_files(self):
write = lambda path: self.tempdir.write(path, '')
isfile = os.path.isfile
dir_ = self.tempdir.makedir('one/two')
file_1 = write('abc/for/PDF/this long name.pdf')
file_2 = write('abc/for/DOCX/y.docx')
file_3 = write('abc/for/DOC/123 54.doc')
file_4 = write('abc/for/none/xw')
file_5 = write('abc/for/PDF/second.pdf')
dir_1 = Folder(os.path.dirname(file_1))
dir_2 = Folder(os.path.dirname(file_2))
dir_3 = Folder(os.path.dirname(file_3))
dir_4 = Folder(os.path.dirname(file_4))
dir_5 = Folder(os.path.dirname(file_5))
dir_6 = Folder(dir_)
with self.subTest(1):
compare([True, True, True, True],
[isfile(file_1), isfile(file_2), isfile(file_3), isfile(file_4)])
dir_1.move_to(dir_)
dir_2.move_to(dir_)
dir_3.move_to(dir_)
dir_4.move_to(dir_)
dir_5.move_to(dir_)
dir_6.move_to(dir_)
with self.subTest(3):
self.tempdir.compare([
'{}/'.format('document'),
'{}/'.format('FOLDERS'),
'{}/{}'.format('document', SORTER_FOLDER_IDENTITY_FILENAME),
'{}/{}'.format('FOLDERS', SORTER_FOLDER_IDENTITY_FILENAME),
'{}/{}/'.format('document', 'PDF'),
'{}/{}/'.format('document', 'DOCX'),
'{}/{}/'.format('document', 'DOC'),
'{}/{}/'.format('FOLDERS', 'none'),
'{}/{}/{}'.format('document', 'PDF',
'this long name.pdf'),
'{}/{}/{}'.format('document', 'PDF', 'second.pdf'),
'{}/{}/{}'.format('document', 'DOCX', 'y.docx'),
'{}/{}/{}'.format('document', 'DOC', '123 54.doc'),
'{}/{}/{}'.format('FOLDERS', 'none', 'xw'),
'{}/{}/{}'.format('document', 'PDF',
SORTER_FOLDER_IDENTITY_FILENAME),
'{}/{}/{}'.format('document', 'DOCX',
SORTER_FOLDER_IDENTITY_FILENAME),
'{}/{}/{}'.format('document', 'DOC',
SORTER_FOLDER_IDENTITY_FILENAME),
], path=dir_)
with self.subTest(4):
compare(os.path.join(dir_, 'document', 'PDF'), dir_5.path)
def test_folder_grouping_with_files_and_existing_destinations(self):
temp_path = self.tempdir.path
write = lambda path: self.tempdir.write(path, '')
isfile = os.path.isfile
dir_ = self.tempdir.makedir('one/two')
self.tempdir.makedir('one/two/document/PDF/')
self.tempdir.makedir('one/two/document/DOCX')
self.tempdir.makedir('one/two/document/DOC')
file_1 = write('abc/for/PDF/this long name.pdf')
file_2 = write('abc/for/DOCX/y.docx')
file_3 = write('abc/for/DOC/123 54.doc')
file_4 = write('abc/for/none/xw')
file_5 = write('abc/for/PDF/second.pdf')
file_6 = write('abc/for/image/JPEG/abc.jpeg')
dir_1 = Folder(os.path.dirname(file_1))
dir_2 = Folder(os.path.dirname(file_2))
dir_3 = Folder(os.path.dirname(file_3))
dir_4 = Folder(os.path.dirname(file_4))
dir_5 = Folder(os.path.dirname(file_5))
dir_6 = Folder(os.path.dirname(os.path.dirname(file_6)))
dir_7 = Folder(dir_)
with self.subTest(1):
compare([True, True, True, True],
[isfile(file_1), isfile(file_2), isfile(file_3), isfile(file_4)])
with self.subTest(2):
compare('document', dir_1.category_folder)
dir_1.move_to(dir_)
dir_2.move_to(dir_)
dir_3.move_to(dir_)
dir_4.move_to(dir_)
dir_5.move_to(dir_)
dir_6.move_to(dir_)
dir_7.move_to(dir_)
with self.subTest(3):
compare(True, dir_1.for_sorter)
with self.subTest(4):
self.tempdir.compare([
'{}/'.format('document'),
'{}/'.format('image'),
'{}/'.format('FOLDERS'),
'{}/{}/'.format('document', 'PDF'),
'{}/{}/'.format('document', 'DOCX'),
'{}/{}/'.format('document', 'DOC'),
'{}/{}/'.format('image', 'JPEG'),
'{}/{}/'.format('FOLDERS', 'none'),
'{}/{}'.format('document', SORTER_FOLDER_IDENTITY_FILENAME),
'{}/{}'.format('image', SORTER_FOLDER_IDENTITY_FILENAME),
'{}/{}'.format('FOLDERS', SORTER_FOLDER_IDENTITY_FILENAME),
'{}/{}/{}'.format('document', 'PDF',
'this long name.pdf'),
'{}/{}/{}'.format('document', 'PDF', 'second.pdf'),
'{}/{}/{}'.format('document', 'DOCX', 'y.docx'),
'{}/{}/{}'.format('document', 'DOC', '123 54.doc'),
'{}/{}/{}'.format('image', 'JPEG', 'abc.jpeg'),
'{}/{}/{}'.format('FOLDERS', 'none', 'xw'),
'{}/{}/{}'.format('document', 'PDF',
SORTER_FOLDER_IDENTITY_FILENAME),
'{}/{}/{}'.format('document', 'DOCX',
SORTER_FOLDER_IDENTITY_FILENAME),
'{}/{}/{}'.format('document', 'DOC',
SORTER_FOLDER_IDENTITY_FILENAME),
], path=dir_)
with self.subTest(5):
compare(os.path.join(dir_, 'document', 'PDF'), dir_5.path)
with self.subTest(6):
compare(True, os.path.isdir(dir_1.path))
with self.subTest(7):
compare(os.path.join(temp_path, dir_, 'document', 'PDF'), dir_1.path)
with self.subTest(8):
compare(True, os.path.isfile(os.path.join(
dir_1.path, SORTER_FOLDER_IDENTITY_FILENAME)))
with self.subTest(9):
compare(False, os.path.exists(os.path.dirname(file_1)))
def test_retuns_false_folder_with_multiple_subfolders_relocation_failse(self):
temp_path = self.tempdir.path
write = lambda path: self.tempdir.write(path, '')
dir_ = self.tempdir.makedir('one/two')
self.tempdir.makedir('one/two/document/PDF/')
file_1 = write('abc/for/PDF/this long name.pdf')
write('abc/for/PDF/somefolder/another/and another/JPEG/abc.jpeg')
dir_1 = Folder(os.path.dirname(file_1))
dir_1.move_to(dir_)
with self.subTest(1):
self.tempdir.compare([
'{}/'.format('document'),
'{}/{}'.format('document', SORTER_FOLDER_IDENTITY_FILENAME),
'{}/{}/'.format('document', 'PDF'),
'{}/{}/{}'.format('document', 'PDF',
SORTER_FOLDER_IDENTITY_FILENAME),
'{}/{}/{}/'.format('document', 'PDF', 'somefolder'),
'{}/{}/{}'.format('document', 'PDF', 'this long name.pdf'),
'{}/{}/{}/{}/'.format('document', 'PDF',
'somefolder', 'another'),
'{}/{}/{}/{}/{}/'.format('document', 'PDF',
'somefolder', 'another', 'and another'),
'{}/{}/{}/{}/{}/{}/'.format('document', 'PDF',
'somefolder', 'another', 'and another', 'JPEG'),
'{}/{}/{}/{}/{}/{}/{}'.format('document', 'PDF', 'somefolder',
'another', 'and another', 'JPEG', 'abc.jpeg'),
], path=dir_)
def test_returns_false_if_folder_relocation_with_ignore_file_succeeds(self):
temp_path = self.tempdir.path
write = lambda path: self.tempdir.write(path, '')
dir_ = self.tempdir.makedir('one/two')
self.tempdir.makedir('one/two/document/PDF/')
file_1 = write('abc/for/PDF/this long name.pdf')
file_2 = write(
'abc/for/PDF/somefolder/another/and another/JPEG/abc.jpeg')
file_3 = write('abc/for/PDF/somefolder/%s' % SORTER_IGNORE_FILENAME)
with self.subTest(1):
compare(True, os.path.isfile(file_3))
with self.subTest(2):
compare(True, os.path.isdir(os.path.dirname(file_2)))
dir_1 = Folder(os.path.dirname(file_1))
dir_1.move_to(dir_)
with self.subTest(3):
self.tempdir.compare([
'{}/'.format('document'),
'{}/{}'.format('document', SORTER_FOLDER_IDENTITY_FILENAME),
'{}/{}/'.format('document', 'PDF'),
'{}/{}/{}'.format('document', 'PDF',
SORTER_FOLDER_IDENTITY_FILENAME),
'{}/{}/{}'.format('document', 'PDF', 'this long name.pdf'),
], path=dir_)
def test_returns_false_if_original_folder_exists(self):
temp_path = self.tempdir.path
write = lambda path: self.tempdir.write(path, '')
isfile = os.path.isfile
dir_ = self.tempdir.makedir('one/two')
self.tempdir.makedir('one/two/document/PDF/')
file_1 = write('abc/for/PDF/this long name.pdf')
dir_1 = Folder(os.path.dirname(file_1))
dir_1.move_to(dir_)
with self.subTest(1):
self.tempdir.compare([
'{}/'.format('document'),
'{}/{}'.format('document', SORTER_FOLDER_IDENTITY_FILENAME),
'{}/{}/'.format('document', 'PDF'),
'{}/{}/{}'.format('document', 'PDF',
SORTER_FOLDER_IDENTITY_FILENAME),
'{}/{}/{}'.format('document', 'PDF', 'this long name.pdf'),
], path=dir_)
with self.subTest(2):
compare(False, os.path.isdir(os.path.dirname(file_1)))
def test_returns_false_if_original_folder_not_exists(self):
temp_path = self.tempdir.path
write = lambda path: self.tempdir.write(path, '')
isfile = os.path.isfile
dir_ = self.tempdir.makedir('one/two')
dir_2 = self.tempdir.makedir('one/two/document/PDF/')
file_1 = write('abc/for/PDF/this long name.pdf')
file_2 = write(
'abc/for/PDF/somefolder/another/and another/JPEG/abc.jpeg')
file_3 = write('abc/for/PDF/somefolder/%s' % SORTER_IGNORE_FILENAME)
with self.subTest(1):
compare(True, os.path.isfile(file_3))
with self.subTest(2):
compare(True, os.path.isdir(os.path.dirname(file_2)))
dir_1 = Folder(os.path.dirname(file_1))
dir_1.move_to(dir_)
with self.subTest(3):
self.tempdir.compare([
'{}/'.format('document'),
'{}/{}'.format('document', SORTER_FOLDER_IDENTITY_FILENAME),
'{}/{}/'.format('document', 'PDF'),
'{}/{}/{}'.format('document', 'PDF',
SORTER_FOLDER_IDENTITY_FILENAME),
'{}/{}/{}'.format('document', 'PDF', 'this long name.pdf'),
], path=dir_)
with self.subTest(4):
compare(True, os.path.isdir(dir_2))
def test_returns_false_if_grouping_in_same_folder_failed(self):
temp_path = self.tempdir.path
write = lambda path: self.tempdir.write(path, '')
isfile = os.path.isfile
dir_ = self.tempdir.makedir('one/two')
file_1 = write('one/two/PDF/this long name.pdf')
file_2 = write(
'one/two/PDF/somefolder/another/and another/JPEG/abc.jpeg')
file_3 = write('one/two/PDF/somefolder/%s' % SORTER_IGNORE_FILENAME)
with self.subTest(1):
compare(True, os.path.isfile(file_3))
with self.subTest(2):
compare(True, os.path.isdir(os.path.dirname(file_2)))
dir_1 = Folder(os.path.dirname(file_1))
dir_1.group(dir_, by_extension=True)
with self.subTest(3):
compare(True, os.path.isfile(os.path.join(temp_path, 'one',
'two', 'document', 'PDF', SORTER_FOLDER_IDENTITY_FILENAME)))
with self.subTest(4):
self.tempdir.compare([
'{}/'.format('document'),
'{}/'.format('PDF'),
'{}/{}'.format('document', SORTER_FOLDER_IDENTITY_FILENAME),
'{}/{}'.format('document', SORTER_IGNORE_FILENAME),
'{}/{}/'.format('document', 'PDF'),
'{}/{}/{}'.format('document', 'PDF',
SORTER_FOLDER_IDENTITY_FILENAME),
'{}/{}/{}'.format('document', 'PDF', 'this long name.pdf'),
'{}/{}/'.format('PDF', 'somefolder'),
'{}/{}/{}'.format('PDF', 'somefolder', SORTER_IGNORE_FILENAME),
'{}/{}/{}/'.format('PDF', 'somefolder', 'another'),
'{}/{}/{}/{}/'.format('PDF', 'somefolder',
'another', 'and another'),
'{}/{}/{}/{}/{}/'.format('PDF', 'somefolder',
'another', 'and another', 'JPEG'),
'{}/{}/{}/{}/{}/{}'.format('PDF', 'somefolder',
'another', 'and another', 'JPEG', 'abc.jpeg')
], path=dir_)
with self.subTest(5):
compare(True, os.path.isdir(os.path.dirname(file_1)))
class test_alignSource(unittest.TestCase):
def setUp(self):
self.dir = TempDirectory()
def tearDown(self):
self.dir.cleanup()
def test_sourcedest(self):
# Get test data
listofstacks, offsets, stackdim = helper_teststack.data(
transformationType.translation)
nstacks = len(listofstacks)
form = "stack%%0%dd" % int(np.floor(np.log10(nstacks)) + 1)
names = [form % i for i in range(nstacks)]
shape = listofstacks[0].shape
nimages = np.take(shape, stackdim)
imgsize = tuple(np.delete(shape, stackdim))
dtype = listofstacks[0].dtype
listofstacks_readonly = [s.copy() for s in listofstacks]
listofstacks2 = [
np.arange(8, dtype=dtype).reshape(2, 2, 2) for s in listofstacks
]
listofstacks3 = [
np.empty(s.shape, dtype=s.dtype) for s in listofstacks
]
listofstacks4 = [
np.zeros(s.shape, dtype=s.dtype) for s in listofstacks
]
# Open HDF5 file
h5name = "rawdata.h5"
h5name2 = "rawdata2.h5"
h5name3 = "rawdata3.h5"
h5name4 = "rawdata4.h5"
absh5name = os.path.join(self.dir.path, h5name)
absh5name2 = os.path.join(self.dir.path, h5name2)
absh5name3 = os.path.join(self.dir.path, h5name3)
absh5name4 = os.path.join(self.dir.path, h5name4)
with h5py.File(absh5name, mode="a") as h5f, h5py.File(
absh5name2, mode="a") as h5f2, h5py.File(
absh5name3, mode="a") as h5f3, h5py.File(absh5name4,
mode="a") as h5f4:
# Write h5 files
for i in range(nstacks):
h5f.create_dataset(names[i], data=listofstacks[i])
h5f2.create_dataset(names[i],
data=listofstacks2[i],
maxshape=shape)
h5f3.create_dataset(names[i], data=listofstacks3[i])
h5f4.create_dataset(names[i], data=listofstacks4[i])
self.dir.compare((h5name, h5name2, h5name3, h5name4))
h5datasets = [h5f[name] for name in names]
h5datasets2 = [h5f2[name] for name in names]
h5datasets3 = [h5f3[name] for name in names]
h5datasets4 = [h5f4[name] for name in names]
# Write edf files
form = "%%s_%%0%dd.edf" % int(np.floor(np.log10(nimages)) + 1)
edfnames = [[form % (name, i) for i in range(nimages)]
for name in names]
absedfnames = [[
os.path.join(self.dir.path, "edfdir", form % (name, i))
for i in range(nimages)
] for name in names]
absedfnames2 = [[
os.path.join(self.dir.path, "edfdir2", form % (name, i))
for i in range(nimages)
] for name in names]
absedfnames3 = [[
os.path.join(self.dir.path, "edfdir3", form % (name, i))
for i in range(nimages)
] for name in names]
absedfnames_1 = [[
os.path.join(self.dir.path, "edfdir_1", form % (name, i))
for i in range(nimages)
] for name in names]
edfdir = os.path.join(self.dir.path, "edfdir")
edfdir2 = os.path.join(self.dir.path, "edfdir2")
edfdir3 = os.path.join(self.dir.path, "edfdir3")
if not os.path.exists(edfdir):
os.makedirs(edfdir)
if not os.path.exists(edfdir2):
os.makedirs(edfdir2)
if not os.path.exists(edfdir3):
os.makedirs(edfdir3)
for i in range(nstacks):
for j in range(nimages):
f = fabio.edfimage.edfimage(data=np.take(
listofstacks_readonly[i], j, axis=stackdim))
f.write(absedfnames[i][j])
f.write(absedfnames2[i][j])
self.dir.compare(tuple(
[item for sublist in edfnames for item in sublist]),
path="edfdir")
self.dir.compare(
tuple([item for sublist in edfnames for item in sublist]),
path="edfdir2",
)
# Check source construction
source = [
alignSource(absh5name, names, stackdim=stackdim),
alignSource(os.path.join(self.dir.path, "edfdir"),
edfnames,
stackdim=stackdim),
alignSource(listofstacks, None, stackdim=stackdim),
alignSource(h5datasets, None, stackdim=stackdim),
]
self.assertTrue(all(nimages == s.nimages for s in source))
self.assertTrue(all(imgsize == s.imgsize for s in source))
# Check source reading
for i in range(nstacks):
for j in range(nimages):
img = np.take(listofstacks_readonly[i], j, axis=stackdim)
for s in source:
np.testing.assert_array_equal(s.readimg(i, j), img)
np.testing.assert_array_equal(
s.readimgas(i, j, np.float32),
img.astype(np.float32))
# Test destination construction
dest_exc = [
alignDest(listofstacks2, None, None,
stackdim=stackdim), # edit datasets with wrong size
alignDest(h5datasets2, None, None, stackdim=stackdim),
] # edit datasets with wrong size
dest = [
alignDest(absh5name, names, "",
stackdim=stackdim), # adds an entry1 subdirectory
alignDest(absh5name2,
names,
"",
stackdim=stackdim,
overwrite=True), # overwrites the datasets
# adds an entry1 subdirectory
alignDest(absh5name3, names, "", stackdim=stackdim),
alignDest(absh5name3, names, ".aligned",
stackdim=stackdim), # adds .aligned to names
# adds .aligned to names in subdirectory entry1
alignDest(absh5name3, names, ".aligned", stackdim=stackdim),
# adds .aligned to names in subdirectory entry2
alignDest(absh5name3, names, ".aligned", stackdim=stackdim),
alignDest(
os.path.join(self.dir.path, "edfdir"),
names,
".edf",
stackdim=stackdim,
), # adds a ..._1 subdirectory
alignDest(
os.path.join(self.dir.path, "edfdir2"),
names,
".edf",
stackdim=stackdim,
overwrite=True,
), # overwrites the files
# creates files in the existing (empty) directory
alignDest(
os.path.join(self.dir.path, "edfdir3"),
names,
".edf",
stackdim=stackdim,
),
alignDest(listofstacks2,
None,
None,
stackdim=stackdim,
overwrite=True), # reform and edit np array
alignDest(listofstacks3, None, None,
stackdim=stackdim), # edit np array
alignDest(h5datasets2,
None,
None,
stackdim=stackdim,
overwrite=True), # reform and edit datasets
alignDest(h5datasets3, None, None, stackdim=stackdim),
] # edit datasets
# Check destination preparation
for d in dest_exc:
try:
d.prepare(nimages, imgsize, dtype) # must throw an error
self.assertTrue(False)
except ValueError:
pass
for d in dest:
d.prepare(nimages, imgsize, dtype)
self.assertEqual(listofstacks2[0].shape, shape)
# Check writing
for i in range(nstacks):
for j in range(nimages):
for d in dest:
img = np.take(listofstacks_readonly[i],
j,
axis=stackdim)
d.writeimg(img, i, j)
# Check written files
self.dir.compare(tuple(
[item for sublist in edfnames for item in sublist]),
path="edfdir")
self.dir.compare(
tuple([item for sublist in edfnames for item in sublist]),
path="edfdir_1",
)
self.dir.compare(
tuple([item for sublist in edfnames for item in sublist]),
path="edfdir2",
)
self.dir.compare(
tuple([item for sublist in edfnames for item in sublist]),
path="edfdir3",
)
# Check written content
h5datasets3_a = [h5f3["/entry1/" + name] for name in names]
h5datasets3_b = [h5f3[name + ".aligned"] for name in names]
h5datasets3_c = [
h5f3["/entry1/" + name + ".aligned"] for name in names
]
h5datasets3_d = [
h5f3["/entry2/" + name + ".aligned"] for name in names
]
for i in range(nstacks):
for j in range(nimages):
img = np.take(listofstacks_readonly[i], j, axis=stackdim)
np.testing.assert_array_equal(
fabio.open(absedfnames[i][j]).data, img)
np.testing.assert_array_equal(
fabio.open(absedfnames2[i][j]).data, img)
np.testing.assert_array_equal(
fabio.open(absedfnames3[i][j]).data, img)
np.testing.assert_array_equal(
fabio.open(absedfnames_1[i][j]).data, img)
np.testing.assert_array_equal(listofstacks_readonly[i],
listofstacks2[i])
np.testing.assert_array_equal(listofstacks_readonly[i],
listofstacks3[i])
np.testing.assert_array_equal(listofstacks_readonly[i],
h5datasets[i])
np.testing.assert_array_equal(listofstacks_readonly[i],
h5datasets2[i])
np.testing.assert_array_equal(listofstacks_readonly[i],
h5datasets3[i])
np.testing.assert_array_equal(listofstacks_readonly[i],
h5datasets3_a[i])
np.testing.assert_array_equal(listofstacks_readonly[i],
h5datasets3_b[i])
np.testing.assert_array_equal(listofstacks_readonly[i],
h5datasets3_c[i])
np.testing.assert_array_equal(listofstacks_readonly[i],
h5datasets3_d[i])
np.testing.assert_array_equal(listofstacks_readonly[i] * 0,
listofstacks4[i])
np.testing.assert_array_equal(listofstacks_readonly[i] * 0,
h5datasets4[i])
def test_teststack(self):
listofstacks, offsets, stackdim = helper_teststack.data(
transformationType.translation)
self.assertIsInstance(listofstacks, list)
self.assertIsInstance(listofstacks[0], np.ndarray)
self.assertEqual(len(listofstacks[0].shape), 3)
self.assertTrue(
all(s.shape == listofstacks[0].shape for s in listofstacks))
class TestDecoratedObjectTypes(MockOpenHelper, TestCase):
def setUp(self):
super(TestDecoratedObjectTypes, self).setUp()
self.dir = TempDirectory()
self.addCleanup(self.dir.cleanup)
def test_normal(self):
base_url = 'https://test.attask-ondemand.com/attask/api/v4.0'
session = Session('test', api_version='v4.0')
self.server.add(
url=base_url+'/metadata',
params='method=GET',
response=json.dumps(dict(data=dict(objects=dict(
SomeThing=dict(objCode='SMTHING', name='SomeThing')
))))
)
expected = dict(
objCode='SMTHING',
name='SomeThing',
stuff='a value'
)
self.server.add(
url=base_url+'/smthing/metadata',
params='method=GET',
response=json.dumps(dict(data=expected))
)
compare(decorated_object_types(session, None),
expected=[('SomeThing', 'SMTHING', expected)])
def test_cache_write(self):
base_url = 'https://test.attask-ondemand.com/attask/api/v4.0'
session = Session('test', api_version='v4.0')
self.server.add(
url=base_url+'/metadata',
params='method=GET',
response=json.dumps(dict(data=dict(objects=dict(
SomeThing=dict(objCode='SMTHING', name='SomeThing')
))))
)
expected = dict(
objCode='SMTHING',
name='SomeThing',
stuff='a value'
)
self.server.add(
url=base_url+'/smthing/metadata',
params='method=GET',
response=json.dumps(dict(data=expected))
)
compare(decorated_object_types(session, self.dir.path),
expected=[('SomeThing', 'SMTHING', expected)])
self.dir.compare(expected=[
'v4.0_metadata.json', 'v4.0_smthing_metadata.json'
])
compare(
json.loads(self.dir.read('v4.0_metadata.json').decode('ascii')),
expected=dict(objects=dict(
SomeThing=dict(objCode='SMTHING', name='SomeThing')
)))
compare(
json.loads(
self.dir.read('v4.0_smthing_metadata.json').decode('ascii')
),
expected=expected
)
def test_cache_read(self):
expected = dict(
objCode='SMTHING',
name='SomeThing',
stuff='a value'
)
self.dir.write('v4.0_metadata.json', json.dumps(dict(objects=dict(
SomeThing=dict(objCode='SMTHING', name='SomeThing')
))), encoding='ascii')
self.dir.write('v4.0_smthing_metadata.json',
json.dumps(expected),
encoding='ascii')
session = Session('test', api_version='v4.0')
compare(decorated_object_types(session, self.dir.path),
expected=[('SomeThing', 'SMTHING', expected)])
def test_unsupported(self):
base_url = 'https://test.attask-ondemand.com/attask/api/unsupported'
session = Session('test')
self.server.add(
url=base_url+'/metadata',
params='method=GET',
response=json.dumps(dict(data=dict(objects=dict(
SomeThing=dict(objCode='SMTHING', name='SomeThing')
))))
)
expected = dict(
objCode='SMTHING',
name='SomeThing',
stuff='a value'
)
self.server.add(
url=base_url+'/smthing/metadata',
params='method=GET',
response=json.dumps(dict(data=expected))
)
compare(decorated_object_types(session, None),
expected=[('SomeThing', 'SMTHING', expected)])
def test_name_override(self):
base_url = 'https://test.attask-ondemand.com/attask/api/unsupported'
session = Session('test')
self.server.add(
url=base_url+'/metadata',
params='method=GET',
response=json.dumps(dict(data=dict(objects=dict(
SomeThing=dict(objCode='OPTASK', name='SomeThing')
))))
)
expected = dict(
objCode='SMTHING',
name='SomeThing',
stuff='a value'
)
self.server.add(
url=base_url+'/optask/metadata',
params='method=GET',
response=json.dumps(dict(data=expected))
)
compare(decorated_object_types(session, None),
expected=[('Issue', 'OPTASK', expected)])
class FunctionalTest(MockOpenHelper, TestCase):
base = 'https://api-cl01.attask-ondemand.com/attask/api/unsupported'
def setUp(self):
super(FunctionalTest, self).setUp()
self.log = LogCapture()
self.addCleanup(self.log.uninstall)
self.dir = TempDirectory()
self.addCleanup(self.dir.cleanup)
self.replace('logging.basicConfig', Mock())
self.replace('workfront.generate.TARGET_ROOT', self.dir.path)
def test_functional(self):
self.replace('sys.argv', ['x'])
self.server.add(
url='/metadata',
params='method=GET',
response=json.dumps(dict(data=dict(objects=dict(
SomeThing=dict(objCode='BAR', name='SomeThing'),
OtherThing=dict(objCode='FOO', name='OtherThing'),
))))
)
self.server.add(
url='/foo/metadata',
params='method=GET',
response=json.dumps(dict(data=dict(
objCode='FOO',
name='OtherThing',
fields={"ID": {}, "anotherField": {}},
references={},
collections={},
actions={},
)))
)
self.server.add(
url='/bar/metadata',
params='method=GET',
response=json.dumps(dict(data=dict(
objCode='BAR',
name='SomeThing',
fields={"ID": {}, "theField": {}},
references={"accessRules": {}},
collections={"assignedTo": {}},
actions={"doSomething": {
"arguments": [
{
"name": "anOption",
"type": "Task"
},
{
"name": "options",
"type": "string[]"
}
],
"resultType": "string",
"label": "doSomething"
}}
)))
)
with OutputCapture() as output:
output.disable()
main()
output.compare("")
self.dir.compare(expected=['unsupported.py'])
compare(self.dir.read('unsupported.py').decode('ascii'), expected=u'''\
# generated from https://api-cl01.attask-ondemand.com/attask/api/unsupported/metadata
from ..meta import APIVersion, Object, Field, Reference, Collection
api = APIVersion('unsupported')
class OtherThing(Object):
code = 'FOO'
another_field = Field('anotherField')
api.register(OtherThing)
class SomeThing(Object):
code = 'BAR'
the_field = Field('theField')
access_rules = Reference('accessRules')
assigned_to = Collection('assignedTo')
def do_something(self, an_option=None, options=None):
"""
The ``doSomething`` action.
:param an_option: anOption (type: ``Task``)
:param options: options (type: ``string[]``)
:return: ``string``
"""
params = {}
if an_option is not None: params['anOption'] = an_option
if options is not None: params['options'] = options
data = self.session.put(self.api_url()+'/doSomething', params)
return data['result']
api.register(SomeThing)
''',
trailing_whitespace=False)
class GradePredictorTestCase(unittest.TestCase):
"""
this class represent all the test cases we will use to test
our API
"""
def setUp(self):
"""this method will make all initialisation for ours tests"""
if os.getenv('CIRCLECI'): #if we are in CIRCLECI environement
self.app = create_app(config_name="CIRCLECI")
else:
self.app = create_app(config_name="testing")
self.app_floder = self.app.root_path
self.client = self.app.test_client
self.directory = TempDirectory(
path=self.app_floder+'/static',
create=False) #initialisation of temporary directory
self.predictives_models = []
self.new_student = {'DIPPERC':0.60, 'SCHOOL_RIGHT':'itfm/bukavu', 'OPTION_RIGHT':'elec indust'}
def test_model_exist(self):
"""
the following testcase will vertify if predictives models
exist in the floder classes
"""
self.directory.compare(sorted([
'FM.pkl',
'FSTA.pkl',
'FD.pkl',
'FSDC.pkl',
'FSEG.pkl',
'FPSE.pkl',
'FT.pkl'
], reverse=True), path='classes/')
def step_1_can_connect_post(self):
"""
Test API can create a (POST request)
"""
res = self.client().post('predictions/predict/', data=json.dumps(self.new_student), content_type='application/json')
self.assertEqual(res.status_code, 200)
#self.assertIn('welcome to grade ', str(res.data))
def step_2_can_load_models(self):
"""
test that the app can read the model from saved floders
here we will try to read all the modeles
and check if there are instances of preictives models class
"""
path = self.app_floder+'/static/classes/'
for filename in os.listdir(path):
model = joblib.load(path+filename)
self.predictives_models.append(model)
self.assertTrue(model.__class__, 'PredictiveModelBuilding' and model.__module__ == 'predictiveModelBuilding')
def step_3_can_predict(self):
"""
this method will test if the app can predict good values
of students grades by goods values I mean grades betweens 30-100
percents .
"""
#check if the model can handle unknow schools
new_student_data = pd.DataFrame(self.new_student, columns=self.new_student.keys(), index=range(1))
for dept in self.predictives_models:
predicted_grades = dept.predict_new(new_student_data)
final_grade = predicted_grades[0]
self.assertTrue((final_grade >= .3 or final_grade < .10) and (new_student_data['DIPPERC'][0] >= 0.50))
def _steps(self):
for name in sorted(dir(self)): #attributes of a object
if name.startswith("step"):
yield name, getattr(self, name)
def test_steps(self):
"""for now I don't know who this works"""
for name, step in self._steps():
try:
step()
except Exception as exception:
self.fail("{} {} failed ({}: {})".format(step, name, type(exception), exception))
def tearDown(self):
"""
class test_xiaedf(unittest.TestCase):
def setUp(self):
# import cProfile
# self.pr = cProfile.Profile()
# self.pr.enable()
self.dir = TempDirectory()
def tearDown(self):
self.dir.cleanup()
# self.pr.disable()
# self.pr.dump_stats("keep.cprof")
def test_nameparsing_special(self):
self.assertEqual(
xiaedf.xianameparser.parse("test_puz_PUZ_xmap_x1_00_0009_0000.edf"),
xiaedf.XiaName(
radix="test_puz",
mapnum=9,
linenum=-1,
label="PUZ_xmap_x1_00",
baselabel="PUZ_xmap_x1",
detector="00",
),
)
self.assertEqual(
xiaedf.xianameparser.parse("samB6_mapa_xmap_x3c_00_0002_0000.edf"),
xiaedf.XiaName(
radix="samB6_mapa",
mapnum=2,
linenum=-1,
label="xmap_x3c_00",
baselabel="xmap_x3c",
detector="00",
),
)
self.assertEqual(
xiaedf.xianameparser.parse("samB6_mapa_xiast_0002_0000_0069.edf"),
xiaedf.XiaName(
radix="samB6_mapa",
mapnum=2,
linenum=69,
label="xiast",
baselabel="xia",
detector="st",
),
)
def test_nameparsing(self):
paths = ["/tmp/a1", "/tmp/a2", "/tmp/b1"]
radix = ["a", "a", "b"]
mapnums = [range(0, 100), range(100, 200), range(0, 50)]
linenums = [range(0, 100), range(0, 100), range(0, 50)]
labels = [["arr_1", "arr_2", "xia00", "xia01", "xiaS0", "xiast"]] * 3
p = zip(paths, radix, mapnums, linenums, labels)
# Grouped
filesgrouped = collections.OrderedDict()
for path, radix, mapnums, linenums, labels in p:
if radix not in filesgrouped:
filesgrouped[radix] = collections.OrderedDict()
for mapnum in mapnums:
if mapnum not in filesgrouped[radix]:
filesgrouped[radix][mapnum] = collections.OrderedDict()
linenum = -1
filesgrouped[radix][mapnum][linenum] = [
os.path.join(
path, xiaedf.xiafilename(radix, mapnum, linenum, label)
)
for label in labels
if "xia" not in label
]
for linenum in linenums:
filesgrouped[radix][mapnum][linenum] = [
os.path.join(
path, xiaedf.xiafilename(radix, mapnum, linenum, label)
)
for label in labels
if "xia" in label
]
# Ordered list
filesordered = [
vline
for radix, vradix in filesgrouped.items()
for mapnum, vmap in vradix.items()
for linenum, vline in vmap.items()
]
filesordered = list(listtools.flatten(filesordered))
# Randomize list of files
files = copy(filesordered)
shuffle(files)
# Check sorting
files2 = sorted(files, key=xiaedf.xiasortkey)
self.assertEqual(filesordered, files2)
# Check grouping
g = xiaedf.xiagroup(files)
self.assertEqual(filesgrouped, g)
# Check map grouping
shuffle(files)
files = [f for f in files if os.path.basename(f).startswith("a")]
g = xiaedf.xiagroupmaps(files)
self.assertEqual(filesgrouped["a"], g)
# Check line grouping
shuffle(files)
files = [f for f in files if "0100_0000" in os.path.basename(f)]
g = xiaedf.xiagrouplines(files)
self.assertEqual(filesgrouped["a"][100], g)
def test_memmap(self):
path = self.dir.path
radix = "memmap"
mapnum = 0
linenum = 0
line = xiaedf.xialine_number(path, radix, mapnum, linenum)
data = np.random.rand(8, 5, 1)
line.save(data, ["xia00"])
emap = edf.edfmemmap(line.datafilenames()[0])
fmap = edf.edfimage(line.datafilenames()[0])
np.testing.assert_array_equal(data[..., 0], fmap.data)
np.testing.assert_array_equal(fmap.data, emap.data)
def _testdata(
self, dataorg, data, stats, ctrs, xiaobject, dshape, sshape, ishape, cshape
):
# data.shape: ....,nchan,ndet
# stats.shape: ....,nstat,ndet
ndim = len(dshape)
# Check data
xiaobject.onlyicrocr(False)
xiaobject.dtcor(False)
xiaobject.detectorsum(False)
xiaobject.globalnorm(None)
xiaobject.exclude_detectors = []
self.assertEqual(xiaobject.dshape, dshape)
self.assertEqual(xiaobject.sshape, sshape)
self.assertEqual(data.shape, xiaobject.dshape)
self.assertEqual(stats.shape, xiaobject.sshape)
self.assertEqual(data.dtype, xiaobject.dtype)
self.assertEqual(stats.dtype, xiaobject.stype)
np.testing.assert_array_equal(data, xiaobject.data)
np.testing.assert_array_equal(stats, xiaobject.stats)
# Check DT correction
xiaobject.onlyicrocr(True)
xiaobject.dtcor(False)
icrocr = xiaobject.stats
icr = icrocr[..., xiaobject.indexicr, :].reshape(ishape)
ocr = icrocr[..., xiaobject.indexocr, :].reshape(ishape)
cor = np.asarray(icr, dtype=xiaobject.CORTYPE) / np.asarray(
ocr, dtype=xiaobject.CORTYPE
)
np.testing.assert_array_equal(dataorg[..., 0], xiaobject.data * cor)
xiaobject.onlyicrocr(False)
xiaobject.dtcor(True)
np.testing.assert_array_equal(dataorg[..., 0], xiaobject.data)
# Test slicing
xiaobject.exclude_detectors = []
indices = genindexing.genindexingn(dshape, advanced=True, eco=True, nmax=50)
for dsum in [False, True]:
xiaobject.detectorsum(dsum)
for norm in [False, True]:
xiaobject.globalnorm("arr_flux" if norm else None)
for dtcor in [False, True]:
xiaobject.dtcor(dtcor)
for onlyicrocr in [False, True]:
xiaobject.onlyicrocr(onlyicrocr)
for together in [False, True]:
for index in indices:
# index = (-10, Ellipsis, slice(8, -3, 2), [True, True, False, False])
if dsum:
index = indexing.replace(
index, ndim, [-1], [slice(None)]
)
logger.debug("\n" * 5)
logger.debug("index = {}".format(index))
logger.debug("sum = {}".format(dsum))
logger.debug("norm = {}".format(norm))
logger.debug("dtcor = {}".format(dtcor))
logger.debug("onlyicrocr = {}".format(onlyicrocr))
logger.debug("together = {}".format(together))
if together:
if ctrs is None:
xiaobject.dataandstats()
ldata, lstats = xiaobject[index]
else:
xiaobject.dataall()
ldata, lstats, lcounters = xiaobject[index]
else:
xiaobject.onlydata()
ldata = xiaobject[index]
xiaobject.onlystats()
lstats = xiaobject[index]
if ctrs is not None:
xiaobject.onlycounters()
lcounters = xiaobject[index]
# Get data directly
if dtcor and norm:
ldata2 = dataorg[..., 2]
elif dtcor:
ldata2 = dataorg[..., 0]
elif norm:
ldata2 = dataorg[..., 1]
else:
ldata2 = data
ldata2 = ldata2[index]
if dsum:
ldata2 = xiaobject.sumdata(
ldata2, xiaobject._getaxis(-1)
)
# Check data
np.testing.assert_allclose(ldata, ldata2)
# Get stats directly
if xiaobject.nstats == 2:
lstats2 = stats[
..., [xiaobject.STICR, xiaobject.STOCR], :
]
else:
lstats2 = stats
lstats2 = lstats2[
indexing.replacefull(index, ndim, [-2])
]
# Check stats
np.testing.assert_array_equal(lstats, lstats2)
# Get counters directly
if ctrs is not None:
try:
lcounters2 = ctrs[
indexing.replacefull(index, ndim, [-2, -1])
]
check = True
except:
# This happens when the MCA channel index cannot be applied to the counter index (same dimension)
check = False
# Check counters
if check:
np.testing.assert_array_equal(
lcounters, lcounters2
)
# Test slicing vs. skip detector
xiaobject.onlyicrocr(False)
xiaobject.dtcor(False)
xiaobject.detectorsum(False)
xiaobject.globalnorm(None)
ndet = dshape[-1]
if ndet > 2:
xiaobject.exclude_detectors = []
ind = range(0, ndet, 2)
xiaobject.dataandstats()
ldata, lstats = xiaobject[..., ind]
np.testing.assert_array_equal(data[..., ind], ldata)
np.testing.assert_array_equal(stats[..., ind], lstats)
xiaobject.exclude_detectors = [0, ndet - 1]
np.testing.assert_array_equal(data[..., 1:-1], xiaobject.data)
np.testing.assert_array_equal(stats[..., 1:-1], xiaobject.stats)
if ndet > 1:
xiaobject.exclude_detectors = []
xiaobject.dataandstats()
ldata, lstats = xiaobject[..., 1:]
np.testing.assert_array_equal(data[..., 1:], ldata)
np.testing.assert_array_equal(stats[..., 1:], lstats)
xiaobject.exclude_detectors = [0]
self.assertEqual(ldata.shape, xiaobject.dshape)
self.assertEqual(lstats.shape, xiaobject.sshape)
np.testing.assert_array_equal(data[..., 1:], xiaobject.data)
np.testing.assert_array_equal(stats[..., 1:], xiaobject.stats)
def _testcopy(self, xiaobject, xiaobjectgen, path):
iscompound = isinstance(xiaobject, xiaedf.xiacompound)
i = 0
for dsum in [False, True]:
xiaobject.detectorsum(dsum)
copystats = not dsum
if dsum:
xialabels = ["xiaS1"]
else:
xialabels = xiaobject.xialabels_used
for norm in [False, True]:
xiaobject.globalnorm("arr_flux" if norm else None)
for dtcor in [False, True]:
xiaobject.dtcor(dtcor)
for onlyicrocr in [False, True]:
xiaobject.onlyicrocr(onlyicrocr)
for copy in [False, True]:
for copyctrs in [True, False]:
for deflabels in [False, True]:
# no counter when not a xiacompound (counters are saved per image)
if copyctrs and not iscompound:
continue
path2 = os.path.join(path, "copy{}".format(i))
i += 1
xiaobject2 = xiaobjectgen(path2)
logger.debug("\n" * 5)
logger.debug("sum = {}".format(dsum))
logger.debug("norm = {}".format(norm))
logger.debug("dtcor = {}".format(dtcor))
logger.debug("onlyicrocr = {}".format(onlyicrocr))
logger.debug("copyctrs = {}".format(copyctrs))
logger.debug("deflabels = {}".format(deflabels))
logger.debug("copy = {}".format(copy))
kwargs = {}
if deflabels:
kwargs["xialabels"] = xialabels
if copy:
kwargs["stats"] = copystats
if iscompound:
kwargs["ctrs"] = copyctrs
if copyctrs:
kwargs[
"ctrnames"
] = xiaobject.counterbasenames()
xiaobject2.save(xiaobject, **kwargs)
else:
if copystats:
kwargs["stats"] = xiaobject.allstats
if copyctrs and iscompound:
kwargs["ctrs"] = xiaobject.counters[..., 0]
kwargs[
"ctrnames"
] = xiaobject.counterbasenames()
xiaobject2.save(xiaobject.data, **kwargs)
np.testing.assert_array_equal(
xiaobject.data, xiaobject2.data
)
if copystats:
np.testing.assert_array_equal(
xiaobject.allstats, xiaobject2.allstats
)
if copyctrs:
np.testing.assert_array_equal(
xiaobject.counters, xiaobject2.counters
)
def _test_line(self, path, radix, mapnum, linenum, ndet, nspec, nchan):
# Generate some spectra + statistics
dataorg, data, stats = xiagen.data(nspec, nchan, ndet)
# Save data
line = xiaedf.xialine_number(path, radix, mapnum, linenum)
xialabels = ["xia{:02d}".format(i) for i in range(ndet)]
line.save(data, xialabels, stats=stats)
# Check saved files
expectedfiles = [
"{}_xia{:02}_{:04}_0000_{:04}.edf".format(radix, det, mapnum, linenum)
for det in range(ndet)
] + ["{}_xiast_{:04}_0000_{:04}.edf".format(radix, mapnum, linenum)]
expectedfiles.sort()
self.dir.compare(expectedfiles, path=path)
# Check static files names
files = xiaedf.xiasearch(path, radix=radix)
line2 = xiaedf.xialine_files(files)
line3 = xiaedf.xialine_number(path, radix, mapnum, linenum)
self.assertEqual(
files,
sorted(
[os.path.join(path, f) for f in expectedfiles], key=xiaedf.xiasortkey
),
)
self.assertEqual(line.statfilename(), line2.statfilename())
self.assertEqual(line.datafilenames(), line2.datafilenames())
self.assertEqual(line.statfilename(), line3.statfilename())
self.assertEqual(line.datafilenames(), line3.datafilenames())
# Check copy
def linegen(x):
return xiaedf.xialine_number(x, radix, mapnum, linenum)
self._testcopy(line, linegen, path)
# Check data
dshape = (nspec, nchan, ndet)
sshape = (nspec, xiaedf.xiadata.NSTATS, ndet)
ishape = (nspec, 1, ndet)
self._testdata(dataorg, data, stats, None, line, dshape, sshape, ishape, None)
# import matplotlib.pyplot as plt
# plt.plot(data[0,:,0])
# plt.show()
def _xiaconfigidcheck(self, xiaobject):
if isinstance(xiaobject, xiaedf.xiacompound):
ret = []
for l in xiaobject._items:
add = self._xiaconfigidcheck(l)
if isinstance(add, list):
ret += add
else:
ret.append(add)
return ret
else:
return (id(xiaobject._xiaconfig), id(xiaobject._cache))
def _test_image(self, path, radix, mapnum, ndet, ncol, nrow, nchan):
# Generate some spectra + statistics
flux = np.linspace(1, 2, nrow * ncol)
ctr1 = np.linspace(10, 20, nrow * ncol)
ctr2 = np.linspace(20, 30, nrow * ncol)
dataorg, data, stats = xiagen.data(nrow * ncol, nchan, ndet, flux=flux)
dataorg = dataorg.reshape(nrow, ncol, nchan, ndet, 3)
data = data.reshape(nrow, ncol, nchan, ndet)
stats = stats.reshape(nrow, ncol, xiaedf.xiadata.NSTATS, ndet)
ctrs = np.stack([ctr1, flux, ctr2], axis=1).reshape(nrow, ncol, 3)
ctrnames = ["arr_ctr1", "arr_flux", "arr_ctr2"]
# Save data
image = xiaedf.xiaimage_number(path, radix, mapnum)
xialabels = ["xia{:02d}".format(i) for i in range(ndet)]
image.save(data, xialabels, stats=stats, ctrs=ctrs, ctrnames=ctrnames)
# Check saved files
expectedfiles = (
[
"{}_xia{:02}_{:04}_0000_{:04}.edf".format(radix, det, mapnum, linenum)
for det in range(ndet)
for linenum in range(nrow)
]
+ [
"{}_xiast_{:04}_0000_{:04}.edf".format(radix, mapnum, linenum)
for linenum in range(nrow)
]
+ ["{}_{}_{:04}_0000.edf".format(radix, k, mapnum) for k in ctrnames]
)
expectedfiles.sort()
self.dir.compare(expectedfiles, path=path)
# Check files names
files = xiaedf.xiasearch(path, radix=radix)
image2 = xiaedf.xiaimage_files(files)
image3 = xiaedf.xiaimage_linenumbers(path, radix, mapnum, range(nrow))
image4 = xiaedf.xiaimage_number(path, radix, mapnum)
self.assertEqual(
files,
sorted(
[os.path.join(path, f) for f in expectedfiles], key=xiaedf.xiasortkey
),
)
self.assertEqual(image.statfilenames(), image2.statfilenames())
self.assertEqual(image.datafilenames(), image2.datafilenames())
self.assertEqual(image.ctrfilenames(), image2.ctrfilenames())
self.assertEqual(image.statfilenames(), image3.statfilenames())
self.assertEqual(image.datafilenames(), image3.datafilenames())
self.assertEqual(image.ctrfilenames(), image3.ctrfilenames())
self.assertEqual(image.statfilenames(), image4.statfilenames())
self.assertEqual(image.datafilenames(), image4.datafilenames())
self.assertEqual(image.ctrfilenames(), image4.ctrfilenames())
# Check only one config
for o in [image, image2, image3, image4]:
tmp = self._xiaconfigidcheck(o)
self.assertEqual(tmp.count(tmp[0]), len(tmp))
# Check copy
def imagegen(x):
return xiaedf.xiaimage_number(x, radix, mapnum)
self._testcopy(image, imagegen, path)
# Check data
dshape = (nrow, ncol, nchan, ndet)
sshape = (nrow, ncol, xiaedf.xiadata.NSTATS, ndet)
ishape = (nrow, ncol, 1, ndet)
cshape = (nrow, ncol, 3, 1)
ctrs = ctrs[:, :, np.argsort(ctrnames)]
ctrs = ctrs[..., np.newaxis]
self._testdata(
dataorg, data, stats, ctrs, image, dshape, sshape, ishape, cshape
)
def _test_stack(self, path, radix, ndet, ncol, nrow, nenergy, nchan):
# Generate some spectra + statistics
flux = np.linspace(1, 2 * nenergy, nrow * ncol * nenergy)
ctr1 = np.linspace(10, 20, nrow * ncol * nenergy)
ctr2 = np.linspace(20, 30, nrow * ncol * nenergy)
dataorg, data, stats = xiagen.data(
nrow * ncol * nenergy, nchan, ndet, flux=flux
)
dataorg = dataorg.reshape(nenergy, nrow, ncol, nchan, ndet, 3)
data = data.reshape(nenergy, nrow, ncol, nchan, ndet)
stats = stats.reshape(nenergy, nrow, ncol, xiaedf.xiadata.NSTATS, ndet)
ctrs = np.stack([ctr1, flux, ctr2], axis=1).reshape(nenergy, nrow, ncol, 3)
ctrnames = ["arr_ctr1", "arr_flux", "arr_ctr2"]
# Save data
stack = xiaedf.xiastack_radix(path, radix)
xialabels = ["xia{:02d}".format(i) for i in range(ndet)]
stack.save(data, xialabels, stats=stats, ctrs=ctrs, ctrnames=ctrnames)
# Check saved files
expectedfiles = (
[
"{}_xia{:02}_{:04}_0000_{:04}.edf".format(radix, det, mapnum, linenum)
for det in range(ndet)
for linenum in range(nrow)
for mapnum in range(nenergy)
]
+ [
"{}_xiast_{:04}_0000_{:04}.edf".format(radix, mapnum, linenum)
for linenum in range(nrow)
for mapnum in range(nenergy)
]
+ [
"{}_{}_{:04}_0000.edf".format(radix, k, mapnum)
for mapnum in range(nenergy)
for k in ctrnames
]
)
expectedfiles.sort()
self.dir.compare(expectedfiles, path=path)
# Check files names
files = xiaedf.xiasearch(path, radix=radix)
stack2 = xiaedf.xiastack_files(files)
stack3 = xiaedf.xiastack_radix(path, radix)
stack4 = xiaedf.xiastack_mapnumbers(path, radix, range(nenergy))
self.assertEqual(
files,
sorted(
[os.path.join(path, f) for f in expectedfiles], key=xiaedf.xiasortkey
),
)
self.assertEqual(stack.statfilenames(), stack2.statfilenames())
self.assertEqual(stack.datafilenames(), stack2.datafilenames())
self.assertEqual(stack.ctrfilenames(), stack2.ctrfilenames())
self.assertEqual(stack.statfilenames(), stack3.statfilenames())
self.assertEqual(stack.datafilenames(), stack3.datafilenames())
self.assertEqual(stack.ctrfilenames(), stack3.ctrfilenames())
self.assertEqual(stack.statfilenames(), stack4.statfilenames())
self.assertEqual(stack.datafilenames(), stack4.datafilenames())
self.assertEqual(stack.ctrfilenames(), stack4.ctrfilenames())
# Check copy
def stackgen(x):
return xiaedf.xiastack_radix(x, radix)
self._testcopy(stack, stackgen, path)
# Check data
dshape = (nenergy, nrow, ncol, nchan, ndet)
sshape = (nenergy, nrow, ncol, xiaedf.xiadata.NSTATS, ndet)
ishape = (nenergy, nrow, ncol, 1, ndet)
cshape = (nenergy, nrow, ncol, 3, 1)
ctrs = ctrs[:, :, :, np.argsort(ctrnames)]
ctrs = ctrs[..., np.newaxis]
self._testdata(
dataorg, data, stats, ctrs, stack, dshape, sshape, ishape, cshape
)
def test_line(self):
mapnum = 2
linenum = 10
i = 0
for ndet in [2, 1]:
for nchan in [8, 1]:
for nspec in [6, 1]:
self._test_line(
os.path.join(self.dir.path, "test_line_{}".format(i)),
"test_line_{}".format(i),
mapnum,
linenum,
ndet,
nspec,
nchan,
)
i += 1
def test_image(self):
mapnum = 2
i = 0
for ndet in [2, 1]:
for nchan in [8, 1]:
for ncol in [6, 1]:
for nrow in [3, 1]:
self._test_image(
os.path.join(self.dir.path, "test_image_{}".format(i)),
"test_image_{}".format(i),
mapnum,
ndet,
ncol,
nrow,
nchan,
)
i += 1
def test_stack(self):
i = 0
for ndet in [2]:
for nchan in [8]:
for ncol in [6]:
for nrow in [3]:
for nenergy in [2]:
self._test_stack(
os.path.join(self.dir.path, "test_stack_{}".format(i)),
"test_stack_{}".format(i),
ndet,
ncol,
nrow,
nenergy,
nchan,
)
i += 1
class TestDecoratedObjectTypes(MockOpenHelper, TestCase):
def setUp(self):
super(TestDecoratedObjectTypes, self).setUp()
self.dir = TempDirectory()
self.addCleanup(self.dir.cleanup)
def test_normal(self):
base_url = 'https://test.attask-ondemand.com/attask/api/v4.0'
session = Session('test', api_version='v4.0')
self.server.add(
url=base_url + '/metadata',
params='method=GET',
response=json.dumps(
dict(data=dict(objects=dict(
SomeThing=dict(objCode='SMTHING', name='SomeThing'))))))
expected = dict(objCode='SMTHING', name='SomeThing', stuff='a value')
self.server.add(url=base_url + '/smthing/metadata',
params='method=GET',
response=json.dumps(dict(data=expected)))
compare(decorated_object_types(session, None),
expected=[('SomeThing', 'SMTHING', expected)])
def test_cache_write(self):
base_url = 'https://test.attask-ondemand.com/attask/api/v4.0'
session = Session('test', api_version='v4.0')
self.server.add(
url=base_url + '/metadata',
params='method=GET',
response=json.dumps(
dict(data=dict(objects=dict(
SomeThing=dict(objCode='SMTHING', name='SomeThing'))))))
expected = dict(objCode='SMTHING', name='SomeThing', stuff='a value')
self.server.add(url=base_url + '/smthing/metadata',
params='method=GET',
response=json.dumps(dict(data=expected)))
compare(decorated_object_types(session, self.dir.path),
expected=[('SomeThing', 'SMTHING', expected)])
self.dir.compare(
expected=['v4.0_metadata.json', 'v4.0_smthing_metadata.json'])
compare(json.loads(
self.dir.read('v4.0_metadata.json').decode('ascii')),
expected=dict(objects=dict(
SomeThing=dict(objCode='SMTHING', name='SomeThing'))))
compare(json.loads(
self.dir.read('v4.0_smthing_metadata.json').decode('ascii')),
expected=expected)
def test_cache_read(self):
expected = dict(objCode='SMTHING', name='SomeThing', stuff='a value')
self.dir.write(
'v4.0_metadata.json',
json.dumps(
dict(objects=dict(
SomeThing=dict(objCode='SMTHING', name='SomeThing')))),
encoding='ascii')
self.dir.write('v4.0_smthing_metadata.json',
json.dumps(expected),
encoding='ascii')
session = Session('test', api_version='v4.0')
compare(decorated_object_types(session, self.dir.path),
expected=[('SomeThing', 'SMTHING', expected)])
def test_unsupported(self):
base_url = 'https://test.attask-ondemand.com/attask/api/unsupported'
session = Session('test')
self.server.add(
url=base_url + '/metadata',
params='method=GET',
response=json.dumps(
dict(data=dict(objects=dict(
SomeThing=dict(objCode='SMTHING', name='SomeThing'))))))
expected = dict(objCode='SMTHING', name='SomeThing', stuff='a value')
self.server.add(url=base_url + '/smthing/metadata',
params='method=GET',
response=json.dumps(dict(data=expected)))
compare(decorated_object_types(session, None),
expected=[('SomeThing', 'SMTHING', expected)])
def test_name_override(self):
base_url = 'https://test.attask-ondemand.com/attask/api/unsupported'
session = Session('test')
self.server.add(
url=base_url + '/metadata',
params='method=GET',
response=json.dumps(
dict(data=dict(objects=dict(
SomeThing=dict(objCode='OPTASK', name='SomeThing'))))))
expected = dict(objCode='SMTHING', name='SomeThing', stuff='a value')
self.server.add(url=base_url + '/optask/metadata',
params='method=GET',
response=json.dumps(dict(data=expected)))
compare(decorated_object_types(session, None),
expected=[('Issue', 'OPTASK', expected)])
class FunctionalTest(MockOpenHelper, TestCase):
base = 'https://api-cl01.attask-ondemand.com/attask/api/unsupported'
def setUp(self):
super(FunctionalTest, self).setUp()
self.log = LogCapture()
self.addCleanup(self.log.uninstall)
self.dir = TempDirectory()
self.addCleanup(self.dir.cleanup)
self.replace('logging.basicConfig', Mock())
self.replace('workfront.generate.TARGET_ROOT', self.dir.path)
def test_functional(self):
self.replace('sys.argv', ['x'])
self.server.add(
url='/metadata',
params='method=GET',
response=json.dumps(
dict(data=dict(objects=dict(
SomeThing=dict(objCode='BAR', name='SomeThing'),
OtherThing=dict(objCode='FOO', name='OtherThing'),
)))))
self.server.add(url='/foo/metadata',
params='method=GET',
response=json.dumps(
dict(data=dict(
objCode='FOO',
name='OtherThing',
fields={
"ID": {},
"anotherField": {}
},
references={},
collections={},
actions={},
))))
self.server.add(
url='/bar/metadata',
params='method=GET',
response=json.dumps(
dict(data=dict(objCode='BAR',
name='SomeThing',
fields={
"ID": {},
"theField": {}
},
references={"accessRules": {}},
collections={"assignedTo": {}},
actions={
"doSomething": {
"arguments": [{
"name": "anOption",
"type": "Task"
}, {
"name": "options",
"type": "string[]"
}],
"resultType":
"string",
"label":
"doSomething"
}
}))))
with OutputCapture() as output:
output.disable()
main()
output.compare("")
self.dir.compare(expected=['unsupported.py'])
compare(self.dir.read('unsupported.py').decode('ascii'),
expected=u'''\
# generated from https://api-cl01.attask-ondemand.com/attask/api/unsupported/metadata
from ..meta import APIVersion, Object, Field, Reference, Collection
api = APIVersion('unsupported')
class OtherThing(Object):
code = 'FOO'
another_field = Field('anotherField')
api.register(OtherThing)
class SomeThing(Object):
code = 'BAR'
the_field = Field('theField')
access_rules = Reference('accessRules')
assigned_to = Collection('assignedTo')
def do_something(self, an_option=None, options=None):
"""
The ``doSomething`` action.
:param an_option: anOption (type: ``Task``)
:param options: options (type: ``string[]``)
:return: ``string``
"""
params = {}
if an_option is not None: params['anOption'] = an_option
if options is not None: params['options'] = options
data = self.session.put(self.api_url()+'/doSomething', params)
return data['result']
api.register(SomeThing)
''',
trailing_whitespace=False)
class TestOperationsTestCase(unittest.TestCase):
def setUp(self):
self.temp = TempDirectory(encoding='utf-8')
self.tempdir = self.temp.path
self.DB_NAME = DB_NAME
self.db_helper = DatabaseHelper(self.DB_NAME)
self.operations = SorterOps(self.db_helper)
def tearDown(self):
self.temp.cleanup()
def test_returns_false_if_default_not_match(self):
with self.subTest(1):
compare('', self.operations.src)
with self.subTest(1):
compare('', self.operations.dst)
with self.subTest(1):
compare('', self.operations.search_string)
with self.subTest(1):
compare('', self.operations.search_string_pattern)
with self.subTest(1):
compare('', self.operations.glob_pattern)
with self.subTest(1):
compare(False, self.operations.group)
with self.subTest(1):
compare(False, self.operations.recursive)
with self.subTest(1):
compare(['*'], self.operations.file_types)
with self.subTest(1):
compare(None, self.operations.status)
with self.subTest(1):
compare(None, self.operations.parser)
with self.subTest(1):
compare({}, self.operations.database_dict)
def test_returns_false_if_search_string_pattern_not_match(self):
search_string = 'one common 2 $ word'
pattern = '*[oO][nN][eE]?[cC][oO][mM][mM][oO][nN]?2?$?[wW][oO][rR][dD]'
with self.subTest(1):
compare(pattern, self.operations.form_search_pattern(search_string))
with self.subTest(2):
compare('', self.operations.form_search_pattern(''))
with self.subTest(3):
compare('', self.operations.form_search_pattern(' '))
with self.subTest(4):
compare('', self.operations.form_search_pattern([]))
@unittest.skipIf(os.name == 'nt', 'Windows systems do not have root folder')
def test_returns_false_if_folder_if_is_writable_fails(self):
dir_1 = self.temp.makedir('one/two')
self.temp.makedir('three/two')
with self.subTest(1):
compare(True, self.operations.is_writable(dir_1))
with self.subTest(2):
compare(False, self.operations.is_writable('/'))
def add_files_to_path(self, path, choice='few', start=None, end=None):
if choice == 'few':
file_list = few_files
if choice == 'many':
file_list = many_files
paths = []
for file_ in file_list[start:end]:
file_path = self.temp.write(os.path.join(path, file_), '')
paths.append(file_path.replace(path + os.sep, ''))
return paths
def test_returns_false_if_files_not_sorted(self):
def messenger(*args, **kwargs):
pass
dir_1 = self.temp.makedir('one/two')
self.add_files_to_path(dir_1)
self.operations.src = dir_1
self.operations.group = False
self.operations.recursive = False
with self.subTest(1):
compare(os.path.join(self.tempdir, 'one', 'two'), dir_1)
self.operations.sort_files(send_message=messenger)
with self.subTest(2):
self.temp.compare([
'{}/'.format('JPEG'),
'{}/'.format('PNG'),
'{}/'.format('PHP'),
'{}/'.format('ZIP'),
'{}/'.format('GZ'),
'{}/'.format('MP4'),
'{}/'.format('APK'),
'{}/'.format('PPTX'),
'{}/{}'.format('JPEG',
'WhatsApp Image 2017-06-10 at 9.53.02 PM.jpeg'),
'{}/{}'.format('PNG', 'unnamed.png'),
'{}/{}'.format('PHP', 'index.php'),
'{}/{}'.format('PHP', 'wp-settings.php'),
'{}/{}'.format('ZIP', 'WallPaper.zip'),
'{}/{}'.format('GZ', 'coverage-4.4.1.tar.gz'),
'{}/{}'.format('MP4', 'Coldplay - The Scientist.mp4'),
'{}/{}'.format('APK',
'GBWAPlus v5.60-2.17.146 @atnfas_hoak.apk'),
'{}/{}'.format('PPTX',
'updates to www.thetruenorth.co.ke 01.pptx'),
'{}/{}'.format('PNG',
'pyinstaller-draft1a-35x35-trans.png'),
'{}/{}'.format('JPEG', SORTER_FOLDER_IDENTITY_FILENAME),
'{}/{}'.format('PNG', SORTER_FOLDER_IDENTITY_FILENAME),
'{}/{}'.format('PHP', SORTER_FOLDER_IDENTITY_FILENAME),
'{}/{}'.format('ZIP', SORTER_FOLDER_IDENTITY_FILENAME),
'{}/{}'.format('GZ', SORTER_FOLDER_IDENTITY_FILENAME),
'{}/{}'.format('MP4', SORTER_FOLDER_IDENTITY_FILENAME),
'{}/{}'.format('APK', SORTER_FOLDER_IDENTITY_FILENAME),
'{}/{}'.format('PPTX', SORTER_FOLDER_IDENTITY_FILENAME),
], path=dir_1)
def test_returns_false_if_files_not_sorted_by_category_with_extension(self):
def messenger(*args, **kwargs):
pass
dir_1 = self.temp.makedir('one/two')
self.add_files_to_path(dir_1)
self.operations.src = dir_1
self.operations.group = True
self.operations.recursive = False
self.operations.by_extension = True
with self.subTest(1):
compare(os.path.join(self.tempdir, 'one', 'two'), dir_1)
self.operations.sort_files(send_message=messenger)
with self.subTest(2):
self.temp.compare([
'{}/'.format('image'),
'{}/'.format('website'),
'{}/'.format('archive'),
'{}/'.format('video'),
'{}/'.format('installer'),
'{}/'.format('presentation'),
'{}/{}/'.format('image', 'JPEG'),
'{}/{}/'.format('image', 'PNG'),
'{}/{}/'.format('website', 'PHP'),
'{}/{}/'.format('archive', 'ZIP'),
'{}/{}/'.format('archive', 'GZ'),
'{}/{}/'.format('video', 'MP4'),
'{}/{}/'.format('installer', 'APK'),
'{}/{}/'.format('presentation', 'PPTX'),
'{}/{}'.format('image', SORTER_FOLDER_IDENTITY_FILENAME),
'{}/{}'.format('website',
SORTER_FOLDER_IDENTITY_FILENAME),
'{}/{}'.format('archive',
SORTER_FOLDER_IDENTITY_FILENAME),
'{}/{}'.format('video', SORTER_FOLDER_IDENTITY_FILENAME),
'{}/{}'.format('installer',
SORTER_FOLDER_IDENTITY_FILENAME),
'{}/{}'.format('presentation',
SORTER_FOLDER_IDENTITY_FILENAME),
'{}/{}'.format('image', SORTER_IGNORE_FILENAME),
'{}/{}'.format('website',
SORTER_IGNORE_FILENAME),
'{}/{}'.format('archive',
SORTER_IGNORE_FILENAME),
'{}/{}'.format('video', SORTER_IGNORE_FILENAME),
'{}/{}'.format('installer',
SORTER_IGNORE_FILENAME),
'{}/{}'.format('presentation',
SORTER_IGNORE_FILENAME),
'{}/{}/{}'.format('image', 'JPEG',
'WhatsApp Image 2017-06-10 at 9.53.02 PM.jpeg'),
'{}/{}/{}'.format('image', 'PNG', 'unnamed.png'),
'{}/{}/{}'.format('website', 'PHP', 'index.php'),
'{}/{}/{}'.format('website', 'PHP', 'wp-settings.php'),
'{}/{}/{}'.format('archive', 'ZIP', 'WallPaper.zip'),
'{}/{}/{}'.format('archive', 'GZ',
'coverage-4.4.1.tar.gz'),
'{}/{}/{}'.format('video', 'MP4',
'Coldplay - The Scientist.mp4'),
'{}/{}/{}'.format('installer', 'APK',
'GBWAPlus v5.60-2.17.146 @atnfas_hoak.apk'),
'{}/{}/{}'.format('presentation', 'PPTX',
'updates to www.thetruenorth.co.ke 01.pptx'),
'{}/{}/{}'.format('image', 'PNG',
'pyinstaller-draft1a-35x35-trans.png'),
'{}/{}/{}'.format('image', 'JPEG',
SORTER_FOLDER_IDENTITY_FILENAME),
'{}/{}/{}'.format('image', 'PNG',
SORTER_FOLDER_IDENTITY_FILENAME),
'{}/{}/{}'.format('website', 'PHP',
SORTER_FOLDER_IDENTITY_FILENAME),
'{}/{}/{}'.format('archive', 'ZIP',
SORTER_FOLDER_IDENTITY_FILENAME),
'{}/{}/{}'.format('archive', 'GZ',
SORTER_FOLDER_IDENTITY_FILENAME),
'{}/{}/{}'.format('video', 'MP4',
SORTER_FOLDER_IDENTITY_FILENAME),
'{}/{}/{}'.format('installer', 'APK',
SORTER_FOLDER_IDENTITY_FILENAME),
'{}/{}/{}'.format('presentation', 'PPTX',
SORTER_FOLDER_IDENTITY_FILENAME),
], path=dir_1)
def test_returns_false_if_files_not_sorted_by_category_without_extension(self):
def messenger(*args, **kwargs):
pass
dir_1 = self.temp.makedir('one/two')
self.add_files_to_path(dir_1)
self.operations.src = dir_1
self.operations.group = True
self.operations.recursive = False
self.operations.group_folder_name = 'sample dir'
self.by_extension = False
with self.subTest(1):
compare(os.path.join(self.tempdir, 'one', 'two'), dir_1)
self.operations.sort_files(send_message=messenger)
with self.subTest(2):
self.temp.compare([
'{}/'.format('sample dir'),
'{}/{}'.format('sample dir',
'WhatsApp Image 2017-06-10 at 9.53.02 PM.jpeg'),
'{}/{}'.format('sample dir', 'unnamed.png'),
'{}/{}'.format('sample dir', 'index.php'),
'{}/{}'.format('sample dir', 'wp-settings.php'),
'{}/{}'.format('sample dir', 'WallPaper.zip'),
'{}/{}'.format('sample dir', 'coverage-4.4.1.tar.gz'),
'{}/{}'.format('sample dir',
'Coldplay - The Scientist.mp4'),
'{}/{}'.format('sample dir',
'GBWAPlus v5.60-2.17.146 @atnfas_hoak.apk'),
'{}/{}'.format('sample dir',
'updates to www.thetruenorth.co.ke 01.pptx'),
'{}/{}'.format('sample dir',
'pyinstaller-draft1a-35x35-trans.png'),
'{}/{}'.format('sample dir',
SORTER_FOLDER_IDENTITY_FILENAME),
'{}/{}'.format('sample dir',
SORTER_IGNORE_FILENAME),
], path=dir_1)
def test_returns_false_if_database_insert_fails(self):
path = self.temp.write('one/three/abc.txt', '')
hash_path = hashlib.md5(path.encode('utf-8')).hexdigest()
self.operations.db_helper.initialise_db(test=True)
obj = self.operations.db_helper.db_file_objects.create(
filename='abc.txt', filepath_hash=hash_path,
last_modified=datetime.now(), added_at=datetime.now())
with self.subTest(1):
compare(1, obj.id)
count = self.operations.db_helper.db_file_objects.count()
with self.subTest(2):
compare(1, count)
def test_returns_false_false_if_start_fails(self):
def messenger(*args, **kwargs):
pass
dir_1 = self.temp.makedir('one/two')
paths = self.add_files_to_path(dir_1)
expected = ['index.php', 'wp-settings.php']
contents = [i for i in paths if os.path.basename(i) not in expected]
self.operations.src = dir_1
self.operations.group = False
self.operations.recursive = False
with self.subTest(1):
compare(os.path.join(self.tempdir, 'one', 'two'), dir_1)
kwargs = {
'search_string': '',
'file_types': ['php'],
'group': False,
'recursive': False,
}
db_ready = self.db_helper.initialise_db(test=True)
with self.subTest(3):
compare(True, db_ready)
with self.subTest(4):
compare(True, os.path.exists(self.db_helper.DB_NAME))
self.operations.start(src=dir_1, dst=dir_1,
send_message=messenger, **kwargs)
with self.subTest(5):
self.temp.compare([
'{}/'.format('PHP'),
'{}/{}'.format('PHP', 'index.php'),
'{}/{}'.format('PHP', 'wp-settings.php'),
'{}/{}'.format('PHP', SORTER_FOLDER_IDENTITY_FILENAME),
] + contents, path=dir_1)
def test_returns_false_false_if_operation_fails_group_true(self):
def messenger(*args, **kwargs):
pass
dir_1 = self.temp.makedir('one/two')
dir_2 = self.temp.makedir('three/two')
self.add_files_to_path(dir_1, 'many')
self.operations.src = dir_1
self.operations.group = False
self.operations.recursive = False
with self.subTest(1):
compare(os.path.join(self.tempdir, 'one', 'two'), dir_1)
kwargs = {
'search_string': 'whatsapp image',
'group': True,
'recursive': False,
}
db_ready = self.db_helper.initialise_db(test=True)
with self.subTest(3):
compare(True, db_ready)
with self.subTest(4):
compare(True, os.path.exists(self.db_helper.DB_NAME))
self.operations.start(src=dir_1, dst=dir_2,
send_message=messenger, **kwargs)
with self.subTest(5):
self.temp.compare([
'{}/'.format('whatsapp image'),
'{}/{}'.format('whatsapp image',
'WhatsApp Image 2017-06-10 at 9.53.02 PM.jpeg'),
'{}/{}'.format('whatsapp image',
'WhatsApp Image 2017-06-05 at 09.19.55.jpeg'),
'{}/{}'.format('whatsapp image',
'WhatsApp Image 2017-06-14 at 2.26.53 PM.jpeg'),
'{}/{}'.format('whatsapp image', 'GBWhatsApp Images.zip'),
'{}/{}'.format('whatsapp image',
SORTER_FOLDER_IDENTITY_FILENAME),
'{}/{}'.format('whatsapp image',
SORTER_IGNORE_FILENAME),
], path=dir_2)
def test_returns_false_false_if_operation_fails_group_true_with_matching_folders(self):
def messenger(*args, **kwargs):
pass
dir_1 = self.temp.makedir('one/two')
dir_2 = self.temp.makedir('three/two')
self.temp.makedir('one/two/new whatsapp images folder')
self.add_files_to_path(dir_1, 'many')
self.operations.src = dir_1
self.operations.group = False
self.operations.recursive = False
with self.subTest(1):
compare(os.path.join(self.tempdir, 'one', 'two'), dir_1)
kwargs = {
'search_string': 'whatsapp image',
'group': True,
'recursive': False,
}
db_ready = self.db_helper.initialise_db(test=True)
with self.subTest(3):
compare(True, db_ready)
with self.subTest(4):
compare(True, os.path.exists(self.db_helper.DB_NAME))
self.operations.start(src=dir_1, dst=dir_2,
send_message=messenger, **kwargs)
with self.subTest(5):
self.temp.compare([
'{}/'.format('whatsapp image'),
'{}/{}/'.format('whatsapp image',
'new whatsapp images folder'),
'{}/{}'.format('whatsapp image',
'WhatsApp Image 2017-06-10 at 9.53.02 PM.jpeg'),
'{}/{}'.format('whatsapp image',
'WhatsApp Image 2017-06-05 at 09.19.55.jpeg'),
'{}/{}'.format('whatsapp image',
'WhatsApp Image 2017-06-14 at 2.26.53 PM.jpeg'),
'{}/{}'.format('whatsapp image', 'GBWhatsApp Images.zip'),
'{}/{}'.format('whatsapp image',
SORTER_FOLDER_IDENTITY_FILENAME),
'{}/{}'.format('whatsapp image',
SORTER_IGNORE_FILENAME),
], path=dir_2)
def test_returns_false_if_recursive_operation_fails(self):
def messenger(*args, **kwargs):
pass
dir_1 = self.temp.makedir('three/two')
dir_2 = self.temp.makedir('one/two')
dir_3 = self.temp.makedir('one/two/new whatsapp images folder')
dir_4 = self.temp.makedir('one/two/new whatsapp images folder/last')
self.add_files_to_path(dir_2, 'few', start=0, end=3)
self.add_files_to_path(dir_3, 'few', start=3, end=6)
self.add_files_to_path(dir_4, 'few', start=6)
with self.subTest(1):
compare([True, True, True, True], [os.path.isdir(dir_1),
os.path.isdir(
dir_2), os.path.isdir(dir_3),
os.path.isdir(dir_4)])
kwargs = {
'group_folder_name': 'sample dir',
'group': True,
'recursive': True,
}
self.operations.start(src=dir_2, dst=dir_1,
send_message=messenger, **kwargs)
with self.subTest(2):
self.temp.compare([
'{}/'.format('sample dir'),
'{}/{}'.format('sample dir',
'WhatsApp Image 2017-06-10 at 9.53.02 PM.jpeg'),
'{}/{}'.format('sample dir', 'unnamed.png'),
'{}/{}'.format('sample dir', 'index.php'),
'{}/{}'.format('sample dir', 'wp-settings.php'),
'{}/{}'.format('sample dir', 'WallPaper.zip'),
'{}/{}'.format('sample dir', 'coverage-4.4.1.tar.gz'),
'{}/{}'.format('sample dir',
'Coldplay - The Scientist.mp4'),
'{}/{}'.format('sample dir',
'GBWAPlus v5.60-2.17.146 @atnfas_hoak.apk'),
'{}/{}'.format('sample dir',
'updates to www.thetruenorth.co.ke 01.pptx'),
'{}/{}'.format('sample dir',
'pyinstaller-draft1a-35x35-trans.png'),
'{}/{}'.format('sample dir',
SORTER_FOLDER_IDENTITY_FILENAME),
'{}/{}'.format('sample dir',
SORTER_IGNORE_FILENAME),
], path=dir_1)
def test_returns_false_if_cleanup_failed(self):
def messenger(*args, **kwargs):
pass
dir_1 = self.temp.makedir('three/two')
dir_2 = self.temp.makedir('one/two')
dir_3 = self.temp.makedir('one/two/new whatsapp images folder')
dir_4 = self.temp.makedir('one/two/new whatsapp images folder/last')
self.add_files_to_path(dir_2, 'few', start=0, end=3)
self.add_files_to_path(dir_3, 'few', start=3, end=6)
self.add_files_to_path(dir_4, 'few', start=6)
with self.subTest(1):
compare([True, True, True, True], [os.path.isdir(dir_1),
os.path.isdir(
dir_2), os.path.isdir(dir_3),
os.path.isdir(dir_4)])
kwargs = {
'group_folder_name': 'sample dir',
'group': True,
'recursive': True,
}
self.operations.start(src=dir_2, dst=dir_1,
send_message=messenger, **kwargs)
self.operations.perform_cleanup(dir_2)
with self.subTest(2):
compare([False, False], [os.path.isdir(dir_3), os.path.isdir(dir_4)])
def test_returns_false_if_cleanup_failed_with_sorter_files(self):
def messenger(*args, **kwargs):
pass
dir_1 = self.temp.makedir('three/two')
dir_2 = self.temp.makedir('one/two')
dir_3 = self.temp.makedir('one/two/new whatsapp images folder')
dir_4 = self.temp.makedir('one/two/new whatsapp images folder/last')
dir_5 = self.temp.makedir('one/two/new/whatsapp/images/folder')
self.temp.write('one/two/new/whatsapp/%s' % SORTER_IGNORE_FILENAME, '')
self.temp.write('one/two/new/whatsapp/images/%s' %
SORTER_FOLDER_IDENTITY_FILENAME, '')
self.add_files_to_path(dir_2, 'few', start=0, end=3)
self.add_files_to_path(dir_3, 'few', start=3, end=6)
self.add_files_to_path(dir_4, 'few', start=6)
with self.subTest(1):
compare([True, True, True, True, True], [os.path.isdir(dir_1),
os.path.isdir(
dir_2), os.path.isdir(dir_3),
os.path.isdir(dir_4), os.path.isdir(dir_5)])
kwargs = {
'group_folder_name': 'sample dir',
'group': True,
'recursive': True,
}
self.operations.start(src=dir_2, dst=dir_1,
send_message=messenger, **kwargs)
self.operations.perform_cleanup(dir_2)
with self.subTest(2):
compare([False, False, False], [os.path.isdir(dir_3),
os.path.isdir(dir_4), os.path.isdir(dir_5)])
