def test_resource_directory_gets_nearest_possible_parent(self):
self._set_data_directory_controller('tmp')
self.project.data.add_child(TestDataDirectoryController(_data_directory(j('tmp','some'))))
self._set_resources(j('tmp','some','resoruces','res.txt'))
assert_equals(len(self.project.data.children), 1)
assert_equals(len(self.project.data.children[0].children), 1)
assert_equals(self.project.data.children[0].children[0].children, [self.project.resources[0]])
def deploy_upstart(ini_path, desc, command_name):
site_upstart_template = open(j(this_dir, 'upstart.conf.template'), "r").read()
site_upstart = site_upstart_template.format(UWSGI_INI_LOCATION=ini_path,
UWSGI_FOR=desc)
with open(j(upstart_dir, '%s.conf' % command_name), "w") as upstart_dest:
upstart_dest.write(site_upstart)
def load(self, fname=None):
import os
if fname is None:
num=1
l_files = os.listdir(PARAMS)
while 'exp%i_params.npy'%num in l_files:
num += 1
num -=1
fname = 'exp%i'%num
from os.path import join as j
if not os.path.exists(j(PARAMS, '%s_params.npy'%fname)):
ename = ''.join(c for c in fname if c.isalpha())
fname = '%s_pretrain'%ename
if not os.path.exists(j(PARAMS, '%s_params.npy'%fname)):
return False
params = np.load(j(PARAMS, '%s_params.npy'%fname))
params_ft = np.load(j(PARAMS, '%s_params_ft.npy'%fname))
for k in range(self.N):
self.layers[k].W.set_value(params[3*k].get_value())
self.layers[k].vbias.set_value(params[3*k+1].get_value())
self.layers[k].hbias.set_value(params[3*k+2].get_value())
self.layers[k].eps_up.set_value(params_ft[2*k].get_value())
self.layers[k].eps_down.set_value(params_ft[2*k+1].get_value())
return True
def test_mixed_case_checksums(self):
bag = bagit.make_bag(self.tmpdir)
hashstr = {}
#Extract entries only for the payload and ignore
# entries from the tagmanifest file
for key in bag.entries.iterkeys():
if key.startswith('data' + os.sep):
hashstr = bag.entries[key]
hashstr = hashstr.itervalues().next()
manifest = open(j(self.tmpdir, "manifest-md5.txt"), "r").read()
manifest = manifest.replace(hashstr, hashstr.upper())
open(j(self.tmpdir, "manifest-md5.txt"),
"w").write(manifest)
#Since manifest-md5.txt file is updated, re-calculate its
# md5 checksum and update it in the tagmanifest-md5.txt file
hasher = hashlib.new('md5')
hasher.update(open(j(self.tmpdir, "manifest-md5.txt"), "r").read())
tagmanifest = open(j(self.tmpdir, "tagmanifest-md5.txt"), "r").read()
tagmanifest = tagmanifest.replace(
bag.entries['manifest-md5.txt']['md5'], hasher.hexdigest())
open(j(self.tmpdir, "tagmanifest-md5.txt"), "w").write(tagmanifest)
bag = bagit.Bag(self.tmpdir)
self.assertTrue(self.validate(bag))
def test_make_bag_sha1_sha256_manifest(self):
bag = bagit.make_bag(self.tmpdir, checksum=['sha1', 'sha256'])
# check that relevant manifests are created
self.assertTrue(os.path.isfile(j(self.tmpdir, 'manifest-sha1.txt')))
self.assertTrue(os.path.isfile(j(self.tmpdir, 'manifest-sha256.txt')))
# check valid with two manifests
self.assertTrue(self.validate(bag, fast=True))
def test_mixed_case_checksums(self):
bag = bagit.make_bag(self.tmpdir, checksums=["md5"])
hashstr = {}
# Extract entries only for the payload and ignore
# entries from the tagmanifest file
for key in bag.entries.keys():
if key.startswith("data" + os.sep):
hashstr = bag.entries[key]
hashstr = next(iter(hashstr.values()))
manifest = slurp_text_file(j(self.tmpdir, "manifest-md5.txt"))
manifest = manifest.replace(hashstr, hashstr.upper())
with open(j(self.tmpdir, "manifest-md5.txt"), "wb") as m:
m.write(manifest.encode("utf-8"))
# Since manifest-md5.txt file is updated, re-calculate its
# md5 checksum and update it in the tagmanifest-md5.txt file
hasher = hashlib.new("md5")
contents = slurp_text_file(j(self.tmpdir, "manifest-md5.txt")).encode("utf-8")
hasher.update(contents)
with open(j(self.tmpdir, "tagmanifest-md5.txt"), "r") as tagmanifest:
tagman_contents = tagmanifest.read()
tagman_contents = tagman_contents.replace(
bag.entries["manifest-md5.txt"]["md5"], hasher.hexdigest()
)
with open(j(self.tmpdir, "tagmanifest-md5.txt"), "w") as tagmanifest:
tagmanifest.write(tagman_contents)
bag = bagit.Bag(self.tmpdir)
self.assertTrue(self.validate(bag))
def test_validate_slow_without_oxum_extra_file(self):
bag = bagit.make_bag(self.tmpdir)
os.remove(j(self.tmpdir, "bag-info.txt"))
with open(j(self.tmpdir, "data", "extra_file"), "w") as ef:
ef.write("foo")
bag = bagit.Bag(self.tmpdir)
self.assertRaises(bagit.BagValidationError, self.validate, bag, fast=False)
def test_filename_unicode_normalization(self):
# We need to handle cases where the Unicode normalization form of a
# filename has changed in-transit. This is hard to do portably in both
# directions because OS X normalizes *all* filenames to an NFD variant
# so we'll start with a basic test which writes the manifest using the
# NFC form and confirm that this does not cause the bag to fail when it
# is written to the filesystem using the NFD form, which will not be
# altered when saved to an HFS+ filesystem:
test_filename = "Núñez Papers.txt"
test_filename_nfd = unicodedata.normalize("NFD", test_filename)
os.makedirs(j(self.tmpdir, "unicode-normalization"))
with open(j(self.tmpdir, "unicode-normalization", test_filename_nfd), "w") as f:
f.write("This is a test filename written using NFD normalization\n")
bag = bagit.make_bag(self.tmpdir)
bag.save()
self.assertTrue(bag.is_valid())
# Now we'll cause the entire manifest file was normalized to NFC:
for m_f in bag.manifest_files():
contents = slurp_text_file(m_f)
normalized_bytes = unicodedata.normalize("NFC", contents).encode("utf-8")
with open(m_f, "wb") as f:
f.write(normalized_bytes)
for alg in bag.algorithms:
bagit._make_tagmanifest_file(alg, bag.path, encoding=bag.encoding)
# Now we'll reload the whole thing:
bag = bagit.Bag(self.tmpdir)
self.assertTrue(bag.is_valid())
def test_make_bag_md5_sha256_manifest(self):
bag = bagit.make_bag(self.tmpdir, checksum=["md5", "sha256"])
# check that relevant manifests are created
self.assertTrue(os.path.isfile(j(self.tmpdir, "manifest-md5.txt")))
self.assertTrue(os.path.isfile(j(self.tmpdir, "manifest-sha256.txt")))
# check valid with two manifests
self.assertTrue(self.validate(bag, fast=True))
def compute_features(ex, package=['db_marche.features']):
cc = 0
feats = {}
pname = os.path.abspath(os.path.join(os.path.dirname(__file__), 'features'))
# List the directories present in the given package
# For each directory with name not starting with _
# We load all the files and add it to the features
# dictionnary
l_dirs = glob.glob(j(pname, '*',''))
for ldir in l_dirs:
dirname = ldir.split(os.sep)[-2]
if dirname[0] != '_':
n, s_feats = compute_features(ex, package+[dirname])
if n > 0:
feats.update(s_feats)
cc += n
# List the module present in the given package
# For each module, compute all the features and
# add them to the features dictionary
for m in glob.glob(j(pname, '*.py')):
fname = m.split(os.sep)[-1][:-3]
if fname[0] != '_':
module = '.'.join(package + [fname])
s_feats = load_feature(ex, module=module)
feats.update(s_feats)
cc += len(s_feats)
return cc, feats
def find_matching(subj, unknown_modality):
"""
Matches between the 'original' folders and the 'renamed' ones
provided to us. There are two ways to match:
a) make sure the list of files match
b) use the prefix (each folder starts with 00X_..., and only the stuff
after the _ was renamed)
the matchine is done with a) and an assertion is used to check b
"""
orig_dir = j(DATA_ROOT, subj, unknown_modality)
filelist = remove_analyze_files(set(os.listdir(orig_dir)))
renamed = j(RENAMED_ROOT, subj)
matching_name = None
for renamed_dir in os.listdir(renamed):
if renamed_dir.startswith('.'):
continue
renamed_filelist = remove_analyze_files(set(os.listdir(j(renamed, renamed_dir))))
if renamed_filelist == filelist:
assert matching_name is None
matching_name = renamed_dir
if matching_name is not None:
assert unknown_modality[:3] == matching_name[:3]
matching_name = matching_name.split('_', 1)[1].lower()
for unwanted_prefix in ['brain', 'acute_strok']:
if matching_name.startswith(unwanted_prefix):
matching_name = matching_name[len(unwanted_prefix):]
else:
raise ValueError("couldn't find a folder with that data")
return matching_name
def test_validation_error_details(self):
bag = bagit.make_bag(self.tmpdir)
readme = j(self.tmpdir, "data", "README")
txt = slurp_text_file(readme)
txt = 'A' + txt[1:]
with open(readme, "w") as r:
r.write(txt)
extra_file = j(self.tmpdir, "data", "extra")
with open(extra_file, "w") as ef:
ef.write('foo')
# remove the bag-info.txt which contains the oxum to force a full
# check of the manifest
os.remove(j(self.tmpdir, "bag-info.txt"))
bag = bagit.Bag(self.tmpdir)
got_exception = False
try:
self.validate(bag)
except bagit.BagValidationError as e:
got_exception = True
self.assertTrue("invalid bag: bag-info.txt exists in manifest but not found on filesystem" in str(e))
self.assertTrue("data/extra exists on filesystem but is not in manifest" in str(e))
self.assertTrue("data/README checksum validation failed (alg=md5 expected=8e2af7a0143c7b8f4de0b3fc90f27354 found=fd41543285d17e7c29cd953f5cf5b955)" in str(e))
self.assertTrue("bag-info.txt checksum validation failed (alg=md5 expected=aeba487217e50cc9c63ac5f90a0b87cb found=%s does not exist)" % j(self.tmpdir, "bag-info.txt"))
self.assertEqual(len(e.details), 4)
error = e.details[0]
self.assertEqual(str(error), "bag-info.txt exists in manifest but not found on filesystem")
self.assertTrue(isinstance(error, bagit.FileMissing))
self.assertEqual(error.path, "bag-info.txt")
error = e.details[1]
self.assertEqual(str(error), "data/extra exists on filesystem but is not in manifest")
self.assertTrue(isinstance(error, bagit.UnexpectedFile))
self.assertEqual(error.path, "data/extra")
if e.details[2].path == 'data/README':
readme_error = e.details[2]
baginfo_error = e.details[3]
else:
readme_error = e.details[3]
baginfo_error = e.details[2]
self.assertEqual(str(readme_error), "data/README checksum validation failed (alg=md5 expected=8e2af7a0143c7b8f4de0b3fc90f27354 found=fd41543285d17e7c29cd953f5cf5b955)")
self.assertTrue(isinstance(readme_error, bagit.ChecksumMismatch))
self.assertEqual(readme_error.algorithm, 'md5')
self.assertEqual(readme_error.path, 'data/README')
self.assertEqual(readme_error.expected, '8e2af7a0143c7b8f4de0b3fc90f27354')
self.assertEqual(readme_error.found, 'fd41543285d17e7c29cd953f5cf5b955')
# cannot test full value of baginfo error as it contains random name for tmp dir
self.assertTrue("bag-info.txt does not exist" in str(baginfo_error))
self.assertTrue(isinstance(baginfo_error, bagit.ChecksumMismatch))
self.assertEqual(baginfo_error.algorithm, 'md5')
self.assertEqual(baginfo_error.path, 'bag-info.txt')
self.assertTrue("bag-info.txt does not exist" in baginfo_error.found)
if not got_exception:
self.fail("didn't get BagValidationError")
def test_load_bagmake_bag_sha1_sha256_manifest(self): bagit.make_bag(self.tmpdir, checksums=['sha1', 'sha256']) bag = update_bag.Repairable_Bag(path = self.tmpdir) # check that relevant manifests are created self.assertTrue(os.path.isfile(j(self.tmpdir, 'manifest-sha1.txt'))) self.assertTrue(os.path.isfile(j(self.tmpdir, 'manifest-sha256.txt'))) # check valid with two manifests self.assertTrue(self.validate(bag, fast=True))
def load(self):
from os.path import join as j
from os.path import exists
if exists(j(RESULT_DIR, '%s_im.npy'%self.exp_name)):
self.im = list(np.load(j(RESULT_DIR,
'%s_im.npy'%self.exp_name)))
self.cost = list(np.load(j(RESULT_DIR,
'%s_cost.npy'%self.exp_name)))
def test_bom_in_bagit_txt(self):
bag = bagit.make_bag(self.tmpdir)
bagfile = codecs.BOM_UTF8
bagfile += open(j(self.tmpdir, "bagit.txt"), "rb").read()
bf = open(j(self.tmpdir, "bagit.txt"), "wb")
bf.write(bagfile)
bf.close()
bag = bagit.Bag(self.tmpdir)
self.assertRaises(bagit.BagValidationError, self.validate, bag)
def check_po_percent(path):
for addon in os.listdir(path):
i18n_path = j(path, addon, 'i18n')
if not os.path.exists(i18n_path):
print("skip '%s'..." % i18n_path)
continue
print("Processing '%s'... in %s" % (addon, i18n_path))
for lang in sorted(filter(lambda x: x.endswith('.po'), os.listdir(i18n_path))):
language = lang.replace('.po', '')
if len(language.split('_')) > 1 and language not in ['zh_CN', 'pt_BR', 'zh_TW']:
# clean only main translations
continue
po_path = j(i18n_path, lang)
po = polib.pofile(po_path, wrapwidth=78)
save = False
for entry in po:
rfrom, rto = False, False
if entry.msgstr and "\\n" in entry.msgstr:
# assert: Translation terms may not include escaped newlines
rfrom, rto = '\\n', '\n'
if entry.msgstr and '%s' in entry.msgid and '%s' not in entry.msgstr:
if '% s' in entry.msgstr:
if ' %s' in entry.msgid:
rfrom, rto = '% s', ' %s'
elif '%s ' in entry.msgid:
rfrom, rto = '% s', '%s '
else:
rfrom, rto = '% s', '%s'
elif '%S' in entry.msgstr:
rfrom = '%S'
elif 'S%' in entry.msgstr:
rfrom = 'S%'
elif 's%' in entry.msgstr:
rfrom = 's%'
elif '%s' in entry.msgid and '%' in entry.msgstr:
rfrom = '%'
rto = '%s'
elif entry.msgid.startswith('%s'):
rfrom = entry.msgstr
rto = '%s '+rfrom
elif entry.msgid.endswith('%s'):
rfrom = entry.msgstr
rto = rfrom + ' %s'
else:
print(f"Bad translation in {po_path} :\n\t{entry.msgid}")
if rfrom:
entry.msgstr = entry.msgstr.replace(rfrom, rto or '%s')
save = True
if save:
po.save()
def test_make_bag_sha256_manifest(self):
bag = bagit.make_bag(self.tmpdir, checksum=['sha256'])
# check manifest
self.assertTrue(os.path.isfile(j(self.tmpdir, 'manifest-sha256.txt')))
manifest_txt = open(j(self.tmpdir, 'manifest-sha256.txt')).read()
self.assertTrue('b6df8058fa818acfd91759edffa27e473f2308d5a6fca1e07a79189b95879953 data/loc/2478433644_2839c5e8b8_o_d.jpg' in manifest_txt)
self.assertTrue('1af90c21e72bb0575ae63877b3c69cfb88284f6e8c7820f2c48dc40a08569da5 data/loc/3314493806_6f1db86d66_o_d.jpg' in manifest_txt)
self.assertTrue('f065a4ae2bc5d47c6d046c3cba5c8cdfd66b07c96ff3604164e2c31328e41c1a data/si/2584174182_ffd5c24905_b_d.jpg' in manifest_txt)
self.assertTrue('45d257c93e59ec35187c6a34c8e62e72c3e9cfbb548984d6f6e8deb84bac41f4 data/si/4011399822_65987a4806_b_d.jpg' in manifest_txt)
def test_make_bag_sha512_manifest(self):
bag = bagit.make_bag(self.tmpdir, checksum=['sha512'])
# check manifest
self.assertTrue(os.path.isfile(j(self.tmpdir, 'manifest-sha512.txt')))
manifest_txt = open(j(self.tmpdir, 'manifest-sha512.txt')).read()
self.assertTrue('51fb9236a23795886cf42d539d580739245dc08f72c3748b60ed8803c9cb0e2accdb91b75dbe7d94a0a461827929d720ef45fe80b825941862fcde4c546a376d data/loc/2478433644_2839c5e8b8_o_d.jpg' in manifest_txt)
self.assertTrue('627c15be7f9aabc395c8b2e4c3ff0b50fd84b3c217ca38044cde50fd4749621e43e63828201fa66a97975e316033e4748fb7a4a500183b571ecf17715ec3aea3 data/loc/3314493806_6f1db86d66_o_d.jpg' in manifest_txt)
self.assertTrue('4cb4dafe39b2539536a9cb31d5addf335734cb91e2d2786d212a9b574e094d7619a84ad53f82bd9421478a7994cf9d3f44fea271d542af09d26ce764edbada46 data/si/2584174182_ffd5c24905_b_d.jpg' in manifest_txt)
self.assertTrue('af1c03483cd1999098cce5f9e7689eea1f81899587508f59ba3c582d376f8bad34e75fed55fd1b1c26bd0c7a06671b85e90af99abac8753ad3d76d8d6bb31ebd data/si/4011399822_65987a4806_b_d.jpg' in manifest_txt)
def parsefiles(datadir):
for installid in os.listdir(datadir):
for queue in os.listdir(j(datadir, installid)):
for date in os.listdir(j(datadir, installid, queue)):
subdir = j(installid, queue, date)
for filename in os.listdir(j(datadir, subdir)):
fullfilename = j(datadir, subdir, filename)
yield subdir, filename.replace('.gz', ''), list(objects(fullfilename))
return
def test_make_bag_sha1_manifest(self):
bag = bagit.make_bag(self.tmpdir, checksum=['sha1'])
# check manifest
self.assertTrue(os.path.isfile(j(self.tmpdir, 'manifest-sha1.txt')))
manifest_txt = open(j(self.tmpdir, 'manifest-sha1.txt')).read()
self.assertTrue('ace19416e605cfb12ab11df4898ca7fd9979ee43 data/README' in manifest_txt)
self.assertTrue('4c0a3da57374e8db379145f18601b159f3cad44b data/loc/2478433644_2839c5e8b8_o_d.jpg' in manifest_txt)
self.assertTrue('62095aeddae2f3207cb77c85937e13c51641ef71 data/loc/3314493806_6f1db86d66_o_d.jpg' in manifest_txt)
self.assertTrue('e592194b3733e25166a631e1ec55bac08066cbc1 data/si/2584174182_ffd5c24905_b_d.jpg' in manifest_txt)
self.assertTrue('db49ef009f85a5d0701829f38d29f8cf9c5df2ea data/si/4011399822_65987a4806_b_d.jpg' in manifest_txt)
def save(self, fname=None):
if fname is None:
import os
l_files = os.listdir(PARAMS)
num = 1
while 'exp%i_params.npy'%num in l_files:
num +=1
fname = 'exp%i'%num
from os.path import join as j
np.save(j(PARAMS, '%s_params.npy'%fname), self.params)
np.save(j(PARAMS, '%s_params_ft.npy'%fname), self.params_ft)
def test_validate_completeness(self):
bag = bagit.make_bag(self.tmpdir)
old_path = j(self.tmpdir, "data", "README")
new_path = j(self.tmpdir, "data", "extra_file")
os.rename(old_path, new_path)
bag = bagit.Bag(self.tmpdir)
self.assertTrue(self.validate(bag, fast=True))
with mock.patch.object(bag, '_validate_entries') as m:
self.assertRaises(bagit.BagValidationError, self.validate, bag,
completeness_only=True)
self.assertEqual(m.call_count, 0)
def __init__(self, filename):
"""
filename
post's filename with extension i.e. "I-am-a-post.md"
"""
self.name = filename[:-len(se)] # post's name i.e. "I-am-a-post"
self.filename = filename
self.src = j(Post.src_dir, self.name + se) # source path
self.out = j(Post.out_dir, self.name + oe) # output path
# post's update time
self.update_at = datetime.fromtimestamp(getmtime(self.src))
def test_bom_in_bagit_txt(self):
bag = bagit.make_bag(self.tmpdir)
BOM = codecs.BOM_UTF8
if sys.version_info[0] >= 3:
BOM = BOM.decode("utf-8")
with open(j(self.tmpdir, "bagit.txt"), "r") as bf:
bagfile = BOM + bf.read()
with open(j(self.tmpdir, "bagit.txt"), "w") as bf:
bf.write(bagfile)
bag = bagit.Bag(self.tmpdir)
self.assertRaises(bagit.BagValidationError, self.validate, bag)
def write(self, datadir, subdir, filename, objects):
outdir = j(self.basedir(datadir), subdir)
try:
os.makedirs(outdir)
except OSError:
pass
f = self.openfile(j(outdir, filename))
try:
for obj in objects:
f.write(self.pack(obj))
finally:
f.close()
def main(sandbox_path=os.getcwd()):
with open(j(sandbox_path, 'Input', 'input.xml'), 'rU') as inputfile:
metadata = xml.parse(inputfile).getroot()
qaoutput = xml.Element("qaQcCoverageOutput")
outputxml = xml.ElementTree(element=qaoutput)
for name, test in qa_tests.items():
xml.SubElement(qaoutput, name).text = str(test(metadata))
with open(j(sandbox_path, 'Output', 'output.xml'), 'w') as outputfile:
d = dom.parseString(xml.tostring(qaoutput))
d.writexml(outputfile, addindent=' ', newl='\n')
def test_make_bag(self):
info = {"Bagging-Date": "1970-01-01", "Contact-Email": "*****@*****.**"}
bagit.make_bag(self.tmpdir, bag_info=info, checksums=["md5"])
# data dir should've been created
self.assertTrue(os.path.isdir(j(self.tmpdir, "data")))
# check bagit.txt
self.assertTrue(os.path.isfile(j(self.tmpdir, "bagit.txt")))
bagit_txt = slurp_text_file(j(self.tmpdir, "bagit.txt"))
self.assertTrue("BagIt-Version: 0.97", bagit_txt)
self.assertTrue("Tag-File-Character-Encoding: UTF-8", bagit_txt)
# check manifest
self.assertTrue(os.path.isfile(j(self.tmpdir, "manifest-md5.txt")))
manifest_txt = slurp_text_file(j(self.tmpdir, "manifest-md5.txt")).splitlines()
self.assertIn("8e2af7a0143c7b8f4de0b3fc90f27354 data/README", manifest_txt)
self.assertIn(
"9a2b89e9940fea6ac3a0cc71b0a933a0 data/loc/2478433644_2839c5e8b8_o_d.jpg",
manifest_txt,
)
self.assertIn(
"6172e980c2767c12135e3b9d246af5a3 data/loc/3314493806_6f1db86d66_o_d.jpg",
manifest_txt,
)
self.assertIn(
"38a84cd1c41de793a0bccff6f3ec8ad0 data/si/2584174182_ffd5c24905_b_d.jpg",
manifest_txt,
)
self.assertIn(
"5580eaa31ad1549739de12df819e9af8 data/si/4011399822_65987a4806_b_d.jpg",
manifest_txt,
)
# check bag-info.txt
self.assertTrue(os.path.isfile(j(self.tmpdir, "bag-info.txt")))
bag_info_txt = slurp_text_file(j(self.tmpdir, "bag-info.txt"))
bag_info_txt = bag_info_txt.splitlines()
self.assertIn("Contact-Email: [email protected]", bag_info_txt)
self.assertIn("Bagging-Date: 1970-01-01", bag_info_txt)
self.assertIn("Payload-Oxum: 991765.5", bag_info_txt)
self.assertIn(
"Bag-Software-Agent: bagit.py v1.5.4 <https://github.com/LibraryOfCongress/bagit-python>",
bag_info_txt,
)
# check tagmanifest-md5.txt
self.assertTrue(os.path.isfile(j(self.tmpdir, "tagmanifest-md5.txt")))
tagmanifest_txt = slurp_text_file(
j(self.tmpdir, "tagmanifest-md5.txt")
).splitlines()
self.assertIn("9e5ad981e0d29adc278f6a294b8c2aca bagit.txt", tagmanifest_txt)
self.assertIn(
"a0ce6631a2a6d1a88e6d38453ccc72a5 manifest-md5.txt", tagmanifest_txt
)
self.assertIn("0a6ffcffe67e9a34e44220f7ebcb4baa bag-info.txt", tagmanifest_txt)
def get_packages_list():
modList = []
modNames = {}
_myDir = j(os.getcwd(), 'packages')
for ii in os.walk(_myDir):
if ii[0] == _myDir:
modNames = ii[1]
break
for name in modNames:
if os.path.exists(j(_myDir, name, '__init__.py')): modList.append(name)
return modList
def test_validate_optional_tagfile_in_directory(self):
bag = bagit.make_bag(self.tmpdir, checksums=["md5"])
tagdir = tempfile.mkdtemp(dir=self.tmpdir)
if not os.path.exists(j(tagdir, "tagfolder")):
os.makedirs(j(tagdir, "tagfolder"))
with open(j(tagdir, "tagfolder", "tagfile"), "w") as tagfile:
tagfile.write("test")
relpath = j(tagdir, "tagfolder", "tagfile").replace(self.tmpdir + os.sep, "")
relpath.replace("\\", "/")
with open(j(self.tmpdir, "tagmanifest-md5.txt"), "w") as tagman:
# Incorrect checksum.
tagman.write("8e2af7a0143c7b8f4de0b3fc90f27354 " + relpath + "\n")
bag = bagit.Bag(self.tmpdir)
self.assertRaises(bagit.BagValidationError, self.validate, bag)
hasher = hashlib.new("md5")
with open(j(tagdir, "tagfolder", "tagfile"), "r") as tf:
contents = tf.read().encode("utf-8")
hasher.update(contents)
with open(j(self.tmpdir, "tagmanifest-md5.txt"), "w") as tagman:
tagman.write(hasher.hexdigest() + " " + relpath + "\n")
bag = bagit.Bag(self.tmpdir)
self.assertTrue(self.validate(bag))
# Missing tagfile.
os.remove(j(tagdir, "tagfolder", "tagfile"))
bag = bagit.Bag(self.tmpdir)
self.assertRaises(bagit.BagValidationError, self.validate, bag)
def merge_po(src_path, new_path, dest_path):
for addons_subpath in ADDONS_SUBPATHS:
if not os.path.exists(j(dest_path, addons_subpath)):
print("skip subpath '%s'..." % j(dest_path, addons_subpath))
continue
for addon in os.listdir(j(dest_path, addons_subpath)):
i18n_path = j(dest_path, addons_subpath, addon, 'i18n')
if not os.path.exists(i18n_path):
print("skip '%s'..." % i18n_path)
continue
print("Processing '%s'..." % addon)
for lang in sorted(filter(lambda x: x.endswith('.po'), os.listdir(i18n_path))):
src = j(src_path, addons_subpath, addon, 'i18n', lang)
new = j(new_path, addons_subpath, addon, 'i18n', lang)
dest = j(dest_path, addons_subpath, addon, 'i18n', lang)
ref = j(dest_path, addons_subpath, addon, 'i18n', addon+'.pot')
language = lang.replace('.po', '')
if not os.path.exists(src):
subprocess.call(['cp', new, dest])
elif not os.path.exists(new) or language not in MASTER_LANG:
subprocess.call(['cp', src, dest])
else:
# cmd = 'msgcat %(src)s %(new)s -o %(dest)s'
subprocess.call(['msgcat', '--use-first', src, new, '-o', dest])
cmd = "msgmerge --no-fuzzy-matching -q %(dest)s %(ref)s | msgattrib --no-obsolete -o %(dest)s" % {'ref': ref, 'dest': dest}
subprocess.call(cmd, shell=True)
class ExternalExe(object):
'''
Main class to derive others that will control the execution of external
programs.
Program configuration is defined in the configSBI.txt file or can be
defined in a file linked to a environment variable called SBI_CONFIG_FILE.
'''
__metaclass__ = ABCMeta
DEFAULT_CONFIG_FILE = j(n(d(__file__)), 'configSBI.txt')
# Executable configuration
_CONFIG = ConfigParser.RawConfigParser(allow_no_value=True)
_EXE = None
def __new__(cls, *args, **kwargs):
cls._CONFIG.read(os.getenv('SBI_CONFIG_FILE', cls.DEFAULT_CONFIG_FILE))
return super(ExternalExe, cls).__new__(cls, *args, **kwargs)
###################
# PRIVATE METHODS #
###################
def _set_default_executable(self, external_id):
'''
Configure the {Executable} of the class according to the default
configuration as defined in the default configSBI.txt or the
SBI_CONFIG_FILE
@param: external_id
@pdef: name of the external program as referred in the
configSBI.txt file.
@ptype: {String}
'''
i, e, p = external_id, 'executable', 'path'
self._EXE = Executable(executable=self._CONFIG.get(i, e),
path=self._CONFIG.get(i, p))
@staticmethod
def _set_dynamic_executable(executable, path):
'''
Manual configuration of the {Executable}.
@param: executable
@pdef: name of the executable file
@ptype: {String}
@param: path
@pdef: path to the executable file
@ptype: {String}
'''
ExternalExe._EXE = Executable(executable=executable, path=path)
def test_make_bag_with_unwritable_source(self):
path_suffixes = ('', 'loc')
for path_suffix in reversed(path_suffixes):
os.chmod(j(self.tmpdir, path_suffix), 0o500)
with self.assertRaises(bagit.BagError) as error_catcher:
bagit.make_bag(self.tmpdir, checksum=['sha256'])
self.assertEqual(
'Missing permissions to move all files and directories',
str(error_catcher.exception))
def test_unicode_bag_info(self):
info = {
'Test-BMP':
u'This element contains a \N{LATIN SMALL LETTER U WITH DIAERESIS}',
'Test-SMP': u'This element contains a \N{LINEAR B SYMBOL B049}',
}
bagit.make_bag(self.tmpdir, bag_info=info, checksums=['md5'])
bag_info_txt = slurp_text_file(j(self.tmpdir, 'bag-info.txt'))
for v in info.values():
self.assertIn(v, bag_info_txt)
def main(start_cmd=None):
args, parser = configure_argparse(rwd=rwd, start_cmd=start_cmd)
# Load default cfg
Setts.refresh(f_pth=j(rwd, Setts.CFG_PATH.value))
# Load users cfg
Setts.refresh(f_pth=args.cfg)
# Load user args
Setts.refresh(cfg=args.__dict__)
init_db() # TODO: Must it be called?
evaluate_arguments()
def ungrab_pm(self, *ignored, **kwignored):
if self.cachedir:
with lockfile(j(self.cachedir, "ifz-lockfile")):
while self.cachemounts:
while ismount(self.cachemounts[-1]):
logger.debug("Unmounting %s", self.cachemounts[-1])
umount(self.cachemounts[-1])
os.rmdir(self.cachemounts[-1])
self.cachemounts.pop()
if self.pkgmgr_config:
self.pkgmgr_config.close()
self.pkgmgr_config = None
def add(version):
tempdir = tempfile.mkdtemp()
files = dict()
metadata = dict()
metadata['id'] = new_id = uuid.uuid4()
#read in files
h = hashlib.md5()
for f in request.files.getlist('file'):
with open(j(tempdir, f.filename), 'wb') as savefile:
bytes = f.stream.read(UPLOAD_BLOCK_SIZE)
h.update(bytes)
savefile.write(bytes)
files[f.filename] = h.hexdigest()
#make celery job
paths = [j(tempdir, fname) for fname in files.keys()]
cleanup = partial(shutil.rmtree, tempdir)
api.add(version=version, paths=paths, metadata=metadata, cleanup=cleanup)
#return id
return jsonify({'id':new_id, 'files_received':files})
def test_open_bag_with_missing_bagit_txt(self):
bagit.make_bag(self.tmpdir)
os.unlink(j(self.tmpdir, "bagit.txt"))
with self.assertRaises(bagit.BagError) as error_catcher:
bagit.Bag(self.tmpdir)
self.assertEqual(
"Expected bagit.txt does not exist: %s/bagit.txt" % self.tmpdir,
str(error_catcher.exception),
)
def test_validate_flipped_bit(self):
bag = bagit.make_bag(self.tmpdir)
readme = j(self.tmpdir, "data", "README")
with open(readme) as r:
txt = r.read()
txt = 'A' + txt[1:]
with open(readme, "w") as r:
r.write(txt)
bag = bagit.Bag(self.tmpdir)
self.assertRaises(bagit.BagValidationError, self.validate, bag)
# fast doesn't catch the flipped bit, since oxsum is the same
self.assertTrue(self.validate(bag, fast=True))
def test_open_bag_with_unknown_encoding(self):
bagit.make_bag(self.tmpdir)
with open(j(self.tmpdir, "bagit.txt"), "w") as f:
f.write(
"BagIt-Version: 0.97\nTag-File-Character-Encoding: WTF-8\n")
with self.assertRaises(bagit.BagError) as error_catcher:
bagit.Bag(self.tmpdir)
self.assertEqual("Unsupported encoding: WTF-8",
str(error_catcher.exception))
def test_open_bag_with_malformed_bagit_txt(self):
bagit.make_bag(self.tmpdir)
with open(j(self.tmpdir, 'bagit.txt'), 'w') as f:
os.ftruncate(f.fileno(), 0)
with self.assertRaises(bagit.BagError) as error_catcher:
bagit.Bag(self.tmpdir)
self.assertEqual(
'Missing required tag in bagit.txt: BagIt-Version, Tag-File-Character-Encoding',
str(error_catcher.exception))
def test_validate_flipped_bit(self):
bag = bagit.make_bag(self.tmpdir)
readme = j(self.tmpdir, "data", "README")
txt = slurp_text_file(readme)
txt = 'A' + txt[1:]
with io.open(readme, "w", newline="\n") as r:
r.write(txt)
bag = bagit.BDBag(self.tmpdir)
self.assertRaises(bagit.BagValidationError, self.validate, bag)
# fast doesn't catch the flipped bit, since oxsum is the same
self.assertTrue(self.validate(bag, fast=True))
self.assertTrue(self.validate(bag, completeness_only=True))
def test_make_bag_sha1_manifest(self):
logger.info(self.getTestHeader(sys._getframe().f_code.co_name))
bagit.make_bag(self.tmpdir, checksums=['sha1'])
# check manifest
self.assertTrue(os.path.isfile(j(self.tmpdir, 'manifest-sha1.txt')))
manifest_txt = slurp_text_file(j(self.tmpdir,
'manifest-sha1.txt')).splitlines()
self.assertIn('ace19416e605cfb12ab11df4898ca7fd9979ee43 data/README',
manifest_txt)
self.assertIn(
'4c0a3da57374e8db379145f18601b159f3cad44b data/loc/2478433644_2839c5e8b8_o_d.jpg',
manifest_txt)
self.assertIn(
'62095aeddae2f3207cb77c85937e13c51641ef71 data/loc/3314493806_6f1db86d66_o_d.jpg',
manifest_txt)
self.assertIn(
'e592194b3733e25166a631e1ec55bac08066cbc1 data/si/2584174182_ffd5c24905_b_d.jpg',
manifest_txt)
self.assertIn(
'db49ef009f85a5d0701829f38d29f8cf9c5df2ea data/si/4011399822_65987a4806_b_d.jpg',
manifest_txt)
def test_open_bag_with_unsupported_version(self):
logger.info(self.getTestHeader(sys._getframe().f_code.co_name))
bagit.make_bag(self.tmpdir)
with open(j(self.tmpdir, 'bagit.txt'), 'w') as f:
f.write('BagIt-Version: 2.0\nTag-File-Character-Encoding: UTF-8\n')
with self.assertRaises(bagit.BagError) as error_catcher:
bagit.BDBag(self.tmpdir)
self.assertEqual('Unsupported bag version: 2.0',
str(error_catcher.exception))
def test_validation_error_details(self):
logger.info(self.getTestHeader(sys._getframe().f_code.co_name))
bag = bagit.make_bag(self.tmpdir,
checksums=['md5'],
bag_info={'Bagging-Date': '1970-01-01'})
readme = j(self.tmpdir, "data", "README")
txt = slurp_text_file(readme)
txt = 'A' + txt[1:]
with io.open(readme, "w", newline="\n") as r:
r.write(txt)
bag = bagit.BDBag(self.tmpdir)
got_exception = False
try:
self.validate(bag)
except bagit.BagValidationError as e:
got_exception = True
exc_str = str(e)
dr = j("data", "README")
self.assertIn(
'%s md5 validation failed: expected="8e2af7a0143c7b8f4de0b3fc90f27354" found="fd41543285d17e7c29cd953f5cf5b955"'
% dr, exc_str)
self.assertEqual(len(e.details), 1)
readme_error = e.details[0]
self.assertEqual(
'%s md5 validation failed: expected="8e2af7a0143c7b8f4de0b3fc90f27354" found="fd41543285d17e7c29cd953f5cf5b955"'
% dr, str(readme_error))
self.assertIsInstance(readme_error, bagit.ChecksumMismatch)
self.assertEqual(readme_error.algorithm, 'md5')
self.assertEqual(readme_error.path, dr)
self.assertEqual(readme_error.expected,
'8e2af7a0143c7b8f4de0b3fc90f27354')
self.assertEqual(readme_error.found,
'fd41543285d17e7c29cd953f5cf5b955')
if not got_exception:
self.fail("didn't get BagValidationError")
def test_validate_optional_tagfile(self):
bag = bagit.make_bag(self.tmpdir, checksums=["md5"])
tagdir = tempfile.mkdtemp(dir=self.tmpdir)
with open(j(tagdir, "tagfile"), "w") as tagfile:
tagfile.write("test")
relpath = j(tagdir, "tagfile").replace(self.tmpdir + os.sep, "")
relpath.replace("\\", "/")
with open(j(self.tmpdir, "tagmanifest-md5.txt"), "w") as tagman:
# Incorrect checksum.
tagman.write("8e2af7a0143c7b8f4de0b3fc90f27354 " + relpath + "\n")
bag = bagit.Bag(self.tmpdir)
self.assertRaises(bagit.BagValidationError, self.validate, bag)
hasher = hashlib.new("md5")
contents = slurp_text_file(j(tagdir, "tagfile")).encode("utf-8")
hasher.update(contents)
with open(j(self.tmpdir, "tagmanifest-md5.txt"), "w") as tagman:
tagman.write(hasher.hexdigest() + " " + relpath + "\n")
bag = bagit.Bag(self.tmpdir)
self.assertTrue(self.validate(bag))
# Missing tagfile.
os.remove(j(tagdir, "tagfile"))
bag = bagit.Bag(self.tmpdir)
self.assertRaises(bagit.BagValidationError, self.validate, bag)
def test_fetch_loader(self):
with open(j(self.tmpdir, 'fetch.txt'), 'w') as fetch_txt:
print(
'https://photojournal.jpl.nasa.gov/jpeg/PIA21390.jpg 143435 data/loc/3314493806_6f1db86d66_o_d.jpg',
file=fetch_txt)
os.remove(j(self.tmpdir, 'data/loc/3314493806_6f1db86d66_o_d.jpg'))
self.bag.save(manifests=True)
self.bag.validate(completeness_only=True)
self.assertListEqual(
[('https://photojournal.jpl.nasa.gov/jpeg/PIA21390.jpg', '143435',
'data/loc/3314493806_6f1db86d66_o_d.jpg')],
list(self.bag.fetch_entries()))
self.assertListEqual(
[j('data', 'loc', '3314493806_6f1db86d66_o_d.jpg')],
list(self.bag.files_to_be_fetched()))
self.assertListEqual(
[j('data', 'loc', '3314493806_6f1db86d66_o_d.jpg')],
list(self.bag.compare_fetch_with_fs()))
def test_filename_unicode_normalization(self):
# We need to handle cases where the Unicode normalization form of a
# filename has changed in-transit. This is hard to do portably in both
# directions because OS X normalizes *all* filenames to an NFD variant
# so we'll start with a basic test which writes the manifest using the
# NFC form and confirm that this does not cause the bag to fail when it
# is written to the filesystem using the NFD form, which will not be
# altered when saved to an HFS+ filesystem:
test_filename = 'Núñez Papers.txt'
test_filename_nfc = unicodedata.normalize('NFC', test_filename)
test_filename_nfd = unicodedata.normalize('NFD', test_filename)
os.makedirs(j(self.tmpdir, 'unicode-normalization'))
with open(j(self.tmpdir, 'unicode-normalization', test_filename_nfd),
'w') as f:
f.write(
'This is a test filename written using NFD normalization\n')
bag = bagit.make_bag(self.tmpdir)
bag.save()
self.assertTrue(bag.is_valid())
# Now we'll cause the entire manifest file was normalized to NFC:
for m_f in bag.manifest_files():
contents = slurp_text_file(m_f)
normalized_bytes = unicodedata.normalize('NFC',
contents).encode('utf-8')
with open(m_f, 'wb') as f:
f.write(normalized_bytes)
for alg in bag.algs:
bagit._make_tagmanifest_file(alg, bag.path, encoding=bag.encoding)
# Now we'll reload the whole thing:
bag = bagit.Bag(self.tmpdir)
self.assertTrue(bag.is_valid())
def test_make_bag_with_unwritable_source(self):
logger.info(self.getTestHeader(sys._getframe().f_code.co_name))
path_suffixes = ('', 'loc')
for path_suffix in reversed(path_suffixes):
os.chmod(j(self.tmpdir, path_suffix), 0o500)
with self.assertRaises(bagit.BagError) as error_catcher:
bagit.make_bag(self.tmpdir, checksums=['sha256'])
self.assertEqual(
'Missing permissions to move all files and directories',
str(error_catcher.exception))
def test_fetch_validation(self):
with open(j(self.tmpdir, 'fetch.txt'), 'w') as fetch_txt:
print(
'https://photojournal.jpl.nasa.gov/jpeg/PIA21390.jpg - data/nasa/PIA21390.jpg',
file=fetch_txt)
self.bag.save(manifests=True)
with mock.patch.object(bagit.Bag, 'validate_fetch') as mock_vf:
self.bag.validate()
self.assertTrue(
mock_vf.called,
msg='Bag.validate() should call Bag.validate_fetch()')
def test_fetch_malformed_url(self):
with open(j(self.tmpdir, 'fetch.txt'), 'w') as fetch_txt:
print('//photojournal.jpl.nasa.gov/jpeg/PIA21390.jpg - data/nasa/PIA21390.jpg',
file=fetch_txt)
self.bag.save(manifests=True)
expected_msg = 'Malformed URL in fetch.txt: //photojournal.jpl.nasa.gov/jpeg/PIA21390.jpg'
with self.assertRaises(bagit.BagError) as cm:
self.bag.validate_fetch()
self.assertEqual(expected_msg, str(cm.exception))
def test_validate_flipped_bit_at_destination(self):
tmp_dir_out = tempfile.mkdtemp(prefix='bagit-test-dest')
bag = bagit.make_bag(self.tmpdir,
dest_dir=tmp_dir_out,
checksums=['sha256', 'sha512'])
readme = j(tmp_dir_out, os.path.basename(self.tmpdir), 'data',
'README')
txt = slurp_text_file(readme)
txt = "A" + txt[1:]
with open(readme, 'w') as r:
r.write(txt)
bag = bagit.Bag(bag.path)
self.assertRaises(bagit.BagValidationError, self.validate, bag)
hasher = hashlib.new('sha256')
contents = slurp_text_file(j(self.tmpdir, 'README')).encode('utf-8')
hasher.update(contents)
self.assertTrue(
hasher.hexdigest() == bag.entries['data/README']['sha256'])
shutil.rmtree(tmp_dir_out)
def test_save_manifests_deleted_files(self):
bag = bagit.make_bag(self.tmpdir)
self.assertTrue(bag.is_valid())
bag.save(manifests=True)
self.assertTrue(bag.is_valid())
os.remove(
j(self.tmpdir, "data", "loc", "2478433644_2839c5e8b8_o_d.jpg"))
self.assertRaises(bagit.BagValidationError,
bag.validate,
bag,
fast=False)
bag.save(manifests=True)
self.assertTrue(bag.is_valid())
def test_make_bag_sha512_manifest(self):
bagit.make_bag(self.tmpdir, checksum=["sha512"])
# check manifest
self.assertTrue(os.path.isfile(j(self.tmpdir, "manifest-sha512.txt")))
manifest_txt = slurp_text_file(j(self.tmpdir,
"manifest-sha512.txt")).splitlines()
self.assertIn(
"51fb9236a23795886cf42d539d580739245dc08f72c3748b60ed8803c9cb0e2accdb91b75dbe7d94a0a461827929d720ef45fe80b825941862fcde4c546a376d data/loc/2478433644_2839c5e8b8_o_d.jpg",
manifest_txt,
)
self.assertIn(
"627c15be7f9aabc395c8b2e4c3ff0b50fd84b3c217ca38044cde50fd4749621e43e63828201fa66a97975e316033e4748fb7a4a500183b571ecf17715ec3aea3 data/loc/3314493806_6f1db86d66_o_d.jpg",
manifest_txt,
)
self.assertIn(
"4cb4dafe39b2539536a9cb31d5addf335734cb91e2d2786d212a9b574e094d7619a84ad53f82bd9421478a7994cf9d3f44fea271d542af09d26ce764edbada46 data/si/2584174182_ffd5c24905_b_d.jpg",
manifest_txt,
)
self.assertIn(
"af1c03483cd1999098cce5f9e7689eea1f81899587508f59ba3c582d376f8bad34e75fed55fd1b1c26bd0c7a06671b85e90af99abac8753ad3d76d8d6bb31ebd data/si/4011399822_65987a4806_b_d.jpg",
manifest_txt,
)
def test_save_manifests(self):
bag = bagit.make_bag(self.tmpdir)
self.assertTrue(bag.is_valid())
bag.save(manifests=True)
self.assertTrue(bag.is_valid())
with open(j(self.tmpdir, "data", "newfile"), "w") as nf:
nf.write("newfile")
self.assertRaises(bagit.BagValidationError,
bag.validate,
bag,
fast=False)
bag.save(manifests=True)
self.assertTrue(bag.is_valid())
def test_make_bag_sha256_manifest(self):
bagit.make_bag(self.tmpdir, checksum=["sha256"])
# check manifest
self.assertTrue(os.path.isfile(j(self.tmpdir, "manifest-sha256.txt")))
manifest_txt = slurp_text_file(j(self.tmpdir,
"manifest-sha256.txt")).splitlines()
self.assertIn(
"b6df8058fa818acfd91759edffa27e473f2308d5a6fca1e07a79189b95879953 data/loc/2478433644_2839c5e8b8_o_d.jpg",
manifest_txt,
)
self.assertIn(
"1af90c21e72bb0575ae63877b3c69cfb88284f6e8c7820f2c48dc40a08569da5 data/loc/3314493806_6f1db86d66_o_d.jpg",
manifest_txt,
)
self.assertIn(
"f065a4ae2bc5d47c6d046c3cba5c8cdfd66b07c96ff3604164e2c31328e41c1a data/si/2584174182_ffd5c24905_b_d.jpg",
manifest_txt,
)
self.assertIn(
"45d257c93e59ec35187c6a34c8e62e72c3e9cfbb548984d6f6e8deb84bac41f4 data/si/4011399822_65987a4806_b_d.jpg",
manifest_txt,
)
def test_save_manifests_deleted_files(self):
logger.info(self.getTestHeader(sys._getframe().f_code.co_name))
bag = bagit.make_bag(self.tmpdir)
self.assertTrue(bag.is_valid())
bag.save(manifests=True)
self.assertTrue(bag.is_valid())
os.remove(
j(self.tmpdir, "data", "loc", "2478433644_2839c5e8b8_o_d.jpg"))
self.assertRaises(bagit.BagValidationError,
bag.validate,
bag,
fast=False)
bag.save(manifests=True)
self.assertTrue(bag.is_valid())
def test_payload_permissions(self):
perms = os.stat(self.tmpdir).st_mode
# our tmpdir should not be writeable by group
self.assertEqual(perms & stat.S_IWOTH, 0)
# but if we make it writeable by the group then resulting
# payload directory should have the same permissions
new_perms = perms | stat.S_IWOTH
self.assertTrue(perms != new_perms)
os.chmod(self.tmpdir, new_perms)
bagit.make_bag(self.tmpdir)
payload_dir = j(self.tmpdir, "data")
self.assertEqual(os.stat(payload_dir).st_mode, new_perms)
def test_save_manifests(self):
logger.info(self.getTestHeader(sys._getframe().f_code.co_name))
bag = bagit.make_bag(self.tmpdir)
self.assertTrue(bag.is_valid())
bag.save(manifests=True)
self.assertTrue(bag.is_valid())
with open(j(self.tmpdir, "data", "newfile"), 'w') as nf:
nf.write('newfile')
self.assertRaises(bagit.BagValidationError,
bag.validate,
bag,
fast=False)
bag.save(manifests=True)
self.assertTrue(bag.is_valid())
def test_fetch_validation(self):
with open(j(self.tmpdir, "fetch.txt"), "w") as fetch_txt:
print(
"https://photojournal.jpl.nasa.gov/jpeg/PIA21390.jpg - data/nasa/PIA21390.jpg",
file=fetch_txt,
)
self.bag.save(manifests=True)
with mock.patch.object(bagit.Bag, "validate_fetch") as mock_vf:
self.bag.validate()
self.assertTrue(
mock_vf.called,
msg="Bag.validate() should call Bag.validate_fetch()")
def test_make_bag_sha1_manifest(self):
bagit.make_bag(self.tmpdir, checksum=["sha1"])
# check manifest
self.assertTrue(os.path.isfile(j(self.tmpdir, "manifest-sha1.txt")))
manifest_txt = slurp_text_file(j(self.tmpdir,
"manifest-sha1.txt")).splitlines()
self.assertIn("ace19416e605cfb12ab11df4898ca7fd9979ee43 data/README",
manifest_txt)
self.assertIn(
"4c0a3da57374e8db379145f18601b159f3cad44b data/loc/2478433644_2839c5e8b8_o_d.jpg",
manifest_txt,
)
self.assertIn(
"62095aeddae2f3207cb77c85937e13c51641ef71 data/loc/3314493806_6f1db86d66_o_d.jpg",
manifest_txt,
)
self.assertIn(
"e592194b3733e25166a631e1ec55bac08066cbc1 data/si/2584174182_ffd5c24905_b_d.jpg",
manifest_txt,
)
self.assertIn(
"db49ef009f85a5d0701829f38d29f8cf9c5df2ea data/si/4011399822_65987a4806_b_d.jpg",
manifest_txt,
)
