def _test_bag_with_remote_file_manifest(self, update=False):
try:
bag_dir = self.test_data_dir if not update else self.test_bag_dir
bag = bdb.make_bag(bag_dir,
algs=["md5", "sha1", "sha256", "sha512"],
update=update,
remote_file_manifest=ospj(
self.test_config_dir,
'test-fetch-manifest.json'))
output = self.stream.getvalue()
self.assertIsInstance(bag, bdbagit.BDBag)
self.assertExpectedMessages([
'Generating remote file references from',
'test-fetch-manifest.json'
], output)
fetch_file = ospj(bag_dir, 'fetch.txt')
self.assertTrue(ospif(fetch_file))
with open(fetch_file) as ff:
fetch_txt = ff.read()
self.assertIn(
'https://raw.githubusercontent.com/fair-research/bdbag/master/test/test-data/test-http/'
'test-fetch-http.txt\t201\tdata/test-fetch-http.txt',
fetch_txt)
self.assertIn(
'ark:/57799/b9dd5t\t223\tdata/test-fetch-identifier.txt',
fetch_txt)
bdb.validate_bag_structure(bag_dir, True)
except Exception as e:
self.fail(bdbag.get_typed_exception(e))
def test_validate_incomplete_bag_structure(self):
logger.info(
self.getTestHeader('test structure validation incomplete bag'))
try:
bdb.validate_bag_structure(self.test_bag_incomplete_dir)
except Exception as e:
self.fail(bdbag.get_typed_exception(e))
def test_update_bag_invalid_fetch_entry(self):
logger.info(self.getTestHeader('update bag with invalid fetch.txt entry with missing length'))
try:
bdb.validate_bag_structure(self.test_bag_update_invalid_fetch_dir)
fetch = ospj(self.test_bag_update_invalid_fetch_dir, 'fetch.txt')
with open(fetch, "w") as ff:
ff.write('https://raw.githubusercontent.com/fair-research/bdbag/master/test/test-data/test-http/'
'test-fetch-http.txt\t-\tdata/test-fetch-http.txt\n')
with self.assertRaises(ValueError) as ar:
bdb.make_bag(self.test_bag_update_invalid_fetch_dir, update=True)
logger.error(get_typed_exception(ar.exception))
except Exception as e:
self.fail(get_typed_exception(e))
def testExportBag(self):
tale = self._create_water_tale()
export_path = '/tale/{}/export'.format(str(tale['_id']))
resp = self.request(path=export_path,
method='GET',
params={'taleFormat': 'bagit'},
isJson=False,
user=self.user)
dirpath = tempfile.mkdtemp()
bag_file = os.path.join(dirpath, "{}.zip".format(str(tale['_id'])))
with open(bag_file, 'wb') as fp:
for content in resp.body:
fp.write(content)
temp_path = bdb.extract_bag(bag_file, temp=True)
try:
bdb.validate_bag_structure(temp_path)
except bagit.BagValidationError:
pass # TODO: Goes without saying that we should not be doing that...
shutil.rmtree(dirpath)
# Test dataSetCitation
resp = self.request(path='/tale/{_id}'.format(**tale),
method='PUT',
type='application/json',
user=self.user,
body=json.dumps({
'dataSet': [],
'imageId': str(tale['imageId']),
'public': tale['public'],
}))
self.assertStatusOk(resp)
tale = resp.json
self.assertEqual(tale['dataSetCitation'], [])
self.model('tale', 'wholetale').remove(tale)
self.model('collection').remove(self.data_collection)
def main():
args, is_bag, is_file = parse_cli()
path = os.path.abspath(args.path)
archive = None
temp_path = None
error = None
result = 0
if not args.quiet:
sys.stderr.write('\n')
try:
if not is_file:
# do not try to create or update the bag if the user just wants to validate or complete an existing bag
if not (
(args.validate or args.validate_profile or args.resolve_fetch)
and not (args.update and bdb.is_bag(path))):
if args.checksum and 'all' in args.checksum:
args.checksum = ['md5', 'sha1', 'sha256', 'sha512']
# create or update the bag depending on the input arguments
bdb.make_bag(path,
algs=args.checksum,
update=args.update,
save_manifests=not args.skip_manifests,
prune_manifests=args.prune_manifests,
metadata=BAG_METADATA if BAG_METADATA else None,
metadata_file=args.metadata_file,
remote_file_manifest=args.remote_file_manifest,
config_file=args.config_file,
ro_metadata_file=args.ro_metadata_file)
# otherwise just extract the bag if it is an archive and no other conflicting options specified
elif not (args.validate or args.validate_profile
or args.resolve_fetch):
bdb.extract_bag(path)
if not args.quiet:
sys.stderr.write('\n')
return result
if args.ro_manifest_generate:
bdb.generate_ro_manifest(
path,
True if args.ro_manifest_generate == "overwrite" else False,
config_file=args.config_file)
if args.resolve_fetch:
if args.validate == 'full':
sys.stderr.write(ASYNC_TRANSFER_VALIDATION_WARNING)
bdb.resolve_fetch(
path,
force=True if args.resolve_fetch == 'all' else False,
keychain_file=args.keychain_file,
config_file=args.config_file,
filter_expr=args.fetch_filter)
if args.validate:
if is_file:
temp_path = bdb.extract_bag(path, temp=True)
if args.validate == 'structure':
bdb.validate_bag_structure(temp_path if temp_path else path)
else:
bdb.validate_bag(
temp_path if temp_path else path,
fast=True if args.validate == 'fast' else False,
config_file=args.config_file)
if args.archiver:
archive = bdb.archive_bag(path, args.archiver)
if archive is None and is_file:
archive = path
if args.validate_profile:
if is_file:
if not temp_path:
temp_path = bdb.extract_bag(path, temp=True)
profile = bdb.validate_bag_profile(
temp_path if temp_path else path)
bdb.validate_bag_serialization(archive if archive else path,
profile)
if args.revert:
bdb.revert_bag(path)
except Exception as e:
result = 1
error = "Error: %s" % get_typed_exception(e)
finally:
if temp_path:
bdb.cleanup_bag(os.path.dirname(temp_path))
if result != 0:
sys.stderr.write("\n%s" % error)
if not args.quiet:
sys.stderr.write('\n')
return result
def test_validate_complete_bag_completeness(self):
logger.info(self.getTestHeader('test completeness validation complete bag'))
try:
bdb.validate_bag_structure(self.test_bag_dir, skip_remote=False)
except Exception as e:
self.fail(get_typed_exception(e))