def test_bad_mediatype():
# this identifier actually has this malformed data
ident = "CIA-RDP96-00789R000700210007-5"
body = '{"metadata": {"mediatype":["texts","texts"]}}'
with IaRequestsMock() as rsps:
rsps.add_metadata_mock(ident, body=body)
# should complete without error
get_item(ident)
def test_download_checksum(tmpdir, caplog):
tmpdir.chdir()
# test overwrite based on checksum.
with IaRequestsMock() as rsps:
rsps.add_metadata_mock('nasa')
rsps.add(responses.GET, DOWNLOAD_URL_RE, body='test content')
rsps.add(responses.GET, DOWNLOAD_URL_RE, body='overwrite based on md5')
nasa_item = get_item('nasa')
nasa_item.download(files='nasa_meta.xml')
nasa_item.download(files='nasa_meta.xml', checksum=True)
assert load_file('nasa/nasa_meta.xml') == 'overwrite based on md5'
# test no overwrite based on checksum.
rsps.reset()
rsps.add(responses.GET,
DOWNLOAD_URL_RE,
body=load_test_data_file('nasa_meta.xml'))
nasa_item.download(files='nasa_meta.xml', checksum=True)
nasa_item.download(files='nasa_meta.xml', checksum=True)
assert 'skipping nasa' in caplog.text
assert 'nasa_meta.xml, file already exists based on checksum.' in caplog.text
def test_upload_checksum(tmpdir, nasa_item):
with IaRequestsMock() as rsps:
rsps.add_metadata_mock('nasa')
nasa_item = get_item('nasa')
_expected_headers = deepcopy(EXPECTED_S3_HEADERS)
del _expected_headers['x-archive-meta00-scanner']
test_file = os.path.join(str(tmpdir), 'checksum_test.txt')
with open(test_file, 'wb') as fh:
fh.write(b'test delete')
# No skip.
rsps.add(responses.PUT, S3_URL_RE, adding_headers=_expected_headers)
resp = nasa_item.upload(test_file,
access_key='a',
secret_key='b',
checksum=True)
for r in resp:
headers = dict((k.lower(), str(v)) for k, v in r.headers.items())
del headers['content-type']
assert headers == _expected_headers
assert r.status_code == 200
# Skip.
nasa_item.item_metadata['files'].append(
dict(name='checksum_test.txt',
md5='33213e7683c1e6d15b2a658f3c567717'))
resp = nasa_item.upload(test_file,
access_key='a',
secret_key='b',
checksum=True)
for r in resp:
headers = dict((k.lower(), str(v)) for k, v in r.headers.items())
assert r.status_code is None
def test_upload_checksum(tmpdir, nasa_item):
with IaRequestsMock() as rsps:
rsps.add_metadata_mock("nasa")
nasa_item = get_item("nasa")
_expected_headers = deepcopy(EXPECTED_S3_HEADERS)
del _expected_headers["x-archive-meta00-scanner"]
test_file = os.path.join(str(tmpdir), "checksum_test.txt")
with open(test_file, "wb") as fh:
fh.write(b"test delete")
# No skip.
rsps.add(responses.PUT, S3_URL_RE, adding_headers=_expected_headers)
resp = nasa_item.upload(test_file, access_key="a", secret_key="b", checksum=True)
for r in resp:
headers = dict((k.lower(), str(v)) for k, v in r.headers.items())
del headers["content-type"]
assert headers == _expected_headers
assert r.status_code == 200
# Skip.
nasa_item.item_metadata["files"].append(dict(name="checksum_test.txt", md5="33213e7683c1e6d15b2a658f3c567717"))
resp = nasa_item.upload(test_file, access_key="a", secret_key="b", checksum=True)
for r in resp:
headers = dict((k.lower(), str(v)) for k, v in r.headers.items())
assert r.status_code is None
def test_upload_checksum(tmpdir, nasa_item):
with IaRequestsMock() as rsps:
rsps.add_metadata_mock('nasa')
nasa_item = get_item('nasa')
_expected_headers = deepcopy(EXPECTED_S3_HEADERS)
del _expected_headers['x-archive-meta00-scanner']
test_file = os.path.join(str(tmpdir), 'checksum_test.txt')
with open(test_file, 'wb') as fh:
fh.write(b'test delete')
# No skip.
rsps.add(responses.PUT, S3_URL_RE, adding_headers=_expected_headers)
resp = nasa_item.upload(test_file,
access_key='a',
secret_key='b',
checksum=True)
for r in resp:
headers = dict((k.lower(), str(v)) for k, v in r.headers.items())
del headers['content-type']
assert headers == _expected_headers
assert r.status_code == 200
# Skip.
nasa_item.item_metadata['files'].append(
dict(name='checksum_test.txt',
md5='33213e7683c1e6d15b2a658f3c567717'))
resp = nasa_item.upload(test_file,
access_key='a',
secret_key='b',
checksum=True)
for r in resp:
headers = dict((k.lower(), str(v)) for k, v in r.headers.items())
assert r.status_code is None
def test_download_checksum(tmpdir, caplog):
tmpdir.chdir()
# test overwrite based on checksum.
with IaRequestsMock() as rsps:
rsps.add_metadata_mock("nasa")
rsps.add(responses.GET, DOWNLOAD_URL_RE, body="test content")
rsps.add(responses.GET, DOWNLOAD_URL_RE, body="overwrite based on md5")
nasa_item = get_item("nasa")
nasa_item.download(files="nasa_meta.xml")
nasa_item.download(files="nasa_meta.xml", checksum=True)
assert load_file("nasa/nasa_meta.xml") == "overwrite based on md5"
# test no overwrite based on checksum.
rsps.add(responses.GET, DOWNLOAD_URL_RE, body=load_test_data_file("nasa_meta.xml"))
nasa_item.download(files="nasa_meta.xml", checksum=True)
nasa_item.download(files="nasa_meta.xml", checksum=True)
assert "skipping nasa" in caplog.text()
assert "nasa_meta.xml, file already exists based on checksum." in caplog.text()
def test_download_checksum(tmpdir, caplog):
tmpdir.chdir()
# test overwrite based on checksum.
with IaRequestsMock() as rsps:
rsps.add_metadata_mock('nasa')
rsps.add(responses.GET, DOWNLOAD_URL_RE, body='test content')
rsps.add(responses.GET, DOWNLOAD_URL_RE, body='overwrite based on md5')
nasa_item = get_item('nasa')
nasa_item.download(files='nasa_meta.xml')
nasa_item.download(files='nasa_meta.xml', checksum=True)
assert load_file('nasa/nasa_meta.xml') == 'overwrite based on md5'
# test no overwrite based on checksum.
rsps.reset()
rsps.add(responses.GET, DOWNLOAD_URL_RE,
body=load_test_data_file('nasa_meta.xml'))
nasa_item.download(files='nasa_meta.xml', checksum=True)
nasa_item.download(files='nasa_meta.xml', checksum=True)
assert 'skipping nasa' in caplog.text
assert 'nasa_meta.xml, file already exists based on checksum.' in caplog.text
def nasa_item(nasa_mocker):
return get_item('nasa')
def nasa_item(nasa_mocker):
return get_item('nasa')