def test_download_checksum(tmpdir, caplog): tmpdir.chdir() # test overwrite based on checksum. with IaRequestsMock() as rsps: rsps.add_metadata_mock('nasa') rsps.add(responses.GET, DOWNLOAD_URL_RE, body='test content') rsps.add(responses.GET, DOWNLOAD_URL_RE, body='overwrite based on md5') nasa_item = get_item('nasa') nasa_item.download(files='nasa_meta.xml') nasa_item.download(files='nasa_meta.xml', checksum=True) assert load_file('nasa/nasa_meta.xml') == 'overwrite based on md5' # test no overwrite based on checksum. rsps.reset() rsps.add(responses.GET, DOWNLOAD_URL_RE, body=load_test_data_file('nasa_meta.xml')) nasa_item.download(files='nasa_meta.xml', checksum=True) nasa_item.download(files='nasa_meta.xml', checksum=True) assert 'skipping nasa' in caplog.text assert 'nasa_meta.xml, file already exists based on checksum.' in caplog.text
def test_ia_search_itemlist(capsys): test_scrape_response = load_test_data_file('scrape_response.json') with responses.RequestsMock(assert_all_requests_are_fired=True) as rsps: url1 = ('{0}//archive.org/services/search/v1/scrape' '?q=collection%3Aattentionkmartshoppers' '&REQUIRE_AUTH=true&count=10000'.format(PROTOCOL)) url2 = ('{0}//archive.org/services/search/v1/scrape?' 'cursor=W3siaWRlbnRpZmllciI6IjE5NjEtTC0wNTkxNCJ9XQ%3D%3D' '&REQUIRE_AUTH=true&q=collection%3Aattentionkmartshoppers' '&count=10000'.format(PROTOCOL)) rsps.add(responses.POST, url1, body=test_scrape_response, match_querystring=True) _j = json.loads(test_scrape_response) del _j['cursor'] _r = json.dumps(_j) rsps.add(responses.POST, url2, body=_r, match_querystring=True) ia_call([ 'ia', 'search', 'collection:attentionkmartshoppers', '--itemlist' ]) out, err = capsys.readouterr() assert len(out.split()) == 200
def test_download_checksum(tmpdir, caplog): tmpdir.chdir() # test overwrite based on checksum. with IaRequestsMock() as rsps: rsps.add_metadata_mock("nasa") rsps.add(responses.GET, DOWNLOAD_URL_RE, body="test content") rsps.add(responses.GET, DOWNLOAD_URL_RE, body="overwrite based on md5") nasa_item = get_item("nasa") nasa_item.download(files="nasa_meta.xml") nasa_item.download(files="nasa_meta.xml", checksum=True) assert load_file("nasa/nasa_meta.xml") == "overwrite based on md5" # test no overwrite based on checksum. rsps.add(responses.GET, DOWNLOAD_URL_RE, body=load_test_data_file("nasa_meta.xml")) nasa_item.download(files="nasa_meta.xml", checksum=True) nasa_item.download(files="nasa_meta.xml", checksum=True) assert "skipping nasa" in caplog.text() assert "nasa_meta.xml, file already exists based on checksum." in caplog.text()
def test_ia_search_itemlist(capsys): test_scrape_response = load_test_data_file('scrape_response.json') with responses.RequestsMock(assert_all_requests_are_fired=True) as rsps: url1 = ('{0}//archive.org/services/search/v1/scrape' '?q=collection%3Aattentionkmartshoppers' '&REQUIRE_AUTH=true&count=10000'.format(PROTOCOL)) url2 = ('{0}//archive.org/services/search/v1/scrape?' 'cursor=W3siaWRlbnRpZmllciI6IjE5NjEtTC0wNTkxNCJ9XQ%3D%3D' '&REQUIRE_AUTH=true&q=collection%3Aattentionkmartshoppers' '&count=10000'.format(PROTOCOL)) rsps.add(responses.POST, url1, body=test_scrape_response, match_querystring=True) _j = json.loads(test_scrape_response) del _j['cursor'] _r = json.dumps(_j) rsps.add(responses.POST, url2, body=_r, match_querystring=True) ia_call(['ia', 'search', 'collection:attentionkmartshoppers', '--itemlist']) out, err = capsys.readouterr() assert len(out.split()) == 200
import json import os import re import responses import six from requests.packages import urllib3 from internetarchive import get_session, get_item, get_files, modify_metadata, upload, \ download, search_items from tests.conftest import NASA_METADATA_PATH, PROTOCOL, IaRequestsMock, \ load_test_data_file, load_file PY3 = six.PY3 TEST_SEARCH_RESPONSE = load_test_data_file('advanced_search_response.json') TEST_SCRAPE_RESPONSE = load_test_data_file('scrape_response.json') _j = json.loads(TEST_SCRAPE_RESPONSE) del _j['cursor'] _j['items'] = [{'identifier': 'nasa'}] TEST_SCRAPE_RESPONSE = json.dumps(_j) def test_get_session_with_config(): s = get_session(config={'s3': {'access': 'key'}}) assert s.access_key == 'key' def test_get_session_with_config_file(tmpdir): tmpdir.chdir() test_conf = """[s3]\naccess = key2"""
# -*- coding: utf-8 -*- import json import responses from tests.conftest import IaRequestsMock, load_test_data_file, ia_call PROTOCOL = 'https:' STATUS_CHECK_RESPONSE = load_test_data_file('s3_status_check.json') def test_ia_upload(tmpdir_ch, caplog): with open('test.txt', 'w') as fh: fh.write('foo') with IaRequestsMock() as rsps: rsps.add_metadata_mock('nasa') rsps.add(responses.PUT, '{0}//s3.us.archive.org/nasa/test.txt'.format(PROTOCOL), body='', content_type='text/plain') ia_call(['ia', '--log', 'upload', 'nasa', 'test.txt']) assert ('uploaded test.txt to {0}//s3.us.archive.org/nasa/' 'test.txt'.format(PROTOCOL)) in caplog.text def test_ia_upload_status_check(capsys): with IaRequestsMock() as rsps: rsps.add(responses.GET, '{0}//s3.us.archive.org'.format(PROTOCOL),
# -*- coding: utf-8 -*- import json import responses from tests.conftest import IaRequestsMock, load_test_data_file, ia_call PROTOCOL = 'https:' STATUS_CHECK_RESPONSE = load_test_data_file('s3_status_check.json') def test_ia_upload(tmpdir_ch, caplog): with open('test.txt', 'w') as fh: fh.write('foo') with IaRequestsMock() as rsps: rsps.add_metadata_mock('nasa') rsps.add(responses.PUT, '{0}//s3.us.archive.org/nasa/test.txt'.format(PROTOCOL), body='', content_type='text/plain') ia_call(['ia', '--log', 'upload', 'nasa', 'test.txt']) assert ('uploaded test.txt to {0}//s3.us.archive.org/nasa/' 'test.txt'.format(PROTOCOL)) in caplog.text def test_ia_upload_status_check(capsys): with IaRequestsMock() as rsps: rsps.add(responses.GET, '{0}//s3.us.archive.org'.format(PROTOCOL), body=STATUS_CHECK_RESPONSE, content_type='application/json')
import json import os import re import responses import six from requests.packages import urllib3 from internetarchive import get_session, get_item, get_files, modify_metadata, upload, \ download, search_items from tests.conftest import NASA_METADATA_PATH, PROTOCOL, IaRequestsMock, \ load_test_data_file, load_file PY3 = six.PY3 TEST_SEARCH_RESPONSE = load_test_data_file('advanced_search_response.json') TEST_SCRAPE_RESPONSE = load_test_data_file('scrape_response.json') _j = json.loads(TEST_SCRAPE_RESPONSE) del _j['cursor'] _j['items'] = [{'identifier': 'nasa'}] TEST_SCRAPE_RESPONSE = json.dumps(_j) def test_get_session_with_config(): s = get_session(config={'s3': {'access': 'key'}, 'gengeral': {'secure': False}}) assert s.access_key == 'key' def test_get_session_with_config_file(tmpdir): tmpdir.chdir() test_conf = """[s3]\naccess = key2"""