def test_import_all_with_data_to_remote(importdb): with open(constants.kImportFile, 'rb') as import_file: metadata = json.load(import_file) for index, percept in enumerate(metadata): percept['locator'] = 'http://' + os.path.join(constants.kExampleBucket + '.' + constants.kS3HostName, str(index) + ".txt") importer = Importer(kConfig, constants.kImportDatabase, metadata, import_data=True) with pytest.raises(NotImplementedError): importer.run()
def test_import_remote_with_data(importdb): metadata = [{ 'source' : 'http://test_url', 'locator': 'file://' + constants.kRepoDirectory, 'format': 'text/plain', }] importer = Importer(kConfig, constants.kImportDatabase, metadata, import_data=True) with pytest.raises(NotImplementedError): importer.run()
def test_import_all_no_data(importdb): importer = Importer(kConfig, constants.kImportDatabase, constants.kImportFile, import_data=False) importer.run() with importdb.get_session() as session: percepts = session.query(Percept).all() assert len(percepts) == 3 for percept in percepts: locator = urlsplit(percept.locator) assert not os.path.exists(locator.path) assert len(percept.annotations) == 1
def test_import_remote_with_data(importdb): metadata = [{ 'source': 'http://test_url', 'locator': 'file://' + constants.kRepoDirectory, 'format': 'text/plain', }] importer = Importer(kConfig, constants.kImportDatabase, metadata, import_data=True) with pytest.raises(NotImplementedError): importer.run()
def test_import_all_with_data_to_remote(importdb): with open(constants.kImportFile, 'rb') as import_file: metadata = json.load(import_file) for index, percept in enumerate(metadata): percept['locator'] = 'http://' + os.path.join( constants.kExampleBucket + '.' + constants.kS3HostName, str(index) + ".txt") importer = Importer(kConfig, constants.kImportDatabase, metadata, import_data=True) with pytest.raises(NotImplementedError): importer.run()
def test_import_all_with_data(importdb): importer = Importer(kConfig, constants.kImportDatabase, constants.kImportFile, import_data=True) importer.run() with importdb.get_session() as session: percepts = session.query(Percept).all() assert len(percepts) == 3 for percept in percepts: locator = urlsplit(percept.locator) assert os.path.exists(locator.path) assert len(percept.annotations) == 1 assert percepts[0].hash == '6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b' assert percepts[1].hash == 'd4735e3a265e16eee03f59718b9b5d03019c07d8b6c51f90da3a666eec13ab35' assert percepts[2].hash == '4e07408562bedb8b60ce05c1decfe3ad16b72230967de01f640b7e4729b49fce'
def test_import_all_with_data_to_s3(importdb): with open(constants.kImportFile, 'rb') as import_file: metadata = json.load(import_file) for index, percept in enumerate(metadata): percept['locator'] = 's3://' + os.path.join(constants.kExampleBucket, str(index) + ".txt") importer = Importer(kConfig, constants.kImportDatabase, metadata, import_data=True) importer.run() ops = PerceptOps(kConfig) with importdb.get_session() as session: percepts = session.query(Percept).all() assert len(percepts) == 3 for percept in percepts: assert ops.read(percept.locator) is not None locator = urlsplit(percept.locator) assert len(percept.annotations) == 1
def test_import_all_with_data(importdb): importer = Importer(kConfig, constants.kImportDatabase, constants.kImportFile, import_data=True) importer.run() with importdb.get_session() as session: percepts = session.query(Percept).all() assert len(percepts) == 3 for percept in percepts: locator = urlsplit(percept.locator) assert os.path.exists(locator.path) assert len(percept.annotations) == 1 assert percepts[ 0].hash == '6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b' assert percepts[ 1].hash == 'd4735e3a265e16eee03f59718b9b5d03019c07d8b6c51f90da3a666eec13ab35' assert percepts[ 2].hash == '4e07408562bedb8b60ce05c1decfe3ad16b72230967de01f640b7e4729b49fce'