def test_import_all_with_data_to_remote(importdb): with open(constants.kImportFile, 'rb') as import_file: metadata = json.load(import_file) for index, percept in enumerate(metadata): percept['locator'] = 'http://' + os.path.join(constants.kExampleBucket + '.' + constants.kS3HostName, str(index) + ".txt") importer = Importer(kConfig, constants.kImportDatabase, metadata, import_data=True) with pytest.raises(NotImplementedError): importer.run()
def test_import_remote_with_data(importdb):
metadata = [{
'source' : 'http://test_url',
'locator': 'file://' + constants.kRepoDirectory,
'format': 'text/plain',
}]
importer = Importer(kConfig, constants.kImportDatabase, metadata, import_data=True)
with pytest.raises(NotImplementedError):
importer.run()
def test_import_all_no_data(importdb): importer = Importer(kConfig, constants.kImportDatabase, constants.kImportFile, import_data=False) importer.run() with importdb.get_session() as session: percepts = session.query(Percept).all() assert len(percepts) == 3 for percept in percepts: locator = urlsplit(percept.locator) assert not os.path.exists(locator.path) assert len(percept.annotations) == 1
def test_import_non_writable_dir(importdb):
source_path = os.path.abspath(os.path.join(constants.kImportDirectory, '01.txt'))
metadata = {
'source' : 'file://' + source_path,
'locator': 'file:////xyz/01.txt',
'format': 'text/plain',
}
importer = Importer(kConfig, constants.kImportDatabase, None, import_data=True)
with pytest.raises(OSError):
importer.import_percept(metadata)
def test_import_single_percept(importdb): importer = Importer(kConfig, constants.kImportDatabase, constants.kImportFile, import_data=False) with importdb.get_session() as session: percepts = session.query(Percept).all() assert len(percepts) == 0 percept_id = importer.import_percept(constants.kExamplePercept) assert percept_id > 0 with importdb.get_session() as session: percepts = session.query(Percept).all() assert len(percepts) == 1
def test_import_remote_with_data(importdb):
metadata = [{
'source': 'http://test_url',
'locator': 'file://' + constants.kRepoDirectory,
'format': 'text/plain',
}]
importer = Importer(kConfig,
constants.kImportDatabase,
metadata,
import_data=True)
with pytest.raises(NotImplementedError):
importer.run()
def test_import_single_percept(importdb):
importer = Importer(kConfig,
constants.kImportDatabase,
constants.kImportFile,
import_data=False)
with importdb.get_session() as session:
percepts = session.query(Percept).all()
assert len(percepts) == 0
percept_id = importer.import_percept(constants.kExamplePercept)
assert percept_id > 0
with importdb.get_session() as session:
percepts = session.query(Percept).all()
assert len(percepts) == 1
def test_import_all_with_data_to_remote(importdb):
with open(constants.kImportFile, 'rb') as import_file:
metadata = json.load(import_file)
for index, percept in enumerate(metadata):
percept['locator'] = 'http://' + os.path.join(
constants.kExampleBucket + '.' + constants.kS3HostName,
str(index) + ".txt")
importer = Importer(kConfig,
constants.kImportDatabase,
metadata,
import_data=True)
with pytest.raises(NotImplementedError):
importer.run()
def test_import_all_no_data(importdb):
importer = Importer(kConfig,
constants.kImportDatabase,
constants.kImportFile,
import_data=False)
importer.run()
with importdb.get_session() as session:
percepts = session.query(Percept).all()
assert len(percepts) == 3
for percept in percepts:
locator = urlsplit(percept.locator)
assert not os.path.exists(locator.path)
assert len(percept.annotations) == 1
def test_import_all_with_data(importdb): importer = Importer(kConfig, constants.kImportDatabase, constants.kImportFile, import_data=True) importer.run() with importdb.get_session() as session: percepts = session.query(Percept).all() assert len(percepts) == 3 for percept in percepts: locator = urlsplit(percept.locator) assert os.path.exists(locator.path) assert len(percept.annotations) == 1 assert percepts[0].hash == '6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b' assert percepts[1].hash == 'd4735e3a265e16eee03f59718b9b5d03019c07d8b6c51f90da3a666eec13ab35' assert percepts[2].hash == '4e07408562bedb8b60ce05c1decfe3ad16b72230967de01f640b7e4729b49fce'
def test_import_non_writable_dir(importdb):
source_path = os.path.abspath(
os.path.join(constants.kImportDirectory, '01.txt'))
metadata = {
'source': 'file://' + source_path,
'locator': 'file:////xyz/01.txt',
'format': 'text/plain',
}
importer = Importer(kConfig,
constants.kImportDatabase,
None,
import_data=True)
with pytest.raises(OSError):
importer.import_percept(metadata)
def test_import_all_with_data_to_s3(importdb): with open(constants.kImportFile, 'rb') as import_file: metadata = json.load(import_file) for index, percept in enumerate(metadata): percept['locator'] = 's3://' + os.path.join(constants.kExampleBucket, str(index) + ".txt") importer = Importer(kConfig, constants.kImportDatabase, metadata, import_data=True) importer.run() ops = PerceptOps(kConfig) with importdb.get_session() as session: percepts = session.query(Percept).all() assert len(percepts) == 3 for percept in percepts: assert ops.read(percept.locator) is not None locator = urlsplit(percept.locator) assert len(percept.annotations) == 1
def test_import_all_with_data_to_s3(importdb):
with open(constants.kImportFile, 'rb') as import_file:
metadata = json.load(import_file)
for index, percept in enumerate(metadata):
percept['locator'] = 's3://' + os.path.join(constants.kExampleBucket,
str(index) + ".txt")
importer = Importer(kConfig,
constants.kImportDatabase,
metadata,
import_data=True)
importer.run()
ops = PerceptOps(kConfig)
with importdb.get_session() as session:
percepts = session.query(Percept).all()
assert len(percepts) == 3
for percept in percepts:
assert ops.read(percept.locator) is not None
locator = urlsplit(percept.locator)
assert len(percept.annotations) == 1
def test_import_all_with_data(importdb):
importer = Importer(kConfig,
constants.kImportDatabase,
constants.kImportFile,
import_data=True)
importer.run()
with importdb.get_session() as session:
percepts = session.query(Percept).all()
assert len(percepts) == 3
for percept in percepts:
locator = urlsplit(percept.locator)
assert os.path.exists(locator.path)
assert len(percept.annotations) == 1
assert percepts[
0].hash == '6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b'
assert percepts[
1].hash == 'd4735e3a265e16eee03f59718b9b5d03019c07d8b6c51f90da3a666eec13ab35'
assert percepts[
2].hash == '4e07408562bedb8b60ce05c1decfe3ad16b72230967de01f640b7e4729b49fce'
