def test_remove_local():
shutil.copy(constants.kExampleImageFile,
constants.kExampleTemporaryImageFile)
assert os.path.exists(constants.kExampleTemporaryImageFile)
ops = PerceptOps(kConfig)
ops.remove(constants.kExampleTemporaryImageFile)
assert not os.path.exists(constants.kExampleTemporaryImageFile)
def main():
parser = argparse.ArgumentParser(description='Deletes percepts from the database by tag and deletes the percept data files.')
parser.add_argument('database', help='Name of database to use')
parser.add_argument('tag', help='Percepts with this tag will be deleted')
parser.add_argument('-c', '--config', type=str, default='~/.rigor.ini', help='Path to .rigor.ini config file. Default: ~/.rigor.ini')
parser.add_argument('--keep-percept-data', action='store_true', default=False, help="Don't remove the percept data files")
parser.add_argument('-n', '--dryrun', action='store_true', default=False, help="Don't actually delete anything")
args = parser.parse_args()
config = RigorDefaultConfiguration(args.config)
db = Database(args.database, config)
ops = PerceptOps(config)
if args.dryrun:
print('DRY RUN')
with db.get_session() as session:
percepts = session.query(Percept).join(PerceptTag).filter(PerceptTag.name == args.tag).all()
if len(percepts) == 0:
print('No percepts have the tag "{}"'.format(args.tag))
for ii, percept in enumerate(percepts):
print('{} of {}: deleting percept with id {}'.format(ii, len(percepts), percept.id))
if args.dryrun:
continue
if args.keep_percept_data:
# only delete database entry, not percept file
session.delete(percept)
else:
# delete percept data file as well as database entries
ops.destroy(percept, session)
if args.dryrun:
print('DRY RUN')
def test_remove_http():
ops = PerceptOps(kConfig)
with pytest.raises(NotImplementedError):
ops.remove(
'http://' + os.path.join(
constants.kExampleBucket + '.' + constants.kS3HostName,
kKeys[1]), None)
def test_read_http():
ops = PerceptOps(kConfig)
result = ops.read(
'http://' + os.path.join(
constants.kExampleBucket + '.' + constants.kS3HostName, kKeys[1]),
None)
with result as text_file:
assert text_file.read() == '1'
def test_fetch(): ops = PerceptOps(kConfig) def read(locator, credentials): return (locator, credentials) ops.read = read percept = Percept() percept.locator = constants.kExampleTextFile result = ops.fetch(percept) assert result == (percept.locator, None)
def __init__(self,
algorithm,
config,
database_name,
parameters=None,
checkpoint=None):
Runner.__init__(self, algorithm, parameters, checkpoint)
self._config = config
self._database = Database(database_name, config)
self._perceptops = PerceptOps(config)
def test_delete_local_by_id(): shutil.copy(constants.kExampleImageFile, constants.kExampleTemporaryImageFile) assert os.path.exists(constants.kExampleTemporaryImageFile) database = db.get_database() ops = PerceptOps(kConfig) with database.get_session() as session: percept = session.query(rigor.types.Percept).get(642924) percept.locator = 'file://' + constants.kExampleTemporaryImageFile ops.destroy(percept.id, session) assert not os.path.exists(constants.kExampleTemporaryImageFile)
def test_fetch():
ops = PerceptOps(kConfig)
def read(locator, credentials):
return (locator, credentials)
ops.read = read
percept = Percept()
percept.locator = constants.kExampleTextFile
result = ops.fetch(percept)
assert result == (percept.locator, None)
def test_delete_local():
shutil.copy(constants.kExampleImageFile,
constants.kExampleTemporaryImageFile)
assert os.path.exists(constants.kExampleTemporaryImageFile)
database = db.get_database()
ops = PerceptOps(kConfig)
with database.get_session() as session:
percept = session.query(rigor.types.Percept).get(642924)
percept.locator = 'file://' + constants.kExampleTemporaryImageFile
ops.destroy(percept, session)
assert not os.path.exists(constants.kExampleTemporaryImageFile)
def main():
parser = argparse.ArgumentParser(
description=
'Deletes percepts from the database by tag and deletes the percept data files.'
)
parser.add_argument('database', help='Name of database to use')
parser.add_argument('tag', help='Percepts with this tag will be deleted')
parser.add_argument(
'-c',
'--config',
type=str,
default='~/.rigor.ini',
help='Path to .rigor.ini config file. Default: ~/.rigor.ini')
parser.add_argument('--keep-percept-data',
action='store_true',
default=False,
help="Don't remove the percept data files")
parser.add_argument('-n',
'--dryrun',
action='store_true',
default=False,
help="Don't actually delete anything")
args = parser.parse_args()
config = RigorDefaultConfiguration(args.config)
db = Database(args.database, config)
ops = PerceptOps(config)
if args.dryrun:
print('DRY RUN')
with db.get_session() as session:
percepts = session.query(Percept).join(PerceptTag).filter(
PerceptTag.name == args.tag).all()
if len(percepts) == 0:
print('No percepts have the tag "{}"'.format(args.tag))
for ii, percept in enumerate(percepts):
print('{} of {}: deleting percept with id {}'.format(
ii, len(percepts), percept.id))
if args.dryrun:
continue
if args.keep_percept_data:
# only delete database entry, not percept file
session.delete(percept)
else:
# delete percept data file as well as database entries
ops.destroy(percept, session)
if args.dryrun:
print('DRY RUN')
def test_import_all_with_data_to_s3(importdb): with open(constants.kImportFile, 'rb') as import_file: metadata = json.load(import_file) for index, percept in enumerate(metadata): percept['locator'] = 's3://' + os.path.join(constants.kExampleBucket, str(index) + ".txt") importer = Importer(kConfig, constants.kImportDatabase, metadata, import_data=True) importer.run() ops = PerceptOps(kConfig) with importdb.get_session() as session: percepts = session.query(Percept).all() assert len(percepts) == 3 for percept in percepts: assert ops.read(percept.locator) is not None locator = urlsplit(percept.locator) assert len(percept.annotations) == 1
class DatabaseRunner(Runner):
"""
Runner class that uses the database to discover percepts for evaluation
:param algorithm: algorithm to run against each percept
:type algorithm: :py:class:`~rigor.algorithm.Algorithm`
:param config: configuration data
:type config: :py:class:`~rigor.config.RigorConfiguration`
:param str database_name: name of the database to use
:param dict parameters: settings for the Runner
:param file checkpoint: open :py:class:`~rigor.checkpoint.Checkpoint` file to resume from
"""
def __init__(self,
algorithm,
config,
database_name,
parameters=None,
checkpoint=None):
Runner.__init__(self, algorithm, parameters, checkpoint)
self._config = config
self._database = Database(database_name, config)
self._perceptops = PerceptOps(config)
def fetch_data(self, percept):
"""
Gets percept data from the repository
:param percept: The percept corresponding to data to fetch
:return: Percept data encapsulated in a :py:func:`~contextlib.contextmanager`
"""
return self._perceptops.fetch(percept)
class DatabaseRunner(Runner): """ Runner class that uses the database to discover percepts for evaluation :param algorithm: algorithm to run against each percept :type algorithm: :py:class:`~rigor.algorithm.Algorithm` :param config: configuration data :type config: :py:class:`~rigor.config.RigorConfiguration` :param str database_name: name of the database to use :param dict parameters: settings for the Runner :param file checkpoint: open :py:class:`~rigor.checkpoint.Checkpoint` file to resume from """ def __init__(self, algorithm, config, database_name, parameters=None, checkpoint=None): Runner.__init__(self, algorithm, parameters, checkpoint) self._config = config self._database = Database(database_name, config) self._perceptops = PerceptOps(config) def fetch_data(self, percept): """ Gets percept data from the repository :param percept: The percept corresponding to data to fetch :return: Percept data encapsulated in a :py:func:`~contextlib.contextmanager` """ return self._perceptops.fetch(percept)
def test_import_all_with_data_to_s3(importdb):
with open(constants.kImportFile, 'rb') as import_file:
metadata = json.load(import_file)
for index, percept in enumerate(metadata):
percept['locator'] = 's3://' + os.path.join(constants.kExampleBucket,
str(index) + ".txt")
importer = Importer(kConfig,
constants.kImportDatabase,
metadata,
import_data=True)
importer.run()
ops = PerceptOps(kConfig)
with importdb.get_session() as session:
percepts = session.query(Percept).all()
assert len(percepts) == 3
for percept in percepts:
assert ops.read(percept.locator) is not None
locator = urlsplit(percept.locator)
assert len(percept.annotations) == 1
def test_remove_s3():
ops = PerceptOps(kConfig)
assert ops.read('s3://' + os.path.join(constants.kExampleBucket, kKeys[0]),
None) is not None
ops.remove('s3://' + os.path.join(constants.kExampleBucket, kKeys[0]),
None)
assert ops.read('s3://' + os.path.join(constants.kExampleBucket, kKeys[0]),
None) is None
def __init__(self, algorithm, config, database_name, parameters=None, checkpoint=None): Runner.__init__(self, algorithm, parameters, checkpoint) self._config = config self._database = Database(database_name, config) self._perceptops = PerceptOps(config)
def test_read_local_file_uri():
ops = PerceptOps(kConfig)
result = ops.read('file://' + constants.kExampleTextFile, None)
with result as text_file:
assert text_file.read() == 'This is a file to test uploading a file to S3'
def test_read_s3():
ops = PerceptOps(kConfig)
result = ops.read('s3://' + os.path.join(constants.kExampleBucket, kKeys[0]), None)
with result as text_file:
assert text_file.read() == '0'
def test_read_http():
ops = PerceptOps(kConfig)
result = ops.read('http://' + os.path.join(constants.kExampleBucket + '.' + constants.kS3HostName, kKeys[1]), None)
with result as text_file:
assert text_file.read() == '1'
def test_remove_local(): shutil.copy(constants.kExampleImageFile, constants.kExampleTemporaryImageFile) assert os.path.exists(constants.kExampleTemporaryImageFile) ops = PerceptOps(kConfig) ops.remove(constants.kExampleTemporaryImageFile) assert not os.path.exists(constants.kExampleTemporaryImageFile)
def test_remove_s3():
ops = PerceptOps(kConfig)
assert ops.read('s3://' + os.path.join(constants.kExampleBucket, kKeys[0]), None) is not None
ops.remove('s3://' + os.path.join(constants.kExampleBucket, kKeys[0]), None)
assert ops.read('s3://' + os.path.join(constants.kExampleBucket, kKeys[0]), None) is None
def test_read_s3():
ops = PerceptOps(kConfig)
result = ops.read(
's3://' + os.path.join(constants.kExampleBucket, kKeys[0]), None)
with result as text_file:
assert text_file.read() == '0'
def test_read_local_file_uri():
ops = PerceptOps(kConfig)
result = ops.read('file://' + constants.kExampleTextFile, None)
with result as text_file:
assert text_file.read(
) == 'This is a file to test uploading a file to S3'
def test_remove_http():
ops = PerceptOps(kConfig)
with pytest.raises(NotImplementedError):
ops.remove('http://' + os.path.join(constants.kExampleBucket + '.' + constants.kS3HostName, kKeys[1]), None)
def test_init():
ops = PerceptOps(kConfig)
