def test_remove_local(): shutil.copy(constants.kExampleImageFile, constants.kExampleTemporaryImageFile) assert os.path.exists(constants.kExampleTemporaryImageFile) ops = PerceptOps(kConfig) ops.remove(constants.kExampleTemporaryImageFile) assert not os.path.exists(constants.kExampleTemporaryImageFile)
def main(): parser = argparse.ArgumentParser(description='Deletes percepts from the database by tag and deletes the percept data files.') parser.add_argument('database', help='Name of database to use') parser.add_argument('tag', help='Percepts with this tag will be deleted') parser.add_argument('-c', '--config', type=str, default='~/.rigor.ini', help='Path to .rigor.ini config file. Default: ~/.rigor.ini') parser.add_argument('--keep-percept-data', action='store_true', default=False, help="Don't remove the percept data files") parser.add_argument('-n', '--dryrun', action='store_true', default=False, help="Don't actually delete anything") args = parser.parse_args() config = RigorDefaultConfiguration(args.config) db = Database(args.database, config) ops = PerceptOps(config) if args.dryrun: print('DRY RUN') with db.get_session() as session: percepts = session.query(Percept).join(PerceptTag).filter(PerceptTag.name == args.tag).all() if len(percepts) == 0: print('No percepts have the tag "{}"'.format(args.tag)) for ii, percept in enumerate(percepts): print('{} of {}: deleting percept with id {}'.format(ii, len(percepts), percept.id)) if args.dryrun: continue if args.keep_percept_data: # only delete database entry, not percept file session.delete(percept) else: # delete percept data file as well as database entries ops.destroy(percept, session) if args.dryrun: print('DRY RUN')
def test_remove_http(): ops = PerceptOps(kConfig) with pytest.raises(NotImplementedError): ops.remove( 'http://' + os.path.join( constants.kExampleBucket + '.' + constants.kS3HostName, kKeys[1]), None)
def test_read_http(): ops = PerceptOps(kConfig) result = ops.read( 'http://' + os.path.join( constants.kExampleBucket + '.' + constants.kS3HostName, kKeys[1]), None) with result as text_file: assert text_file.read() == '1'
def test_fetch(): ops = PerceptOps(kConfig) def read(locator, credentials): return (locator, credentials) ops.read = read percept = Percept() percept.locator = constants.kExampleTextFile result = ops.fetch(percept) assert result == (percept.locator, None)
def __init__(self, algorithm, config, database_name, parameters=None, checkpoint=None): Runner.__init__(self, algorithm, parameters, checkpoint) self._config = config self._database = Database(database_name, config) self._perceptops = PerceptOps(config)
def test_delete_local_by_id(): shutil.copy(constants.kExampleImageFile, constants.kExampleTemporaryImageFile) assert os.path.exists(constants.kExampleTemporaryImageFile) database = db.get_database() ops = PerceptOps(kConfig) with database.get_session() as session: percept = session.query(rigor.types.Percept).get(642924) percept.locator = 'file://' + constants.kExampleTemporaryImageFile ops.destroy(percept.id, session) assert not os.path.exists(constants.kExampleTemporaryImageFile)
def test_delete_local(): shutil.copy(constants.kExampleImageFile, constants.kExampleTemporaryImageFile) assert os.path.exists(constants.kExampleTemporaryImageFile) database = db.get_database() ops = PerceptOps(kConfig) with database.get_session() as session: percept = session.query(rigor.types.Percept).get(642924) percept.locator = 'file://' + constants.kExampleTemporaryImageFile ops.destroy(percept, session) assert not os.path.exists(constants.kExampleTemporaryImageFile)
def main(): parser = argparse.ArgumentParser( description= 'Deletes percepts from the database by tag and deletes the percept data files.' ) parser.add_argument('database', help='Name of database to use') parser.add_argument('tag', help='Percepts with this tag will be deleted') parser.add_argument( '-c', '--config', type=str, default='~/.rigor.ini', help='Path to .rigor.ini config file. Default: ~/.rigor.ini') parser.add_argument('--keep-percept-data', action='store_true', default=False, help="Don't remove the percept data files") parser.add_argument('-n', '--dryrun', action='store_true', default=False, help="Don't actually delete anything") args = parser.parse_args() config = RigorDefaultConfiguration(args.config) db = Database(args.database, config) ops = PerceptOps(config) if args.dryrun: print('DRY RUN') with db.get_session() as session: percepts = session.query(Percept).join(PerceptTag).filter( PerceptTag.name == args.tag).all() if len(percepts) == 0: print('No percepts have the tag "{}"'.format(args.tag)) for ii, percept in enumerate(percepts): print('{} of {}: deleting percept with id {}'.format( ii, len(percepts), percept.id)) if args.dryrun: continue if args.keep_percept_data: # only delete database entry, not percept file session.delete(percept) else: # delete percept data file as well as database entries ops.destroy(percept, session) if args.dryrun: print('DRY RUN')
def test_import_all_with_data_to_s3(importdb): with open(constants.kImportFile, 'rb') as import_file: metadata = json.load(import_file) for index, percept in enumerate(metadata): percept['locator'] = 's3://' + os.path.join(constants.kExampleBucket, str(index) + ".txt") importer = Importer(kConfig, constants.kImportDatabase, metadata, import_data=True) importer.run() ops = PerceptOps(kConfig) with importdb.get_session() as session: percepts = session.query(Percept).all() assert len(percepts) == 3 for percept in percepts: assert ops.read(percept.locator) is not None locator = urlsplit(percept.locator) assert len(percept.annotations) == 1
class DatabaseRunner(Runner): """ Runner class that uses the database to discover percepts for evaluation :param algorithm: algorithm to run against each percept :type algorithm: :py:class:`~rigor.algorithm.Algorithm` :param config: configuration data :type config: :py:class:`~rigor.config.RigorConfiguration` :param str database_name: name of the database to use :param dict parameters: settings for the Runner :param file checkpoint: open :py:class:`~rigor.checkpoint.Checkpoint` file to resume from """ def __init__(self, algorithm, config, database_name, parameters=None, checkpoint=None): Runner.__init__(self, algorithm, parameters, checkpoint) self._config = config self._database = Database(database_name, config) self._perceptops = PerceptOps(config) def fetch_data(self, percept): """ Gets percept data from the repository :param percept: The percept corresponding to data to fetch :return: Percept data encapsulated in a :py:func:`~contextlib.contextmanager` """ return self._perceptops.fetch(percept)
def test_remove_s3(): ops = PerceptOps(kConfig) assert ops.read('s3://' + os.path.join(constants.kExampleBucket, kKeys[0]), None) is not None ops.remove('s3://' + os.path.join(constants.kExampleBucket, kKeys[0]), None) assert ops.read('s3://' + os.path.join(constants.kExampleBucket, kKeys[0]), None) is None
def test_read_local_file_uri(): ops = PerceptOps(kConfig) result = ops.read('file://' + constants.kExampleTextFile, None) with result as text_file: assert text_file.read() == 'This is a file to test uploading a file to S3'
def test_read_s3(): ops = PerceptOps(kConfig) result = ops.read('s3://' + os.path.join(constants.kExampleBucket, kKeys[0]), None) with result as text_file: assert text_file.read() == '0'
def test_read_http(): ops = PerceptOps(kConfig) result = ops.read('http://' + os.path.join(constants.kExampleBucket + '.' + constants.kS3HostName, kKeys[1]), None) with result as text_file: assert text_file.read() == '1'
def test_read_s3(): ops = PerceptOps(kConfig) result = ops.read( 's3://' + os.path.join(constants.kExampleBucket, kKeys[0]), None) with result as text_file: assert text_file.read() == '0'
def test_read_local_file_uri(): ops = PerceptOps(kConfig) result = ops.read('file://' + constants.kExampleTextFile, None) with result as text_file: assert text_file.read( ) == 'This is a file to test uploading a file to S3'
def test_remove_http(): ops = PerceptOps(kConfig) with pytest.raises(NotImplementedError): ops.remove('http://' + os.path.join(constants.kExampleBucket + '.' + constants.kS3HostName, kKeys[1]), None)
def test_init(): ops = PerceptOps(kConfig)