def test_run(self): conn = get_fakeredis() operation = "lala" worker = CountingWorker(conn=conn, stages=[operation]) worker.sync() assert worker.test_done == 0, worker.test_done job = Job.create(conn, "test") stage = job.get_stage(operation) task = stage.queue({}, {}) assert not job.is_done() assert worker.test_done == 0, worker.test_done worker.sync() assert worker.test_done == 1, worker.test_done assert job.is_done() worker.retry(task) assert not job.is_done() worker.sync() assert job.is_done() assert worker.exit_code == 0, worker.exit_code assert worker.test_done == 1, worker.test_done worker._handle_signal(5, None) assert worker.exit_code == 5, worker.exit_code worker.retry(task) worker.run(blocking=False) assert job.is_done() assert worker.exit_code == 0, worker.exit_code worker.num_threads = None worker.retry(task) worker.run(blocking=False) assert job.is_done() assert worker.exit_code == 0, worker.exit_code
def test_rate(self): conn = get_fakeredis() limit = RateLimit(conn, 'banana', limit=10) assert limit.check() limit.update() assert limit.check() for _ in range(13): limit.update() assert not limit.check()
def test_rate(self): conn = get_fakeredis() limit = RateLimit(conn, "banana", limit=10) assert limit.check() limit.update() assert limit.check() for num in range(13): assert num + 2 == limit.update() assert not limit.check()
def debug(path, dataset, languages=None): """Debug the ingest for the given path.""" conn = get_fakeredis() settings.sts.DATABASE_URI = 'sqlite://' db = get_dataset(dataset, OP_INGEST) _ingest_path(db, conn, dataset, path, languages=languages) worker = IngestWorker(conn=conn, stages=STAGES) worker.sync() for entity in db.iterate(): pprint(entity.to_dict())
def debug(path, dataset, languages=None): """Debug the ingest for the given path.""" conn = get_fakeredis() db = balkhash.init(dataset) db.delete() _ingest_path(conn, dataset, path, languages=languages) worker = IngestWorker(conn=conn, stages=STAGES) worker.sync() for entity in db.iterate(): pprint(entity.to_dict())
def debug(path, languages=None): """Debug the ingest for the given path.""" conn = get_fakeredis() settings.fts.DATABASE_URI = "sqlite:////tmp/debug.sqlite3" db = get_dataset("debug", origin=OP_INGEST, database_uri=settings.fts.DATABASE_URI) db.delete() _ingest_path(db, conn, "debug", path, languages=languages) worker = IngestWorker(conn=conn, stages=STAGES) worker.sync() for entity in db.iterate(): pprint(entity.to_dict())
def setUp(self): # Force tests to use fake configuration service_settings.REDIS_URL = None service_settings.ARCHIVE_TYPE = 'file' service_settings.ARCHIVE_PATH = mkdtemp() balkhash_settings.BACKEND = 'LEVELDB' balkhash_settings.LEVELDB_PATH = mkdtemp() conn = get_fakeredis() self.queue = ServiceQueue(conn, ServiceQueue.OP_INGEST, 'test') self.manager = Manager(self.queue, {}) self.manager.entities = [] self.manager.emit_entity = types.MethodType(emit_entity, self.manager) self.manager.queue_entity = types.MethodType(queue_entity, self.manager) # noqa self.archive = init_archive() self.manager._archive = self.archive
def setUp(self): # Force tests to use fake configuration ingestors_settings.TESTING = True service_settings.REDIS_URL = None service_settings.ARCHIVE_TYPE = 'file' service_settings.ARCHIVE_PATH = mkdtemp() ftmstore_settings.DATABASE_URI = 'sqlite://' conn = get_fakeredis() job = Job.create(conn, 'test') stage = Stage(job, OP_INGEST) dataset = get_dataset(job.dataset.name, OP_INGEST) self.manager = Manager(dataset, stage, {}) self.manager.entities = [] self.manager.emit_entity = types.MethodType(emit_entity, self.manager) self.manager.queue_entity = types.MethodType(queue_entity, self.manager) # noqa self.archive = init_archive() self.manager._archive = self.archive
def setUp(self): self.conn = get_fakeredis() self.conn.flushall() self.dataset = 'test_1'
def connect_redis(): if settings.TESTING: return get_fakeredis() return get_redis()