def debug(path, dataset, languages=None):
"""Debug the ingest for the given path."""
conn = get_fakeredis()
settings.sts.DATABASE_URI = 'sqlite://'
db = get_dataset(dataset, OP_INGEST)
_ingest_path(db, conn, dataset, path, languages=languages)
worker = IngestWorker(conn=conn, stages=STAGES)
worker.sync()
for entity in db.iterate():
pprint(entity.to_dict())
def debug(path, dataset, languages=None):
"""Debug the ingest for the given path."""
conn = get_fakeredis()
db = balkhash.init(dataset)
db.delete()
_ingest_path(conn, dataset, path, languages=languages)
worker = IngestWorker(conn=conn, stages=STAGES)
worker.sync()
for entity in db.iterate():
pprint(entity.to_dict())
def debug(path, languages=None):
"""Debug the ingest for the given path."""
conn = get_fakeredis()
settings.fts.DATABASE_URI = "sqlite:////tmp/debug.sqlite3"
db = get_dataset("debug",
origin=OP_INGEST,
database_uri=settings.fts.DATABASE_URI)
db.delete()
_ingest_path(db, conn, "debug", path, languages=languages)
worker = IngestWorker(conn=conn, stages=STAGES)
worker.sync()
for entity in db.iterate():
pprint(entity.to_dict())
def process(sync):
"""Start the queue and process tasks as they come. Blocks while waiting"""
worker = IngestWorker(stages=[Stage.INGEST])
if sync:
worker.sync()
else:
worker.run()
def process(sync):
setup_debugpy_server(settings.DEBUGPY_PROCESS,
settings.DEBUGPY_WAIT_FOR_CLIENT)
"""Start the queue and process tasks as they come. Blocks while waiting"""
worker = IngestWorker(stages=STAGES)
if sync:
worker.sync()
else:
worker.run()
def process(sync):
"""Start the queue and process tasks as they come. Blocks while waiting"""
num_threads = None if sync else sl_settings.WORKER_THREADS
worker = IngestWorker(stages=STAGES, num_threads=num_threads)
code = worker.run()
sys.exit(code)