def test_import_queued_submissions(conn): insert_meta(conn, {'track': 'Foo'}) insert_submission(conn, { 'fingerprint': TEST_1_FP_RAW, 'length': TEST_1_LENGTH, 'bitrate': 192, 'source_id': 1, 'format_id': 1, 'meta_id': 1, }) insert_submission(conn, { 'fingerprint': TEST_2_FP_RAW, 'length': TEST_2_LENGTH, 'bitrate': 192, 'source_id': 1, 'format_id': 1, }) insert_submission(conn, { 'fingerprint': TEST_1_FP_RAW, 'length': TEST_1_LENGTH, 'bitrate': 192, 'source_id': 1, 'format_id': 1, }) import_queued_submissions(conn) count = conn.execute("SELECT count(*) FROM fingerprint WHERE id IN (1,2,3)").scalar() assert_equals(2, count) count = conn.execute("SELECT count(*) FROM track WHERE id IN (5,6,7)").scalar() assert_equals(2, count)
def do_import(script): # type: (Script) -> None count = 1 retries = 0 max_retries = 3 while count > 0: try: with script.context() as ctx: ingest_db = ctx.db.get_ingest_db() app_db = ctx.db.get_app_db() fingerprint_db = ctx.db.get_fingerprint_db() timeout_ms = 20 * 1000 ingest_db.execute("SET LOCAL statement_timeout TO {}".format(timeout_ms)) app_db.execute("SET LOCAL statement_timeout TO {}".format(timeout_ms)) fingerprint_db.execute("SET LOCAL statement_timeout TO {}".format(timeout_ms)) count = import_queued_submissions(ingest_db, app_db, fingerprint_db, ctx.index, limit=10) ctx.db.session.commit() retries = 0 except IndexClientError: if retries > max_retries: raise retries += 1 continue
def do_import(script, index_first=False, only_index=False): with closing(script.engine.connect()) as db: if index_first: update_fingerprint_index(db, script.index) if not only_index: while True: count = import_queued_submissions(db, script.index, limit=10) if not count: break update_fingerprint_index(db, script.index)
def do_import(script): # type: (Script) -> None count = 1 while count > 0: with script.context() as ctx: ingest_db = ctx.db.get_ingest_db() app_db = ctx.db.get_app_db() fingerprint_db = ctx.db.get_fingerprint_db() count = import_queued_submissions(ingest_db, app_db, fingerprint_db, ctx.index, limit=10) ctx.db.session.commit()
def test_import_queued_submissions(ctx): # type: (ScriptContext) -> None ingest_db = ctx.db.get_ingest_db() app_db = ctx.db.get_app_db() fingerprint_db = ctx.db.get_fingerprint_db() insert_submission( ingest_db, { 'fingerprint': TEST_1_FP_RAW, 'length': TEST_1_LENGTH, 'bitrate': 192, 'source_id': 1, 'format_id': 1, 'meta': { 'track': 'Foo' }, }) insert_submission( ingest_db, { 'fingerprint': TEST_2_FP_RAW, 'length': TEST_2_LENGTH, 'bitrate': 192, 'source_id': 1, 'format_id': 1, }) insert_submission( ingest_db, { 'fingerprint': TEST_1_FP_RAW, 'length': TEST_1_LENGTH, 'bitrate': 192, 'source_id': 1, 'format_id': 1, }) import_queued_submissions(ingest_db, app_db, fingerprint_db, ctx.index) count = fingerprint_db.execute( "SELECT count(*) FROM fingerprint WHERE id IN (1,2,3)").scalar() assert_equals(2, count) count = fingerprint_db.execute( "SELECT count(*) FROM track WHERE id IN (5,6,7)").scalar() assert_equals(2, count)
def main(script, opts, args): conn = script.engine.connect() with conn.begin(): import_queued_submissions(conn)
def main(script, opts, args): conn = script.engine.connect() import_queued_submissions(conn, limit=300, index=script.index)