def do_load(logger): # there are 2 things to load from db: account and backends # accounts also point to backends associated with them # accounts also have rules about filtering matches c, rs = yield db_query(LOAD_ACCOUNTS) _, rs2 = yield db_query(ACCOUNT_BACKENDS) _, rs3 = yield db_query(LOAD_RULES) acc_backs = defaultdict(lambda: []) for r in rs2: # associate backends with accounts acc_backs[r.account].append(r) acc_rules = defaultdict(lambda: set()) for r in rs3: # parse rules, and collect into dict if r.k == 'VIDEO_ONLY' and r.value == '0': acc_rules[r.account].add('drop-video') elif r.k == 'AUDIO_ONLY' and r.value == '0': acc_rules[r.account].add('drop-audio') elif r.k == 'AUDIO_VIDEO_ONLY' and r.value == '0': acc_rules[r.account].add('drop-both') elif r.k == 'ONLY_THE_BEST_RESULT' and r.value == '1': # r.value is compared with 0 here, because the rule is the reverse # of the key acc_rules[r.account].add('best-match') _, rs4 = yield db_query(LOAD_BACKENDS) backends = {} for r in rs4: backends[r.id] = r accounts = {} if c > 0: a = namedtuple('_', rs[0]._fields + ('backends', 'rules')) for r in rs: good = True for be in acc_backs[r.id]: if not be.backend in backends: logger.error('backend %s not found for working account %s', be.backend, r.id) good = False break if r.user_deleted == 'true': logger.error('backend user \'%s\' deleted for working account %s', r.backend_user, r.id) good = False if r.hot == 'true' and r.hot_user_deleted == 'true': logger.error( 'hot backend user \'%s\' deleted for working ' + 'account %s', r.hot_user, r.id) good = False if r.hot == 'true' and r.hot_user == None: logger.error( 'hot backend user not set for account with hot query ' + 'enabled %s', r.id) good = False if good: accounts[r.id] = a(*(list(r) + [acc_backs[r.id], acc_rules[r.id]])) yield db_result(accounts, backends)
def do_load(logger): # there are 2 things to load from db: account and backends # accounts also point to backends associated with them # accounts also have rules about filtering matches c, rs = yield db_query(LOAD_ACCOUNTS) _, rs2 = yield db_query(ACCOUNT_BACKENDS) _, rs3 = yield db_query(LOAD_RULES) acc_backs = defaultdict(lambda: []) for r in rs2: # associate backends with accounts acc_backs[r.account].append(r) acc_rules = defaultdict(lambda: set()) for r in rs3: # parse rules, and collect into dict if r.k == 'VIDEO_ONLY' and r.value == '0': acc_rules[r.account].add('drop-video') elif r.k == 'AUDIO_ONLY' and r.value == '0': acc_rules[r.account].add('drop-audio') elif r.k == 'AUDIO_VIDEO_ONLY' and r.value == '0': acc_rules[r.account].add('drop-both') elif r.k == 'ONLY_THE_BEST_RESULT' and r.value == '1': # r.value is compared with 0 here, because the rule is the reverse # of the key acc_rules[r.account].add('best-match') _, rs4 = yield db_query(LOAD_BACKENDS) backends = {} for r in rs4: backends[r.id] = r accounts = {} if c > 0: a = namedtuple('_', rs[0]._fields + ('backends', 'rules')) for r in rs: good = True for be in acc_backs[r.id]: if not be.backend in backends: logger.error('backend %s not found for working account %s', be.backend, r.id) good = False break if r.user_deleted == 'true': logger.error('backend user \'%s\' deleted for working account %s', r.backend_user, r.id) good = False if r.hot == 'true' and r.hot_user_deleted == 'true': logger.error('hot backend user \'%s\' deleted for working ' + 'account %s', r.hot_user, r.id) good = False if r.hot == 'true' and r.hot_user == None: logger.error('hot backend user not set for account with hot query ' + 'enabled %s', r.id) good = False if good: accounts[r.id] = a(*(list(r) + [acc_backs[r.id], acc_rules[r.id]])) yield db_result(accounts, backends)
def do_check(): # NOTE: the algorithm is to use the record locking and auto txn rollback # as a method of synchronization between distributed processes # with where conditions, only 1 contender can update the original # row, and when he updates it with his process info, he's the king # insertion works likewise (with module name as unique key) # first, try to sit on the throne which looks like deserted, or like # owned by us rc, _ = yield db_execute(USURP, host, port, module, host, port, timeout) if rc == 1: # yes we sat down yield db_result(True) # we couldn't sit down, is there actually a throne? rc, _ = yield db_query(CHECK_THRONE, module) if rc == 1: # yes there is, it's just occupied yield db_result(False) # the throne is not there yet, make one and sit on it! rc, _ = yield db_execute(MAKE_THRONE, module, host, port) # the throne belongs to its maker! yield db_result(rc == 1)
def do_setup(acc, user): print acc, user uid = str(uuid4()) yield db_execute('''INSERT INTO task(task_identification, status, site_asset_id, company_id, task_priority, user_id, clip_duration, clip_format) VALUES (%s, 'query', UUID(), %s, 128, %s, 600, 'mp4')''', uid, acc, user) _, r = yield db_insert('''INSERT INTO taskQueryHis(task_identification) VALUES (%s)''', uid) yield db_result(uid, r)
def fetch_for_account(acc, p): c, rs = yield db_query(FETCH_TASKS, acc, p.max_buf) ts = [] if c > 0: t_ = namedtuple('_', rs[0]._fields + ('scope',)) for r in rs: t = r._asdict() if r.queued_at == None or r.queued_at == 0: t['queued_at'] = r.created_at if r.submit_at == None or r.submit_at == 0: t['submit_at'] = r.created_at if (r.deadline == None or r.deadline == 0) and \ (p.time_limit != None and p.time_limit != 0): t['deadline'] = t['submit_at'] + p.time_limit rc2, rs2 = yield db_query(FETCH_SCOPE, r.id) if rc2 > VDDB_SCOPE_LIMIT: t['scope'] = None else: t['scope'] = [r2.content for r2 in rs2] ts.append(t_(**t)) yield db_result(ts)
def update_row(): row = yield insert_row() row_count, _ = yield db_execute('UPDATE test SET a = 3 WHERE id = ?', row) assert row_count == 1 yield db_result(row)
def do_renew(): rc, _ = yield db_execute(RENEW_TASKS) yield db_result(rc)
def genUUID(): rc, res = yield db_query(GEN_UUID) yield db_result(res)
def fetchCompanyId(user_id): rc, res = yield db_query(GET_COMPANYID, user_id) yield db_result(res)
def load_task_status(uuid): rc, rs = yield db_query(LOAD_TASK_STATUS, uuid) yield db_result(rs[0])
def fetchCompanyId(user_id): rc,res = yield db_query(GET_COMPANYID, user_id ) yield db_result(res)
def insert_row(): row_count, row_id = yield db_insert('INSERT INTO test(a, b) VALUES(?, ?)', 1, '2') assert row_count == 1 yield db_result(row_id)
def check_matches(t): # check for previous matches rc, rs = yield db_query(CHECK_MATCHES, t.uuid) yield db_result(rc > 0 and rs[0][0] > 1)
def query_row(row): row_count, rows = yield db_query('SELECT a, b FROM test WHERE id = ?', row) assert row_count == 1 r = rows[0] yield delete_row(row) yield db_result(r.a, r.b)