def do_load(logger):
# there are 2 things to load from db: account and backends
# accounts also point to backends associated with them
# accounts also have rules about filtering matches
c, rs = yield db_query(LOAD_ACCOUNTS)
_, rs2 = yield db_query(ACCOUNT_BACKENDS)
_, rs3 = yield db_query(LOAD_RULES)
acc_backs = defaultdict(lambda: [])
for r in rs2: # associate backends with accounts
acc_backs[r.account].append(r)
acc_rules = defaultdict(lambda: set())
for r in rs3: # parse rules, and collect into dict
if r.k == 'VIDEO_ONLY' and r.value == '0':
acc_rules[r.account].add('drop-video')
elif r.k == 'AUDIO_ONLY' and r.value == '0':
acc_rules[r.account].add('drop-audio')
elif r.k == 'AUDIO_VIDEO_ONLY' and r.value == '0':
acc_rules[r.account].add('drop-both')
elif r.k == 'ONLY_THE_BEST_RESULT' and r.value == '1':
# r.value is compared with 0 here, because the rule is the reverse
# of the key
acc_rules[r.account].add('best-match')
_, rs4 = yield db_query(LOAD_BACKENDS)
backends = {}
for r in rs4:
backends[r.id] = r
accounts = {}
if c > 0:
a = namedtuple('_', rs[0]._fields + ('backends', 'rules'))
for r in rs:
good = True
for be in acc_backs[r.id]:
if not be.backend in backends:
logger.error('backend %s not found for working account %s',
be.backend, r.id)
good = False
break
if r.user_deleted == 'true':
logger.error('backend user \'%s\' deleted for working account %s',
r.backend_user, r.id)
good = False
if r.hot == 'true' and r.hot_user_deleted == 'true':
logger.error(
'hot backend user \'%s\' deleted for working ' + 'account %s',
r.hot_user, r.id)
good = False
if r.hot == 'true' and r.hot_user == None:
logger.error(
'hot backend user not set for account with hot query ' +
'enabled %s', r.id)
good = False
if good:
accounts[r.id] = a(*(list(r) + [acc_backs[r.id], acc_rules[r.id]]))
yield db_result(accounts, backends)
def do_load(logger):
# there are 2 things to load from db: account and backends
# accounts also point to backends associated with them
# accounts also have rules about filtering matches
c, rs = yield db_query(LOAD_ACCOUNTS)
_, rs2 = yield db_query(ACCOUNT_BACKENDS)
_, rs3 = yield db_query(LOAD_RULES)
acc_backs = defaultdict(lambda: [])
for r in rs2: # associate backends with accounts
acc_backs[r.account].append(r)
acc_rules = defaultdict(lambda: set())
for r in rs3: # parse rules, and collect into dict
if r.k == 'VIDEO_ONLY' and r.value == '0':
acc_rules[r.account].add('drop-video')
elif r.k == 'AUDIO_ONLY' and r.value == '0':
acc_rules[r.account].add('drop-audio')
elif r.k == 'AUDIO_VIDEO_ONLY' and r.value == '0':
acc_rules[r.account].add('drop-both')
elif r.k == 'ONLY_THE_BEST_RESULT' and r.value == '1':
# r.value is compared with 0 here, because the rule is the reverse
# of the key
acc_rules[r.account].add('best-match')
_, rs4 = yield db_query(LOAD_BACKENDS)
backends = {}
for r in rs4:
backends[r.id] = r
accounts = {}
if c > 0:
a = namedtuple('_', rs[0]._fields + ('backends', 'rules'))
for r in rs:
good = True
for be in acc_backs[r.id]:
if not be.backend in backends:
logger.error('backend %s not found for working account %s',
be.backend, r.id)
good = False
break
if r.user_deleted == 'true':
logger.error('backend user \'%s\' deleted for working account %s',
r.backend_user, r.id)
good = False
if r.hot == 'true' and r.hot_user_deleted == 'true':
logger.error('hot backend user \'%s\' deleted for working ' +
'account %s', r.hot_user, r.id)
good = False
if r.hot == 'true' and r.hot_user == None:
logger.error('hot backend user not set for account with hot query ' +
'enabled %s', r.id)
good = False
if good:
accounts[r.id] = a(*(list(r) + [acc_backs[r.id], acc_rules[r.id]]))
yield db_result(accounts, backends)
def do_check():
# NOTE: the algorithm is to use the record locking and auto txn rollback
# as a method of synchronization between distributed processes
# with where conditions, only 1 contender can update the original
# row, and when he updates it with his process info, he's the king
# insertion works likewise (with module name as unique key)
# first, try to sit on the throne which looks like deserted, or like
# owned by us
rc, _ = yield db_execute(USURP, host, port, module, host, port,
timeout)
if rc == 1: # yes we sat down
yield db_result(True)
# we couldn't sit down, is there actually a throne?
rc, _ = yield db_query(CHECK_THRONE, module)
if rc == 1: # yes there is, it's just occupied
yield db_result(False)
# the throne is not there yet, make one and sit on it!
rc, _ = yield db_execute(MAKE_THRONE, module, host, port)
# the throne belongs to its maker!
yield db_result(rc == 1)
def do_setup(acc, user):
print acc, user
uid = str(uuid4())
yield db_execute('''INSERT INTO task(task_identification, status,
site_asset_id, company_id,
task_priority, user_id, clip_duration,
clip_format)
VALUES (%s, 'query', UUID(), %s, 128, %s, 600,
'mp4')''', uid, acc, user)
_, r = yield db_insert('''INSERT INTO taskQueryHis(task_identification)
VALUES (%s)''', uid)
yield db_result(uid, r)
def do_check():
# NOTE: the algorithm is to use the record locking and auto txn rollback
# as a method of synchronization between distributed processes
# with where conditions, only 1 contender can update the original
# row, and when he updates it with his process info, he's the king
# insertion works likewise (with module name as unique key)
# first, try to sit on the throne which looks like deserted, or like
# owned by us
rc, _ = yield db_execute(USURP, host, port, module, host, port,
timeout)
if rc == 1: # yes we sat down
yield db_result(True)
# we couldn't sit down, is there actually a throne?
rc, _ = yield db_query(CHECK_THRONE, module)
if rc == 1: # yes there is, it's just occupied
yield db_result(False)
# the throne is not there yet, make one and sit on it!
rc, _ = yield db_execute(MAKE_THRONE, module, host, port)
# the throne belongs to its maker!
yield db_result(rc == 1)
def fetch_for_account(acc, p):
c, rs = yield db_query(FETCH_TASKS, acc, p.max_buf)
ts = []
if c > 0:
t_ = namedtuple('_', rs[0]._fields + ('scope',))
for r in rs:
t = r._asdict()
if r.queued_at == None or r.queued_at == 0:
t['queued_at'] = r.created_at
if r.submit_at == None or r.submit_at == 0:
t['submit_at'] = r.created_at
if (r.deadline == None or r.deadline == 0) and \
(p.time_limit != None and p.time_limit != 0):
t['deadline'] = t['submit_at'] + p.time_limit
rc2, rs2 = yield db_query(FETCH_SCOPE, r.id)
if rc2 > VDDB_SCOPE_LIMIT:
t['scope'] = None
else:
t['scope'] = [r2.content for r2 in rs2]
ts.append(t_(**t))
yield db_result(ts)
def update_row():
row = yield insert_row()
row_count, _ = yield db_execute('UPDATE test SET a = 3 WHERE id = ?', row)
assert row_count == 1
yield db_result(row)
def do_renew():
rc, _ = yield db_execute(RENEW_TASKS)
yield db_result(rc)
def genUUID():
rc, res = yield db_query(GEN_UUID)
yield db_result(res)
def fetchCompanyId(user_id):
rc, res = yield db_query(GET_COMPANYID, user_id)
yield db_result(res)
def load_task_status(uuid):
rc, rs = yield db_query(LOAD_TASK_STATUS, uuid)
yield db_result(rs[0])
def genUUID():
rc, res = yield db_query(GEN_UUID)
yield db_result(res)
def update_row():
row = yield insert_row()
row_count, _ = yield db_execute('UPDATE test SET a = 3 WHERE id = ?', row)
assert row_count == 1
yield db_result(row)
def fetchCompanyId(user_id):
rc,res = yield db_query(GET_COMPANYID, user_id )
yield db_result(res)
def insert_row():
row_count, row_id = yield db_insert('INSERT INTO test(a, b) VALUES(?, ?)',
1, '2')
assert row_count == 1
yield db_result(row_id)
def load_task_status(uuid):
rc, rs = yield db_query(LOAD_TASK_STATUS, uuid)
yield db_result(rs[0])
def check_matches(t):
# check for previous matches
rc, rs = yield db_query(CHECK_MATCHES, t.uuid)
yield db_result(rc > 0 and rs[0][0] > 1)
def insert_row():
row_count, row_id = yield db_insert('INSERT INTO test(a, b) VALUES(?, ?)',
1, '2')
assert row_count == 1
yield db_result(row_id)
def query_row(row):
row_count, rows = yield db_query('SELECT a, b FROM test WHERE id = ?', row)
assert row_count == 1
r = rows[0]
yield delete_row(row)
yield db_result(r.a, r.b)
def query_row(row):
row_count, rows = yield db_query('SELECT a, b FROM test WHERE id = ?', row)
assert row_count == 1
r = rows[0]
yield delete_row(row)
yield db_result(r.a, r.b)
def check_matches(t):
# check for previous matches
rc, rs = yield db_query(CHECK_MATCHES, t.uuid)
yield db_result(rc > 0 and rs[0][0] > 1)
