def do_load(logger):
# there are 2 things to load from db: account and backends
# accounts also point to backends associated with them
# accounts also have rules about filtering matches
c, rs = yield db_query(LOAD_ACCOUNTS)
_, rs2 = yield db_query(ACCOUNT_BACKENDS)
_, rs3 = yield db_query(LOAD_RULES)
acc_backs = defaultdict(lambda: [])
for r in rs2: # associate backends with accounts
acc_backs[r.account].append(r)
acc_rules = defaultdict(lambda: set())
for r in rs3: # parse rules, and collect into dict
if r.k == 'VIDEO_ONLY' and r.value == '0':
acc_rules[r.account].add('drop-video')
elif r.k == 'AUDIO_ONLY' and r.value == '0':
acc_rules[r.account].add('drop-audio')
elif r.k == 'AUDIO_VIDEO_ONLY' and r.value == '0':
acc_rules[r.account].add('drop-both')
elif r.k == 'ONLY_THE_BEST_RESULT' and r.value == '1':
# r.value is compared with 0 here, because the rule is the reverse
# of the key
acc_rules[r.account].add('best-match')
_, rs4 = yield db_query(LOAD_BACKENDS)
backends = {}
for r in rs4:
backends[r.id] = r
accounts = {}
if c > 0:
a = namedtuple('_', rs[0]._fields + ('backends', 'rules'))
for r in rs:
good = True
for be in acc_backs[r.id]:
if not be.backend in backends:
logger.error('backend %s not found for working account %s',
be.backend, r.id)
good = False
break
if r.user_deleted == 'true':
logger.error('backend user \'%s\' deleted for working account %s',
r.backend_user, r.id)
good = False
if r.hot == 'true' and r.hot_user_deleted == 'true':
logger.error(
'hot backend user \'%s\' deleted for working ' + 'account %s',
r.hot_user, r.id)
good = False
if r.hot == 'true' and r.hot_user == None:
logger.error(
'hot backend user not set for account with hot query ' +
'enabled %s', r.id)
good = False
if good:
accounts[r.id] = a(*(list(r) + [acc_backs[r.id], acc_rules[r.id]]))
yield db_result(accounts, backends)
def do_load(logger):
# there are 2 things to load from db: account and backends
# accounts also point to backends associated with them
# accounts also have rules about filtering matches
c, rs = yield db_query(LOAD_ACCOUNTS)
_, rs2 = yield db_query(ACCOUNT_BACKENDS)
_, rs3 = yield db_query(LOAD_RULES)
acc_backs = defaultdict(lambda: [])
for r in rs2: # associate backends with accounts
acc_backs[r.account].append(r)
acc_rules = defaultdict(lambda: set())
for r in rs3: # parse rules, and collect into dict
if r.k == 'VIDEO_ONLY' and r.value == '0':
acc_rules[r.account].add('drop-video')
elif r.k == 'AUDIO_ONLY' and r.value == '0':
acc_rules[r.account].add('drop-audio')
elif r.k == 'AUDIO_VIDEO_ONLY' and r.value == '0':
acc_rules[r.account].add('drop-both')
elif r.k == 'ONLY_THE_BEST_RESULT' and r.value == '1':
# r.value is compared with 0 here, because the rule is the reverse
# of the key
acc_rules[r.account].add('best-match')
_, rs4 = yield db_query(LOAD_BACKENDS)
backends = {}
for r in rs4:
backends[r.id] = r
accounts = {}
if c > 0:
a = namedtuple('_', rs[0]._fields + ('backends', 'rules'))
for r in rs:
good = True
for be in acc_backs[r.id]:
if not be.backend in backends:
logger.error('backend %s not found for working account %s',
be.backend, r.id)
good = False
break
if r.user_deleted == 'true':
logger.error('backend user \'%s\' deleted for working account %s',
r.backend_user, r.id)
good = False
if r.hot == 'true' and r.hot_user_deleted == 'true':
logger.error('hot backend user \'%s\' deleted for working ' +
'account %s', r.hot_user, r.id)
good = False
if r.hot == 'true' and r.hot_user == None:
logger.error('hot backend user not set for account with hot query ' +
'enabled %s', r.id)
good = False
if good:
accounts[r.id] = a(*(list(r) + [acc_backs[r.id], acc_rules[r.id]]))
yield db_result(accounts, backends)
def storeTaskMysql(task):
rc, res = yield db_execute(STORE_TASK, str(task['task_uuid']),
str(task['external_id']), task['task_priority'],
task['created_at'], task['dna'],
task['company_id'], str(task['site_asset_id']))
if rc:
if not len(task['query_scope']) == 0:
for i in task['query_scope']:
try:
r, re = yield db_query(CHECK_SCOPE, i)
#r= yield db_result(r)
if r:
rc, res = yield db_execute(STORE_SCOPE, task_id, i)
else:
self.logger.info(
'query_scope is not in vddbMetaContent'
' table, task_uuid: %s, site_asset_id: %s,query_scope: %s',
str(task['task_uuid']), str(task['site_asset_id']),
i)
except:
self.logger.error(
'failed to store scope ,task_uuid: %s,'
'site_asset_id, query_scope :%s ',
str(task['task_uuid']), str(task['site_asset_id']), i)
raise
def do_doom(self, t, code, queue_at, deadline):
logger = logging.getLogger('mwtm_cleaner')
if queue_at != None and (deadline == None or deadline == 0 or \
deadline > queue_at):
logger.debug('to retry task %s, queue at %s', t.uuid, queue_at)
yield db_execute(RETRY_TASK, queue_at, code, t.id)
#yield db_execute(RENEW_EVENT, t.uuid, 'retry')
g_logger.info(
trans2json(message="task_uuid:%s, "
"site_asset_id:%s, deadline:%s, external_id:%s " %
(t.uuid, t.site_asset_id, deadline, t.external_id),
action="retry task"))
else:
logger.debug('to fail task %s', t.uuid)
g_logger.info(
trans2json(message="task_uuid:%s, "
"site_asset_id:%s, external_id:%s" %
(t.uuid, t.site_asset_id, t.external_id),
action="to fail task"))
rc, _ = yield db_query(CHECK_TASK, t.id)
if rc <= 0:
yield db_execute(FAIL_TASK, code, t.id)
self.send_matches(t, unrecognized=True)
task_status = db_txn(self.pool,
partial(self.load_task_status, t.uuid))
self.update_hbase_task(task_status)
stats.incr(QUERY_FAILED, 1)
def storeScope(pool, task):
if not len(task['query_scope']) == 0:
for i in task['query_scope']:
try :
r, re = yield db_query(CHECK_SCOPE, meta)
#r= yield db_result(r)
if r:
rc, res = yield db_execute(STORE_SCOPE, task_id, meta_uuid)
else:
logger.info('query_scope is not in vddbMetaContent table, task_uuid: %s,\
query_scope: %s', task['task_uuid'], i)
except:
logger.error('failed to store scope ,task_uuid: %s, query_scope :%s ',
task['task_uuid'], i)
def fetch_for_account(acc, p):
c, rs = yield db_query(FETCH_TASKS, acc, p.max_buf)
ts = []
if c > 0:
t_ = namedtuple('_', rs[0]._fields + ('scope',))
for r in rs:
t = r._asdict()
if r.queued_at == None or r.queued_at == 0:
t['queued_at'] = r.created_at
if r.submit_at == None or r.submit_at == 0:
t['submit_at'] = r.created_at
if (r.deadline == None or r.deadline == 0) and \
(p.time_limit != None and p.time_limit != 0):
t['deadline'] = t['submit_at'] + p.time_limit
rc2, rs2 = yield db_query(FETCH_SCOPE, r.id)
if rc2 > VDDB_SCOPE_LIMIT:
t['scope'] = None
else:
t['scope'] = [r2.content for r2 in rs2]
ts.append(t_(**t))
yield db_result(ts)
def storeTaskMysql(task):
rc, res = yield db_execute(STORE_TASK, task['task_uuid'] ,
task['task_priority'], task['created_at'],
task['dna'],
task['company_id'], str(task['site_asset_ids']))
if rc:
if not len(task['query_scope']) == 0:
for i in task['query_scope']:
try :
r, re = yield db_query(CHECK_SCOPE, meta)
#r= yield db_result(r)
if r:
rc, res = yield db_execute(STORE_SCOPE, task_id, meta_uuid)
else:
logger.info('query_scope is not in vddbMetaContent table, task_uuid: %s,\
query_scope: %s', task['task_uuid'], i)
except:
logger.error('failed to store scope ,task_uuid: %s, query_scope :%s ',
task['task_uuid'], i)
def do_check():
# NOTE: the algorithm is to use the record locking and auto txn rollback
# as a method of synchronization between distributed processes
# with where conditions, only 1 contender can update the original
# row, and when he updates it with his process info, he's the king
# insertion works likewise (with module name as unique key)
# first, try to sit on the throne which looks like deserted, or like
# owned by us
rc, _ = yield db_execute(USURP, host, port, module, host, port,
timeout)
if rc == 1: # yes we sat down
yield db_result(True)
# we couldn't sit down, is there actually a throne?
rc, _ = yield db_query(CHECK_THRONE, module)
if rc == 1: # yes there is, it's just occupied
yield db_result(False)
# the throne is not there yet, make one and sit on it!
rc, _ = yield db_execute(MAKE_THRONE, module, host, port)
# the throne belongs to its maker!
yield db_result(rc == 1)
def do_check():
# NOTE: the algorithm is to use the record locking and auto txn rollback
# as a method of synchronization between distributed processes
# with where conditions, only 1 contender can update the original
# row, and when he updates it with his process info, he's the king
# insertion works likewise (with module name as unique key)
# first, try to sit on the throne which looks like deserted, or like
# owned by us
rc, _ = yield db_execute(USURP, host, port, module, host, port,
timeout)
if rc == 1: # yes we sat down
yield db_result(True)
# we couldn't sit down, is there actually a throne?
rc, _ = yield db_query(CHECK_THRONE, module)
if rc == 1: # yes there is, it's just occupied
yield db_result(False)
# the throne is not there yet, make one and sit on it!
rc, _ = yield db_execute(MAKE_THRONE, module, host, port)
# the throne belongs to its maker!
yield db_result(rc == 1)
def do_doom(self, t, code, queue_at, deadline):
logger = logging.getLogger('mwtm_cleaner')
if queue_at != None and (deadline == None or deadline == 0 or \
deadline > queue_at):
logger.debug('to retry task %s, queue at %s', t.uuid, queue_at)
yield db_execute(RETRY_TASK, queue_at, code, t.id)
#yield db_execute(RENEW_EVENT, t.uuid, 'retry')
g_logger.info(trans2json(message="task_uuid:%s, "
"site_asset_id:%s, deadline:%s, external_id:%s " % (t.uuid,
t.site_asset_id, deadline, t.external_id),
action="retry task"))
else:
logger.debug('to fail task %s', t.uuid)
g_logger.info(trans2json(message="task_uuid:%s, "
"site_asset_id:%s, external_id:%s" % (t.uuid,
t.site_asset_id, t.external_id), action="to fail task"))
rc, _ = yield db_query(CHECK_TASK, t.id)
if rc <= 0:
yield db_execute(FAIL_TASK, code, t.id)
self.send_matches(t, unrecognized=True)
task_status = db_txn(self.pool, partial(self.load_task_status, t.uuid))
self.update_hbase_task(task_status)
stats.incr(QUERY_FAILED, 1)
def genUUID():
rc, res = yield db_query(GEN_UUID)
yield db_result(res)
def fetchCompanyId(user_id):
rc, res = yield db_query(GET_COMPANYID, user_id)
yield db_result(res)
def load_task_status(uuid):
rc, rs = yield db_query(LOAD_TASK_STATUS, uuid)
yield db_result(rs[0])
def load_task_status(uuid):
rc, rs = yield db_query(LOAD_TASK_STATUS, uuid)
yield db_result(rs[0])
def check_matches(t):
# check for previous matches
rc, rs = yield db_query(CHECK_MATCHES, t.uuid)
yield db_result(rc > 0 and rs[0][0] > 1)
def fetchCompanyId(user_id):
rc,res = yield db_query(GET_COMPANYID, user_id )
yield db_result(res)
def query_row(row):
row_count, rows = yield db_query('SELECT a, b FROM test WHERE id = ?', row)
assert row_count == 1
r = rows[0]
yield delete_row(row)
yield db_result(r.a, r.b)
def check_matches(t):
# check for previous matches
rc, rs = yield db_query(CHECK_MATCHES, t.uuid)
yield db_result(rc > 0 and rs[0][0] > 1)
def query_row(row):
row_count, rows = yield db_query('SELECT a, b FROM test WHERE id = ?', row)
assert row_count == 1
r = rows[0]
yield delete_row(row)
yield db_result(r.a, r.b)
def genUUID():
rc, res = yield db_query(GEN_UUID)
yield db_result(res)