def announce_hittask_links(pid, qid, listeners):
# get url.
sqlstmt = "select msg from CS_event where action='gotoHIT' and pid={0} and qid={1}".format(pid, qid)
content = mdlib.get_single_mysql_data(cs_dbhost, cs_dbname, sqlstmt)
if content != None:
# send it to the listeners
emails = listeners.split(',')
for i in range(len(emails)):
mdlib.send_email("Medusa HIT task has been posted", content[0], emails[i])
mdlib.log("* Weblink for HIT task has been announced: {0}".format(listeners))
else:
mdlib.log("! No registered link, please verify Worker Manager is running properly")
def announce_hittask_links(pid, qid, listeners):
# get url.
sqlstmt = "select msg from CS_event where action='gotoHIT' and pid={0} and qid={1}".format(pid, qid)
content = mdlib.get_single_mysql_data(cs_dbhost, cs_dbname, sqlstmt)
if content != None:
# send it to the listeners
emails = listeners.split(',')
for i in range(len(emails)):
mdlib.send_email("Medusa HIT task has been posted", content[0], emails[i])
mdlib.log("* Weblink for HIT task has been announced: {0}".format(listeners))
else:
mdlib.log("! No registered link, please verify Worker Manager is running properly")
def task_main(taskpath):
global data_base_url, spc_host, spc_urlloc, hit_host, hit_urlloc
global wait4res_duration, poll4res_interval
global failover_interval, failover_retrylimit
global debug, anonymity, clock, ENV
global soc2, cs_dbhost, cs_dbname
global task_tracker_id
mdlib.log("********************************")
mdlib.log("* Task-Flow Server *")
mdlib.log("********************************")
mdlib.log("* starting task-flow server(TFS).")
pid = os.getpid()
print "++++pid+++++++" + str(pid)
msend(soc2, str(pid) + " " + task_tracker_id)
######################################################
# Program Start
#
# load envvars
vdict = get_env_values(
"BASE-URL-DATA,SPC-HOST,SPC-URI,HIT-HOST,HIT-URI,FAIL-SLEEP-INTERVAL,FAIL-RETRY-CNT,WAIT-DURATION-FOR-STAGE,CHECK-PERIOD-FOR-STAGE-OUTPUT"
)
# Server configurations.
data_base_url = vdict["BASE-URL-DATA"]
spc_host = vdict["SPC-HOST"]
spc_urlloc = vdict["SPC-URI"]
hit_host = vdict["HIT-HOST"]
hit_urlloc = vdict["HIT-URI"]
# parameters on failover
wait4res_duration = float(vdict["WAIT-DURATION-FOR-STAGE"])
poll4res_interval = float(vdict["CHECK-PERIOD-FOR-STAGE-OUTPUT"])
failover_interval = float(vdict["FAIL-SLEEP-INTERVAL"])
failover_retrylimit = float(vdict["FAIL-RETRY-CNT"])
# declare global data structures.
cur_stages = []
cur_stage = None
# parse input arguments
#fname = parse_args()
fname = taskpath
dict_stages, dict_conns, cur_stage = parse_xml(fname)
cur_stages.append(cur_stage)
mdlib.log('* parsing completed.')
mdlib.log('* starting parameter type checking.')
mdchecker.check(dict_stages, dict_conns, cur_stage, ENV)
mdlib.log("* [overhead] task interpretation delay: {0}".format(
clock.stop()))
#mdlib.log("* [overhead] type-checking delay: {0}".format(clock.stop()))
mdlib.log('* done for type checking.')
#pid = os.getpid()
#send back pid
#soc2.send(str(pid))
mdlib.log('* pid={pidval}'.format(pidval=pid))
# update program state
if debug:
mdlib.update_program_state(cs_dbhost, cs_dbname, pid, "startTFS",
ENV["APP_NAME"])
# run program
params_dict = dict()
newdevice = ""
if ENV["G_VAR"] != None:
g_vars = ENV["G_VAR"].split(",")
for i in range(len(g_vars)):
g_kv = g_vars[i].split("=")
params_dict[g_kv[0]] = g_kv[1]
mdlib.log("Global input var name {0} is {1}".format(
g_kv[0], g_kv[1]))
while len(cur_stages) > 0:
# initialize
exec_result = [[]] * len(cur_stages)
qid = [[]] * len(cur_stages)
respstr = [[]] * len(cur_stages)
# generate qid
for i in range(len(cur_stages)):
exec_result[i] = False
qid[i] = random.randint(1000, 9999)
mdlib.log('* qid[{idx}]={qidval}'.format(idx=i, qidval=qid[i]))
mid = random.randint(1000, 9999)
cur_stage = cur_stages[
0] # assume that all stages in the same level has the same type.
mdlib.log(
"* [overhead] task tracker latency amonst stages: {0}".format(
clock.stop()))
########################################
# Human Intelligence Task
#
if cur_stage.type == "HIT":
mdlib.log('* invoking HIT [' + cur_stage.name + ']')
# process amt credentials.
if cur_stage.rid == None:
rid = ENV["R_RID"]
rkey = ENV["R_RKEY"]
else:
rid = ENV[cur_stage.rid]
rkey = ENV[cur_stage.rkey]
if cur_stage.wid == None:
wid = ENV["W_WID"]
else:
wid = ENV[
cur_stage.wid] if cur_stage.wid in ENV else cur_stage.wid
req_params = [[]] * len(cur_stages)
# set data-owner
if len(params_dict) > 0:
dataowner = ENV["W_WID"]
else:
dataowner = 'None'
# set data..
if cur_stage.config_input in params_dict:
data = params_dict[cur_stage.config_input]
else:
data = 'None'
# make requests (aggregated)
for i in range(len(cur_stages)):
params = {
'pid': pid,
'qid': qid[i],
'action': 'regHIT',
'qtype': cur_stage.stype,
'rid': rid,
'rkey': rkey,
'wid': wid,
'query': cur_stage.config_stmt,
'expiration': cur_stage.config_expiration,
'reward': cur_stage.config_reward,
'numusers': cur_stage.config_numusers,
'dataowner': dataowner,
'data': data
}
req_params[i] = params.copy()
# wait for the response
for i in range(len(cur_stages)):
respstr[i], exec_result[i] = wait_for_hit_response(
pid, qid[i], cur_stages[i], req_params[i])
mdlib.log("* [overhead] HIT task execution time: {0}".format(
clock.stop()))
#########################################
# SPC (Sensing, Processing, Communicaion)
#
elif cur_stage.type == "SPC":
rp_dict = [[]] * len(cur_stages)
# request route
for i in range(len(cur_stages)):
mdlib.log('* SPC Task [' + cur_stages[i].name + ']')
if i > 0:
time.sleep(3)
rp_dict[i] = dict()
emails = None
cnt = 0
# if <device> tag is set by email address, anonymity will be gone.
# commanding emails will be delivered to the phones directly.
if cur_stages[i].device != None:
type, addrs = cur_stages[i].device.split("=")
mdlib.log("* <device> tag is set, content={0}".format(
cur_stages[i].device))
if type != "EMAIL":
mdlib.log(
"! <device> config error. it should start with either 'EMAIL=' or 'AMT='"
)
else:
emails = addrs.split('|')
if len(emails) > 0:
anonymity = False
userids = emails
# make requests to mobile users
uid = ENV["W_WID"]
req_params = mst_request_params(pid, qid[i], cur_stages[i],
uid, params_dict)
if len(cur_stages) > 1:
req_params[
'custom'] = "<multi>{0}|{1}/{2}</multi>{3}".format(
mid, i, len(cur_stages), req_params['custom'])
rp_dict[i] = req_params
if anonymity == True:
# case I) req. to AMT
resp = mdlib.httpsend(spc_host, spc_urlloc, req_params,
clock)
if resp[0] == 200:
cnt = cnt + 1
else:
mdlib.log("! mst_request_http_error code={0}".format(
resp[0]))
exit()
else:
# case II) req. via direct email
req_params['custom'] = "<xml><pid>{0}</pid><qid>{1}</qid><amtid>{2}</amtid>{3}</xml>" \
.format(pid, qid[i], uid, req_params['custom'])
#print req_params['custom']
mdlib.send_email('Medusa commanding message',
req_params['custom'], uid)
# wait for job completions
mdlib.log("* sms cmd msg has been sent to userid={0}".format(uid))
ID = 0
info = []
while ID < len(cur_stages):
msg = mrecv(soc2)
print "####recieved msg : " + msg
info = msg.split(',')
if info[0] == 'completeTask':
ID = ID + 1
print "8888" + str(info[2]) + " " + str(qid)
if int(info[2]) in qid:
print "8888 in if"
respstr[i], exec_result[i] = pullout_db(
info[1], info[2])
print "***" + str(respstr[i]) + " " + str(
exec_result[i])
if (cur_stage.notification != None):
print "batched notifcation ++++ : " + cur_stage.notification
sqlstmt = "select path,uid,muid from CS_data where pid = {0} and qid = {1}".format(
info[1], info[2])
con, cur = mdlib.get_raw_mysql_data(
cs_dbhost, cs_dbname, sqlstmt)
path = ''
row = cur.fetchone()
file_ids = row[1] + '=' if row != None else ''
while row != None:
path = path + row[0] + ","
file_ids += row[2] + '|'
row = cur.fetchone()
con.close()
path = path[:-1] if len(path) > 0 else path
file_ids = file_ids[:-1] if len(
file_ids) > 0 else file_ids
if file_ids == '':
file_ids = ENV["W_WID"]
d = [(file_ids, path)]
print d
req = urllib2.Request(cur_stage.notification,
urllib.urlencode(d))
u = urllib2.urlopen(req)
# from here, notification for MediaScope, 20130326
# elif info[0] == 'file':
# d= [(info[3], info[4])]
# print "got single file:" +str(d)
# if (cur_stage.notification != None):
# req = urllib2.Request(cur_stage.notification, urllib.urlencode(d))
# u = urllib2.urlopen(req)
# end of notification for MediaScope, 20130326
else:
print "error, find out reason! " + msg
# if (cur_stage.notification != None):
# print "notifcation ++++ : "+cur_stage.notification
# sqlstmt = "select path,uid,muid from CS_data where pid = {0} and qid = {1}"\
# .format(info[1], info[2])
# con, cur = mdlib.get_raw_mysql_data(cs_dbhost, cs_dbname, sqlstmt)
# path = ''
# row = cur.fetchone()
# file_ids = row[1]+'=' if row != None else ''
# while row != None:
# path = path + row[0]+","
# file_ids += row[2]+'|'
# row = cur.fetchone()
# con.close()
# path = path[:-1] if len(path)>0 else path
# file_ids = file_ids[:-1] if len(file_ids)>0 else file_ids
#dat = cur.fetchall()
#path = ''
#for row in dat:
# path = path+row[0]+', '
#path = path[:-2]
# d= [(file_ids, path)]
# print d
# req = urllib2.Request(cur_stage.notification, urllib.urlencode(d))
# u = urllib2.urlopen(req)
# for i in range(len(cur_stages)):
# respstr[i], exec_result[i] = wait_for_mst_response(pid, qid[i], rp_dict[i], cur_stages[i].config_output)
mdlib.log("* [overhead] SPC task execution time: {0}".format(
clock.stop()))
else:
mdlib.log("! unknown stage type: " + cur_stage.type)
exit()
# judge if the operation was successful
exec_resall = True
for i in range(len(cur_stages)):
if exec_result[i] == False:
exec_resall = False
#
# augment passing parameter set.
# { 'amt_wid' : { 'var_name' : 'uidlist', 'var_name' : 'uidlist', ... }, ... }
#
if exec_resall == True:
params_dict = interpret_result(respstr, cur_stages, params_dict)
update_amt_cred(pid)
else:
params_dict = dict()
mdlib.log(params_dict)
# decide next stage.
if cur_stage.name in dict_conns:
cstages = []
for sg in cur_stages:
for c in dict_conns[sg.name]:
if exec_resall == True:
st = dict_stages[c.dst_success]
else:
st = dict_stages[c.dst_failure]
if st not in cstages:
cstages.append(st)
cur_stages = cstages
else:
cur_stages = []
# end of while cur_stage:
#
mdlib.log("* shutting down task-flow server..")
# update program state
if debug:
mdlib.update_program_state(cs_dbhost, cs_dbname, pid, "exitTFS",
ENV["APP_NAME"])
#soc1.close()
soc2.close()
def task_main(taskpath):
global data_base_url, spc_host, spc_urlloc, hit_host, hit_urlloc
global wait4res_duration, poll4res_interval
global failover_interval, failover_retrylimit
global debug, anonymity, clock, ENV
global soc2, cs_dbhost, cs_dbname
global task_tracker_id
mdlib.log("********************************")
mdlib.log("* Task-Flow Server *")
mdlib.log("********************************")
mdlib.log("* starting task-flow server(TFS).")
pid = os.getpid()
print "++++pid+++++++" + str(pid)
msend(soc2, str(pid) + " " + task_tracker_id)
######################################################
# Program Start
#
# load envvars
vdict = get_env_values("BASE-URL-DATA,SPC-HOST,SPC-URI,HIT-HOST,HIT-URI,FAIL-SLEEP-INTERVAL,FAIL-RETRY-CNT,WAIT-DURATION-FOR-STAGE,CHECK-PERIOD-FOR-STAGE-OUTPUT")
# Server configurations.
data_base_url = vdict["BASE-URL-DATA"]
spc_host = vdict["SPC-HOST"]
spc_urlloc = vdict["SPC-URI"]
hit_host = vdict["HIT-HOST"]
hit_urlloc = vdict["HIT-URI"]
# parameters on failover
wait4res_duration = float(vdict["WAIT-DURATION-FOR-STAGE"])
poll4res_interval = float(vdict["CHECK-PERIOD-FOR-STAGE-OUTPUT"])
failover_interval = float(vdict["FAIL-SLEEP-INTERVAL"])
failover_retrylimit = float(vdict["FAIL-RETRY-CNT"])
# declare global data structures.
cur_stages = []
cur_stage = None
# parse input arguments
#fname = parse_args()
fname = taskpath
dict_stages, dict_conns, cur_stage = parse_xml(fname)
cur_stages.append(cur_stage)
mdlib.log('* parsing completed.')
mdlib.log('* starting parameter type checking.')
mdchecker.check(dict_stages, dict_conns, cur_stage, ENV)
mdlib.log("* [overhead] task interpretation delay: {0}".format(clock.stop()))
#mdlib.log("* [overhead] type-checking delay: {0}".format(clock.stop()))
mdlib.log('* done for type checking.')
#pid = os.getpid()
#send back pid
#soc2.send(str(pid))
mdlib.log('* pid={pidval}'.format(pidval=pid))
# update program state
if debug:
mdlib.update_program_state(cs_dbhost, cs_dbname, pid, "startTFS", ENV["APP_NAME"])
# run program
params_dict = dict()
newdevice = ""
if ENV["G_VAR"] != None:
g_vars = ENV["G_VAR"].split(",")
for i in range(len(g_vars)):
g_kv = g_vars[i].split("=")
params_dict[g_kv[0]] = g_kv[1]
mdlib.log("Global input var name {0} is {1}".format(g_kv[0], g_kv[1]))
while len(cur_stages) > 0:
# initialize
exec_result = [[]] * len(cur_stages)
qid = [[]] * len(cur_stages)
respstr = [[]] * len(cur_stages)
# generate qid
for i in range(len(cur_stages)):
exec_result[i] = False
qid[i] = random.randint(1000, 9999)
mdlib.log('* qid[{idx}]={qidval}'.format(idx=i, qidval=qid[i]))
mid = random.randint(1000,9999)
cur_stage = cur_stages[0] # assume that all stages in the same level has the same type.
mdlib.log("* [overhead] task tracker latency amonst stages: {0}".format(clock.stop()))
########################################
# Human Intelligence Task
#
if cur_stage.type == "HIT":
mdlib.log('* invoking HIT ['+cur_stage.name+']')
# process amt credentials.
if cur_stage.rid == None:
rid = ENV["R_RID"]
rkey = ENV["R_RKEY"]
else:
rid = ENV[cur_stage.rid]
rkey = ENV[cur_stage.rkey]
if cur_stage.wid == None:
wid = ENV["W_WID"]
else:
wid = ENV[cur_stage.wid] if cur_stage.wid in ENV else cur_stage.wid
req_params = [[]] * len(cur_stages)
# set data-owner
if len(params_dict) > 0:
dataowner = ENV["W_WID"]
else:
dataowner = 'None'
# set data..
if cur_stage.config_input in params_dict:
data = params_dict[cur_stage.config_input]
else:
data = 'None'
# make requests (aggregated)
for i in range(len(cur_stages)):
params = {'pid' : pid, 'qid' : qid[i], 'action' : 'regHIT', 'qtype' : cur_stage.stype
, 'rid' : rid, 'rkey' : rkey, 'wid' : wid
, 'query' : cur_stage.config_stmt, 'expiration' : cur_stage.config_expiration
, 'reward' : cur_stage.config_reward, 'numusers' : cur_stage.config_numusers
, 'dataowner' : dataowner, 'data' : data }
req_params[i] = params.copy()
# wait for the response
for i in range(len(cur_stages)):
respstr[i], exec_result[i] = wait_for_hit_response(pid, qid[i], cur_stages[i], req_params[i])
mdlib.log("* [overhead] HIT task execution time: {0}".format(clock.stop()))
#########################################
# SPC (Sensing, Processing, Communicaion)
#
elif cur_stage.type == "SPC":
rp_dict = [[]] * len(cur_stages)
# request route
for i in range(len(cur_stages)):
mdlib.log('* SPC Task [' + cur_stages[i].name + ']')
if i > 0:
time.sleep(3)
rp_dict[i] = dict()
emails = None
cnt = 0
# if <device> tag is set by email address, anonymity will be gone.
# commanding emails will be delivered to the phones directly.
if cur_stages[i].device != None:
type, addrs = cur_stages[i].device.split("=")
mdlib.log("* <device> tag is set, content={0}".format(cur_stages[i].device))
if type != "EMAIL":
mdlib.log("! <device> config error. it should start with either 'EMAIL=' or 'AMT='")
else:
emails = addrs.split('|')
if len(emails) > 0:
anonymity = False
userids = emails
# make requests to mobile users
uid = ENV["W_WID"]
req_params = mst_request_params(pid, qid[i], cur_stages[i], uid, params_dict)
if len(cur_stages) > 1:
req_params['custom'] = "<multi>{0}|{1}/{2}</multi>{3}".format(mid, i, len(cur_stages), req_params['custom'])
rp_dict[i] = req_params
if anonymity == True:
# case I) req. to AMT
resp = mdlib.httpsend(spc_host, spc_urlloc, req_params, clock)
if resp[0] == 200:
cnt = cnt+1
else:
mdlib.log("! mst_request_http_error code={0}".format(resp[0]))
exit()
else:
# case II) req. via direct email
req_params['custom'] = "<xml><pid>{0}</pid><qid>{1}</qid><amtid>{2}</amtid>{3}</xml>" \
.format(pid, qid[i], uid, req_params['custom'])
#print req_params['custom']
mdlib.send_email('Medusa commanding message', req_params['custom'], uid)
# wait for job completions
mdlib.log("* sms cmd msg has been sent to userid={0}".format(uid))
ID = 0
info = []
while ID < len(cur_stages):
msg = mrecv(soc2)
print "####recieved msg : "+msg
info = msg.split(',')
if info[0] == 'completeTask':
ID = ID + 1
print "8888"+str(info[2])+" "+str(qid)
if int(info[2]) in qid:
print "8888 in if"
respstr[i], exec_result[i] = pullout_db(info[1],info[2])
print "***"+str(respstr[i])+" "+str(exec_result[i])
if (cur_stage.notification != None):
print "batched notifcation ++++ : "+cur_stage.notification
sqlstmt = "select path,uid,muid from CS_data where pid = {0} and qid = {1}".format(info[1], info[2])
con, cur = mdlib.get_raw_mysql_data(cs_dbhost, cs_dbname, sqlstmt)
path = ''
row = cur.fetchone()
file_ids = row[1]+'=' if row != None else ''
while row != None:
path = path + row[0]+","
file_ids += row[2]+'|'
row = cur.fetchone()
con.close()
path = path[:-1] if len(path)>0 else path
file_ids = file_ids[:-1] if len(file_ids)>0 else file_ids
if file_ids == '':
file_ids = ENV["W_WID"]
d= [(file_ids, path)]
print d
req = urllib2.Request(cur_stage.notification, urllib.urlencode(d))
u = urllib2.urlopen(req)
# from here, notification for MediaScope, 20130326
# elif info[0] == 'file':
# d= [(info[3], info[4])]
# print "got single file:" +str(d)
# if (cur_stage.notification != None):
# req = urllib2.Request(cur_stage.notification, urllib.urlencode(d))
# u = urllib2.urlopen(req)
# end of notification for MediaScope, 20130326
else:
print "error, find out reason! "+msg
# if (cur_stage.notification != None):
# print "notifcation ++++ : "+cur_stage.notification
# sqlstmt = "select path,uid,muid from CS_data where pid = {0} and qid = {1}"\
# .format(info[1], info[2])
# con, cur = mdlib.get_raw_mysql_data(cs_dbhost, cs_dbname, sqlstmt)
# path = ''
# row = cur.fetchone()
# file_ids = row[1]+'=' if row != None else ''
# while row != None:
# path = path + row[0]+","
# file_ids += row[2]+'|'
# row = cur.fetchone()
# con.close()
# path = path[:-1] if len(path)>0 else path
# file_ids = file_ids[:-1] if len(file_ids)>0 else file_ids
#dat = cur.fetchall()
#path = ''
#for row in dat:
# path = path+row[0]+', '
#path = path[:-2]
# d= [(file_ids, path)]
# print d
# req = urllib2.Request(cur_stage.notification, urllib.urlencode(d))
# u = urllib2.urlopen(req)
# for i in range(len(cur_stages)):
# respstr[i], exec_result[i] = wait_for_mst_response(pid, qid[i], rp_dict[i], cur_stages[i].config_output)
mdlib.log("* [overhead] SPC task execution time: {0}".format(clock.stop()))
else:
mdlib.log("! unknown stage type: " + cur_stage.type)
exit()
# judge if the operation was successful
exec_resall = True
for i in range(len(cur_stages)):
if exec_result[i] == False:
exec_resall = False
#
# augment passing parameter set.
# { 'amt_wid' : { 'var_name' : 'uidlist', 'var_name' : 'uidlist', ... }, ... }
#
if exec_resall == True:
params_dict = interpret_result(respstr, cur_stages, params_dict)
update_amt_cred(pid)
else:
params_dict = dict();
mdlib.log(params_dict)
# decide next stage.
if cur_stage.name in dict_conns:
cstages = []
for sg in cur_stages:
for c in dict_conns[sg.name]:
if exec_resall == True:
st = dict_stages[c.dst_success]
else:
st = dict_stages[c.dst_failure]
if st not in cstages:
cstages.append(st)
cur_stages = cstages
else:
cur_stages = []
# end of while cur_stage:
#
mdlib.log("* shutting down task-flow server..")
# update program state
if debug:
mdlib.update_program_state(cs_dbhost, cs_dbname, pid, "exitTFS", ENV["APP_NAME"])
#soc1.close()
soc2.close()