def announce_hittask_links(pid, qid, listeners): # get url. sqlstmt = "select msg from CS_event where action='gotoHIT' and pid={0} and qid={1}".format(pid, qid) content = mdlib.get_single_mysql_data(cs_dbhost, cs_dbname, sqlstmt) if content != None: # send it to the listeners emails = listeners.split(',') for i in range(len(emails)): mdlib.send_email("Medusa HIT task has been posted", content[0], emails[i]) mdlib.log("* Weblink for HIT task has been announced: {0}".format(listeners)) else: mdlib.log("! No registered link, please verify Worker Manager is running properly")
def announce_hittask_links(pid, qid, listeners): # get url. sqlstmt = "select msg from CS_event where action='gotoHIT' and pid={0} and qid={1}".format(pid, qid) content = mdlib.get_single_mysql_data(cs_dbhost, cs_dbname, sqlstmt) if content != None: # send it to the listeners emails = listeners.split(',') for i in range(len(emails)): mdlib.send_email("Medusa HIT task has been posted", content[0], emails[i]) mdlib.log("* Weblink for HIT task has been announced: {0}".format(listeners)) else: mdlib.log("! No registered link, please verify Worker Manager is running properly")
def task_main(taskpath): global data_base_url, spc_host, spc_urlloc, hit_host, hit_urlloc global wait4res_duration, poll4res_interval global failover_interval, failover_retrylimit global debug, anonymity, clock, ENV global soc2, cs_dbhost, cs_dbname global task_tracker_id mdlib.log("********************************") mdlib.log("* Task-Flow Server *") mdlib.log("********************************") mdlib.log("* starting task-flow server(TFS).") pid = os.getpid() print "++++pid+++++++" + str(pid) msend(soc2, str(pid) + " " + task_tracker_id) ###################################################### # Program Start # # load envvars vdict = get_env_values( "BASE-URL-DATA,SPC-HOST,SPC-URI,HIT-HOST,HIT-URI,FAIL-SLEEP-INTERVAL,FAIL-RETRY-CNT,WAIT-DURATION-FOR-STAGE,CHECK-PERIOD-FOR-STAGE-OUTPUT" ) # Server configurations. data_base_url = vdict["BASE-URL-DATA"] spc_host = vdict["SPC-HOST"] spc_urlloc = vdict["SPC-URI"] hit_host = vdict["HIT-HOST"] hit_urlloc = vdict["HIT-URI"] # parameters on failover wait4res_duration = float(vdict["WAIT-DURATION-FOR-STAGE"]) poll4res_interval = float(vdict["CHECK-PERIOD-FOR-STAGE-OUTPUT"]) failover_interval = float(vdict["FAIL-SLEEP-INTERVAL"]) failover_retrylimit = float(vdict["FAIL-RETRY-CNT"]) # declare global data structures. cur_stages = [] cur_stage = None # parse input arguments #fname = parse_args() fname = taskpath dict_stages, dict_conns, cur_stage = parse_xml(fname) cur_stages.append(cur_stage) mdlib.log('* parsing completed.') mdlib.log('* starting parameter type checking.') mdchecker.check(dict_stages, dict_conns, cur_stage, ENV) mdlib.log("* [overhead] task interpretation delay: {0}".format( clock.stop())) #mdlib.log("* [overhead] type-checking delay: {0}".format(clock.stop())) mdlib.log('* done for type checking.') #pid = os.getpid() #send back pid #soc2.send(str(pid)) mdlib.log('* pid={pidval}'.format(pidval=pid)) # update program state if debug: mdlib.update_program_state(cs_dbhost, cs_dbname, pid, "startTFS", ENV["APP_NAME"]) # run program params_dict = dict() newdevice = "" if ENV["G_VAR"] != None: g_vars = ENV["G_VAR"].split(",") for i in range(len(g_vars)): g_kv = g_vars[i].split("=") params_dict[g_kv[0]] = g_kv[1] mdlib.log("Global input var name {0} is {1}".format( g_kv[0], g_kv[1])) while len(cur_stages) > 0: # initialize exec_result = [[]] * len(cur_stages) qid = [[]] * len(cur_stages) respstr = [[]] * len(cur_stages) # generate qid for i in range(len(cur_stages)): exec_result[i] = False qid[i] = random.randint(1000, 9999) mdlib.log('* qid[{idx}]={qidval}'.format(idx=i, qidval=qid[i])) mid = random.randint(1000, 9999) cur_stage = cur_stages[ 0] # assume that all stages in the same level has the same type. mdlib.log( "* [overhead] task tracker latency amonst stages: {0}".format( clock.stop())) ######################################## # Human Intelligence Task # if cur_stage.type == "HIT": mdlib.log('* invoking HIT [' + cur_stage.name + ']') # process amt credentials. if cur_stage.rid == None: rid = ENV["R_RID"] rkey = ENV["R_RKEY"] else: rid = ENV[cur_stage.rid] rkey = ENV[cur_stage.rkey] if cur_stage.wid == None: wid = ENV["W_WID"] else: wid = ENV[ cur_stage.wid] if cur_stage.wid in ENV else cur_stage.wid req_params = [[]] * len(cur_stages) # set data-owner if len(params_dict) > 0: dataowner = ENV["W_WID"] else: dataowner = 'None' # set data.. if cur_stage.config_input in params_dict: data = params_dict[cur_stage.config_input] else: data = 'None' # make requests (aggregated) for i in range(len(cur_stages)): params = { 'pid': pid, 'qid': qid[i], 'action': 'regHIT', 'qtype': cur_stage.stype, 'rid': rid, 'rkey': rkey, 'wid': wid, 'query': cur_stage.config_stmt, 'expiration': cur_stage.config_expiration, 'reward': cur_stage.config_reward, 'numusers': cur_stage.config_numusers, 'dataowner': dataowner, 'data': data } req_params[i] = params.copy() # wait for the response for i in range(len(cur_stages)): respstr[i], exec_result[i] = wait_for_hit_response( pid, qid[i], cur_stages[i], req_params[i]) mdlib.log("* [overhead] HIT task execution time: {0}".format( clock.stop())) ######################################### # SPC (Sensing, Processing, Communicaion) # elif cur_stage.type == "SPC": rp_dict = [[]] * len(cur_stages) # request route for i in range(len(cur_stages)): mdlib.log('* SPC Task [' + cur_stages[i].name + ']') if i > 0: time.sleep(3) rp_dict[i] = dict() emails = None cnt = 0 # if <device> tag is set by email address, anonymity will be gone. # commanding emails will be delivered to the phones directly. if cur_stages[i].device != None: type, addrs = cur_stages[i].device.split("=") mdlib.log("* <device> tag is set, content={0}".format( cur_stages[i].device)) if type != "EMAIL": mdlib.log( "! <device> config error. it should start with either 'EMAIL=' or 'AMT='" ) else: emails = addrs.split('|') if len(emails) > 0: anonymity = False userids = emails # make requests to mobile users uid = ENV["W_WID"] req_params = mst_request_params(pid, qid[i], cur_stages[i], uid, params_dict) if len(cur_stages) > 1: req_params[ 'custom'] = "<multi>{0}|{1}/{2}</multi>{3}".format( mid, i, len(cur_stages), req_params['custom']) rp_dict[i] = req_params if anonymity == True: # case I) req. to AMT resp = mdlib.httpsend(spc_host, spc_urlloc, req_params, clock) if resp[0] == 200: cnt = cnt + 1 else: mdlib.log("! mst_request_http_error code={0}".format( resp[0])) exit() else: # case II) req. via direct email req_params['custom'] = "<xml><pid>{0}</pid><qid>{1}</qid><amtid>{2}</amtid>{3}</xml>" \ .format(pid, qid[i], uid, req_params['custom']) #print req_params['custom'] mdlib.send_email('Medusa commanding message', req_params['custom'], uid) # wait for job completions mdlib.log("* sms cmd msg has been sent to userid={0}".format(uid)) ID = 0 info = [] while ID < len(cur_stages): msg = mrecv(soc2) print "####recieved msg : " + msg info = msg.split(',') if info[0] == 'completeTask': ID = ID + 1 print "8888" + str(info[2]) + " " + str(qid) if int(info[2]) in qid: print "8888 in if" respstr[i], exec_result[i] = pullout_db( info[1], info[2]) print "***" + str(respstr[i]) + " " + str( exec_result[i]) if (cur_stage.notification != None): print "batched notifcation ++++ : " + cur_stage.notification sqlstmt = "select path,uid,muid from CS_data where pid = {0} and qid = {1}".format( info[1], info[2]) con, cur = mdlib.get_raw_mysql_data( cs_dbhost, cs_dbname, sqlstmt) path = '' row = cur.fetchone() file_ids = row[1] + '=' if row != None else '' while row != None: path = path + row[0] + "," file_ids += row[2] + '|' row = cur.fetchone() con.close() path = path[:-1] if len(path) > 0 else path file_ids = file_ids[:-1] if len( file_ids) > 0 else file_ids if file_ids == '': file_ids = ENV["W_WID"] d = [(file_ids, path)] print d req = urllib2.Request(cur_stage.notification, urllib.urlencode(d)) u = urllib2.urlopen(req) # from here, notification for MediaScope, 20130326 # elif info[0] == 'file': # d= [(info[3], info[4])] # print "got single file:" +str(d) # if (cur_stage.notification != None): # req = urllib2.Request(cur_stage.notification, urllib.urlencode(d)) # u = urllib2.urlopen(req) # end of notification for MediaScope, 20130326 else: print "error, find out reason! " + msg # if (cur_stage.notification != None): # print "notifcation ++++ : "+cur_stage.notification # sqlstmt = "select path,uid,muid from CS_data where pid = {0} and qid = {1}"\ # .format(info[1], info[2]) # con, cur = mdlib.get_raw_mysql_data(cs_dbhost, cs_dbname, sqlstmt) # path = '' # row = cur.fetchone() # file_ids = row[1]+'=' if row != None else '' # while row != None: # path = path + row[0]+"," # file_ids += row[2]+'|' # row = cur.fetchone() # con.close() # path = path[:-1] if len(path)>0 else path # file_ids = file_ids[:-1] if len(file_ids)>0 else file_ids #dat = cur.fetchall() #path = '' #for row in dat: # path = path+row[0]+', ' #path = path[:-2] # d= [(file_ids, path)] # print d # req = urllib2.Request(cur_stage.notification, urllib.urlencode(d)) # u = urllib2.urlopen(req) # for i in range(len(cur_stages)): # respstr[i], exec_result[i] = wait_for_mst_response(pid, qid[i], rp_dict[i], cur_stages[i].config_output) mdlib.log("* [overhead] SPC task execution time: {0}".format( clock.stop())) else: mdlib.log("! unknown stage type: " + cur_stage.type) exit() # judge if the operation was successful exec_resall = True for i in range(len(cur_stages)): if exec_result[i] == False: exec_resall = False # # augment passing parameter set. # { 'amt_wid' : { 'var_name' : 'uidlist', 'var_name' : 'uidlist', ... }, ... } # if exec_resall == True: params_dict = interpret_result(respstr, cur_stages, params_dict) update_amt_cred(pid) else: params_dict = dict() mdlib.log(params_dict) # decide next stage. if cur_stage.name in dict_conns: cstages = [] for sg in cur_stages: for c in dict_conns[sg.name]: if exec_resall == True: st = dict_stages[c.dst_success] else: st = dict_stages[c.dst_failure] if st not in cstages: cstages.append(st) cur_stages = cstages else: cur_stages = [] # end of while cur_stage: # mdlib.log("* shutting down task-flow server..") # update program state if debug: mdlib.update_program_state(cs_dbhost, cs_dbname, pid, "exitTFS", ENV["APP_NAME"]) #soc1.close() soc2.close()
def task_main(taskpath): global data_base_url, spc_host, spc_urlloc, hit_host, hit_urlloc global wait4res_duration, poll4res_interval global failover_interval, failover_retrylimit global debug, anonymity, clock, ENV global soc2, cs_dbhost, cs_dbname global task_tracker_id mdlib.log("********************************") mdlib.log("* Task-Flow Server *") mdlib.log("********************************") mdlib.log("* starting task-flow server(TFS).") pid = os.getpid() print "++++pid+++++++" + str(pid) msend(soc2, str(pid) + " " + task_tracker_id) ###################################################### # Program Start # # load envvars vdict = get_env_values("BASE-URL-DATA,SPC-HOST,SPC-URI,HIT-HOST,HIT-URI,FAIL-SLEEP-INTERVAL,FAIL-RETRY-CNT,WAIT-DURATION-FOR-STAGE,CHECK-PERIOD-FOR-STAGE-OUTPUT") # Server configurations. data_base_url = vdict["BASE-URL-DATA"] spc_host = vdict["SPC-HOST"] spc_urlloc = vdict["SPC-URI"] hit_host = vdict["HIT-HOST"] hit_urlloc = vdict["HIT-URI"] # parameters on failover wait4res_duration = float(vdict["WAIT-DURATION-FOR-STAGE"]) poll4res_interval = float(vdict["CHECK-PERIOD-FOR-STAGE-OUTPUT"]) failover_interval = float(vdict["FAIL-SLEEP-INTERVAL"]) failover_retrylimit = float(vdict["FAIL-RETRY-CNT"]) # declare global data structures. cur_stages = [] cur_stage = None # parse input arguments #fname = parse_args() fname = taskpath dict_stages, dict_conns, cur_stage = parse_xml(fname) cur_stages.append(cur_stage) mdlib.log('* parsing completed.') mdlib.log('* starting parameter type checking.') mdchecker.check(dict_stages, dict_conns, cur_stage, ENV) mdlib.log("* [overhead] task interpretation delay: {0}".format(clock.stop())) #mdlib.log("* [overhead] type-checking delay: {0}".format(clock.stop())) mdlib.log('* done for type checking.') #pid = os.getpid() #send back pid #soc2.send(str(pid)) mdlib.log('* pid={pidval}'.format(pidval=pid)) # update program state if debug: mdlib.update_program_state(cs_dbhost, cs_dbname, pid, "startTFS", ENV["APP_NAME"]) # run program params_dict = dict() newdevice = "" if ENV["G_VAR"] != None: g_vars = ENV["G_VAR"].split(",") for i in range(len(g_vars)): g_kv = g_vars[i].split("=") params_dict[g_kv[0]] = g_kv[1] mdlib.log("Global input var name {0} is {1}".format(g_kv[0], g_kv[1])) while len(cur_stages) > 0: # initialize exec_result = [[]] * len(cur_stages) qid = [[]] * len(cur_stages) respstr = [[]] * len(cur_stages) # generate qid for i in range(len(cur_stages)): exec_result[i] = False qid[i] = random.randint(1000, 9999) mdlib.log('* qid[{idx}]={qidval}'.format(idx=i, qidval=qid[i])) mid = random.randint(1000,9999) cur_stage = cur_stages[0] # assume that all stages in the same level has the same type. mdlib.log("* [overhead] task tracker latency amonst stages: {0}".format(clock.stop())) ######################################## # Human Intelligence Task # if cur_stage.type == "HIT": mdlib.log('* invoking HIT ['+cur_stage.name+']') # process amt credentials. if cur_stage.rid == None: rid = ENV["R_RID"] rkey = ENV["R_RKEY"] else: rid = ENV[cur_stage.rid] rkey = ENV[cur_stage.rkey] if cur_stage.wid == None: wid = ENV["W_WID"] else: wid = ENV[cur_stage.wid] if cur_stage.wid in ENV else cur_stage.wid req_params = [[]] * len(cur_stages) # set data-owner if len(params_dict) > 0: dataowner = ENV["W_WID"] else: dataowner = 'None' # set data.. if cur_stage.config_input in params_dict: data = params_dict[cur_stage.config_input] else: data = 'None' # make requests (aggregated) for i in range(len(cur_stages)): params = {'pid' : pid, 'qid' : qid[i], 'action' : 'regHIT', 'qtype' : cur_stage.stype , 'rid' : rid, 'rkey' : rkey, 'wid' : wid , 'query' : cur_stage.config_stmt, 'expiration' : cur_stage.config_expiration , 'reward' : cur_stage.config_reward, 'numusers' : cur_stage.config_numusers , 'dataowner' : dataowner, 'data' : data } req_params[i] = params.copy() # wait for the response for i in range(len(cur_stages)): respstr[i], exec_result[i] = wait_for_hit_response(pid, qid[i], cur_stages[i], req_params[i]) mdlib.log("* [overhead] HIT task execution time: {0}".format(clock.stop())) ######################################### # SPC (Sensing, Processing, Communicaion) # elif cur_stage.type == "SPC": rp_dict = [[]] * len(cur_stages) # request route for i in range(len(cur_stages)): mdlib.log('* SPC Task [' + cur_stages[i].name + ']') if i > 0: time.sleep(3) rp_dict[i] = dict() emails = None cnt = 0 # if <device> tag is set by email address, anonymity will be gone. # commanding emails will be delivered to the phones directly. if cur_stages[i].device != None: type, addrs = cur_stages[i].device.split("=") mdlib.log("* <device> tag is set, content={0}".format(cur_stages[i].device)) if type != "EMAIL": mdlib.log("! <device> config error. it should start with either 'EMAIL=' or 'AMT='") else: emails = addrs.split('|') if len(emails) > 0: anonymity = False userids = emails # make requests to mobile users uid = ENV["W_WID"] req_params = mst_request_params(pid, qid[i], cur_stages[i], uid, params_dict) if len(cur_stages) > 1: req_params['custom'] = "<multi>{0}|{1}/{2}</multi>{3}".format(mid, i, len(cur_stages), req_params['custom']) rp_dict[i] = req_params if anonymity == True: # case I) req. to AMT resp = mdlib.httpsend(spc_host, spc_urlloc, req_params, clock) if resp[0] == 200: cnt = cnt+1 else: mdlib.log("! mst_request_http_error code={0}".format(resp[0])) exit() else: # case II) req. via direct email req_params['custom'] = "<xml><pid>{0}</pid><qid>{1}</qid><amtid>{2}</amtid>{3}</xml>" \ .format(pid, qid[i], uid, req_params['custom']) #print req_params['custom'] mdlib.send_email('Medusa commanding message', req_params['custom'], uid) # wait for job completions mdlib.log("* sms cmd msg has been sent to userid={0}".format(uid)) ID = 0 info = [] while ID < len(cur_stages): msg = mrecv(soc2) print "####recieved msg : "+msg info = msg.split(',') if info[0] == 'completeTask': ID = ID + 1 print "8888"+str(info[2])+" "+str(qid) if int(info[2]) in qid: print "8888 in if" respstr[i], exec_result[i] = pullout_db(info[1],info[2]) print "***"+str(respstr[i])+" "+str(exec_result[i]) if (cur_stage.notification != None): print "batched notifcation ++++ : "+cur_stage.notification sqlstmt = "select path,uid,muid from CS_data where pid = {0} and qid = {1}".format(info[1], info[2]) con, cur = mdlib.get_raw_mysql_data(cs_dbhost, cs_dbname, sqlstmt) path = '' row = cur.fetchone() file_ids = row[1]+'=' if row != None else '' while row != None: path = path + row[0]+"," file_ids += row[2]+'|' row = cur.fetchone() con.close() path = path[:-1] if len(path)>0 else path file_ids = file_ids[:-1] if len(file_ids)>0 else file_ids if file_ids == '': file_ids = ENV["W_WID"] d= [(file_ids, path)] print d req = urllib2.Request(cur_stage.notification, urllib.urlencode(d)) u = urllib2.urlopen(req) # from here, notification for MediaScope, 20130326 # elif info[0] == 'file': # d= [(info[3], info[4])] # print "got single file:" +str(d) # if (cur_stage.notification != None): # req = urllib2.Request(cur_stage.notification, urllib.urlencode(d)) # u = urllib2.urlopen(req) # end of notification for MediaScope, 20130326 else: print "error, find out reason! "+msg # if (cur_stage.notification != None): # print "notifcation ++++ : "+cur_stage.notification # sqlstmt = "select path,uid,muid from CS_data where pid = {0} and qid = {1}"\ # .format(info[1], info[2]) # con, cur = mdlib.get_raw_mysql_data(cs_dbhost, cs_dbname, sqlstmt) # path = '' # row = cur.fetchone() # file_ids = row[1]+'=' if row != None else '' # while row != None: # path = path + row[0]+"," # file_ids += row[2]+'|' # row = cur.fetchone() # con.close() # path = path[:-1] if len(path)>0 else path # file_ids = file_ids[:-1] if len(file_ids)>0 else file_ids #dat = cur.fetchall() #path = '' #for row in dat: # path = path+row[0]+', ' #path = path[:-2] # d= [(file_ids, path)] # print d # req = urllib2.Request(cur_stage.notification, urllib.urlencode(d)) # u = urllib2.urlopen(req) # for i in range(len(cur_stages)): # respstr[i], exec_result[i] = wait_for_mst_response(pid, qid[i], rp_dict[i], cur_stages[i].config_output) mdlib.log("* [overhead] SPC task execution time: {0}".format(clock.stop())) else: mdlib.log("! unknown stage type: " + cur_stage.type) exit() # judge if the operation was successful exec_resall = True for i in range(len(cur_stages)): if exec_result[i] == False: exec_resall = False # # augment passing parameter set. # { 'amt_wid' : { 'var_name' : 'uidlist', 'var_name' : 'uidlist', ... }, ... } # if exec_resall == True: params_dict = interpret_result(respstr, cur_stages, params_dict) update_amt_cred(pid) else: params_dict = dict(); mdlib.log(params_dict) # decide next stage. if cur_stage.name in dict_conns: cstages = [] for sg in cur_stages: for c in dict_conns[sg.name]: if exec_resall == True: st = dict_stages[c.dst_success] else: st = dict_stages[c.dst_failure] if st not in cstages: cstages.append(st) cur_stages = cstages else: cur_stages = [] # end of while cur_stage: # mdlib.log("* shutting down task-flow server..") # update program state if debug: mdlib.update_program_state(cs_dbhost, cs_dbname, pid, "exitTFS", ENV["APP_NAME"]) #soc1.close() soc2.close()