def submit_task():
app = cm.load_configs("production.conf")
uid = str(uuid.uuid1())
data = {
"job_id":
uid,
"username":
"******",
"jobtype":
"doc2vec",
"inputs": [{
"src": "https://s3.amazonaws.com/klab-jobs/inputs/test.txt",
"dest": "test.txt"
}],
"outputs": [{
"src": "doc_mat.pkl",
"dest": "klab-jobs/outputs/{0}/doc_mat.pkl".format(uid)
}, {
"src": "word_mat.pkl",
"dest": "klab-jobs/outputs/{0}/word_mat.pkl".format(uid)
}, {
"src": "mdl.pkl",
"dest": "klab-jobs/outputs/{0}/mdl.pkl".format(uid)
}],
"submit_time":
int(time.time()),
"status":
"pending"
}
dutils.dynamodb_update(app.config["dyno.conn"], data)
sns_sqs.publish(app.config["sns.conn"],
app.config["instance.tags"]["JobsSNSTopicARN"],
json.dumps(data))
def submit_task(app, task_desc_file):
uid = str(uuid.uuid1())
t = int(time.time())
tstamp = str(time.strftime("%Y-%m-%d %H:%M:%S"))
task_desc = template(task_desc_file, uid=uid, time=t, tstamp=tstamp)
print "-" * 50
print task_desc
print "-" * 50
data = ast.literal_eval(task_desc)
dutils.dynamodb_update(app.config["dyno.conn"], data)
sns_sqs.publish(app.config["sns.conn"], app.config["instance.tags"]["JobsSNSTopicARN"], data)
return uid
def submit_task():
app = cm.load_configs("production.conf")
uid = str(uuid.uuid1())
data = {"job_id" : uid,
"username" : "yadu",
"jobtype" : "doc2vec",
"inputs" : [{"src": "https://s3.amazonaws.com/klab-jobs/inputs/test.txt", "dest": "test.txt" }],
"outputs" : [{"src": "doc_mat.pkl", "dest": "klab-jobs/outputs/{0}/doc_mat.pkl".format(uid)},
{"src": "word_mat.pkl", "dest": "klab-jobs/outputs/{0}/word_mat.pkl".format(uid)},
{"src": "mdl.pkl", "dest": "klab-jobs/outputs/{0}/mdl.pkl".format(uid)}],
"submit_time" : int(time.time()),
"status" : "pending"
}
dutils.dynamodb_update(app.config["dyno.conn"], data)
sns_sqs.publish(app.config["sns.conn"], app.config["instance.tags"]["JobsSNSTopicARN"],
json.dumps(data))
def submit_task(app, task_desc_file):
uid = str(uuid.uuid1())
t = int(time.time())
tstamp = str(time.strftime('%Y-%m-%d %H:%M:%S'))
task_desc = template(task_desc_file,
uid=uid,
time=t,
tstamp=tstamp)
print "-"*50
print task_desc
print "-"*50
data = ast.literal_eval(task_desc)
dutils.dynamodb_update(app.config["dyno.conn"], data)
sns_sqs.publish(app.config["sns.conn"], app.config["instance.tags"]["JobsSNSTopicARN"],
data)
return uid
def _submit_task(request, session):
user_id = session["user_id"]
input_url = request.POST.get('input_url')
jobtype = request.POST.get('jobtype').strip()
outputs = request.POST.get('outputs', None)
uid = str(uuid.uuid4())
queue = request.POST.get('queue')
results = request.app.config["dyno.conn"].scan(i_user_id__eq=user_id)
results = identity.get_wos_dbinfo(request.app, user_id)
data = {"job_id" : uid,
"username" : session["username"],
"i_user_id" : session["user_id"],
"i_wosuser" : results.get("wos_user", "None"),
"i_wospasswd" : results.get("wos_passwd", "None"),
"jobname" : request.POST.get('jobname', uid),
"i_user_role" : "arn:aws:iam::{0}:role/{1}".format(request.app.config["iam.project"], session["user_role"]),
"user_email" : session["email"],
"submit_time" : int(time.time()),
"submit_stamp" : str(time.strftime('%Y-%m-%d %H:%M:%S')),
"walltime" : int(request.POST.get('walltime')) * 60,
"queue" : queue,
"jobtype" : jobtype,
"status" : "pending"
}
##############################################################################################################
# Doc_to_vec specific attributes
##############################################################################################################
if jobtype == "doc_to_vec":
print "Inputs : ", [{"type": "doc", "src": input_url, "dest": input_url.split('/')[-1] }]
data["inputs"] = [{"type": "doc", "src": input_url, "dest": input_url.split('/')[-1] }]
data["outputs"] = [{"src": "doc_mat.pkl", "dest": "klab-jobs/outputs/{0}/doc_mat.pkl".format(uid)},
{"src": "word_mat.pkl", "dest": "klab-jobs/outputs/{0}/word_mat.pkl".format(uid)},
{"src": "mdl.pkl", "dest": "klab-jobs/outputs/{0}/mdl.pkl".format(uid)},
{"src": "STDOUT.txt", "dest": "klab-jobs/outputs/{0}/STDOUT.txt".format(uid)},
{"src": "STDERR.txt", "dest": "klab-jobs/outputs/{0}/STDERR.txt".format(uid)},
{"src": "pipeline.log", "dest": "klab-jobs/outputs/{0}/pipeline.log".format(uid)}]
model_url = request.POST.get('model_url')
if model_url :
data["inputs"].extend([{"type": "model", "src": model_url, "dest": model_url.split('/')[-1]}])
params_url = request.POST.get('params_url')
if params_url :
data["inputs"].extend([{"type": "params", "src": params_url, "dest": params_url.split('/')[-1]}])
elif jobtype == "script":
data["executable"] = request.POST.get('executable')
data["args"] = request.POST.get('args', '')
data["i_script"] = request.POST.get('script').rstrip('\r')
data["i_script_name"] = request.POST.get('script_name')
data["outputs"] = []
data["inputs"] = []
data["outputs"].extend([{"src" : data["i_script_name"],
"dest": "klab-jobs/outputs/{0}/{1}".format(uid, data["i_script_name"])}])
for k in request.POST.keys():
#print "Key : {0}".format(k)
if k.startswith('input_url'):
input_url = request.POST.get(k)
data["inputs"].extend([{"src" : input_url,
"dest": input_url.split('/')[-1]}])
elif k == "inputs" :
inputs = request.POST.get(k)
if inputs :
inp = [{"src":x.strip(), "dest":x.strip().split('/')[-1]} for x in inputs.split(',')]
data["inputs"].extend(inp)
elif k == "outputs":
if not outputs :
continue
for outfile in outputs.split(','):
outfile = outfile.lstrip().rstrip()
data["outputs"].extend([{"src" : outfile,
"dest": "klab-jobs/outputs/{0}/{1}".format(uid, outfile)}])
elif k.startswith('output_'):
output_file = request.POST.get(k)
print "Outfile : ", output_file
data["outputs"].extend([{"src" : output_file,
"dest": "klab-jobs/outputs/{0}/{1}".format(uid, output_file)}])
#print "*" * 50
#for k in data:
# print "{0:20} | {1:20}".format(k, data.get(k))
#print "--" * 40
enable_mock = False
#enable_mock = True
if enable_mock :
return template("./views/submit_confirm.tpl",
job_id="MOCK-{0}".format(uid),
title="Task Confirmation - MOCK",
session=session)
dutils.dynamodb_update(request.app.config["dyno.conn"], data)
qname = "TestJobsSNSTopicARN"
if queue in ["Test", "Prod"]:
qname = queue + "JobsSNSTopicARN"
else:
raise Exception("Queue : [{0}] is not valid".format(queue))
sns_sqs.publish(request.app.config["sns.conn"], request.app.config["instance.tags"][qname],
json.dumps(data))
return uid
def _submit_task(request, session):
user_id = session["user_id"]
input_url = request.POST.get('input_url')
jobtype = request.POST.get('jobtype').strip()
outputs = request.POST.get('outputs', None)
uid = str(uuid.uuid4())
queue = request.POST.get('queue')
results = request.app.config["dyno.conn"].scan(i_user_id__eq=user_id)
results = identity.get_wos_dbinfo(request.app, user_id)
data = {
"job_id":
uid,
"username":
session["username"],
"i_user_id":
session["user_id"],
"i_wosuser":
results.get("wos_user", "None"),
"i_wospasswd":
results.get("wos_passwd", "None"),
"jobname":
request.POST.get('jobname', uid),
"i_user_role":
"arn:aws:iam::{0}:role/{1}".format(request.app.config["iam.project"],
session["user_role"]),
"user_email":
session["email"],
"submit_time":
int(time.time()),
"submit_stamp":
str(time.strftime('%Y-%m-%d %H:%M:%S')),
"walltime":
int(request.POST.get('walltime')) * 60,
"queue":
queue,
"jobtype":
jobtype,
"status":
"pending"
}
##############################################################################################################
# Doc_to_vec specific attributes
##############################################################################################################
if jobtype == "doc_to_vec":
print "Inputs : ", [{
"type": "doc",
"src": input_url,
"dest": input_url.split('/')[-1]
}]
data["inputs"] = [{
"type": "doc",
"src": input_url,
"dest": input_url.split('/')[-1]
}]
data["outputs"] = [{
"src":
"doc_mat.pkl",
"dest":
"klab-jobs/outputs/{0}/doc_mat.pkl".format(uid)
}, {
"src":
"word_mat.pkl",
"dest":
"klab-jobs/outputs/{0}/word_mat.pkl".format(uid)
}, {
"src": "mdl.pkl",
"dest": "klab-jobs/outputs/{0}/mdl.pkl".format(uid)
}, {
"src":
"STDOUT.txt",
"dest":
"klab-jobs/outputs/{0}/STDOUT.txt".format(uid)
}, {
"src":
"STDERR.txt",
"dest":
"klab-jobs/outputs/{0}/STDERR.txt".format(uid)
}, {
"src":
"pipeline.log",
"dest":
"klab-jobs/outputs/{0}/pipeline.log".format(uid)
}]
model_url = request.POST.get('model_url')
if model_url:
data["inputs"].extend([{
"type": "model",
"src": model_url,
"dest": model_url.split('/')[-1]
}])
params_url = request.POST.get('params_url')
if params_url:
data["inputs"].extend([{
"type": "params",
"src": params_url,
"dest": params_url.split('/')[-1]
}])
elif jobtype == "script":
data["executable"] = request.POST.get('executable')
data["args"] = request.POST.get('args', '')
data["i_script"] = request.POST.get('script').rstrip('\r')
data["i_script_name"] = request.POST.get('script_name')
data["outputs"] = []
data["inputs"] = []
data["outputs"].extend([{
"src":
data["i_script_name"],
"dest":
"klab-jobs/outputs/{0}/{1}".format(uid, data["i_script_name"])
}])
for k in request.POST.keys():
#print "Key : {0}".format(k)
if k.startswith('input_url'):
input_url = request.POST.get(k)
data["inputs"].extend([{
"src": input_url,
"dest": input_url.split('/')[-1]
}])
elif k == "inputs":
inputs = request.POST.get(k)
if inputs:
inp = [{
"src": x.strip(),
"dest": x.strip().split('/')[-1]
} for x in inputs.split(',')]
data["inputs"].extend(inp)
elif k == "outputs":
if not outputs:
continue
for outfile in outputs.split(','):
outfile = outfile.lstrip().rstrip()
data["outputs"].extend([{
"src":
outfile,
"dest":
"klab-jobs/outputs/{0}/{1}".format(uid, outfile)
}])
elif k.startswith('output_'):
output_file = request.POST.get(k)
print "Outfile : ", output_file
data["outputs"].extend([{
"src":
output_file,
"dest":
"klab-jobs/outputs/{0}/{1}".format(uid, output_file)
}])
#print "*" * 50
#for k in data:
# print "{0:20} | {1:20}".format(k, data.get(k))
#print "--" * 40
enable_mock = False
#enable_mock = True
if enable_mock:
return template("./views/submit_confirm.tpl",
job_id="MOCK-{0}".format(uid),
title="Task Confirmation - MOCK",
session=session)
dutils.dynamodb_update(request.app.config["dyno.conn"], data)
qname = "TestJobsSNSTopicARN"
if queue in ["Test", "Prod"]:
qname = queue + "JobsSNSTopicARN"
else:
raise Exception("Queue : [{0}] is not valid".format(queue))
sns_sqs.publish(request.app.config["sns.conn"],
request.app.config["instance.tags"][qname],
json.dumps(data))
return uid