Beispiel #1
0
def submit_task():
    app = cm.load_configs("production.conf")
    uid = str(uuid.uuid1())
    data = {
        "job_id":
        uid,
        "username":
        "******",
        "jobtype":
        "doc2vec",
        "inputs": [{
            "src": "https://s3.amazonaws.com/klab-jobs/inputs/test.txt",
            "dest": "test.txt"
        }],
        "outputs": [{
            "src": "doc_mat.pkl",
            "dest": "klab-jobs/outputs/{0}/doc_mat.pkl".format(uid)
        }, {
            "src": "word_mat.pkl",
            "dest": "klab-jobs/outputs/{0}/word_mat.pkl".format(uid)
        }, {
            "src": "mdl.pkl",
            "dest": "klab-jobs/outputs/{0}/mdl.pkl".format(uid)
        }],
        "submit_time":
        int(time.time()),
        "status":
        "pending"
    }

    dutils.dynamodb_update(app.config["dyno.conn"], data)
    sns_sqs.publish(app.config["sns.conn"],
                    app.config["instance.tags"]["JobsSNSTopicARN"],
                    json.dumps(data))
Beispiel #2
0
def submit_task(app, task_desc_file):

    uid = str(uuid.uuid1())
    t = int(time.time())
    tstamp = str(time.strftime("%Y-%m-%d %H:%M:%S"))

    task_desc = template(task_desc_file, uid=uid, time=t, tstamp=tstamp)
    print "-" * 50
    print task_desc
    print "-" * 50
    data = ast.literal_eval(task_desc)

    dutils.dynamodb_update(app.config["dyno.conn"], data)
    sns_sqs.publish(app.config["sns.conn"], app.config["instance.tags"]["JobsSNSTopicARN"], data)
    return uid
Beispiel #3
0
def submit_task():
    app = cm.load_configs("production.conf")
    uid = str(uuid.uuid1())
    data = {"job_id"           : uid,
            "username"         : "yadu",
            "jobtype"          : "doc2vec",
            "inputs"           : [{"src": "https://s3.amazonaws.com/klab-jobs/inputs/test.txt", "dest": "test.txt" }],
            "outputs"          : [{"src": "doc_mat.pkl",  "dest": "klab-jobs/outputs/{0}/doc_mat.pkl".format(uid)},
                                  {"src": "word_mat.pkl", "dest": "klab-jobs/outputs/{0}/word_mat.pkl".format(uid)},
                                  {"src": "mdl.pkl",      "dest": "klab-jobs/outputs/{0}/mdl.pkl".format(uid)}],
            "submit_time"      : int(time.time()),
            "status"           : "pending"
    }


    dutils.dynamodb_update(app.config["dyno.conn"], data)
    sns_sqs.publish(app.config["sns.conn"], app.config["instance.tags"]["JobsSNSTopicARN"],
                    json.dumps(data))
Beispiel #4
0
def submit_task(app, task_desc_file):

    uid = str(uuid.uuid1())
    t   = int(time.time())
    tstamp = str(time.strftime('%Y-%m-%d %H:%M:%S'))

    task_desc = template(task_desc_file,
                         uid=uid,
                         time=t,
                         tstamp=tstamp)
    print "-"*50
    print task_desc
    print "-"*50
    data = ast.literal_eval(task_desc)

    dutils.dynamodb_update(app.config["dyno.conn"], data)
    sns_sqs.publish(app.config["sns.conn"], app.config["instance.tags"]["JobsSNSTopicARN"],
                    data)
    return uid
Beispiel #5
0
def _submit_task(request, session):

    user_id   = session["user_id"]
    input_url = request.POST.get('input_url')
    jobtype   = request.POST.get('jobtype').strip()
    outputs   = request.POST.get('outputs', None)
    uid       = str(uuid.uuid4())
    queue     = request.POST.get('queue')
    results   = request.app.config["dyno.conn"].scan(i_user_id__eq=user_id)

    results   = identity.get_wos_dbinfo(request.app, user_id)

    data      = {"job_id"           : uid,
                 "username"         : session["username"],
                 "i_user_id"        : session["user_id"],
                 "i_wosuser"        : results.get("wos_user", "None"),
                 "i_wospasswd"      : results.get("wos_passwd", "None"),
                 "jobname"          : request.POST.get('jobname', uid),
                 "i_user_role"      : "arn:aws:iam::{0}:role/{1}".format(request.app.config["iam.project"], session["user_role"]),
                 "user_email"       : session["email"],
                 "submit_time"      : int(time.time()),
                 "submit_stamp"     : str(time.strftime('%Y-%m-%d %H:%M:%S')),
                 "walltime"         : int(request.POST.get('walltime')) * 60,
                 "queue"            : queue,
                 "jobtype"          : jobtype,
                 "status"           : "pending"                
            }

    ##############################################################################################################
    # Doc_to_vec specific attributes
    ##############################################################################################################
    if jobtype == "doc_to_vec":

        print "Inputs : ", [{"type": "doc", "src": input_url, "dest": input_url.split('/')[-1] }]

        data["inputs"]   =  [{"type": "doc", "src": input_url, "dest": input_url.split('/')[-1] }]
        data["outputs"]  =  [{"src": "doc_mat.pkl",  "dest": "klab-jobs/outputs/{0}/doc_mat.pkl".format(uid)},
                             {"src": "word_mat.pkl", "dest": "klab-jobs/outputs/{0}/word_mat.pkl".format(uid)},
                             {"src": "mdl.pkl",      "dest": "klab-jobs/outputs/{0}/mdl.pkl".format(uid)},
                             {"src": "STDOUT.txt",   "dest": "klab-jobs/outputs/{0}/STDOUT.txt".format(uid)},
                             {"src": "STDERR.txt",   "dest": "klab-jobs/outputs/{0}/STDERR.txt".format(uid)},
                             {"src": "pipeline.log", "dest": "klab-jobs/outputs/{0}/pipeline.log".format(uid)}]

        model_url = request.POST.get('model_url')
        if model_url :
            data["inputs"].extend([{"type": "model", "src": model_url, "dest": model_url.split('/')[-1]}])

        params_url = request.POST.get('params_url')
        if params_url :
            data["inputs"].extend([{"type": "params", "src": params_url, "dest": params_url.split('/')[-1]}])

    elif jobtype == "script":

        data["executable"]    = request.POST.get('executable')
        data["args"]          = request.POST.get('args', '')
        data["i_script"]      = request.POST.get('script').rstrip('\r')
        data["i_script_name"] = request.POST.get('script_name')
        data["outputs"]       = []
        data["inputs"]        = []

        data["outputs"].extend([{"src" : data["i_script_name"], 
                                 "dest": "klab-jobs/outputs/{0}/{1}".format(uid, data["i_script_name"])}])
        
        for k in request.POST.keys():
            #print "Key : {0}".format(k)
            if k.startswith('input_url'):
                input_url =  request.POST.get(k)
                data["inputs"].extend([{"src" : input_url, 
                                        "dest": input_url.split('/')[-1]}])
            elif k == "inputs" :
                inputs = request.POST.get(k)
                if inputs :
                    inp = [{"src":x.strip(), "dest":x.strip().split('/')[-1]} for x in inputs.split(',')]
                    data["inputs"].extend(inp)
                
            elif k == "outputs":
                if not outputs :
                    continue
                for outfile in outputs.split(','):
                    outfile = outfile.lstrip().rstrip()                    
                    data["outputs"].extend([{"src" : outfile, 
                                             "dest": "klab-jobs/outputs/{0}/{1}".format(uid, outfile)}])

            elif k.startswith('output_'):
                output_file = request.POST.get(k)
                print "Outfile : ", output_file
                data["outputs"].extend([{"src" : output_file, 
                                         "dest": "klab-jobs/outputs/{0}/{1}".format(uid, output_file)}])

        #print "*" * 50
        #for k in data:
        #    print "{0:20} | {1:20}".format(k, data.get(k))
        #print "--" * 40
        
       
    enable_mock = False
    #enable_mock = True

    if enable_mock :
        return template("./views/submit_confirm.tpl",
                        job_id="MOCK-{0}".format(uid),
                        title="Task Confirmation - MOCK",
                        session=session)


    dutils.dynamodb_update(request.app.config["dyno.conn"], data)
    qname = "TestJobsSNSTopicARN"
    if queue in ["Test", "Prod"]:
        qname = queue + "JobsSNSTopicARN"
    else:
        raise Exception("Queue : [{0}] is not valid".format(queue))

    sns_sqs.publish(request.app.config["sns.conn"], request.app.config["instance.tags"][qname],
                    json.dumps(data))

    return uid
Beispiel #6
0
def _submit_task(request, session):

    user_id = session["user_id"]
    input_url = request.POST.get('input_url')
    jobtype = request.POST.get('jobtype').strip()
    outputs = request.POST.get('outputs', None)
    uid = str(uuid.uuid4())
    queue = request.POST.get('queue')
    results = request.app.config["dyno.conn"].scan(i_user_id__eq=user_id)

    results = identity.get_wos_dbinfo(request.app, user_id)

    data = {
        "job_id":
        uid,
        "username":
        session["username"],
        "i_user_id":
        session["user_id"],
        "i_wosuser":
        results.get("wos_user", "None"),
        "i_wospasswd":
        results.get("wos_passwd", "None"),
        "jobname":
        request.POST.get('jobname', uid),
        "i_user_role":
        "arn:aws:iam::{0}:role/{1}".format(request.app.config["iam.project"],
                                           session["user_role"]),
        "user_email":
        session["email"],
        "submit_time":
        int(time.time()),
        "submit_stamp":
        str(time.strftime('%Y-%m-%d %H:%M:%S')),
        "walltime":
        int(request.POST.get('walltime')) * 60,
        "queue":
        queue,
        "jobtype":
        jobtype,
        "status":
        "pending"
    }

    ##############################################################################################################
    # Doc_to_vec specific attributes
    ##############################################################################################################
    if jobtype == "doc_to_vec":

        print "Inputs : ", [{
            "type": "doc",
            "src": input_url,
            "dest": input_url.split('/')[-1]
        }]

        data["inputs"] = [{
            "type": "doc",
            "src": input_url,
            "dest": input_url.split('/')[-1]
        }]
        data["outputs"] = [{
            "src":
            "doc_mat.pkl",
            "dest":
            "klab-jobs/outputs/{0}/doc_mat.pkl".format(uid)
        }, {
            "src":
            "word_mat.pkl",
            "dest":
            "klab-jobs/outputs/{0}/word_mat.pkl".format(uid)
        }, {
            "src": "mdl.pkl",
            "dest": "klab-jobs/outputs/{0}/mdl.pkl".format(uid)
        }, {
            "src":
            "STDOUT.txt",
            "dest":
            "klab-jobs/outputs/{0}/STDOUT.txt".format(uid)
        }, {
            "src":
            "STDERR.txt",
            "dest":
            "klab-jobs/outputs/{0}/STDERR.txt".format(uid)
        }, {
            "src":
            "pipeline.log",
            "dest":
            "klab-jobs/outputs/{0}/pipeline.log".format(uid)
        }]

        model_url = request.POST.get('model_url')
        if model_url:
            data["inputs"].extend([{
                "type": "model",
                "src": model_url,
                "dest": model_url.split('/')[-1]
            }])

        params_url = request.POST.get('params_url')
        if params_url:
            data["inputs"].extend([{
                "type": "params",
                "src": params_url,
                "dest": params_url.split('/')[-1]
            }])

    elif jobtype == "script":

        data["executable"] = request.POST.get('executable')
        data["args"] = request.POST.get('args', '')
        data["i_script"] = request.POST.get('script').rstrip('\r')
        data["i_script_name"] = request.POST.get('script_name')
        data["outputs"] = []
        data["inputs"] = []

        data["outputs"].extend([{
            "src":
            data["i_script_name"],
            "dest":
            "klab-jobs/outputs/{0}/{1}".format(uid, data["i_script_name"])
        }])

        for k in request.POST.keys():
            #print "Key : {0}".format(k)
            if k.startswith('input_url'):
                input_url = request.POST.get(k)
                data["inputs"].extend([{
                    "src": input_url,
                    "dest": input_url.split('/')[-1]
                }])
            elif k == "inputs":
                inputs = request.POST.get(k)
                if inputs:
                    inp = [{
                        "src": x.strip(),
                        "dest": x.strip().split('/')[-1]
                    } for x in inputs.split(',')]
                    data["inputs"].extend(inp)

            elif k == "outputs":
                if not outputs:
                    continue
                for outfile in outputs.split(','):
                    outfile = outfile.lstrip().rstrip()
                    data["outputs"].extend([{
                        "src":
                        outfile,
                        "dest":
                        "klab-jobs/outputs/{0}/{1}".format(uid, outfile)
                    }])

            elif k.startswith('output_'):
                output_file = request.POST.get(k)
                print "Outfile : ", output_file
                data["outputs"].extend([{
                    "src":
                    output_file,
                    "dest":
                    "klab-jobs/outputs/{0}/{1}".format(uid, output_file)
                }])

        #print "*" * 50
        #for k in data:
        #    print "{0:20} | {1:20}".format(k, data.get(k))
        #print "--" * 40

    enable_mock = False
    #enable_mock = True

    if enable_mock:
        return template("./views/submit_confirm.tpl",
                        job_id="MOCK-{0}".format(uid),
                        title="Task Confirmation - MOCK",
                        session=session)

    dutils.dynamodb_update(request.app.config["dyno.conn"], data)
    qname = "TestJobsSNSTopicARN"
    if queue in ["Test", "Prod"]:
        qname = queue + "JobsSNSTopicARN"
    else:
        raise Exception("Queue : [{0}] is not valid".format(queue))

    sns_sqs.publish(request.app.config["sns.conn"],
                    request.app.config["instance.tags"][qname],
                    json.dumps(data))

    return uid