예제 #1
0
def submit_task():
    app = cm.load_configs("production.conf")
    uid = str(uuid.uuid1())
    data = {
        "job_id":
        uid,
        "username":
        "******",
        "jobtype":
        "doc2vec",
        "inputs": [{
            "src": "https://s3.amazonaws.com/klab-jobs/inputs/test.txt",
            "dest": "test.txt"
        }],
        "outputs": [{
            "src": "doc_mat.pkl",
            "dest": "klab-jobs/outputs/{0}/doc_mat.pkl".format(uid)
        }, {
            "src": "word_mat.pkl",
            "dest": "klab-jobs/outputs/{0}/word_mat.pkl".format(uid)
        }, {
            "src": "mdl.pkl",
            "dest": "klab-jobs/outputs/{0}/mdl.pkl".format(uid)
        }],
        "submit_time":
        int(time.time()),
        "status":
        "pending"
    }

    dutils.dynamodb_update(app.config["dyno.conn"], data)
    sns_sqs.publish(app.config["sns.conn"],
                    app.config["instance.tags"]["JobsSNSTopicARN"],
                    json.dumps(data))
예제 #2
0
def testing():
    import config_manager as cm
    app = cm.load_configs("production.conf")
    cmd = {"job_id"     : 123123,
           "executable" : "/bin/echo",
           "args"       : "hello"}
       
    job_id = "ce19ede4-da29-48e5-abcf-2eff53778333"
    update_usage_stats(app, job_id)
    update_usage_stats(app, job_id)
    
    status = execute(app, "/bin/doo Hello World", 5, None)
    if status == 127 :
        print "Pass"
    else:
        print "Failed test"

    status = execute(app, "/bin/sleep 0", 5, None)
    if status == 127 :
        print "Pass"
    else:
        print "Failed test"

    status = execute(app, "/bin/sleep 60", 1, None)
    if status == 127 :
        print "Pass"
    else:
        print "Failed test"

    status = execute(app, '/bin/echo "Hello World"; sleep 8', 10, None)
    if status == 0 :
        print "Pass"
    else:
        print "Failed test"
        
    cmd = {"job_id"     : 123123,
           "executable" : "aws",
           "args"       : "s3 cp {0} {1}".format("./dummy50m", "s3://klab-jobs/yadu/data/dummy50m") }

    print execute_wait(app, cmd, 50, "asdsada")

    cmd = {"job_id"     : 123123,
           "executable" : "aws",
           "args"       : "s3 cp {0} {1}".format("./dummy500m", "s3://klab-jobs/yadu/data/dummy500m") }

    print execute_wait(app, cmd, 50, "asdsada")

    cmd = {"job_id"     : 123123,
           "executable" : "aws",
           "args"       : "s3 cp {0} {1}".format("./dummy1g", "s3://klab-jobs/yadu/data/dummy1g") }

    print execute_wait(app, cmd, 50, "asdsada")

    cmd = {"job_id"     : 123123,
           "executable" : "aws",
           "args"       : "s3 cp {0} {1}".format("./shuf.txt", "s3://klab-jobs/yadu/data/dummy1g") }

    print execute_wait(app, cmd, 50, "asdsada")
예제 #3
0
def submit_task():
    app = cm.load_configs("production.conf")
    uid = str(uuid.uuid1())
    data = {"job_id"           : uid,
            "username"         : "yadu",
            "jobtype"          : "doc2vec",
            "inputs"           : [{"src": "https://s3.amazonaws.com/klab-jobs/inputs/test.txt", "dest": "test.txt" }],
            "outputs"          : [{"src": "doc_mat.pkl",  "dest": "klab-jobs/outputs/{0}/doc_mat.pkl".format(uid)},
                                  {"src": "word_mat.pkl", "dest": "klab-jobs/outputs/{0}/word_mat.pkl".format(uid)},
                                  {"src": "mdl.pkl",      "dest": "klab-jobs/outputs/{0}/mdl.pkl".format(uid)}],
            "submit_time"      : int(time.time()),
            "status"           : "pending"
    }


    dutils.dynamodb_update(app.config["dyno.conn"], data)
    sns_sqs.publish(app.config["sns.conn"], app.config["instance.tags"]["JobsSNSTopicARN"],
                    json.dumps(data))
예제 #4
0
def test_1():
    import config_manager as cm
    import time
    import uuid
    app = cm.load_configs("production.conf")
    uid = str(uuid.uuid1())

    
    data = {"job_id"           : uid,
            "username"         : "yadu",
            "jobtype"          : "doc2vec",
            "inputs"           : [{"src": "https://s3.amazonaws.com/klab-jobs/inputs/test.txt", "dest": "test.txt" }],
            "outputs"          : [{"src": "doc_mat.pkl",  "dest": "klab-jobs/outputs/{0}/doc_mat.pkl".format(uid)},
                                  {"src": "word_mat.pkl", "dest": "klab-jobs/outputs/{0}/word_mat.pkl".format(uid)},
                                  {"src": "mdl.pkl",      "dest": "klab-jobs/outputs/{0}/mdl.pkl".format(uid)}],
            "submit_time"      : int(time.time()),
            "status"           : "pending"
    }
    uid = "27013a48-9164-11e5-a61b-0edd34be4cf3"
    #dynamodb_update(app.config["dyno.conn"], data)
    dynamodb_get(app.config["dyno.conn"], uid)
예제 #5
0
def test_2():
    import config_manager as cm
    app = cm.load_configs("production.conf")

    results = {}
    users = find_all_users(app)
    for user in users:
        print user["user_id"], user["name"]
        all_jobs = request.app.config["dyno.conn"].scan(i_user_id__eq=user["user_id"])
        
        results[user["name"]] = {"Prod" : {"count" : 0,
                                           "walltime" : 0},
                                 "Test" : {"count" : 0,
                                           "walltime" : 0}}
        
        count = 0
        for j in all_jobs:
            #walltime += j["walltime"]            
            #count += 1
            print "User:{0} Count: "
     
    return
예제 #6
0
                                         message_attributes=['All'])
        if not messages:
            return None
        for msg in messages:
            if msg.message_attributes["instance_id"]["string_value"] == attr[
                    "instance_id"]["string_value"] and msg.message_attributes[
                        "job_id"]["string_value"] == attr["job_id"][
                            "string_value"]:
                active_q.delete_message(msg)
                return 1

    return None


if __name__ == "__main__":
    app = cm.load_configs("production.conf")

    sqs_conn = app.config["sqs.conn"]
    pending = app.config["instance.tags"]["JobsQueueName"]
    active = app.config["instance.tags"]["ActiveQueueName"]
    pending_q = sqs_conn.get_queue(pending)
    active_q = sqs_conn.get_queue(active)

    msg = {"job_id": "fooo", "walltime": "aaaa"}
    attr, msg = post_message_to_active(app, active_q, json.dumps(msg),
                                       get_uuid())
    print "Posted mesage : ", msg, attr

    time.sleep(10)
    st = delete_message_from_active(sqs_conn, active_q, attr)
예제 #7
0
                                                                                      total,
                                                                                      visible,
                                                                                      inflight))
            print r
        time.sleep(60)

########################################################################################################
if __name__ == "__main__":

   parser   = argparse.ArgumentParser()
   parser.add_argument("-v", "--verbose", default="DEBUG", help="set level of verbosity, DEBUG, INFO, WARN")
   parser.add_argument("-l", "--logfile", default="queue_watcher.log", help="Logfile path. Defaults to ./task_executor.log")
   parser.add_argument("-c", "--conffile", default="test.conf", help="Config file path. Defaults to ./test.conf")
   parser.add_argument("-j", "--jobid", type=str, action='append')
   parser.add_argument("-i", "--workload_id", default=None)
   args   = parser.parse_args()

   if args.verbose not in conf_man.log_levels :
      print "Unknown verbosity level : {0}".format(args.verbose)
      print "Cannot proceed. Exiting"
      exit(-1)

   logging.basicConfig(filename=args.logfile, level=conf_man.log_levels[args.verbose],
                       format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',
                       datefmt='%m-%d %H:%M')
   logging.getLogger('boto').setLevel(logging.CRITICAL)

   logging.debug("\n{0}\nStarting task_executor\n{0}\n".format("*"*50))
   app = conf_man.load_configs(args.conffile);
   watch_loop(app)
예제 #8
0
                        "--verbose",
                        default="DEBUG",
                        help="set level of verbosity, DEBUG, INFO, WARN")
    parser.add_argument("-l",
                        "--logfile",
                        default="queue_watcher.log",
                        help="Logfile path. Defaults to ./task_executor.log")
    parser.add_argument("-c",
                        "--conffile",
                        default="test.conf",
                        help="Config file path. Defaults to ./test.conf")
    parser.add_argument("-j", "--jobid", type=str, action='append')
    parser.add_argument("-i", "--workload_id", default=None)
    args = parser.parse_args()

    if args.verbose not in conf_man.log_levels:
        print "Unknown verbosity level : {0}".format(args.verbose)
        print "Cannot proceed. Exiting"
        exit(-1)

    logging.basicConfig(
        filename=args.logfile,
        level=conf_man.log_levels[args.verbose],
        format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',
        datefmt='%m-%d %H:%M')
    logging.getLogger('boto').setLevel(logging.CRITICAL)

    logging.debug("\n{0}\nStarting task_executor\n{0}\n".format("*" * 50))
    app = conf_man.load_configs(args.conffile)
    watch_loop(app)
예제 #9
0
            s3conn  = app.config["s3conn"]            
            bucket  = s3conn.get_bucket(bucket_name, validate=False)
            key     = bucket.get_key(prefix)
            #key.metadata.update({'last_accessed' : str(time.strftime('%Y-%m-%d %H:%M:%S'))})
            #nkey    = key.copy(key.bucket.name, key.name, key.metadata, preserve_acl=True)
            #nkey.metadata = key.metadata
            key.set_metadata({'last_accessed' : str(time.strftime('%Y-%m-%d %H:%M:%S'))})

    except Exception, e :
        print "ERROR: Failed to get data/update metadata : ", e
        raise

    
if __name__ == "__main__":
    import config_manager as cm
    app = cm.load_configs("production.conf")
    import sts
    import s3_utils as s3
    rolestring  = '' # Left out due to security concerns

    if not rolestring :
        print "Fill out rolestring to continue tests"

    creds = sts.get_temp_creds(rolestring)
    s3conn  = get_s3_conn( creds["AccessKeyId"],
                           creds["SecretAccessKey"],
                           creds["SessionToken"] )
    
    
    bucket_name = "klab-webofscience"
    prefix = 'raw_zipfiles/1976_DSSHPSH.zip'
예제 #10
0
    parser = argparse.ArgumentParser()
    parser.add_argument("-c",
                        "--conffile",
                        default="production.conf",
                        help="Config file path. Defaults to ./test.conf")
    parser.add_argument("-u",
                        "--user_id",
                        required=True,
                        help="Amazon user id")
    parser.add_argument("-n", "--name", help="Name of new user,in quotes")
    parser.add_argument("-e", "--email", help="Email address of new user")
    parser.add_argument("-r", "--role", help="Role to assign to new user")
    args = parser.parse_args()

    app = cm.load_configs(args.conffile)
    print args.user_id
    print args.name
    print args.email
    print args.role

    dyno = connect(app)

    user = find_user_role(app, dyno, args.user_id)
    if user != None:
        print "User exists "
        if args.name != None and args.name != user["name"]:
            print "Mismatch in Name : Arg:{0}  DB:{1}".format(
                args.name, user["name"])
            user["name"] = args.name
예제 #11
0
    update_db(app, cmd)
    return 

    
if __name__ == "__main__":

    parser   = argparse.ArgumentParser()
    parser.add_argument("-c", "--conffile", default="production.conf", help="Config file path. Defaults to ./test.conf")
    parser.add_argument("-w", "--webofscience", default=None, help="Config file path for webofscience")
    parser.add_argument("-u", "--user_id", required=True, help="Amazon user id");
    parser.add_argument("-n", "--name", help="Name of new user,in quotes");
    parser.add_argument("-e", "--email", help="Email address of new user");
    parser.add_argument("-r", "--role", help="Role to assign to new user");   
    args   = parser.parse_args()

    app = cm.load_configs(args.conffile)

    if args.webofscience :
        w = load_config(args.webofscience)


    app.config["wos_user"]   = args.name.replace(' ', '').lower()
    app.config["wos_passwd"] = str(uuid.uuid4()).replace('-', '')[0:20]

    dyno = connect(app)

    user = find_user_role(app, dyno, args.user_id)
    if user != None:
        print "User exists "
        if args.name != None and args.name != user["name"]:
            print "Mismatch in Name : Arg:{0}  DB:{1}".format(args.name, user["name"])
예제 #12
0
def testing():
    import config_manager as cm
    app = cm.load_configs("production.conf")
    cmd = {"job_id": 123123, "executable": "/bin/echo", "args": "hello"}

    job_id = "ce19ede4-da29-48e5-abcf-2eff53778333"
    update_usage_stats(app, job_id)
    update_usage_stats(app, job_id)

    status = execute(app, "/bin/doo Hello World", 5, None)
    if status == 127:
        print "Pass"
    else:
        print "Failed test"

    status = execute(app, "/bin/sleep 0", 5, None)
    if status == 127:
        print "Pass"
    else:
        print "Failed test"

    status = execute(app, "/bin/sleep 60", 1, None)
    if status == 127:
        print "Pass"
    else:
        print "Failed test"

    status = execute(app, '/bin/echo "Hello World"; sleep 8', 10, None)
    if status == 0:
        print "Pass"
    else:
        print "Failed test"

    cmd = {
        "job_id":
        123123,
        "executable":
        "aws",
        "args":
        "s3 cp {0} {1}".format("./dummy50m",
                               "s3://klab-jobs/yadu/data/dummy50m")
    }

    print execute_wait(app, cmd, 50, "asdsada")

    cmd = {
        "job_id":
        123123,
        "executable":
        "aws",
        "args":
        "s3 cp {0} {1}".format("./dummy500m",
                               "s3://klab-jobs/yadu/data/dummy500m")
    }

    print execute_wait(app, cmd, 50, "asdsada")

    cmd = {
        "job_id":
        123123,
        "executable":
        "aws",
        "args":
        "s3 cp {0} {1}".format("./dummy1g", "s3://klab-jobs/yadu/data/dummy1g")
    }

    print execute_wait(app, cmd, 50, "asdsada")

    cmd = {
        "job_id":
        123123,
        "executable":
        "aws",
        "args":
        "s3 cp {0} {1}".format("./shuf.txt",
                               "s3://klab-jobs/yadu/data/dummy1g")
    }

    print execute_wait(app, cmd, 50, "asdsada")