def submit_task():
app = cm.load_configs("production.conf")
uid = str(uuid.uuid1())
data = {
"job_id":
uid,
"username":
"******",
"jobtype":
"doc2vec",
"inputs": [{
"src": "https://s3.amazonaws.com/klab-jobs/inputs/test.txt",
"dest": "test.txt"
}],
"outputs": [{
"src": "doc_mat.pkl",
"dest": "klab-jobs/outputs/{0}/doc_mat.pkl".format(uid)
}, {
"src": "word_mat.pkl",
"dest": "klab-jobs/outputs/{0}/word_mat.pkl".format(uid)
}, {
"src": "mdl.pkl",
"dest": "klab-jobs/outputs/{0}/mdl.pkl".format(uid)
}],
"submit_time":
int(time.time()),
"status":
"pending"
}
dutils.dynamodb_update(app.config["dyno.conn"], data)
sns_sqs.publish(app.config["sns.conn"],
app.config["instance.tags"]["JobsSNSTopicARN"],
json.dumps(data))
def testing():
import config_manager as cm
app = cm.load_configs("production.conf")
cmd = {"job_id" : 123123,
"executable" : "/bin/echo",
"args" : "hello"}
job_id = "ce19ede4-da29-48e5-abcf-2eff53778333"
update_usage_stats(app, job_id)
update_usage_stats(app, job_id)
status = execute(app, "/bin/doo Hello World", 5, None)
if status == 127 :
print "Pass"
else:
print "Failed test"
status = execute(app, "/bin/sleep 0", 5, None)
if status == 127 :
print "Pass"
else:
print "Failed test"
status = execute(app, "/bin/sleep 60", 1, None)
if status == 127 :
print "Pass"
else:
print "Failed test"
status = execute(app, '/bin/echo "Hello World"; sleep 8', 10, None)
if status == 0 :
print "Pass"
else:
print "Failed test"
cmd = {"job_id" : 123123,
"executable" : "aws",
"args" : "s3 cp {0} {1}".format("./dummy50m", "s3://klab-jobs/yadu/data/dummy50m") }
print execute_wait(app, cmd, 50, "asdsada")
cmd = {"job_id" : 123123,
"executable" : "aws",
"args" : "s3 cp {0} {1}".format("./dummy500m", "s3://klab-jobs/yadu/data/dummy500m") }
print execute_wait(app, cmd, 50, "asdsada")
cmd = {"job_id" : 123123,
"executable" : "aws",
"args" : "s3 cp {0} {1}".format("./dummy1g", "s3://klab-jobs/yadu/data/dummy1g") }
print execute_wait(app, cmd, 50, "asdsada")
cmd = {"job_id" : 123123,
"executable" : "aws",
"args" : "s3 cp {0} {1}".format("./shuf.txt", "s3://klab-jobs/yadu/data/dummy1g") }
print execute_wait(app, cmd, 50, "asdsada")
def submit_task():
app = cm.load_configs("production.conf")
uid = str(uuid.uuid1())
data = {"job_id" : uid,
"username" : "yadu",
"jobtype" : "doc2vec",
"inputs" : [{"src": "https://s3.amazonaws.com/klab-jobs/inputs/test.txt", "dest": "test.txt" }],
"outputs" : [{"src": "doc_mat.pkl", "dest": "klab-jobs/outputs/{0}/doc_mat.pkl".format(uid)},
{"src": "word_mat.pkl", "dest": "klab-jobs/outputs/{0}/word_mat.pkl".format(uid)},
{"src": "mdl.pkl", "dest": "klab-jobs/outputs/{0}/mdl.pkl".format(uid)}],
"submit_time" : int(time.time()),
"status" : "pending"
}
dutils.dynamodb_update(app.config["dyno.conn"], data)
sns_sqs.publish(app.config["sns.conn"], app.config["instance.tags"]["JobsSNSTopicARN"],
json.dumps(data))
def test_1():
import config_manager as cm
import time
import uuid
app = cm.load_configs("production.conf")
uid = str(uuid.uuid1())
data = {"job_id" : uid,
"username" : "yadu",
"jobtype" : "doc2vec",
"inputs" : [{"src": "https://s3.amazonaws.com/klab-jobs/inputs/test.txt", "dest": "test.txt" }],
"outputs" : [{"src": "doc_mat.pkl", "dest": "klab-jobs/outputs/{0}/doc_mat.pkl".format(uid)},
{"src": "word_mat.pkl", "dest": "klab-jobs/outputs/{0}/word_mat.pkl".format(uid)},
{"src": "mdl.pkl", "dest": "klab-jobs/outputs/{0}/mdl.pkl".format(uid)}],
"submit_time" : int(time.time()),
"status" : "pending"
}
uid = "27013a48-9164-11e5-a61b-0edd34be4cf3"
#dynamodb_update(app.config["dyno.conn"], data)
dynamodb_get(app.config["dyno.conn"], uid)
def test_2():
import config_manager as cm
app = cm.load_configs("production.conf")
results = {}
users = find_all_users(app)
for user in users:
print user["user_id"], user["name"]
all_jobs = request.app.config["dyno.conn"].scan(i_user_id__eq=user["user_id"])
results[user["name"]] = {"Prod" : {"count" : 0,
"walltime" : 0},
"Test" : {"count" : 0,
"walltime" : 0}}
count = 0
for j in all_jobs:
#walltime += j["walltime"]
#count += 1
print "User:{0} Count: "
return
message_attributes=['All'])
if not messages:
return None
for msg in messages:
if msg.message_attributes["instance_id"]["string_value"] == attr[
"instance_id"]["string_value"] and msg.message_attributes[
"job_id"]["string_value"] == attr["job_id"][
"string_value"]:
active_q.delete_message(msg)
return 1
return None
if __name__ == "__main__":
app = cm.load_configs("production.conf")
sqs_conn = app.config["sqs.conn"]
pending = app.config["instance.tags"]["JobsQueueName"]
active = app.config["instance.tags"]["ActiveQueueName"]
pending_q = sqs_conn.get_queue(pending)
active_q = sqs_conn.get_queue(active)
msg = {"job_id": "fooo", "walltime": "aaaa"}
attr, msg = post_message_to_active(app, active_q, json.dumps(msg),
get_uuid())
print "Posted mesage : ", msg, attr
time.sleep(10)
st = delete_message_from_active(sqs_conn, active_q, attr)
total,
visible,
inflight))
print r
time.sleep(60)
########################################################################################################
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("-v", "--verbose", default="DEBUG", help="set level of verbosity, DEBUG, INFO, WARN")
parser.add_argument("-l", "--logfile", default="queue_watcher.log", help="Logfile path. Defaults to ./task_executor.log")
parser.add_argument("-c", "--conffile", default="test.conf", help="Config file path. Defaults to ./test.conf")
parser.add_argument("-j", "--jobid", type=str, action='append')
parser.add_argument("-i", "--workload_id", default=None)
args = parser.parse_args()
if args.verbose not in conf_man.log_levels :
print "Unknown verbosity level : {0}".format(args.verbose)
print "Cannot proceed. Exiting"
exit(-1)
logging.basicConfig(filename=args.logfile, level=conf_man.log_levels[args.verbose],
format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',
datefmt='%m-%d %H:%M')
logging.getLogger('boto').setLevel(logging.CRITICAL)
logging.debug("\n{0}\nStarting task_executor\n{0}\n".format("*"*50))
app = conf_man.load_configs(args.conffile);
watch_loop(app)
"--verbose",
default="DEBUG",
help="set level of verbosity, DEBUG, INFO, WARN")
parser.add_argument("-l",
"--logfile",
default="queue_watcher.log",
help="Logfile path. Defaults to ./task_executor.log")
parser.add_argument("-c",
"--conffile",
default="test.conf",
help="Config file path. Defaults to ./test.conf")
parser.add_argument("-j", "--jobid", type=str, action='append')
parser.add_argument("-i", "--workload_id", default=None)
args = parser.parse_args()
if args.verbose not in conf_man.log_levels:
print "Unknown verbosity level : {0}".format(args.verbose)
print "Cannot proceed. Exiting"
exit(-1)
logging.basicConfig(
filename=args.logfile,
level=conf_man.log_levels[args.verbose],
format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',
datefmt='%m-%d %H:%M')
logging.getLogger('boto').setLevel(logging.CRITICAL)
logging.debug("\n{0}\nStarting task_executor\n{0}\n".format("*" * 50))
app = conf_man.load_configs(args.conffile)
watch_loop(app)
s3conn = app.config["s3conn"]
bucket = s3conn.get_bucket(bucket_name, validate=False)
key = bucket.get_key(prefix)
#key.metadata.update({'last_accessed' : str(time.strftime('%Y-%m-%d %H:%M:%S'))})
#nkey = key.copy(key.bucket.name, key.name, key.metadata, preserve_acl=True)
#nkey.metadata = key.metadata
key.set_metadata({'last_accessed' : str(time.strftime('%Y-%m-%d %H:%M:%S'))})
except Exception, e :
print "ERROR: Failed to get data/update metadata : ", e
raise
if __name__ == "__main__":
import config_manager as cm
app = cm.load_configs("production.conf")
import sts
import s3_utils as s3
rolestring = '' # Left out due to security concerns
if not rolestring :
print "Fill out rolestring to continue tests"
creds = sts.get_temp_creds(rolestring)
s3conn = get_s3_conn( creds["AccessKeyId"],
creds["SecretAccessKey"],
creds["SessionToken"] )
bucket_name = "klab-webofscience"
prefix = 'raw_zipfiles/1976_DSSHPSH.zip'
parser = argparse.ArgumentParser()
parser.add_argument("-c",
"--conffile",
default="production.conf",
help="Config file path. Defaults to ./test.conf")
parser.add_argument("-u",
"--user_id",
required=True,
help="Amazon user id")
parser.add_argument("-n", "--name", help="Name of new user,in quotes")
parser.add_argument("-e", "--email", help="Email address of new user")
parser.add_argument("-r", "--role", help="Role to assign to new user")
args = parser.parse_args()
app = cm.load_configs(args.conffile)
print args.user_id
print args.name
print args.email
print args.role
dyno = connect(app)
user = find_user_role(app, dyno, args.user_id)
if user != None:
print "User exists "
if args.name != None and args.name != user["name"]:
print "Mismatch in Name : Arg:{0} DB:{1}".format(
args.name, user["name"])
user["name"] = args.name
update_db(app, cmd)
return
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("-c", "--conffile", default="production.conf", help="Config file path. Defaults to ./test.conf")
parser.add_argument("-w", "--webofscience", default=None, help="Config file path for webofscience")
parser.add_argument("-u", "--user_id", required=True, help="Amazon user id");
parser.add_argument("-n", "--name", help="Name of new user,in quotes");
parser.add_argument("-e", "--email", help="Email address of new user");
parser.add_argument("-r", "--role", help="Role to assign to new user");
args = parser.parse_args()
app = cm.load_configs(args.conffile)
if args.webofscience :
w = load_config(args.webofscience)
app.config["wos_user"] = args.name.replace(' ', '').lower()
app.config["wos_passwd"] = str(uuid.uuid4()).replace('-', '')[0:20]
dyno = connect(app)
user = find_user_role(app, dyno, args.user_id)
if user != None:
print "User exists "
if args.name != None and args.name != user["name"]:
print "Mismatch in Name : Arg:{0} DB:{1}".format(args.name, user["name"])
def testing():
import config_manager as cm
app = cm.load_configs("production.conf")
cmd = {"job_id": 123123, "executable": "/bin/echo", "args": "hello"}
job_id = "ce19ede4-da29-48e5-abcf-2eff53778333"
update_usage_stats(app, job_id)
update_usage_stats(app, job_id)
status = execute(app, "/bin/doo Hello World", 5, None)
if status == 127:
print "Pass"
else:
print "Failed test"
status = execute(app, "/bin/sleep 0", 5, None)
if status == 127:
print "Pass"
else:
print "Failed test"
status = execute(app, "/bin/sleep 60", 1, None)
if status == 127:
print "Pass"
else:
print "Failed test"
status = execute(app, '/bin/echo "Hello World"; sleep 8', 10, None)
if status == 0:
print "Pass"
else:
print "Failed test"
cmd = {
"job_id":
123123,
"executable":
"aws",
"args":
"s3 cp {0} {1}".format("./dummy50m",
"s3://klab-jobs/yadu/data/dummy50m")
}
print execute_wait(app, cmd, 50, "asdsada")
cmd = {
"job_id":
123123,
"executable":
"aws",
"args":
"s3 cp {0} {1}".format("./dummy500m",
"s3://klab-jobs/yadu/data/dummy500m")
}
print execute_wait(app, cmd, 50, "asdsada")
cmd = {
"job_id":
123123,
"executable":
"aws",
"args":
"s3 cp {0} {1}".format("./dummy1g", "s3://klab-jobs/yadu/data/dummy1g")
}
print execute_wait(app, cmd, 50, "asdsada")
cmd = {
"job_id":
123123,
"executable":
"aws",
"args":
"s3 cp {0} {1}".format("./shuf.txt",
"s3://klab-jobs/yadu/data/dummy1g")
}
print execute_wait(app, cmd, 50, "asdsada")