def get(self):
if users.is_current_user_capable("mr_api"):
currentVal = memcache.get("currentVal")
currentTask = memcache.get("currentTask")
if currentVal is None:
currentVal = ""
currentTask = ""
power = memcache.get("power-" + currentVal)
result = None
if currentTask == "runMRJob":
result = memcache.get("result-" + currentVal)
elif currentTask == "timing":
result = memcache.get("timing-" + currentVal)
if power is None:
power = "(Not run yet)"
if result is None:
result = "No result data yet."
else:
power = ""
result = """You are not authorized to run MapReduce jobs. Please get
authorization on your user account for MapReduce jobs and try again.
"""
self.response.out.write(template.render('mapreduce.html',
{'power': power,
'result': result}))
def runMRJob(mapper, reducer, inputLoc, outputLoc, config={}):
if users.is_current_user_capable("mr_api") == False:
return "this user cannot call the mapreduce api"
mapper = urllib.unquote(mapper)
reducer = urllib.unquote(reducer)
inputLoc = urllib.unquote(inputLoc)
outputLoc = urllib.unquote(outputLoc)
regex = r"[^\w\d/\.-]"
pattern = re.compile(regex)
mydir = os.getcwd() + "/"
mapper = "\"" + getLang(mapper) + " " + mydir + pattern.sub('', mapper) + "\""
reducer = "\"" + getLang(reducer) + " " + mydir + pattern.sub('', reducer) + "\""
inputLoc = pattern.sub('', inputLoc)
outputLoc = pattern.sub('', outputLoc)
removeOutput = HADOOP_BIN + " fs -rmr " + outputLoc
sys.stderr.write(removeOutput + "\n")
os.system(removeOutput)
formattedConfig = ""
for key in config:
formattedConfig = formattedConfig + " -D " + key + "=" + config[key]
command = HADOOP_BIN + " jar " + HADOOP_STREAMING +" " + formattedConfig + " -input " + inputLoc + " -output " + outputLoc + " -mapper " + mapper + " -reducer " + reducer
sys.stderr.write("\n" + command + "\n")
start = time.time()
os.system(command)
end = time.time()
sys.stderr.write("\nTime elapsed = " + str(end - start) + "seconds\n")
def ec2_delete_keypair(options = {}):
if users.is_current_user_capable("ec2_api") == False:
return "this user cannot call the ec2 api"
errors = ec2_set_environment()
if errors is not None:
return errors
options = sanitize(options)
if "key" not in options:
return "EC2_API Error: Options must have a 'key' field with the keys to remove"
key = options["key"]
if not isinstance(key, str):
return "EC2_API Error: 'key' must be a string"
cert, pk, ec2_url, s3_url, ec2_access_key, ec2_secret_key = get_credentials()
command = "EC2_CERT=" + cert + " EC2_PRIVATE_KEY=" + pk + " EC2_URL=" + ec2_url + " S3_URL=" + s3_url + " EC2_ACCESS_KEY=" + ec2_access_key + " EC2_SECRET_KEY=" + ec2_secret_key + " /usr/local/bin/euca-delete-keypair "
for k, v in options.iteritems():
if k != "key":
command = command + " -" + k + " " + v
command = command + " " + key + " 2>&1"
result = os.popen(command).read()
return result
def get(self):
if users.is_current_user_capable("mr_api"):
currentVal = memcache.get("currentVal")
currentTask = memcache.get("currentTask")
if currentVal is None:
currentVal = ""
currentTask = ""
power = memcache.get("power-" + currentVal)
result = None
if currentTask == "runMRJob":
result = memcache.get("result-" + currentVal)
elif currentTask == "timing":
result = memcache.get("timing-" + currentVal)
if power is None:
power = "(Not run yet)"
if result is None:
result = "No result data yet."
else:
power = ""
result = """You are not authorized to run MapReduce jobs. Please get
authorization on your user account for MapReduce jobs and try again.
"""
self.response.out.write(
template.render('mapreduce.html', {
'power': power,
'result': result
}))
def remove_ec2_creds():
if users.is_current_user_capable("ec2_api") == False:
return "this user cannot call the ec2 api"
user = get_current_user()
if not user:
raise NameError('Cannot call remove_ec2_creds while not logged in.')
cred_dir = "/tmp/ec2/" + user.nickname()
os.system("rm -rf " + cred_dir)
def getNumOfNodes():
if users.is_current_user_capable("mr_api") == False:
return "this user cannot call the mapreduce api"
num_of_nodes = 0
fileLoc = APPSCALE_HOME + "/.appscale/num_of_nodes"
if os.path.exists(fileLoc):
f = open(fileLoc)
num_of_nodes = int(f.read())
return num_of_nodes
def getAllIPs():
if users.is_current_user_capable("mr_api") == False:
return "this user cannot call the mapreduce api"
all_ips = []
fileLoc = APPSCALE_HOME + "/.appscale/all_ips"
if os.path.exists(fileLoc):
f = open(fileLoc)
text = f.read()
all_ips = text.split("\n")
return all_ips
def write_neptune_job_params(params):
if users.is_current_user_capable("neptune_api") == False:
return "this user cannot call the neptune api"
neptune_code = "puts neptune(:boo => 2, \n"
for k, v in params.iteritems():
neptune_code += " " + str(k) + " => " + str(v) + ",\n"
neptune_code += " :baz => 2).inspect"
return write_neptune_job_code(neptune_code)
def ensure_user_is_ec2_authorized():
"""
Uses the Users API to check the currently logged in user's authorizations,
raising an exception if they are not authorized to use the EC2 API.
Raises:
EC2Exception: If the currently logged in user is not authorized to use the
EC2 API.
Returns:
None: If the currently logged in user is authorized to use the EC2 API.
"""
if not users.is_current_user_capable("ec2_api"):
raise EC2Exception("this user cannot call the ec2 api")
def ensure_user_is_ec2_authorized():
"""
Uses the Users API to check the currently logged in user's authorizations,
raising an exception if they are not authorized to use the EC2 API.
Raises:
EC2Exception: If the currently logged in user is not authorized to use the
EC2 API.
Returns:
None: If the currently logged in user is authorized to use the EC2 API.
"""
if not users.is_current_user_capable("ec2_api"):
raise EC2Exception("this user cannot call the ec2 api")
def writeTempFile(suffix, data):
if users.is_current_user_capable("mr_api") == False:
return "this user cannot call the mapreduce api"
suffix = urllib.unquote(suffix)
regex = r"[^\w\d/\.-]"
pattern = re.compile(regex)
suffix = pattern.sub('', suffix)
fileLoc = "/tmp/" + suffix
f = open(fileLoc, "w+")
f.write(data)
f.close()
return fileLoc
def ensure_user_is_mapreduce_authorized():
"""
Uses the Users API to check the currently logged in user's authorizations,
raising an exception if they are not authorized to use the AppScale
MapReduce API.
Raises:
MapReduceException: If the currently logged in user is not authorized to
use the AppScale MapReduce API.
Returns:
None: If the currently logged in user is authorized to use the AppScale
MapReduce API.
"""
if not users.is_current_user_capable("mapreduce_api"):
raise MapReduceException("this user cannot call the mapreduce api")
def write_neptune_job_code(code):
if users.is_current_user_capable("neptune_api") == False:
return "this user cannot call the neptune api"
neptune_dir = "/tmp/neptune/"
mkdir_p = "mkdir -p " + neptune_dir
os.system(mkdir_p)
file_location = neptune_dir + "job-" + str(random.randint(0,1000000)) + ".rb"
f = open(file_location, "w+")
f.write(code)
f.close()
sys.stderr.write("Wrote Neptune code to " + file_location)
return file_location
def run_neptune_job(file_location):
if users.is_current_user_capable("neptune_api") == False:
return "this user cannot call the neptune api"
file_location = urllib.unquote(file_location)
regex = r"[^\w\d/\.-]"
pattern = re.compile(regex)
file_location = pattern.sub('', file_location)
result = "file not found"
if os.path.exists(file_location):
cmd = "neptune " + file_location
result = os.popen(cmd).read()
sys.stderr.write(result)
return result
def ensure_user_is_mapreduce_authorized():
"""
Uses the Users API to check the currently logged in user's authorizations,
raising an exception if they are not authorized to use the AppScale
MapReduce API.
Raises:
MapReduceException: If the currently logged in user is not authorized to
use the AppScale MapReduce API.
Returns:
None: If the currently logged in user is authorized to use the AppScale
MapReduce API.
"""
if not users.is_current_user_capable("mapreduce_api"):
raise MapReduceException("this user cannot call the mapreduce api")
def get_ec2_secret_key():
if users.is_current_user_capable("ec2_api") == False:
return "this user cannot call the ec2 api"
user = get_current_user()
if not user:
raise NameError('Cannot call get_ec2_secret_key while not logged in.')
ec2_secret_key_loc = "/tmp/ec2/" + user.nickname() + "/ec2secretkey"
contents = None
if os.path.exists(ec2_secret_key_loc):
f = open(ec2_secret_key_loc, "r")
contents = f.read()
f.close()
return contents
def ec2_describe_instances(options = {}):
if users.is_current_user_capable("ec2_api") == False:
return "this user cannot call the ec2 api"
errors = ec2_set_environment()
if errors is not None:
return errors
options = sanitize(options)
cert, pk, ec2_url, s3_url, ec2_access_key, ec2_secret_key = get_credentials()
command = "EC2_CERT=" + cert + " EC2_PRIVATE_KEY=" + pk + " EC2_URL=" + ec2_url + " S3_URL=" + s3_url + " EC2_ACCESS_KEY=" + ec2_access_key + " EC2_SECRET_KEY=" + ec2_secret_key + " /usr/local/bin/euca-describe-instances "
for k, v in options.iteritems():
command = command + " -" + k + " " + v
command = command + " 2>&1"
result = os.popen(command).read()
return result
def get_credentials():
if users.is_current_user_capable("ec2_api") == False:
return "this user cannot call the ec2 api"
user = get_current_user()
if not user:
raise NameError("Cannot call get_credentials while not logged in")
cred_dir = "/tmp/ec2/" + user.nickname()
cert = cred_dir + "/cert.pem"
pk = cred_dir + "/pk.pem"
ec2_url = get_ec2_url()
s3_url = get_s3_url()
ec2_access_key = get_ec2_access_key()
ec2_secret_key = get_ec2_secret_key()
return (cert, pk, ec2_url, s3_url, ec2_access_key, ec2_secret_key)
def get(self):
user = users.get_current_user()
if user is not None:
response = "You are logged in as " + user.nickname() + ".<br />"
response += "Is that <a href='" + users.create_logout_url("/") + "'>not you?</a><br /><br />"
else:
response = "You are not logged in yet.<br />"
response += "Would you <a href='" + users.create_login_url("/") + "'>like to?</a><br /><br />"
if users.is_current_user_capable("ec2_api"):
response += "You are authorized to use the EC2 API."
else:
response += "You are NOT authorized to use the EC2 API."
self.response.out.write(template.render('index.html',
{'tool': 'Welcome!',
'result': response}))
def ec2_run_instances(options = {}):
if users.is_current_user_capable("ec2_api") == False:
return "this user cannot call the ec2 api"
errors = ec2_set_environment()
if errors is not None:
return errors
options = sanitize(options)
if "machine" not in options:
return "EC2_API Error: Options must have a 'machine' field with the AMI to spawn"
cert, pk, ec2_url, s3_url, ec2_access_key, ec2_secret_key = get_credentials()
command = "EC2_CERT=" + cert + " EC2_PRIVATE_KEY=" + pk + " EC2_URL=" + ec2_url + " S3_URL=" + s3_url + " EC2_ACCESS_KEY=" + ec2_access_key + " EC2_SECRET_KEY=" + ec2_secret_key + " /usr/local/bin/euca-run-instances " + options['machine']
for k, v in options.iteritems():
if k != "machine":
command = command + " -" + k + " " + v
command = command + " 2>&1"
result = os.popen(command).read()
return result
def ec2_terminate_instances(options = {}):
if users.is_current_user_capable("ec2_api") == False:
return "this user cannot call the ec2 api"
errors = ec2_set_environment()
if errors is not None:
return errors
options = sanitize(options)
if "ids" not in options:
return "EC2_API Error: Options must have a 'ids' field with the IDs to kill"
ids = options['ids']
# later, make sure that ids is an array
cert, pk, ec2_url, s3_url, ec2_access_key, ec2_secret_key = get_credentials()
command = "EC2_CERT=" + cert + " EC2_PRIVATE_KEY=" + pk + " EC2_URL=" + ec2_url + " S3_URL=" + s3_url + " EC2_ACCESS_KEY=" + ec2_access_key + " EC2_SECRET_KEY=" + ec2_secret_key + " /usr/local/bin/euca-terminate-instances " + " ".join(ids)
for k, v in options.iteritems():
if k != "ids":
command = command + " -" + k + " " + v
command = command + " 2>&1"
result = os.popen(command).read()
return result
def putMRInput(data, inputLoc):
if users.is_current_user_capable("mr_api") == False:
return "this user cannot call the mapreduce api"
inputLoc = urllib.unquote(inputLoc)
regex = r"[^\w\d/\.-]"
pattern = re.compile(regex)
inputLoc = pattern.sub('', inputLoc)
fileLoc = "/tmp/" + inputLoc
f = open(fileLoc, "w+")
f.write(data)
f.close()
removeInput = HADOOP_BIN + " fs -rmr " + inputLoc
sys.stderr.write(removeInput + "\n")
os.system(removeInput)
put = HADOOP_BIN + " fs -put " + fileLoc + " " + inputLoc
os.system(put)
return
def write_ec2_creds(cert, pk, ec2_url, s3_url, ec2_access_key, ec2_secret_key):
if users.is_current_user_capable("ec2_api") == False:
return "this user cannot call the ec2 api"
user = get_current_user()
if not user:
raise NameError('Cannot call write_ec2_creds while not logged in.')
cred_dir = "/tmp/ec2/" + user.nickname()
os.system("mkdir -p " + cred_dir)
ec2_cert_loc = cred_dir + "/cert.pem"
f = open(ec2_cert_loc, "w+")
f.write(cert)
f.close()
ec2_pk_loc = cred_dir + "/pk.pem"
f = open(ec2_pk_loc, "w+")
f.write(pk)
f.close()
ec2_url_loc = cred_dir + "/ec2url"
f = open(ec2_url_loc, "w+")
f.write(ec2_url)
f.close()
s3_url_loc = cred_dir + "/s3url"
f = open(s3_url_loc, "w+")
f.write(s3_url)
f.close()
ec2_access_key_loc = cred_dir + "/ec2accesskey"
f = open(ec2_access_key_loc, "w+")
f.write(ec2_access_key)
f.close()
ec2_secret_key_loc = cred_dir + "/ec2secretkey"
f = open(ec2_secret_key_loc, "w+")
f.write(ec2_secret_key)
f.close()
def getMRLogs(outputLoc):
if users.is_current_user_capable("mr_api") == False:
return "this user cannot call the mapreduce api"
outputLoc = urllib.unquote(outputLoc)
regex = r"[^\w\d/\.-]"
pattern = re.compile(regex)
outputLoc = pattern.sub('', outputLoc)
fileLoc = "/tmp/" + outputLoc
rmr = "rm -rf " + fileLoc
os.system(rmr)
get = APPSCALE_HOME + "/AppDB/hadoop-0.20.2/bin/hadoop fs -get " + outputLoc + " " + fileLoc
os.system(get)
contents = "no logs"
if os.path.exists(fileLoc):
cmd = "cat " + fileLoc + "/_logs/history/*"
contents = os.popen(cmd).read()
sys.stderr.write(contents)
return contents
