def get(self): if users.is_current_user_capable("mr_api"): currentVal = memcache.get("currentVal") currentTask = memcache.get("currentTask") if currentVal is None: currentVal = "" currentTask = "" power = memcache.get("power-" + currentVal) result = None if currentTask == "runMRJob": result = memcache.get("result-" + currentVal) elif currentTask == "timing": result = memcache.get("timing-" + currentVal) if power is None: power = "(Not run yet)" if result is None: result = "No result data yet." else: power = "" result = """You are not authorized to run MapReduce jobs. Please get authorization on your user account for MapReduce jobs and try again. """ self.response.out.write(template.render('mapreduce.html', {'power': power, 'result': result}))
def runMRJob(mapper, reducer, inputLoc, outputLoc, config={}): if users.is_current_user_capable("mr_api") == False: return "this user cannot call the mapreduce api" mapper = urllib.unquote(mapper) reducer = urllib.unquote(reducer) inputLoc = urllib.unquote(inputLoc) outputLoc = urllib.unquote(outputLoc) regex = r"[^\w\d/\.-]" pattern = re.compile(regex) mydir = os.getcwd() + "/" mapper = "\"" + getLang(mapper) + " " + mydir + pattern.sub('', mapper) + "\"" reducer = "\"" + getLang(reducer) + " " + mydir + pattern.sub('', reducer) + "\"" inputLoc = pattern.sub('', inputLoc) outputLoc = pattern.sub('', outputLoc) removeOutput = HADOOP_BIN + " fs -rmr " + outputLoc sys.stderr.write(removeOutput + "\n") os.system(removeOutput) formattedConfig = "" for key in config: formattedConfig = formattedConfig + " -D " + key + "=" + config[key] command = HADOOP_BIN + " jar " + HADOOP_STREAMING +" " + formattedConfig + " -input " + inputLoc + " -output " + outputLoc + " -mapper " + mapper + " -reducer " + reducer sys.stderr.write("\n" + command + "\n") start = time.time() os.system(command) end = time.time() sys.stderr.write("\nTime elapsed = " + str(end - start) + "seconds\n")
def ec2_delete_keypair(options = {}): if users.is_current_user_capable("ec2_api") == False: return "this user cannot call the ec2 api" errors = ec2_set_environment() if errors is not None: return errors options = sanitize(options) if "key" not in options: return "EC2_API Error: Options must have a 'key' field with the keys to remove" key = options["key"] if not isinstance(key, str): return "EC2_API Error: 'key' must be a string" cert, pk, ec2_url, s3_url, ec2_access_key, ec2_secret_key = get_credentials() command = "EC2_CERT=" + cert + " EC2_PRIVATE_KEY=" + pk + " EC2_URL=" + ec2_url + " S3_URL=" + s3_url + " EC2_ACCESS_KEY=" + ec2_access_key + " EC2_SECRET_KEY=" + ec2_secret_key + " /usr/local/bin/euca-delete-keypair " for k, v in options.iteritems(): if k != "key": command = command + " -" + k + " " + v command = command + " " + key + " 2>&1" result = os.popen(command).read() return result
def get(self): if users.is_current_user_capable("mr_api"): currentVal = memcache.get("currentVal") currentTask = memcache.get("currentTask") if currentVal is None: currentVal = "" currentTask = "" power = memcache.get("power-" + currentVal) result = None if currentTask == "runMRJob": result = memcache.get("result-" + currentVal) elif currentTask == "timing": result = memcache.get("timing-" + currentVal) if power is None: power = "(Not run yet)" if result is None: result = "No result data yet." else: power = "" result = """You are not authorized to run MapReduce jobs. Please get authorization on your user account for MapReduce jobs and try again. """ self.response.out.write( template.render('mapreduce.html', { 'power': power, 'result': result }))
def remove_ec2_creds(): if users.is_current_user_capable("ec2_api") == False: return "this user cannot call the ec2 api" user = get_current_user() if not user: raise NameError('Cannot call remove_ec2_creds while not logged in.') cred_dir = "/tmp/ec2/" + user.nickname() os.system("rm -rf " + cred_dir)
def getNumOfNodes(): if users.is_current_user_capable("mr_api") == False: return "this user cannot call the mapreduce api" num_of_nodes = 0 fileLoc = APPSCALE_HOME + "/.appscale/num_of_nodes" if os.path.exists(fileLoc): f = open(fileLoc) num_of_nodes = int(f.read()) return num_of_nodes
def getAllIPs(): if users.is_current_user_capable("mr_api") == False: return "this user cannot call the mapreduce api" all_ips = [] fileLoc = APPSCALE_HOME + "/.appscale/all_ips" if os.path.exists(fileLoc): f = open(fileLoc) text = f.read() all_ips = text.split("\n") return all_ips
def write_neptune_job_params(params): if users.is_current_user_capable("neptune_api") == False: return "this user cannot call the neptune api" neptune_code = "puts neptune(:boo => 2, \n" for k, v in params.iteritems(): neptune_code += " " + str(k) + " => " + str(v) + ",\n" neptune_code += " :baz => 2).inspect" return write_neptune_job_code(neptune_code)
def ensure_user_is_ec2_authorized(): """ Uses the Users API to check the currently logged in user's authorizations, raising an exception if they are not authorized to use the EC2 API. Raises: EC2Exception: If the currently logged in user is not authorized to use the EC2 API. Returns: None: If the currently logged in user is authorized to use the EC2 API. """ if not users.is_current_user_capable("ec2_api"): raise EC2Exception("this user cannot call the ec2 api")
def writeTempFile(suffix, data): if users.is_current_user_capable("mr_api") == False: return "this user cannot call the mapreduce api" suffix = urllib.unquote(suffix) regex = r"[^\w\d/\.-]" pattern = re.compile(regex) suffix = pattern.sub('', suffix) fileLoc = "/tmp/" + suffix f = open(fileLoc, "w+") f.write(data) f.close() return fileLoc
def ensure_user_is_mapreduce_authorized(): """ Uses the Users API to check the currently logged in user's authorizations, raising an exception if they are not authorized to use the AppScale MapReduce API. Raises: MapReduceException: If the currently logged in user is not authorized to use the AppScale MapReduce API. Returns: None: If the currently logged in user is authorized to use the AppScale MapReduce API. """ if not users.is_current_user_capable("mapreduce_api"): raise MapReduceException("this user cannot call the mapreduce api")
def write_neptune_job_code(code): if users.is_current_user_capable("neptune_api") == False: return "this user cannot call the neptune api" neptune_dir = "/tmp/neptune/" mkdir_p = "mkdir -p " + neptune_dir os.system(mkdir_p) file_location = neptune_dir + "job-" + str(random.randint(0,1000000)) + ".rb" f = open(file_location, "w+") f.write(code) f.close() sys.stderr.write("Wrote Neptune code to " + file_location) return file_location
def run_neptune_job(file_location): if users.is_current_user_capable("neptune_api") == False: return "this user cannot call the neptune api" file_location = urllib.unquote(file_location) regex = r"[^\w\d/\.-]" pattern = re.compile(regex) file_location = pattern.sub('', file_location) result = "file not found" if os.path.exists(file_location): cmd = "neptune " + file_location result = os.popen(cmd).read() sys.stderr.write(result) return result
def get_ec2_secret_key(): if users.is_current_user_capable("ec2_api") == False: return "this user cannot call the ec2 api" user = get_current_user() if not user: raise NameError('Cannot call get_ec2_secret_key while not logged in.') ec2_secret_key_loc = "/tmp/ec2/" + user.nickname() + "/ec2secretkey" contents = None if os.path.exists(ec2_secret_key_loc): f = open(ec2_secret_key_loc, "r") contents = f.read() f.close() return contents
def ec2_describe_instances(options = {}): if users.is_current_user_capable("ec2_api") == False: return "this user cannot call the ec2 api" errors = ec2_set_environment() if errors is not None: return errors options = sanitize(options) cert, pk, ec2_url, s3_url, ec2_access_key, ec2_secret_key = get_credentials() command = "EC2_CERT=" + cert + " EC2_PRIVATE_KEY=" + pk + " EC2_URL=" + ec2_url + " S3_URL=" + s3_url + " EC2_ACCESS_KEY=" + ec2_access_key + " EC2_SECRET_KEY=" + ec2_secret_key + " /usr/local/bin/euca-describe-instances " for k, v in options.iteritems(): command = command + " -" + k + " " + v command = command + " 2>&1" result = os.popen(command).read() return result
def get_credentials(): if users.is_current_user_capable("ec2_api") == False: return "this user cannot call the ec2 api" user = get_current_user() if not user: raise NameError("Cannot call get_credentials while not logged in") cred_dir = "/tmp/ec2/" + user.nickname() cert = cred_dir + "/cert.pem" pk = cred_dir + "/pk.pem" ec2_url = get_ec2_url() s3_url = get_s3_url() ec2_access_key = get_ec2_access_key() ec2_secret_key = get_ec2_secret_key() return (cert, pk, ec2_url, s3_url, ec2_access_key, ec2_secret_key)
def get(self): user = users.get_current_user() if user is not None: response = "You are logged in as " + user.nickname() + ".<br />" response += "Is that <a href='" + users.create_logout_url("/") + "'>not you?</a><br /><br />" else: response = "You are not logged in yet.<br />" response += "Would you <a href='" + users.create_login_url("/") + "'>like to?</a><br /><br />" if users.is_current_user_capable("ec2_api"): response += "You are authorized to use the EC2 API." else: response += "You are NOT authorized to use the EC2 API." self.response.out.write(template.render('index.html', {'tool': 'Welcome!', 'result': response}))
def ec2_run_instances(options = {}): if users.is_current_user_capable("ec2_api") == False: return "this user cannot call the ec2 api" errors = ec2_set_environment() if errors is not None: return errors options = sanitize(options) if "machine" not in options: return "EC2_API Error: Options must have a 'machine' field with the AMI to spawn" cert, pk, ec2_url, s3_url, ec2_access_key, ec2_secret_key = get_credentials() command = "EC2_CERT=" + cert + " EC2_PRIVATE_KEY=" + pk + " EC2_URL=" + ec2_url + " S3_URL=" + s3_url + " EC2_ACCESS_KEY=" + ec2_access_key + " EC2_SECRET_KEY=" + ec2_secret_key + " /usr/local/bin/euca-run-instances " + options['machine'] for k, v in options.iteritems(): if k != "machine": command = command + " -" + k + " " + v command = command + " 2>&1" result = os.popen(command).read() return result
def ec2_terminate_instances(options = {}): if users.is_current_user_capable("ec2_api") == False: return "this user cannot call the ec2 api" errors = ec2_set_environment() if errors is not None: return errors options = sanitize(options) if "ids" not in options: return "EC2_API Error: Options must have a 'ids' field with the IDs to kill" ids = options['ids'] # later, make sure that ids is an array cert, pk, ec2_url, s3_url, ec2_access_key, ec2_secret_key = get_credentials() command = "EC2_CERT=" + cert + " EC2_PRIVATE_KEY=" + pk + " EC2_URL=" + ec2_url + " S3_URL=" + s3_url + " EC2_ACCESS_KEY=" + ec2_access_key + " EC2_SECRET_KEY=" + ec2_secret_key + " /usr/local/bin/euca-terminate-instances " + " ".join(ids) for k, v in options.iteritems(): if k != "ids": command = command + " -" + k + " " + v command = command + " 2>&1" result = os.popen(command).read() return result
def putMRInput(data, inputLoc): if users.is_current_user_capable("mr_api") == False: return "this user cannot call the mapreduce api" inputLoc = urllib.unquote(inputLoc) regex = r"[^\w\d/\.-]" pattern = re.compile(regex) inputLoc = pattern.sub('', inputLoc) fileLoc = "/tmp/" + inputLoc f = open(fileLoc, "w+") f.write(data) f.close() removeInput = HADOOP_BIN + " fs -rmr " + inputLoc sys.stderr.write(removeInput + "\n") os.system(removeInput) put = HADOOP_BIN + " fs -put " + fileLoc + " " + inputLoc os.system(put) return
def write_ec2_creds(cert, pk, ec2_url, s3_url, ec2_access_key, ec2_secret_key): if users.is_current_user_capable("ec2_api") == False: return "this user cannot call the ec2 api" user = get_current_user() if not user: raise NameError('Cannot call write_ec2_creds while not logged in.') cred_dir = "/tmp/ec2/" + user.nickname() os.system("mkdir -p " + cred_dir) ec2_cert_loc = cred_dir + "/cert.pem" f = open(ec2_cert_loc, "w+") f.write(cert) f.close() ec2_pk_loc = cred_dir + "/pk.pem" f = open(ec2_pk_loc, "w+") f.write(pk) f.close() ec2_url_loc = cred_dir + "/ec2url" f = open(ec2_url_loc, "w+") f.write(ec2_url) f.close() s3_url_loc = cred_dir + "/s3url" f = open(s3_url_loc, "w+") f.write(s3_url) f.close() ec2_access_key_loc = cred_dir + "/ec2accesskey" f = open(ec2_access_key_loc, "w+") f.write(ec2_access_key) f.close() ec2_secret_key_loc = cred_dir + "/ec2secretkey" f = open(ec2_secret_key_loc, "w+") f.write(ec2_secret_key) f.close()
def getMRLogs(outputLoc): if users.is_current_user_capable("mr_api") == False: return "this user cannot call the mapreduce api" outputLoc = urllib.unquote(outputLoc) regex = r"[^\w\d/\.-]" pattern = re.compile(regex) outputLoc = pattern.sub('', outputLoc) fileLoc = "/tmp/" + outputLoc rmr = "rm -rf " + fileLoc os.system(rmr) get = APPSCALE_HOME + "/AppDB/hadoop-0.20.2/bin/hadoop fs -get " + outputLoc + " " + fileLoc os.system(get) contents = "no logs" if os.path.exists(fileLoc): cmd = "cat " + fileLoc + "/_logs/history/*" contents = os.popen(cmd).read() sys.stderr.write(contents) return contents