def yourFunction(request, context): dbEndpoint = os.getenv('databaseEndpoint') if 'dbEndpoint' in request: dbEndpoint = request['dbEndpoint'] dbName = os.getenv('databaseName') if 'dbName' in request: dbName = request['dbName'] key = str(request['key']) bucketname = str(request['bucketname']) tablename = str(request['tablename']) batchSize = request['batchSize'] # import the module and collect data inspector = Inspector() inspector.inspectAll() inspector.addAttribute("endpoint", dbEndpoint) inspector.addAttribute("bucketname", bucketname) inspector.addAttribute("key", key) s3 = boto3.client('s3') csvfile = s3.get_object(Bucket=bucketname, Key=key) csvcontent = csvfile['Body'].read().split(b'\n') i = 0 for line in csvcontent: csvcontent[i] = line.decode("utf-8") i += 1 content = read_csv(csvcontent, inspector) write_output(content, batchSize, tablename, dbEndpoint, dbName, inspector) inspector.inspectAllDeltas() return inspector.finish()
def yourFunction(request): # Import the module and collect data inspector = Inspector() inspector.inspectAll() inspector.addTimeStamp("frameworkRuntime") calcs = request['calcs'] sleepTime = request['sleep'] loops = request['loops'] operand_a = [0] * calcs operand_b = [0] * calcs operand_c = [0] * calcs for k in range(0, loops): for i in range(0, calcs): j = randint(0, calcs) operand_a[j] = randint(0, 99999) operand_b[j] = randint(0, 99999) operand_c[j] = randint(0, 99999) mult = operand_a[j] * operand_b[j] div = mult / operand_c[j] inspector.inspectCPUDelta() return inspector.finish()
def yourFunction(request, context): # Import the module and collect data inspector = Inspector() inspector.inspectAll() inspector.addTimeStamp("frameworkRuntime") bucketname = str(request['bucketname']) key = str(request['key']) s3 = boto3.client('s3') csvfile = s3.get_object(Bucket=bucketname, Key=key) csvcontent = csvfile['Body'].read().split(b'\n') i = 0 for line in csvcontent: csvcontent[i] = line.decode("utf-8") i = i + 1 csv_data = csv.DictReader(csvcontent) #csv_input = pd.read_csv('input.csv') #csv_input['Berries'] = csv_input['Name'] #csv_input.to_csv('output.csv', index=False) #test_str=csv_data["Region"] test_val = "" # Add custom message and finish the function if ('key' in request): inspector.addAttribute( "bucketname", "bucketname " + str(request['bucketname']) + "!") inspector.addAttribute("key", str(request['key'])) inspector.addAttribute("test val", csvcontent[0]) inspector.inspectCPUDelta() return inspector.finish()
def yourFunction(request, context): dbEndpoint = os.getenv('databaseEndpoint') if 'dbEndpoint' in request: dbEndpoint = request['dbEndpoint'] dbName = os.getenv('databaseName') if 'dbName' in request: dbName = request['dbName'] key = str(request['key']) bucketname = str(request['bucketname']) tablename = str(request['tablename']) stressTestLoops = request['stressTestLoops'] # import the module and collect data inspector = Inspector() inspector.inspectAll() inspector.addAttribute("endpoint", dbEndpoint) inspector.addAttribute("bucketname", bucketname) inspector.addAttribute("key", key) s3 = boto3.client('s3') request['filterBy'] = dict() request['filterBy']["Region"] = ["Australia and Oceania"] request['filterBy']["Item Type"] = ["Office Supplies"] request['filterBy']["Sales Channel"] = ["Office Supplies"] request['filterBy']["Order Priority"] = ["Offline"] request['filterBy']["Country"] = ["Fiji"] request['aggregateBy'] = dict() request['aggregateBy']["max"] = ["Units Sold"] request['aggregateBy']["min"] = ["Units Sold"] request['aggregateBy']["avg"] = [ "Order Processing Time", "Gross Margin", "Units Sold" ] request['aggregateBy']["sum"] = [ "Units Sold", "Total Revenue", "Total Profit" ] query_string = contstruct_query_string(request['filterBy'], request['aggregateBy'], request['tablename']) query_result = exexute_query(query_string, dbEndpoint, dbName) stressTest(stressTestLoops, tablename, dbEndpoint, dbName) key_split = str(request['key']).split('_')[0] result_key = "{0}_results.csv".format(key_split) csv_content = convert_rs_to_csv(query_result) s3.put_object(Bucket=bucketname, Key=result_key, Body=(csv_content)) inspector.inspectAllDeltas() return inspector.finish()
def yourFunction(request, context): # Import the module and collect data inspector = Inspector() inspector.inspectAll() inspector.addTimeStamp("frameworkRuntime") bucketname = str(request['bucketname']) key = str(request['key']) s3 = boto3.client('s3') csvfile = s3.get_object(Bucket=bucketname, Key=key) csvcontent = csvfile['Body'].read().split(b'\n') i = 0 for line in csvcontent: csvcontent[i] = line.decode("utf-8") i = i + 1 csv_data = csv.DictReader(csvcontent) test_val = "" query_string = contstruct_query_string(request['filterBy'], request['aggregateBy'], request['tablename']) query_result = exexute_query(query_string) json_result = convert_query_to_json(query_result) csv_result = convert_json_csv(query_result) #dest_object_name = "newjson.txt" #my_bytes = bytes(json_result.encode('UTF-8')) #s3.put_object(Bucket=bucketname, Key=dest_object_name,Body=(my_bytes)) my_bytes = bytes(csv_result.getvalue()) key_split = str(request['key']).split('_')[0] result_key = "{0}_results.csv".format(key_split) s3.put_object(Bucket=bucketname, Key=result_key, Body=(my_bytes)) #bytes = csv_result.getvalue() # Add custom message and finish the function if ('key' in request): inspector.addAttribute( "bucketname", "bucketname " + str(request['bucketname']) + "!") inspector.addAttribute("key", str(request['key'])) inspector.addAttribute("test val", csvcontent[0]) inspector.inspectCPUDelta() return inspector.finish()
def yourFunction(request): # Import the module and collect data inspector = Inspector() inspector.inspectAll() inspector.addTimeStamp("frameworkRuntime") # Add custom message and finish the function inspector.addAttribute("message", "Hello " + request['name'] + "!") inspector.inspectCPUDelta() return inspector.finish()
def yourFunction(request, context): # Import the module and collect data inspector = Inspector() inspector.inspectAll() # Add custom message and finish the function if ('name' in request): inspector.addAttribute("message", "Pello " + str(request['name'])) else: inspector.addAttribute("message", "Pello World!") inspector.inspectAllDeltas() return inspector.finish()
def handler(event, context): s3 = boto3.client('s3') inspector = Inspector() inspector.inspectAll() inspector.addTimeStamp("FrameWorkRuntime") bucket = event.get("bucketname") key = event.get("filename") data_path = '/tmp/target.csv' db_path = '/tmp/' + key.split('.')[0] + '.db' s3.download_file(bucket, key, data_path) load.database_init(data_path, db_path, logger) s3.upload_file(db_path, bucket, 'target.db') inspector.inspectCPUDelta() inspector.addAttribute("DatabaseName", "target.db") return inspector.finish()
def handler(event, context): s3 = boto3.client('s3') inspector = Inspector() inspector.inspectAll() inspector.addTimeStamp("FrameWorkRuntime") service = event.get("service") bucket = event.get("bucketname") key = event.get("filename") if service == 1: filename = '/tmp/target.csv' processed_file = '/tmp/processed.csv' upload_key = 'transform.csv' s3.download_file(bucket, key, filename) processed_data = trans.process(filename) processed_data.to_csv(processed_file, index=False) s3.upload_file(processed_file, bucket, upload_key) inspector.addAttribute("numLine", processed_data.shape[0]) inspector.addAttribute("outputFile", upload_key) elif service == 2: logger = logging.getLogger() logger.setLevel(logging.INFO) data_path = '/tmp/target.csv' db_path = '/tmp/' + key.split('.')[0] + '.db' s3.download_file = 'transform.csv' load.database_init(data_path, db_path, logger) s3.upload_file(db_path, bucket, 'target.db') inspector.addAttribute("DatabaseName", "target.db") elif service == 3: where_statement = event.get("where") group_statement = event.get("group") db_path = '/tmp/target.db' if not os.path.isfile(db_path): s3.download_file(bucket, key, db_path) result, lines = query.search(db_path, where_statement, group_statement) inspector.addAttribute("data", result) inspector.addAttribute("numLine", lines) else: raise NameError("There is no such service") inspector.inspectCPUDelta() return inspector.finish()
def yourFunction(request, context): # Import the module and collect data inspector = Inspector() inspector.inspectAll() inspector.addTimeStamp("frameworkRuntime") bucketname = str(request['bucketname']) key = str(request['key']) s3 = boto3.client('s3') csvfile = s3.get_object(Bucket=bucketname, Key=key) csvcontent = csvfile['Body'].read().decode('utf-8').split("\n") test_val = "" content = read_csv(csvcontent) output = write_csv(content) bytes = output.getvalue() record_size = str(request['key']).split("_") dest_object_name = "{0}_newdata.csv".format(record_size[0]) s3.put_object(Bucket=bucketname, Key=dest_object_name, Body=bytes) # Add custom message and finish the function if ('key' in request): inspector.addAttribute( "bucketname", "bucketname " + str(request['bucketname']) + "!") inspector.addAttribute("key", str(request['key'])) inspector.addAttribute("test val", csvcontent[0]) inspector.inspectCPUDelta() return inspector.finish()
def handler(event, context): s3_client = boto3.client('s3') inspector = Inspector() inspector.inspectAll() inspector.addTimeStamp("FramworkRuntime") bucket = event.get("bucketname") key = event.get("filename") filename = '/tmp/target.csv' processed_file = '/tmp/processed.csv' upload_key = 'transform.csv' s3_client.download_file(bucket, key, filename) processed_data = trans.process(filename) processed_data.to_csv(processed_file, index=False) s3_client.upload_file(processed_file, bucket, upload_key) inspector.inspectCPUDelta() inspector.addAttribute("outputFile", upload_key) inspector.addAttribute("numLine", processed_data.shape[0]) return inspector.finish()
def handler(event, context): s3_client = boto3.client('s3') # import the module and collect data inspector = Inspector() inspector.inspectAll() inspector.addTimeStamp("frameworkRuntime") # detector bucket = event.get("bucketname") key = event.get("filename") #image = os.mkdir(os.path.join('/tmp', key)) image = '/tmp/target.jpg' print(image) s3_client.download_file(bucket, key, image) box, profile = dc.getBox(image) inspector.addAttribute("box", str(box)) inspector.addAttribute("Detector_profile", profile) inspector.inspectCPUDelta() return inspector.finish()
def yourFunction(request, context): key = str(request['key']) bucketname = str(request['bucketname']) # Import the module and collect data inspector = Inspector() inspector.inspectAll() inspector.addAttribute("bucketname", bucketname) inspector.addAttribute("key", key) s3 = boto3.client('s3') csvfile = s3.get_object(Bucket=bucketname, Key=key) csvcontent = csvfile['Body'].read().decode('utf-8').split("\n") content = read_csv(csvcontent) output = write_csv(content) _bytes = output.getvalue() record_size = key.split("_") dest_object_name = "edited_{0}_Sales_Records.csv".format(record_size[0]) s3.put_object(Bucket=bucketname, Key=dest_object_name, Body=_bytes) inspector.inspectAllDeltas() return inspector.finish()
printBanner = False Banner = "CMS Preliminary" verbose = False # check options option,args = parse(__doc__) if not args and not option: exit() if option.batch: ROOT.gROOT.SetBatch() if option.verbose: verbose = True if option.list: ins = Inspector.Inspector() ins.Verbose(True) ins.createXML(False) ins.SetFilename(option.list) ins.GetListObjects() sys.exit() if option.create: createXML = Inspector.Inspector() createXML.Verbose(False) createXML.createXML(True) if option.tag: createXML.SetTag(option.tag) createXML.SetFilename(option.create) createXML.GetListObjects() sys.exit()
def yourFunction(request, context): # Import the module and collect data inspector = Inspector() accessKey = request['accessKey'] secretKey = request['secretKey'] sessionToken = request['sessionToken'] jobID = request['jobID'] task = request['task'] arguments = request['arguments'] if ('episodes' in arguments): arguments['episodes'] += 1 instanceID = "" if ('instanceID' in arguments): instanceID = arguments['instanceID'] if (instanceID is not None): jobID += str(instanceID) if ('gitHubURL' not in arguments): arguments['gitHubURL'] = githubDefaultRepo arguments['gitHubBranch'] = githubDefaultBranch continuousTraining = False if ("continuousTraining" in arguments): continuousTraining = arguments["continuousTraining"] modelName = "model.bin" botoSession = boto3.Session( aws_access_key_id=accessKey, aws_secret_access_key=secretKey, aws_session_token=sessionToken, region_name=awsRegion ) inspector.addAttribute("instanceStateText", "Loading...") if 'instanceType' in arguments: try: inspector.addAttribute("cost", instanceInfo[arguments['instanceType']]['cost']) inspector.addAttribute("info", instanceInfo[arguments['instanceType']]) except: pass if (task == "poll"): ec2Client = botoSession.client('ec2') ec2Resource = botoSession.resource('ec2') s3Resource = botoSession.resource('s3') try: ourInstance = findOurInstance(ec2Client, jobID, inspector) inspector.addAttribute("validCredentials", 1) except: inspector.addAttribute("validCredentials", 0) return inspector.finish() if (ourInstance is None): createInstance(ec2Client, ec2Resource, jobID, arguments, inspector) inspector.addAttribute("message", "creating instance") inspector.addAttribute("instanceState", "booting") inspector.addAttribute("instanceStateText", "Booting") else: # Check if it is ready to SSH... try: ip = ourInstance['PublicIpAddress'] inspector.addAttribute("ip", ip) ssh = paramiko.SSHClient() ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) ssh.connect(ip, username='******', password='******') ssh_stdin, ssh_stdout, ssh_stderr = ssh.exec_command( "echo test") stdout = ssh_stdout.readlines() except: inspector.addAttribute( "error", "Problem creating ssh connection to " + str(ip) + " try again") return inspector.finish() if (stdout[0] == "test\n"): ssh_stdin, ssh_stdout, ssh_stderr = ssh.exec_command( "cat easyRL-v0/lambda/version_check1.txt") instanceData = ssh_stdout.readlines() # Has the version check? If not update if (instanceData == []): ssh_stdin, ssh_stdout, ssh_stderr = ssh.exec_command( "mv easyRL-v0/ OLD" + str(random.randint(1,10000000)) + "/") stdout = ssh_stdout.readlines() if (sessionToken == ""): ssh_stdin, ssh_stdout, ssh_stderr = ssh.exec_command( "sleep " + str(arguments['killTime']) + " && python3.7 easyRL-v0/lambda/killSelf.py " + jobID + " " + accessKey + " " + secretKey + " &") else: ssh_stdin, ssh_stdout, ssh_stderr = ssh.exec_command( "sleep " + str(arguments['killTime']) + " && python3.7 easyRL-v0/lambda/killSelf.py " + jobID + " " + accessKey + " " + secretKey + " " + sessionToken + " &") ssh_stdin, ssh_stdout, ssh_stderr = ssh.exec_command( "git clone --branch " + arguments['gitHubBranch'] + " " + arguments['gitHubURL']) stdout = ssh_stdout.readlines() # DO NOT REMOVE stderr = ssh_stderr.readlines() # DO NOT REMOVE ssh_stdin, ssh_stdout, ssh_stderr = ssh.exec_command( "echo " + arguments['instanceType'] + str(arguments['killTime']) + " > tag.txt") inspector.addAttribute("instanceState", "updated") inspector.addAttribute("instanceStateText", "Cloned Repository") else: # Instance type match the tag? If not reboot... ssh_stdin, ssh_stdout, ssh_stderr = ssh.exec_command( "cat tag.txt") instanceData = ssh_stdout.readlines() tag = arguments['instanceType'] + str(arguments['killTime']) if (instanceData == [] or tag not in instanceData[0]): terminateInstance( ec2Client, ec2Resource, ourInstance, inspector) createInstance(ec2Client, ec2Resource, jobID, arguments, inspector) try: bucket = s3Resource.Bucket('easyrl-' + jobID) bucket.objects.all().delete() except: pass inspector.addAttribute('instanceState', "rebooting") inspector.addAttribute("instanceStateText", "Recreating") else: # Is job running? If it is get progress. Else return idle. ssh_stdin, ssh_stdout, ssh_stderr = ssh.exec_command( "ps -aux | grep EasyRL.py") stdout = ssh_stdout.readlines() results = "" for line in stdout: results += line if ("terminal" in results): inspector.addAttribute( 'instanceState', "runningJob") inspector.addAttribute("instanceStateText", "Running Task") ssh_stdin, ssh_stdout, ssh_stderr = ssh.exec_command( "cat ./arguments.json") stdout = ssh_stdout.readlines() #inspector.addAttribute("Test", str(stdout)) #return inspctor.finish() if (stdout != []): jobArguments = json.loads(stdout[0]) inspector.addAttribute( "jobArguments", jobArguments) if continuousTraining and jobArguments != arguments: inspector.addAttribute('instanceState', "changingJob") inspector.addAttribute("instanceStateText", "Changing Task") task = "haltJob" ssh_stdin, ssh_stdout, ssh_stderr = ssh.exec_command( "cat ./data.json") stdout = ssh_stdout.readlines() if (stdout != []): try: inspector.addAttribute( "progress", json.loads(stdout[0])) except: inspector.addAttribute("progress", "waiting") else: inspector.addAttribute("progress", "waiting") else: inspector.addAttribute('instanceState', "idle") inspector.addAttribute("instanceStateText", "Idle") if continuousTraining: task = "runJob" inspector.addAttribute('instanceState', "startingJob") inspector.addAttribute("instanceStateText", "Starting Task") else: inspector.addAttribute('instanceState', "initializing") inspector.addAttribute("instanceStateText", "Initializing") ssh.close() if (task == "runJob"): ec2Client = botoSession.client('ec2') ec2Resource = botoSession.resource('ec2') s3Resource = botoSession.resource('s3') try: bucket = s3Resource.Bucket('easyrl-' + jobID) bucket.objects.all().delete() except: pass ourInstance = findOurInstance(ec2Client, jobID, inspector) if (ourInstance is not None): ip = ourInstance['PublicIpAddress'] inspector.addAttribute("ip", str(ip)) try: ssh = paramiko.SSHClient() ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) ssh.connect(ip, username='******', password='******') ssh_stdin, ssh_stdout, ssh_stderr = ssh.exec_command( "ps -aux | grep EasyRL.py") stdout = ssh_stdout.readlines() except: inspector.addAttribute( "error", "Problem creating ssh connection to " + str(ip) + " try again") return inspector.finish() results = "" for line in stdout: results += line if ("terminal" not in results): # Error Checking if (str(arguments['agent']) in paraMap): missingAttributes = [] outOfRange = [] valid = True envIndex = str(arguments['environment']) agentIndex = str(arguments['agent']) if envMap[envIndex]['type'] not in agentMap[agentIndex]['supportedEnvs']: inspector.addAttribute("error", "Incompatible agent/environment pair!") return inspector.finish() for pp in paraMap[str(arguments['agent'])]: pp = str(pp) if pp not in arguments: missingAttributes.append(pp) else: val = arguments[pp] if (val < paramConditions[pp]['min'] or val > paramConditions[pp]['max']): outOfRange.append(pp) if len(missingAttributes) > 0: inspector.addAttribute("error-Missing", "Missing hyperparameters for agent: " + str(missingAttributes)) valid = False if len(outOfRange) > 0: errorMessage = "Attributes with invalid value: " for error in outOfRange: errorMessage += error + " min: " + str(paramConditions[error]['min']) + " max: " + str(paramConditions[error]['max']) + " used: " + str(arguments[error]) + " " inspector.addAttribute("error-Range", errorMessage) valid = False if (valid == False): return inspector.finish() else: inspector.addAttribute("error", "Unknown Agent " + str(arguments['agent'])) return inspector.finish() ssh_stdin, ssh_stdout, ssh_stderr = ssh.exec_command( "echo \'" + json.dumps(arguments) + "\' > arguments.json") stdout = ssh_stdout.readlines() command = 'printf "' command += str(arguments['environment']) + '\n' command += str(arguments['agent']) + '\n' command += '1\n' paramList = paraMap[str(arguments['agent'])] for param in paramList: command += str(arguments[param]) + '\n' command += '4\n' command += modelName + '\n' command += '5\n' if (sessionToken != ""): command += '" | python3.7 ./easyRL-v0/EasyRL.py --terminal --secretKey ' + secretKey + \ ' --accessKey ' + accessKey + ' --sessionToken ' + \ sessionToken + ' --jobID ' + jobID else: command += '" | python3.7 ./easyRL-v0/EasyRL.py --terminal --secretKey ' + \ secretKey + ' --accessKey ' + accessKey + ' --jobID ' + jobID command += ' &> lastJobLog.txt & sleep 1' #inspector.addAttribute("command", command) ssh_stdin, ssh_stdout, ssh_stderr = ssh.exec_command(command) stdout = ssh_stdout.readlines() #inspector.addAttribute("stdout", stdout) ssh.close() inspector.addAttribute("message", "Job started") else: inspector.addAttribute("message", "Job already running") else: inspector.addAttribute('error', 'Instance not found.') if (task == "runTest"): ec2Client = botoSession.client('ec2') ec2Resource = botoSession.resource('ec2') s3Resource = botoSession.resource('s3') try: bucket = s3Resource.Bucket('easyrl-' + jobID) bucket.objects.all().delete() except: pass ourInstance = findOurInstance(ec2Client, jobID, inspector) if (ourInstance is not None): ip = ourInstance['PublicIpAddress'] inspector.addAttribute("ip", str(ip)) try: ssh = paramiko.SSHClient() ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) ssh.connect(ip, username='******', password='******') ssh_stdin, ssh_stdout, ssh_stderr = ssh.exec_command( "ps -aux | grep EasyRL.py") stdout = ssh_stdout.readlines() except: inspector.addAttribute( "error", "Problem creating ssh connection to " + str(ip) + " try again") return inspector.finish() results = "" for line in stdout: results += line if ("terminal" not in results): # Error Checking if (str(arguments['agent']) in paraMap): missingAttributes = [] outOfRange = [] valid = True envIndex = str(arguments['environment']) agentIndex = str(arguments['agent']) if envMap[envIndex]['type'] not in agentMap[agentIndex]['supportedEnvs']: inspector.addAttribute("error", "Incompatible agent/environment pair!") return inspector.finish() for pp in paraMap[str(arguments['agent'])]: pp = str(pp) if pp not in arguments: missingAttributes.append(pp) else: val = arguments[pp] if (val < paramConditions[pp]['min'] or val > paramConditions[pp]['max']): outOfRange.append(pp) if len(missingAttributes) > 0: inspector.addAttribute("error-Missing", "Missing hyperparameters for agent: " + str(missingAttributes)) valid = False if len(outOfRange) > 0: errorMessage = "Attributes with invalid value: " for error in outOfRange: errorMessage += error + " min: " + str(paramConditions[error]['min']) + " max: " + str(paramConditions[error]['max']) + " used: " + str(arguments[error]) + " " inspector.addAttribute("error-Range", errorMessage) valid = False if (valid == False): return inspector.finish() else: inspector.addAttribute("error", "Unknown Agent " + str(arguments['agent'])) return inspector.finish() ssh_stdin, ssh_stdout, ssh_stderr = ssh.exec_command( "echo \'" + json.dumps(arguments) + "\' > arguments.json") stdout = ssh_stdout.readlines() ssh_stdin, ssh_stdout, ssh_stderr = ssh.exec_command( "md5sum " + modelName) instanceData = ssh_stdout.readlines() # Has the tag? If not update if (instanceData != []): command = 'printf "' command += str(arguments['environment']) + '\n' command += str(arguments['agent']) + '\n' command += '2\n' command += modelName + '\n' command += '3\n' paramList = paraMap[str(arguments['agent'])] for param in paramList: command += str(arguments[param]) + '\n' command += '5\n' if (sessionToken != ""): command += '" | python3.7 ./easyRL-v0/EasyRL.py --terminal --secretKey ' + secretKey + \ ' --accessKey ' + accessKey + ' --sessionToken ' + \ sessionToken + ' --jobID ' + jobID else: command += '" | python3.7 ./easyRL-v0/EasyRL.py --terminal --secretKey ' + \ secretKey + ' --accessKey ' + accessKey + ' --jobID ' + jobID command += ' &> lastJobLog.txt & sleep 1' #inspector.addAttribute("command", command) ssh_stdin, ssh_stdout, ssh_stderr = ssh.exec_command(command) stdout = ssh_stdout.readlines() #inspector.addAttribute("stdout", stdout) ssh.close() inspector.addAttribute("message", "Test started") else: ssh.close() inspector.addAttribute("error", "No trained agent found") else: inspector.addAttribute("message", "Test already running") else: inspector.addAttribute('error', 'Instance not found.') if (task == "haltJob"): ec2Client = botoSession.client('ec2') ec2Resource = botoSession.resource('ec2') ourInstance = findOurInstance(ec2Client, jobID, inspector) if (ourInstance is not None): ip = ourInstance['PublicIpAddress'] #inspector.addAttribute("ip", str(ip)) ssh = paramiko.SSHClient() ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) ssh.connect(ip, username='******', password='******') command = "pkill python3.7" #inspector.addAttribute("command", command) ssh_stdin, ssh_stdout, ssh_stderr = ssh.exec_command(command) stdout = ssh_stdout.readlines() #inspector.addAttribute("stdout", stdout) ssh.close() inspector.addAttribute("message", "Job halted.") else: inspector.addAttribute("error", "Instance not found.") if (task == "exportModel"): ec2Client = botoSession.client('ec2') ec2Resource = botoSession.resource('ec2') ourInstance = findOurInstance(ec2Client, jobID, inspector) if (ourInstance is not None): ip = ourInstance['PublicIpAddress'] #inspector.addAttribute("ip", str(ip)) ssh = paramiko.SSHClient() ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) ssh.connect(ip, username='******', password='******') ssh_stdin, ssh_stdout, ssh_stderr = ssh.exec_command( "md5sum " + modelName) instanceData = ssh_stdout.readlines() # Has the tag? If not update if (instanceData != []): if (sessionToken == ""): command = "python3.7 easyRL-v0/lambda/upload.py " + modelName + " " + jobID + " " + accessKey + " " + secretKey else: command = "python3.7 easyRL-v0/lambda/upload.py " + modelName + " " + jobID + " " + accessKey + " " + secretKey + " " + sessionToken ssh_stdin, ssh_stdout, ssh_stderr = ssh.exec_command(command) stdout = ssh_stdout.readlines() inspector.addAttribute("url", "https://easyrl-" + str(jobID) + ".s3.amazonaws.com/" + modelName) else: inspector.addAttribute("error", "Model not trained yet!") ssh.close() else: inspector.addAttribute("error", "Instance not found.") if (task == "import"): ec2Client = botoSession.client('ec2') ec2Resource = botoSession.resource('ec2') ourInstance = findOurInstance(ec2Client, jobID, inspector) if (ourInstance is not None): ip = ourInstance['PublicIpAddress'] ssh = paramiko.SSHClient() ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) ssh.connect(ip, username='******', password='******') if (sessionToken == ""): command = "python3.7 easyRL-v0/lambda/download.py " + modelName + " " + jobID + " " + accessKey + " " + secretKey else: command = "python3.7 easyRL-v0/lambda/download.py " + modelName + " " + jobID + " " + accessKey + " " + secretKey + " " + sessionToken ssh_stdin, ssh_stdout, ssh_stderr = ssh.exec_command(command) stdout = ssh_stdout.readlines() inspector.addAttribute("error", stdout) ssh.close() else: inspector.addAttribute("error", "Instance not found.") if (task == "jobLog"): ec2Client = botoSession.client('ec2') ec2Resource = botoSession.resource('ec2') ourInstance = findOurInstance(ec2Client, jobID, inspector) if (ourInstance is not None): ip = ourInstance['PublicIpAddress'] #inspector.addAttribute("ip", str(ip)) ssh = paramiko.SSHClient() ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) ssh.connect(ip, username='******', password='******') if (sessionToken == ""): command = "python3.7 easyRL-v0/lambda/upload.py lastJobLog.txt " + jobID + " " + accessKey + " " + secretKey else: command = "python3.7 easyRL-v0/lambda/upload.py lastJobLog.txt " + jobID + " " + accessKey + " " + secretKey + " " + sessionToken ssh_stdin, ssh_stdout, ssh_stderr = ssh.exec_command(command) stdout = ssh_stdout.readlines() inspector.addAttribute("url", "https://easyrl-" + str(jobID) + ".s3.amazonaws.com/lastJobLog.txt") ssh.close() else: inspector.addAttribute("error", "Instance not found.") if (task == "terminateInstance"): ec2Client = botoSession.client('ec2') ec2Resource = botoSession.resource('ec2') s3Resource = botoSession.resource('s3') try: bucket = s3Resource.Bucket('easyrl-' + jobID) bucket.objects.all().delete() except: pass ourInstance = findOurInstance(ec2Client, jobID, inspector) terminateInstance(ec2Client, ec2Resource, ourInstance, inspector) if (task == "info"): inspector.addAttribute("environments", envList) inspector.addAttribute("environmentsMap", envMap) inspector.addAttribute("parameters", paramConditions) combinedAgents = [] for agent in agentList: agent['parameters'] = paraMap[agent['index']] combinedAgents.append(agent) combinedAgentsMap = {} for aa in combinedAgents: combinedAgentsMap[aa['index']] = aa inspector.addAttribute("agents", combinedAgents) inspector.addAttribute("agentsMap", combinedAgentsMap) return inspector.finish()