def handler_ensemble(event, context): """ Newest version of handler that logs outputs to a subfolder of the result folder that is indexed by the job submission date and the submit name. Update 05/25: first check the config file to see if this is predict mode or train mode. """ for record in event['Records']: time = record['eventTime'] bucket_name = record['s3']['bucket']['name'] key = record['s3']['object']['key'] submit_file = utilsparams3.load_json(bucket_name, key) configpath = submit_file["configname"] try: configfile = utilsparams3.load_json(bucket_name, configpath) except ValueError: try: configfile = utilsparams3.load_yaml(bucket_name, configpath) except Exception: raise Exception("Config is not json or yaml.") print("Processing in {} mode".format(configfile["mode"])) if configfile["mode"] == "train": #print("handler_params",bucket_name,key,time) #print(event,context,'event, context') exitcode = process_upload_ensemble(bucket_name, key, time) print("processing returned exit code {}".format(exitcode)) else: exitcode = process_upload_dev(bucket_name, key, time) print("processing returned exit code {}".format(exitcode)) return exitcode
def parse_config(self): """ Parse the config file given for specific neurocaas parameters. In particular, the *duration* of the job, and the *dataset size* TODO: check for type in these configuration files. """ extension = os.path.splitext(self.config_name)[-1] if extension == ".json": passed_config = utilsparams3.load_json(self.bucket_name, self.config_name) elif extension == ".yaml": passed_config = utilsparams3.load_yaml(self.bucket_name, self.config_name) try: self.jobduration = passed_config["__duration__"] except KeyError: self.logger.append( "parameter __duration__ not given, proceeding with standard compute launch." ) self.logger.write() self.jobduration = None try: self.jobsize = passed_config["__dataset_size__"] except KeyError: self.logger.append( "parameter __dataset_size__ is not given, proceeding with standard compute launch." ) self.logger.write() self.jobsize = None
def parse_config(self): """Parse the config file for the number of entries to include in this ensemble in addition to other neurocaas parameters. Additionally writes a copy of the config for each copy with just the jobnumber changed. Internally, creates a dictionary self.confignames and changes the self.filenames list to be duplicated, once for each item. """ super().parse_config() extension = os.path.splitext(self.config_name)[-1] if extension == ".json": passed_config = utilsparams3.load_json(self.bucket_name, self.config_name) elif extension == ".yaml": passed_config = utilsparams3.load_yaml(self.bucket_name, self.config_name) try: self.ensemble_size = passed_config["ensemble_size"] except KeyError: raise KeyError( "Ensemble size (ensemble_size) parameter not given.") ## Now change the filenames parameter accordingly. self.filenames = self.filenames * self.ensemble_size preconfigs = [ dict(passed_config.items()) for i in range(self.ensemble_size) ] [pc.update({"jobnb": i + 1}) for i, pc in enumerate(preconfigs)] configdir = os.path.dirname(self.config_name) ## As is, this actually does not separate by jobname. Leads to overwrites and issues in processing :( self.ensembleconfigs = { os.path.join(configdir, "{}inst{}config.json".format(self.jobname, i + 1)): preconfigs[i] for i in range(self.ensemble_size) } ## need to start at 1 because this parameter is parsed in analysis later. for cfig in self.ensembleconfigs: utilsparams3.put_json(self.bucket_name, cfig, self.ensembleconfigs[cfig])
def load_config(self): """Load in an existing config file from the results area to help us fill in a new one. We assume that the config files in question will be called inst{modelindex}config.json. If the ensemble is successfully trained, we should expect at most max(ensemble_size,9) models. """ ## assume that ensemble is of size >=1, so we get inst1config.json jobdirname = os.path.basename(os.path.normpath(self.jobdir)) configpath = os.path.join(self.jobdir,"process_results","{}inst1config.json".format(jobdirname)) configdict = utilsparams3.load_json(self.bucket,configpath) return configdict
def get_costmonitoring(self): """ Gets the cost incurred by a given group so far by looking at the logs bucket of the appropriate s3 folder. """ ## first get the path to the log folder we should be looking at. group_name = self.path assert len(group_name) > 0 "group_name must exist." logfolder_path = "logs/{}/".format(group_name) full_reportpath = os.path.join(logfolder_path, "i-") ## now get all of the computereport filenames: all_files = utilsparams3.ls_name(self.bucket_name, full_reportpath) ## for each, we extract the contents: jobdata = {} cost = 0 ## now calculate the cost: for jobfile in all_files: instanceid = jobfile.split(full_reportpath)[1].split(".json")[0] jobdata = utilsparams3.load_json(self.bucket_name, jobfile) price = jobdata["price"] start = jobdata["start"] end = jobdata["end"] try: starttime = datetime.strptime(start, "%Y-%m-%dT%H:%M:%SZ") endtime = datetime.strptime(end, "%Y-%m-%dT%H:%M:%SZ") diff = endtime - starttime duration = abs(diff.seconds) cost = price * duration / 3600. except TypeError: ## In rare cases it seems one or the other of these things don't actually have entries. This is a problem. for now, charge for the hour: cost = price cost += cost ## Now compare with budget: budget = float(os.environ["MAXCOST"]) if cost < budget: message = "Incurred cost so far: ${}. Remaining budget: ${}".format( cost, budget - cost) self.logger.append(message) self.logger.write() validjob = True elif cost >= budget: message = "Incurred cost so far: ${}. Over budget (${}), cancelling job. Contact administrator.".format( cost, budget) self.logger.append(message) self.logger.write() validjob = False return validjob
def parse_config(self): """ Parse the config file given for specific neurocaas parameters. In particular, the *duration* of the job, and the *dataset size* """ passed_config = utilsparams3.load_json(self.bucket_name,self.config_name) print(passed_config,"passedconfig") try: self.jobduration = passed_config["__duration__"] except KeyError: self.logger.append("parameter __duration__ not given, proceeding with standard compute launch.") self.logger.write() self.jobduration = None try: self.jobsize = passed_config["__dataset_size__"] except KeyError: self.logger.append("parameter __dataset_size__ is not given, proceeding with standard compute launch." ) self.logger.write() self.jobsize = None
def __init__(self, bucket_name, key, time): ## Initialize as before: # Get Upload Location Information self.bucket_name = bucket_name ## Get directory above the input directory. self.path = re.findall('.+?(?=/' + os.environ["SUBMITDIR"] + ')', key)[0] ## Now add in the time parameter: self.time = time ## We will index by the submit file name prefix if it exists: submit_search = re.findall('.+?(?=/submit.json)', os.path.basename(key)) try: submit_name = submit_search[0] except IndexError as e: ## If the filename is just "submit.json, we just don't append anything to the job name. " submit_name = "" #### Parse submit file submit_file = utilsparams3.load_json(bucket_name, key) ## Machine formatted fields (error only available in lambda) ## These next three fields check that the submit file is correctly formatted try: self.timestamp = submit_file["timestamp"] ## KEY: Now set up logging in the input folder too: except KeyError as ke: ## Now raise an exception to halt processing, because this is a catastrophic error. raise ValueError("Missing timestamp when data was uploaded.") ## Initialize s3 directory for this job. self.jobname = "job_{}_{}_{}".format(submit_name, bucket_name, self.timestamp) jobpath = os.path.join(self.path, os.environ['OUTDIR'], self.jobname) self.jobpath = jobpath ## And create a corresponding directory in the submit area. create_jobdir = utilsparams3.mkdir( self.bucket_name, os.path.join(self.path, os.environ['OUTDIR']), self.jobname) ## Create a logging object and write to it. ## a logger for the submit area. self.logger = utilsparams3.JobLogger_demo(self.bucket_name, self.jobpath) self.logger.append("Unique analysis version id: {}".format( os.environ['versionid'].split("\n")[0])) self.logger.append("Initializing analysis.") self.logger.write() ######################## ## Now parse the rest of the file. try: self.instance_type = submit_file[ 'instance_type'] # TODO default option from config except KeyError as ke: msg = "Using default instance type {} from config file".format( os.environ["INSTANCE_TYPE"]) self.instance_type = os.environ["INSTANCE_TYPE"] # Log this message self.logger.append(msg) self.logger.write() ## Check that we have a dataname field: submit_errmsg = "INPUT ERROR: Submit file does not contain field {}, needed to analyze data." try: self.data_name = submit_file[ 'dataname'] # TODO validate extensions except KeyError as ke: print(submit_errmsg.format(ke)) ## Write to logger self.logger.append(submit_errmsg.format(ke)) self.logger.write() ## Now raise an exception to halt processing, because this is a catastrophic error. raise ValueError("Missing data name to analyze") try: self.config_name = submit_file["configname"] self.logger.assign_config(self.config_name) except KeyError as ke: print(submit_errmsg.format(ke)) ## Write to logger self.logger.append(submit_errmsg.format(ke)) self.logger.write() ## Now raise an exception to halt processing, because this is a catastrophic error. raise ValueError(os.environ["MISSING_CONFIG_ERROR"]) self.logger.append( "Analysis request detected with dataset(s): {}, config file {}. Reading analysis blueprint." .format(self.data_name, self.config_name)) self.logger.write() ########################## ## Check for the existence of the corresponding data and config in s3. ## Check that we have the actual data in the bucket. exists_errmsg = "INPUT ERROR: S3 Bucket does not contain {}" if type(self.data_name) is str: check_data_exists = utilsparams3.exists(self.bucket_name, self.data_name) elif type(self.data_name) is list: check_data_exists = all([ utilsparams3.exists(self.bucket_name, name) for name in self.data_name ]) else: raise TypeError("dataname should be string or list.") if not check_data_exists: msg = exists_errmsg.format(self.data_name) self.logger.append(msg) self.logger.write() raise ValueError("dataname given does not exist in bucket.") elif not utilsparams3.exists(self.bucket_name, self.config_name): msg = exists_errmsg.format(self.config_name) self.logger.append(msg) self.logger.write() raise ValueError("configname given does not exist in bucket.") ########################### ## Now get the actual paths to relevant data from the foldername: if type(self.data_name) is str: self.filenames = utilsparams3.extract_files(self.bucket_name, self.data_name, ext=None) elif type(self.data_name) is list: self.filenames = self.data_name assert len(self.filenames) > 0, "we must have data to analyze."
def __init__(self, bucket_name, key, time): #### Declare basic parameters: # Get Upload Location Information self.bucket_name = bucket_name ## Important paths: ## Get directory above the input directory where the job was submitted. self.path = re.findall('.+?(?=/' + os.environ["INDIR"] + ')', key)[0] ## The other important directory is the actual base directory of the input bucket itself. ## Now add in the time parameter: self.time = time #### Set up basic logging so we can get a trace when errors happen. ## We will index by the submit file name prefix if it exists: submit_search = re.findall('.+?(?=/submit.json)', os.path.basename(key)) try: submit_name = submit_search[0] except IndexError as e: ## If the filename is just "submit.json, we just don't append anything to the job name. " submit_name = "" #### Parse submit file submit_file = utilsparams3.load_json(bucket_name, key) ## These next three fields check that the submit file is correctly formatted try: self.timestamp = submit_file["timestamp"] ## KEY: Now set up logging in the input folder too: except KeyError as ke: ## Now raise an exception to halt processing, because this is a catastrophic error. raise ValueError("Missing timestamp when data was uploaded.") ## Now we're going to get the path to the results directory in the submit folder: self.jobname = "job_{}_{}_{}".format(submit_name, bucket_name, self.timestamp) jobpath = os.path.join(self.path, os.environ['OUTDIR'], self.jobname) self.jobpath_submit = jobpath ## And create a corresponding directory in the submit area. create_jobdir = utilsparams3.mkdir( self.bucket_name, os.path.join(self.path, os.environ['OUTDIR']), self.jobname) ## a logger for the submit area. self.logger = utilsparams3.JobLogger_demo(self.bucket_name, self.jobpath) self.logger.append( "Initializing EPI analysis: Parameter search for 2D LDS.") self.logger.write() try: self.instance_type = submit_file[ 'instance_type'] # TODO default option from config except KeyError as ke: msg = "Using default instance type {} from config file".format( os.environ["INSTANCE_TYPE"]) self.instance_type = os.environ["INSTANCE_TYPE"] ## Check that we have a dataname field: submit_errmsg = "INPUT ERROR: Submit file does not contain field {}, needed to analyze data." try: self.input_bucket_name = submit_file["bucketname"] ## KEY: Now set up logging in the input folder too: self.inputlogger = utilsparams3.JobLogger( self.input_bucket_name, os.path.join(os.environ['OUTDIR'], self.jobname) ) ##TODO: this relies upon "OUTDIR" being the same in the submit and input buckets. Make sure to alter this later. except KeyError as ke: print(submit_errmsg.format(ke)) ## Write to logger self.submitlogger.append(submit_errmsg.format(ke)) self.submitlogger.write() ## Now raise an exception to halt processing, because this is a catastrophic error. raise ValueError("Missing bucket name where data is located.") try: self.data_name = submit_file[ 'dataname'] # TODO validate extensions except KeyError as ke: print(submit_errmsg.format(ke)) ## Write to logger self.submitlogger.append(submit_errmsg.format(ke)) self.submitlogger.write() self.inputlogger.append(submit_errmsg.format(ke)) self.inputlogger.write() ## Now raise an exception to halt processing, because this is a catastrophic error. raise ValueError("Missing data name to analyze") try: self.config_name = submit_file["configname"] self.submitlogger.assign_config(self.config_name) except KeyError as ke: print(submit_errmsg.format(ke)) ## Write to logger self.submitlogger.append(submit_errmsg.format(ke)) self.submitlogger.write() self.inputlogger.append(submit_errmsg.format(ke)) self.inputlogger.write() ## Now raise an exception to halt processing, because this is a catastrophic error. raise ValueError(os.environ["MISSING_CONFIG_ERROR"]) ## Check that we have the actual data in the bucket. exists_errmsg = "INPUT ERROR: S3 Bucket does not contain {}" if not utilsparams3.exists(self.input_bucket_name, self.data_name): msg = exists_errmsg.format(self.data_name) self.submitlogger.append(msg) self.submitlogger.write() self.inputlogger.append(msg) self.inputlogger.write() raise ValueError("dataname given does not exist in bucket.") elif not utilsparams3.exists(self.input_bucket_name, self.config_name): msg = exists_errmsg.format(self.config_name) self.submitlogger.append(msg) self.submitlogger.write() self.inputlogger.append(msg) self.inputlogger.write() raise ValueError("configname given does not exist in bucket.") ## Check what instance we should use. try: self.instance_type = submit_file['instance_type'] except KeyError as ke: msg = "Instance type {} does not exist, using default from config file".format( ke) self.instance_type = os.environ["INSTANCE_TYPE"] ## Log this message. self.submitlogger.append(msg) self.submitlogger.write() self.inputlogger.append(msg) self.inputlogger.write() ########################### ## Now get the actual paths to relevant data from the foldername: self.filenames = utilsparams3.extract_files(self.input_bucket_name, self.data_name, ext=None) assert len(self.filenames) > 0, "we must have data to analyze."
def __init__(self,bucket_name,key,time): ## Initialize as before: # Get Upload Location Information self.bucket_name = bucket_name ## Get directory above the input directory. self.path = re.findall('.+?(?=/'+os.environ["INDIR"]+')',key)[0] ## Now add in the time parameter: self.time = time ## We will index by the submit file name prefix if it exists: submit_search = re.findall('.+?(?=/submit.json)',os.path.basename(key)) try: submit_name = submit_search[0] except IndexError as e: ## If the filename is just "submit.json, we just don't append anything to the job name. " submit_name = "" ## Now we're going to get the path to the results directory: self.jobname = "job"+submit_name+self.time jobpath = os.path.join(self.path,os.environ['OUTDIR'],self.jobname) self.jobpath = jobpath create_jobdir = utilsparams3.mkdir(self.bucket_name, os.path.join(self.path,os.environ['OUTDIR']),self.jobname) print(self.path,'path') self.logger = utilsparams3.JobLogger(self.bucket_name, self.jobpath) #self.out_path = utilsparams3.mkdir(self.bucket_name, self.path, config.OUTDIR) #self.in_path = utilsparams3.mkdir(self.bucket_name, self.path, config.INDIR) # Load Content Of Submit File submit_file = utilsparams3.load_json(bucket_name, key) ## Check what instance we should use. try: self.instance_type = submit_file['instance_type'] # TODO default option from config except KeyError as ke: msg = "Instance type {} does not exist, using default from config file".format(ke) self.instance_type = os.environ["INSTANCE_TYPE"] ## Log this message. self.logger.append(msg) self.logger.write() ## These next two check that the submit file is correctly formatted ## Check that we have a dataname field: submit_errmsg = "INPUT ERROR: Submit file does not contain field {}, needed to analyze data." try: self.data_name = submit_file['dataname'] # TODO validate extensions except KeyError as ke: print(submit_errmsg.format(ke)) ## Write to logger self.logger.append(submit_errmsg.format(ke)) self.logger.write() ## Now raise an exception to halt processing, because this is a catastrophic error. raise ValueError("Missing data name to analyze") try: self.config_name = submit_file["configname"] self.logger.assign_config(self.config_name) except KeyError as ke: print(submit_errmsg.format(ke)) ## Write to logger self.logger.append(submit_errmsg.format(ke)) self.logger.write() ## Now raise an exception to halt processing, because this is a catastrophic error. raise ValueError(os.environ["MISSING_CONFIG_ERROR"]) ## Check that we have the actual data in the bucket. exists_errmsg = "INPUT ERROR: S3 Bucket does not contain {}" if not utilsparams3.exists(self.bucket_name,self.data_name): msg = exists_errmsg.format(self.data_name) self.logger.append(msg) self.logger.write() raise ValueError("dataname given does not exist in bucket.") elif not utilsparams3.exists(self.bucket_name,self.config_name): msg = exists_errmsg.format(self.config_name) self.logger.append(msg) self.logger.write() raise ValueError("configname given does not exist in bucket.") ########################### ## Now get the actual paths to relevant data from the foldername: self.filenames = utilsparams3.extract_files(self.bucket_name,self.data_name,ext = None) assert len(self.filenames) > 0, "we must have data to analyze."
def __init__(self,bucket_name,key,time): ## Initialize as before: # Get Upload Location Information self.bucket_name = bucket_name ## Get directory above the input directory. self.path = re.findall('.+?(?=/'+os.environ["SUBMITDIR"]+')',key)[0] ## Now add in the time parameter: self.time = time ## We will index by the submit file name prefix if it exists: submit_search = re.findall('.+?(?=/submit.json)',os.path.basename(key)) try: submit_name = submit_search[0] except IndexError as e: ## If the filename is just "submit.json, we just don't append anything to the job name. " submit_name = "" try: #### Parse submit file submit_file = utilsparams3.load_json(bucket_name, key) except ClientError as e: print(e.response["Error"]) raise ClientError("[JOB TERMINATE REASON] 'json not loaded.'") ## Machine formatted fields (error only available in lambda) ## These next three fields check that the submit file is correctly formatted try: self.timestamp = submit_file["timestamp"] ## KEY: Now set up logging in the input folder too: except KeyError as ke: ## Now raise an exception to halt processing, because this is a catastrophic error. raise ValueError("[JOB TERMINATE REASON] 'timestamp' field not given in submit.json file.") ## Initialize s3 directory for this job. self.jobname = "job_{}_{}_{}".format(submit_name,bucket_name,self.timestamp) jobpath = os.path.join(self.path,os.environ['OUTDIR'],self.jobname) self.jobpath = jobpath try: ## And create a corresponding directory in the submit area. create_jobdir = utilsparams3.mkdir(self.bucket_name, os.path.join(self.path,os.environ['OUTDIR']),self.jobname) ## Create a logging object and write to it. ## a logger for the submit area. self.logger = utilsparams3.JobLogger_demo(self.bucket_name, self.jobpath) msg = "REQUEST START TIME: {} (GMT)".format(str(self.logger.basetime)[:-4]) self.logger.append(msg) self.logger.printlatest() self.logger.write() msg = "ANALYSIS VERSION ID: {}".format(os.environ['versionid'].split("\n")[0]) self.logger.append(msg) self.logger.printlatest() self.logger.write() msg = "JOB ID: {}".format(self.timestamp) self.logger.append(msg) self.logger.printlatest() self.logger.write() self.logger._logs.append("\n ") msg = "[Job Manager] Detected new job: starting up." self.logger.append(msg) self.logger.printlatest() self.logger.write() msg = " [Internal (init)] Initializing job manager." self.logger.append(msg) self.logger.printlatest() self.logger.write() ######################## ## Now parse the rest of the file. print("finished logging setup.") except ClientError as e: print("error with logging:", e.response["Error"]) try: self.instance_type = submit_file['instance_type'] # TODO default option from config except KeyError as ke: msg = " [Internal (init)] Using default instance type {} from config file.".format(os.environ["INSTANCE_TYPE"]) self.instance_type = os.environ["INSTANCE_TYPE"] # Log this message self.logger.append(msg) self.logger.printlatest() self.logger.write() ## Check that we have a dataname field: submit_errmsg = " [Internal (init)] INPUT ERROR: Submit file does not contain field {}, needed to analyze data." try: self.data_name = submit_file['dataname'] # TODO validate extensions except KeyError as ke: ## Write to logger self.logger.append(submit_errmsg.format(ke)) self.logger.printlatest() self.logger.write() ## Now raise an exception to halt processing, because this is a catastrophic error. raise ValueError("[JOB TERMINATE REASON] 'dataname' field not given in submit.json file") try: self.config_name = submit_file["configname"] self.logger.assign_config(self.config_name) except KeyError as ke: ## Write to logger self.logger.append(submit_errmsg.format(ke)) self.logger.printlatest() self.logger.write() ## Now raise an exception to halt processing, because this is a catastrophic error. raise ValueError("[JOB TERMINATE REASON] 'configname' field not given in submit.json file") msg = " [Internal (init)] Analysis request with dataset(s): {}, config file {}".format(self.data_name,self.config_name) self.logger.append(msg) self.logger.printlatest() self.logger.write()
def get_costmonitoring(self): """ Gets the cost incurred by a given group so far by looking at the logs bucket of the appropriate s3 folder. """ ## first get the path to the log folder we should be looking at. group_name = self.path assert len(group_name) > 0 "[JOB TERMINATE REASON] Can't locate the group that triggered analysis, making it impossible to determine incurred cost." logfolder_path = "logs/{}/".format(group_name) full_reportpath = os.path.join(logfolder_path, "i-") ## now get all of the computereport filenames: all_files = utilsparams3.ls_name(self.bucket_name, full_reportpath) ## for each, we extract the contents: jobdata = {} cost = 0 ## now calculate the cost: for jobfile in all_files: instanceid = jobfile.split(full_reportpath)[1].split(".json")[0] jobdata = utilsparams3.load_json(self.bucket_name, jobfile) price = jobdata["price"] start = jobdata["start"] end = jobdata["end"] try: starttime = datetime.strptime(start, "%Y-%m-%dT%H:%M:%SZ") endtime = datetime.strptime(end, "%Y-%m-%dT%H:%M:%SZ") diff = endtime - starttime duration = abs(diff.seconds) instcost = price * duration / 3600. except TypeError: ## In rare cases it seems one or the other of these things don't actually have entries. This is a problem. for now, charge for the hour: message = " [Internal (get_costmonitoring)] Duration of past jobs not found. Pricing for an hour" self.logger.append(message) self.logger.printlatest() instcost = price cost += instcost ## Now compare against the cost of the job you're currently running: ## need duration from config (self.parse_config), self.instance_type, and self.nb_instances ## By assuming they're all standard instances we upper bound the cost. try: price = utilsparampricing.get_price( utilsparampricing.get_region_name(utilsparampricing.region_id), self.instance_type, os="Linux") nb_instances = len(self.filenames) if self.jobduration is None: duration = defaultduration / 60 ## in hours. else: duration = self.jobduration / 60 jobpricebound = duration * price * nb_instances cost += jobpricebound except Exception as e: print(e) raise Exception( " [Internal (get_costmonitoring)] Unexpected Error: Unable to estimate cost of current job." ) ## Now compare agains the expected cost of instances with the current ami: try: ami = os.environ["AMI"] total_activeprice = self.prices_active_instances_ami(ami) except Exception as e: print(e) try: activeprice = utilsparampricing.get_price( utilsparampricing.get_region_name( utilsparampricing.region_id), self.instance_type, os="Linux") number = len( [i for i in utilsparamec2.get_active_instances_ami(ami)]) activeduration = defaultduration * number / 60 ## default to the default duration instead if not given. total_activeprice = activeprice * activeduration except Exception as e: print(e) raise Exception( " [Internal (get_costmonitoring)] Unexpected Error: Unable to estimate cost of active jobs." ) cost += total_activeprice ## Now compare with budget: try: budget = float( utilsparamssm.get_budget_parameter(self.path, self.bucket_name)) except ClientError as e: try: assert e.response["Error"]["Code"] == "ParameterNotFound" budget = float(os.environ["MAXCOST"]) message = " [Internal (get_costmonitoring)] Customized budget not found. Using default budget value of {}".format( budget) self.logger.append(message) self.logger.printlatest() except: raise Exception( " [Internal (get_costmonitoring)] Unexpected Error: Unable to get budget." ) except Exception: raise Exception( " [Internal (get_costmonitoring)] Unexpected Error: Unable to get budget." ) if cost < budget: message = " [Internal (get_costmonitoring)] Projected total costs: ${}. Remaining budget: ${}".format( cost, budget - cost) self.logger.append(message) self.logger.printlatest() self.logger.write() validjob = True elif cost >= budget: message = " [Internal (get_costmonitoring)] Projected total costs: ${}. Over budget (${}), cancelling job. Contact administrator.".format( cost, budget) self.logger.append(message) self.logger.printlatest() self.logger.write() validjob = False return validjob