def handler_ensemble(event, context):
"""
Newest version of handler that logs outputs to a subfolder of the result folder that is indexed by the job submission date and the submit name.
Update 05/25: first check the config file to see if this is predict mode or train mode.
"""
for record in event['Records']:
time = record['eventTime']
bucket_name = record['s3']['bucket']['name']
key = record['s3']['object']['key']
submit_file = utilsparams3.load_json(bucket_name, key)
configpath = submit_file["configname"]
try:
configfile = utilsparams3.load_json(bucket_name, configpath)
except ValueError:
try:
configfile = utilsparams3.load_yaml(bucket_name, configpath)
except Exception:
raise Exception("Config is not json or yaml.")
print("Processing in {} mode".format(configfile["mode"]))
if configfile["mode"] == "train":
#print("handler_params",bucket_name,key,time)
#print(event,context,'event, context')
exitcode = process_upload_ensemble(bucket_name, key, time)
print("processing returned exit code {}".format(exitcode))
else:
exitcode = process_upload_dev(bucket_name, key, time)
print("processing returned exit code {}".format(exitcode))
return exitcode
def parse_config(self):
"""
Parse the config file given for specific neurocaas parameters. In particular, the *duration* of the job, and the *dataset size*
TODO: check for type in these configuration files.
"""
extension = os.path.splitext(self.config_name)[-1]
if extension == ".json":
passed_config = utilsparams3.load_json(self.bucket_name,
self.config_name)
elif extension == ".yaml":
passed_config = utilsparams3.load_yaml(self.bucket_name,
self.config_name)
try:
self.jobduration = passed_config["__duration__"]
except KeyError:
self.logger.append(
"parameter __duration__ not given, proceeding with standard compute launch."
)
self.logger.write()
self.jobduration = None
try:
self.jobsize = passed_config["__dataset_size__"]
except KeyError:
self.logger.append(
"parameter __dataset_size__ is not given, proceeding with standard compute launch."
)
self.logger.write()
self.jobsize = None
def parse_config(self):
"""Parse the config file for the number of entries to include in this ensemble in addition to other neurocaas parameters. Additionally writes a copy of the config for each copy with just the jobnumber changed. Internally, creates a dictionary self.confignames and changes the self.filenames list to be duplicated, once for each item.
"""
super().parse_config()
extension = os.path.splitext(self.config_name)[-1]
if extension == ".json":
passed_config = utilsparams3.load_json(self.bucket_name,
self.config_name)
elif extension == ".yaml":
passed_config = utilsparams3.load_yaml(self.bucket_name,
self.config_name)
try:
self.ensemble_size = passed_config["ensemble_size"]
except KeyError:
raise KeyError(
"Ensemble size (ensemble_size) parameter not given.")
## Now change the filenames parameter accordingly.
self.filenames = self.filenames * self.ensemble_size
preconfigs = [
dict(passed_config.items()) for i in range(self.ensemble_size)
]
[pc.update({"jobnb": i + 1}) for i, pc in enumerate(preconfigs)]
configdir = os.path.dirname(self.config_name)
## As is, this actually does not separate by jobname. Leads to overwrites and issues in processing :(
self.ensembleconfigs = {
os.path.join(configdir,
"{}inst{}config.json".format(self.jobname, i + 1)):
preconfigs[i]
for i in range(self.ensemble_size)
} ## need to start at 1 because this parameter is parsed in analysis later.
for cfig in self.ensembleconfigs:
utilsparams3.put_json(self.bucket_name, cfig,
self.ensembleconfigs[cfig])
def load_config(self):
"""Load in an existing config file from the results area to help us fill in a new one.
We assume that the config files in question will be called inst{modelindex}config.json.
If the ensemble is successfully trained, we should expect at most max(ensemble_size,9) models.
"""
## assume that ensemble is of size >=1, so we get inst1config.json
jobdirname = os.path.basename(os.path.normpath(self.jobdir))
configpath = os.path.join(self.jobdir,"process_results","{}inst1config.json".format(jobdirname))
configdict = utilsparams3.load_json(self.bucket,configpath)
return configdict
def get_costmonitoring(self):
"""
Gets the cost incurred by a given group so far by looking at the logs bucket of the appropriate s3 folder.
"""
## first get the path to the log folder we should be looking at.
group_name = self.path
assert len(group_name) > 0
"group_name must exist."
logfolder_path = "logs/{}/".format(group_name)
full_reportpath = os.path.join(logfolder_path, "i-")
## now get all of the computereport filenames:
all_files = utilsparams3.ls_name(self.bucket_name, full_reportpath)
## for each, we extract the contents:
jobdata = {}
cost = 0
## now calculate the cost:
for jobfile in all_files:
instanceid = jobfile.split(full_reportpath)[1].split(".json")[0]
jobdata = utilsparams3.load_json(self.bucket_name, jobfile)
price = jobdata["price"]
start = jobdata["start"]
end = jobdata["end"]
try:
starttime = datetime.strptime(start, "%Y-%m-%dT%H:%M:%SZ")
endtime = datetime.strptime(end, "%Y-%m-%dT%H:%M:%SZ")
diff = endtime - starttime
duration = abs(diff.seconds)
cost = price * duration / 3600.
except TypeError:
## In rare cases it seems one or the other of these things don't actually have entries. This is a problem. for now, charge for the hour:
cost = price
cost += cost
## Now compare with budget:
budget = float(os.environ["MAXCOST"])
if cost < budget:
message = "Incurred cost so far: ${}. Remaining budget: ${}".format(
cost, budget - cost)
self.logger.append(message)
self.logger.write()
validjob = True
elif cost >= budget:
message = "Incurred cost so far: ${}. Over budget (${}), cancelling job. Contact administrator.".format(
cost, budget)
self.logger.append(message)
self.logger.write()
validjob = False
return validjob
def parse_config(self):
"""
Parse the config file given for specific neurocaas parameters. In particular, the *duration* of the job, and the *dataset size*
"""
passed_config = utilsparams3.load_json(self.bucket_name,self.config_name)
print(passed_config,"passedconfig")
try:
self.jobduration = passed_config["__duration__"]
except KeyError:
self.logger.append("parameter __duration__ not given, proceeding with standard compute launch.")
self.logger.write()
self.jobduration = None
try:
self.jobsize = passed_config["__dataset_size__"]
except KeyError:
self.logger.append("parameter __dataset_size__ is not given, proceeding with standard compute launch." )
self.logger.write()
self.jobsize = None
def __init__(self, bucket_name, key, time):
## Initialize as before:
# Get Upload Location Information
self.bucket_name = bucket_name
## Get directory above the input directory.
self.path = re.findall('.+?(?=/' + os.environ["SUBMITDIR"] + ')',
key)[0]
## Now add in the time parameter:
self.time = time
## We will index by the submit file name prefix if it exists:
submit_search = re.findall('.+?(?=/submit.json)',
os.path.basename(key))
try:
submit_name = submit_search[0]
except IndexError as e:
## If the filename is just "submit.json, we just don't append anything to the job name. "
submit_name = ""
#### Parse submit file
submit_file = utilsparams3.load_json(bucket_name, key)
## Machine formatted fields (error only available in lambda)
## These next three fields check that the submit file is correctly formatted
try:
self.timestamp = submit_file["timestamp"]
## KEY: Now set up logging in the input folder too:
except KeyError as ke:
## Now raise an exception to halt processing, because this is a catastrophic error.
raise ValueError("Missing timestamp when data was uploaded.")
## Initialize s3 directory for this job.
self.jobname = "job_{}_{}_{}".format(submit_name, bucket_name,
self.timestamp)
jobpath = os.path.join(self.path, os.environ['OUTDIR'], self.jobname)
self.jobpath = jobpath
## And create a corresponding directory in the submit area.
create_jobdir = utilsparams3.mkdir(
self.bucket_name, os.path.join(self.path, os.environ['OUTDIR']),
self.jobname)
## Create a logging object and write to it.
## a logger for the submit area.
self.logger = utilsparams3.JobLogger_demo(self.bucket_name,
self.jobpath)
self.logger.append("Unique analysis version id: {}".format(
os.environ['versionid'].split("\n")[0]))
self.logger.append("Initializing analysis.")
self.logger.write()
########################
## Now parse the rest of the file.
try:
self.instance_type = submit_file[
'instance_type'] # TODO default option from config
except KeyError as ke:
msg = "Using default instance type {} from config file".format(
os.environ["INSTANCE_TYPE"])
self.instance_type = os.environ["INSTANCE_TYPE"]
# Log this message
self.logger.append(msg)
self.logger.write()
## Check that we have a dataname field:
submit_errmsg = "INPUT ERROR: Submit file does not contain field {}, needed to analyze data."
try:
self.data_name = submit_file[
'dataname'] # TODO validate extensions
except KeyError as ke:
print(submit_errmsg.format(ke))
## Write to logger
self.logger.append(submit_errmsg.format(ke))
self.logger.write()
## Now raise an exception to halt processing, because this is a catastrophic error.
raise ValueError("Missing data name to analyze")
try:
self.config_name = submit_file["configname"]
self.logger.assign_config(self.config_name)
except KeyError as ke:
print(submit_errmsg.format(ke))
## Write to logger
self.logger.append(submit_errmsg.format(ke))
self.logger.write()
## Now raise an exception to halt processing, because this is a catastrophic error.
raise ValueError(os.environ["MISSING_CONFIG_ERROR"])
self.logger.append(
"Analysis request detected with dataset(s): {}, config file {}. Reading analysis blueprint."
.format(self.data_name, self.config_name))
self.logger.write()
##########################
## Check for the existence of the corresponding data and config in s3.
## Check that we have the actual data in the bucket.
exists_errmsg = "INPUT ERROR: S3 Bucket does not contain {}"
if type(self.data_name) is str:
check_data_exists = utilsparams3.exists(self.bucket_name,
self.data_name)
elif type(self.data_name) is list:
check_data_exists = all([
utilsparams3.exists(self.bucket_name, name)
for name in self.data_name
])
else:
raise TypeError("dataname should be string or list.")
if not check_data_exists:
msg = exists_errmsg.format(self.data_name)
self.logger.append(msg)
self.logger.write()
raise ValueError("dataname given does not exist in bucket.")
elif not utilsparams3.exists(self.bucket_name, self.config_name):
msg = exists_errmsg.format(self.config_name)
self.logger.append(msg)
self.logger.write()
raise ValueError("configname given does not exist in bucket.")
###########################
## Now get the actual paths to relevant data from the foldername:
if type(self.data_name) is str:
self.filenames = utilsparams3.extract_files(self.bucket_name,
self.data_name,
ext=None)
elif type(self.data_name) is list:
self.filenames = self.data_name
assert len(self.filenames) > 0, "we must have data to analyze."
def __init__(self, bucket_name, key, time):
#### Declare basic parameters:
# Get Upload Location Information
self.bucket_name = bucket_name
## Important paths:
## Get directory above the input directory where the job was submitted.
self.path = re.findall('.+?(?=/' + os.environ["INDIR"] + ')', key)[0]
## The other important directory is the actual base directory of the input bucket itself.
## Now add in the time parameter:
self.time = time
#### Set up basic logging so we can get a trace when errors happen.
## We will index by the submit file name prefix if it exists:
submit_search = re.findall('.+?(?=/submit.json)',
os.path.basename(key))
try:
submit_name = submit_search[0]
except IndexError as e:
## If the filename is just "submit.json, we just don't append anything to the job name. "
submit_name = ""
#### Parse submit file
submit_file = utilsparams3.load_json(bucket_name, key)
## These next three fields check that the submit file is correctly formatted
try:
self.timestamp = submit_file["timestamp"]
## KEY: Now set up logging in the input folder too:
except KeyError as ke:
## Now raise an exception to halt processing, because this is a catastrophic error.
raise ValueError("Missing timestamp when data was uploaded.")
## Now we're going to get the path to the results directory in the submit folder:
self.jobname = "job_{}_{}_{}".format(submit_name, bucket_name,
self.timestamp)
jobpath = os.path.join(self.path, os.environ['OUTDIR'], self.jobname)
self.jobpath_submit = jobpath
## And create a corresponding directory in the submit area.
create_jobdir = utilsparams3.mkdir(
self.bucket_name, os.path.join(self.path, os.environ['OUTDIR']),
self.jobname)
## a logger for the submit area.
self.logger = utilsparams3.JobLogger_demo(self.bucket_name,
self.jobpath)
self.logger.append(
"Initializing EPI analysis: Parameter search for 2D LDS.")
self.logger.write()
try:
self.instance_type = submit_file[
'instance_type'] # TODO default option from config
except KeyError as ke:
msg = "Using default instance type {} from config file".format(
os.environ["INSTANCE_TYPE"])
self.instance_type = os.environ["INSTANCE_TYPE"]
## Check that we have a dataname field:
submit_errmsg = "INPUT ERROR: Submit file does not contain field {}, needed to analyze data."
try:
self.input_bucket_name = submit_file["bucketname"]
## KEY: Now set up logging in the input folder too:
self.inputlogger = utilsparams3.JobLogger(
self.input_bucket_name,
os.path.join(os.environ['OUTDIR'], self.jobname)
) ##TODO: this relies upon "OUTDIR" being the same in the submit and input buckets. Make sure to alter this later.
except KeyError as ke:
print(submit_errmsg.format(ke))
## Write to logger
self.submitlogger.append(submit_errmsg.format(ke))
self.submitlogger.write()
## Now raise an exception to halt processing, because this is a catastrophic error.
raise ValueError("Missing bucket name where data is located.")
try:
self.data_name = submit_file[
'dataname'] # TODO validate extensions
except KeyError as ke:
print(submit_errmsg.format(ke))
## Write to logger
self.submitlogger.append(submit_errmsg.format(ke))
self.submitlogger.write()
self.inputlogger.append(submit_errmsg.format(ke))
self.inputlogger.write()
## Now raise an exception to halt processing, because this is a catastrophic error.
raise ValueError("Missing data name to analyze")
try:
self.config_name = submit_file["configname"]
self.submitlogger.assign_config(self.config_name)
except KeyError as ke:
print(submit_errmsg.format(ke))
## Write to logger
self.submitlogger.append(submit_errmsg.format(ke))
self.submitlogger.write()
self.inputlogger.append(submit_errmsg.format(ke))
self.inputlogger.write()
## Now raise an exception to halt processing, because this is a catastrophic error.
raise ValueError(os.environ["MISSING_CONFIG_ERROR"])
## Check that we have the actual data in the bucket.
exists_errmsg = "INPUT ERROR: S3 Bucket does not contain {}"
if not utilsparams3.exists(self.input_bucket_name, self.data_name):
msg = exists_errmsg.format(self.data_name)
self.submitlogger.append(msg)
self.submitlogger.write()
self.inputlogger.append(msg)
self.inputlogger.write()
raise ValueError("dataname given does not exist in bucket.")
elif not utilsparams3.exists(self.input_bucket_name, self.config_name):
msg = exists_errmsg.format(self.config_name)
self.submitlogger.append(msg)
self.submitlogger.write()
self.inputlogger.append(msg)
self.inputlogger.write()
raise ValueError("configname given does not exist in bucket.")
## Check what instance we should use.
try:
self.instance_type = submit_file['instance_type']
except KeyError as ke:
msg = "Instance type {} does not exist, using default from config file".format(
ke)
self.instance_type = os.environ["INSTANCE_TYPE"]
## Log this message.
self.submitlogger.append(msg)
self.submitlogger.write()
self.inputlogger.append(msg)
self.inputlogger.write()
###########################
## Now get the actual paths to relevant data from the foldername:
self.filenames = utilsparams3.extract_files(self.input_bucket_name,
self.data_name,
ext=None)
assert len(self.filenames) > 0, "we must have data to analyze."
def __init__(self,bucket_name,key,time):
## Initialize as before:
# Get Upload Location Information
self.bucket_name = bucket_name
## Get directory above the input directory.
self.path = re.findall('.+?(?=/'+os.environ["INDIR"]+')',key)[0]
## Now add in the time parameter:
self.time = time
## We will index by the submit file name prefix if it exists:
submit_search = re.findall('.+?(?=/submit.json)',os.path.basename(key))
try:
submit_name = submit_search[0]
except IndexError as e:
## If the filename is just "submit.json, we just don't append anything to the job name. "
submit_name = ""
## Now we're going to get the path to the results directory:
self.jobname = "job"+submit_name+self.time
jobpath = os.path.join(self.path,os.environ['OUTDIR'],self.jobname)
self.jobpath = jobpath
create_jobdir = utilsparams3.mkdir(self.bucket_name, os.path.join(self.path,os.environ['OUTDIR']),self.jobname)
print(self.path,'path')
self.logger = utilsparams3.JobLogger(self.bucket_name, self.jobpath)
#self.out_path = utilsparams3.mkdir(self.bucket_name, self.path, config.OUTDIR)
#self.in_path = utilsparams3.mkdir(self.bucket_name, self.path, config.INDIR)
# Load Content Of Submit File
submit_file = utilsparams3.load_json(bucket_name, key)
## Check what instance we should use.
try:
self.instance_type = submit_file['instance_type'] # TODO default option from config
except KeyError as ke:
msg = "Instance type {} does not exist, using default from config file".format(ke)
self.instance_type = os.environ["INSTANCE_TYPE"]
## Log this message.
self.logger.append(msg)
self.logger.write()
## These next two check that the submit file is correctly formatted
## Check that we have a dataname field:
submit_errmsg = "INPUT ERROR: Submit file does not contain field {}, needed to analyze data."
try:
self.data_name = submit_file['dataname'] # TODO validate extensions
except KeyError as ke:
print(submit_errmsg.format(ke))
## Write to logger
self.logger.append(submit_errmsg.format(ke))
self.logger.write()
## Now raise an exception to halt processing, because this is a catastrophic error.
raise ValueError("Missing data name to analyze")
try:
self.config_name = submit_file["configname"]
self.logger.assign_config(self.config_name)
except KeyError as ke:
print(submit_errmsg.format(ke))
## Write to logger
self.logger.append(submit_errmsg.format(ke))
self.logger.write()
## Now raise an exception to halt processing, because this is a catastrophic error.
raise ValueError(os.environ["MISSING_CONFIG_ERROR"])
## Check that we have the actual data in the bucket.
exists_errmsg = "INPUT ERROR: S3 Bucket does not contain {}"
if not utilsparams3.exists(self.bucket_name,self.data_name):
msg = exists_errmsg.format(self.data_name)
self.logger.append(msg)
self.logger.write()
raise ValueError("dataname given does not exist in bucket.")
elif not utilsparams3.exists(self.bucket_name,self.config_name):
msg = exists_errmsg.format(self.config_name)
self.logger.append(msg)
self.logger.write()
raise ValueError("configname given does not exist in bucket.")
###########################
## Now get the actual paths to relevant data from the foldername:
self.filenames = utilsparams3.extract_files(self.bucket_name,self.data_name,ext = None)
assert len(self.filenames) > 0, "we must have data to analyze."
def __init__(self,bucket_name,key,time):
## Initialize as before:
# Get Upload Location Information
self.bucket_name = bucket_name
## Get directory above the input directory.
self.path = re.findall('.+?(?=/'+os.environ["SUBMITDIR"]+')',key)[0]
## Now add in the time parameter:
self.time = time
## We will index by the submit file name prefix if it exists:
submit_search = re.findall('.+?(?=/submit.json)',os.path.basename(key))
try:
submit_name = submit_search[0]
except IndexError as e:
## If the filename is just "submit.json, we just don't append anything to the job name. "
submit_name = ""
try:
#### Parse submit file
submit_file = utilsparams3.load_json(bucket_name, key)
except ClientError as e:
print(e.response["Error"])
raise ClientError("[JOB TERMINATE REASON] 'json not loaded.'")
## Machine formatted fields (error only available in lambda)
## These next three fields check that the submit file is correctly formatted
try:
self.timestamp = submit_file["timestamp"]
## KEY: Now set up logging in the input folder too:
except KeyError as ke:
## Now raise an exception to halt processing, because this is a catastrophic error.
raise ValueError("[JOB TERMINATE REASON] 'timestamp' field not given in submit.json file.")
## Initialize s3 directory for this job.
self.jobname = "job_{}_{}_{}".format(submit_name,bucket_name,self.timestamp)
jobpath = os.path.join(self.path,os.environ['OUTDIR'],self.jobname)
self.jobpath = jobpath
try:
## And create a corresponding directory in the submit area.
create_jobdir = utilsparams3.mkdir(self.bucket_name, os.path.join(self.path,os.environ['OUTDIR']),self.jobname)
## Create a logging object and write to it.
## a logger for the submit area.
self.logger = utilsparams3.JobLogger_demo(self.bucket_name, self.jobpath)
msg = "REQUEST START TIME: {} (GMT)".format(str(self.logger.basetime)[:-4])
self.logger.append(msg)
self.logger.printlatest()
self.logger.write()
msg = "ANALYSIS VERSION ID: {}".format(os.environ['versionid'].split("\n")[0])
self.logger.append(msg)
self.logger.printlatest()
self.logger.write()
msg = "JOB ID: {}".format(self.timestamp)
self.logger.append(msg)
self.logger.printlatest()
self.logger.write()
self.logger._logs.append("\n ")
msg = "[Job Manager] Detected new job: starting up."
self.logger.append(msg)
self.logger.printlatest()
self.logger.write()
msg = " [Internal (init)] Initializing job manager."
self.logger.append(msg)
self.logger.printlatest()
self.logger.write()
########################
## Now parse the rest of the file.
print("finished logging setup.")
except ClientError as e:
print("error with logging:", e.response["Error"])
try:
self.instance_type = submit_file['instance_type'] # TODO default option from config
except KeyError as ke:
msg = " [Internal (init)] Using default instance type {} from config file.".format(os.environ["INSTANCE_TYPE"])
self.instance_type = os.environ["INSTANCE_TYPE"]
# Log this message
self.logger.append(msg)
self.logger.printlatest()
self.logger.write()
## Check that we have a dataname field:
submit_errmsg = " [Internal (init)] INPUT ERROR: Submit file does not contain field {}, needed to analyze data."
try:
self.data_name = submit_file['dataname'] # TODO validate extensions
except KeyError as ke:
## Write to logger
self.logger.append(submit_errmsg.format(ke))
self.logger.printlatest()
self.logger.write()
## Now raise an exception to halt processing, because this is a catastrophic error.
raise ValueError("[JOB TERMINATE REASON] 'dataname' field not given in submit.json file")
try:
self.config_name = submit_file["configname"]
self.logger.assign_config(self.config_name)
except KeyError as ke:
## Write to logger
self.logger.append(submit_errmsg.format(ke))
self.logger.printlatest()
self.logger.write()
## Now raise an exception to halt processing, because this is a catastrophic error.
raise ValueError("[JOB TERMINATE REASON] 'configname' field not given in submit.json file")
msg = " [Internal (init)] Analysis request with dataset(s): {}, config file {}".format(self.data_name,self.config_name)
self.logger.append(msg)
self.logger.printlatest()
self.logger.write()
def get_costmonitoring(self):
"""
Gets the cost incurred by a given group so far by looking at the logs bucket of the appropriate s3 folder.
"""
## first get the path to the log folder we should be looking at.
group_name = self.path
assert len(group_name) > 0
"[JOB TERMINATE REASON] Can't locate the group that triggered analysis, making it impossible to determine incurred cost."
logfolder_path = "logs/{}/".format(group_name)
full_reportpath = os.path.join(logfolder_path, "i-")
## now get all of the computereport filenames:
all_files = utilsparams3.ls_name(self.bucket_name, full_reportpath)
## for each, we extract the contents:
jobdata = {}
cost = 0
## now calculate the cost:
for jobfile in all_files:
instanceid = jobfile.split(full_reportpath)[1].split(".json")[0]
jobdata = utilsparams3.load_json(self.bucket_name, jobfile)
price = jobdata["price"]
start = jobdata["start"]
end = jobdata["end"]
try:
starttime = datetime.strptime(start, "%Y-%m-%dT%H:%M:%SZ")
endtime = datetime.strptime(end, "%Y-%m-%dT%H:%M:%SZ")
diff = endtime - starttime
duration = abs(diff.seconds)
instcost = price * duration / 3600.
except TypeError:
## In rare cases it seems one or the other of these things don't actually have entries. This is a problem. for now, charge for the hour:
message = " [Internal (get_costmonitoring)] Duration of past jobs not found. Pricing for an hour"
self.logger.append(message)
self.logger.printlatest()
instcost = price
cost += instcost
## Now compare against the cost of the job you're currently running:
## need duration from config (self.parse_config), self.instance_type, and self.nb_instances
## By assuming they're all standard instances we upper bound the cost.
try:
price = utilsparampricing.get_price(
utilsparampricing.get_region_name(utilsparampricing.region_id),
self.instance_type,
os="Linux")
nb_instances = len(self.filenames)
if self.jobduration is None:
duration = defaultduration / 60 ## in hours.
else:
duration = self.jobduration / 60
jobpricebound = duration * price * nb_instances
cost += jobpricebound
except Exception as e:
print(e)
raise Exception(
" [Internal (get_costmonitoring)] Unexpected Error: Unable to estimate cost of current job."
)
## Now compare agains the expected cost of instances with the current ami:
try:
ami = os.environ["AMI"]
total_activeprice = self.prices_active_instances_ami(ami)
except Exception as e:
print(e)
try:
activeprice = utilsparampricing.get_price(
utilsparampricing.get_region_name(
utilsparampricing.region_id),
self.instance_type,
os="Linux")
number = len(
[i for i in utilsparamec2.get_active_instances_ami(ami)])
activeduration = defaultduration * number / 60 ## default to the default duration instead if not given.
total_activeprice = activeprice * activeduration
except Exception as e:
print(e)
raise Exception(
" [Internal (get_costmonitoring)] Unexpected Error: Unable to estimate cost of active jobs."
)
cost += total_activeprice
## Now compare with budget:
try:
budget = float(
utilsparamssm.get_budget_parameter(self.path,
self.bucket_name))
except ClientError as e:
try:
assert e.response["Error"]["Code"] == "ParameterNotFound"
budget = float(os.environ["MAXCOST"])
message = " [Internal (get_costmonitoring)] Customized budget not found. Using default budget value of {}".format(
budget)
self.logger.append(message)
self.logger.printlatest()
except:
raise Exception(
" [Internal (get_costmonitoring)] Unexpected Error: Unable to get budget."
)
except Exception:
raise Exception(
" [Internal (get_costmonitoring)] Unexpected Error: Unable to get budget."
)
if cost < budget:
message = " [Internal (get_costmonitoring)] Projected total costs: ${}. Remaining budget: ${}".format(
cost, budget - cost)
self.logger.append(message)
self.logger.printlatest()
self.logger.write()
validjob = True
elif cost >= budget:
message = " [Internal (get_costmonitoring)] Projected total costs: ${}. Over budget (${}), cancelling job. Contact administrator.".format(
cost, budget)
self.logger.append(message)
self.logger.printlatest()
self.logger.write()
validjob = False
return validjob
