def create_structure(self):
"""
Creates the necessary model input data in the database to ingest against.
:return: An integer representing the replication created.
"""
myrun = DimRun()
myrun.name = "AUTO SCOTTY TEST RUN"
myrun.models_key_id = 1
myrun.save()
mychannel = DimChannel.objects.get(pk=228441)
myrun.dimchannel_set.add(mychannel)
myex = DimExecution()
myex.run_key_id = myrun.id
myex.save()
myrep = DimReplication()
myrep.execution_key_id = myex.id
myrep.seed_used = 99
myrep.series_id = 100
myrep.save()
self.model_input.append([myrun.id, myex.id, myrep.id, mychannel.id])
return myrep.id
def handle(self, *args, **options):
print "BEGIN TEST"
# Create execution, replication, and run
myrun = DimRun()
myrun.name = "AUTO SCOTTY TEST RUN"
myrun.models_key_id = 1
myrun.save()
mychannel = DimChannel.objects.get(pk=228441)
myrun.dimchannel_set.add(mychannel)
myex = DimExecution()
myex.run_key_id = myrun.id
myex.save()
myrep = DimReplication()
myrep.execution_key_id = myex.id
myrep.seed_used = 99
myrep.series_id = 100
myrep.save()
replication_number = myrep.id
print "REPLICATION NUMBER", replication_number
print "CHANNEL NUMBER", mychannel.id
print "RUN NUMBER", myrun.id
print "EXECUTION NUMBER", myex.id
# rename the test zip_file
oldfilename = options['filename']
oldlist = oldfilename.split(".")
newfilename = oldlist[0] + "-" + str(replication_number) + "." + oldlist[1]
shutil.copyfile(oldfilename, newfilename)
# make sure this replication output data doesn't already exist
results = [i for i in BaseFactData.objects.filter(replication_key_id=replication_number,
channel_key_id=mychannel.id, run_key_id=myrun.id)]
count = len(results)
if count != 0:
print "There currently exists data in BaseFactData for the chosen replication, channel, and run." \
"Please choose a replication, channel, and run which have not previously been ingested."
# Curl the files to AutoScotty
print "Beam us up, Scotty.\nAye, Sir."
print "File: ", newfilename
f = open(newfilename, 'rb')
files = {'zip_file': f}
urlname = options['urlname']
model_type = options['modeltype']
if options['hashname']:
r = requests.post(urlname, files=files, data={'model_type': model_type, 'sync': True,
'zip_file_hash': options['hashname']})
else:
myhash = hashlib.sha1()
myhash.update(f.read())
f.seek(0)
r = requests.post(urlname, files=files, data={'model_type': model_type, 'sync': True,
'zip_file_hash': myhash.hexdigest()})
# remove the temporary copy of the file
os.remove(newfilename)
if r.status_code != 200:
print "There was an ingestion error at the server. The HTTP Response code is: ", r.status_code
print "Please check the celery and apache logs on the server to determine the source of the error."
# fetch the ingested results
results = BaseFactData.objects.filter(replication_key_id=replication_number, channel_key_id=mychannel.id,
run_key_id=myrun.id).aggregate(Count("timestep"))
count = results['timestep__count']
if count == 10950:
print "The anticipated number of entries in BaseFactData were present. The data was successfully ingested."
else:
print "The count of " + str(count) + " is not the expected number of entries. Please make sure channel " \
"228441 exists, and that you used the 'autoscottytest.zip' file."
# remove the ingested data
BaseFactData.objects.filter(replication_key_id=replication_number, channel_key_id=mychannel.id,
run_key_id=myrun.id).delete()
myrun.dimchannel_set.remove(mychannel)
myrep.delete()
myex.delete()
myrun.delete()
# make sure data was deleted
results = BaseFactData.objects.filter(replication_key_id=replication_number, channel_key_id=mychannel.id,
run_key_id=myrun.id).aggregate(Count("timestep"))
count = results['timestep__count']
if count == 0:
print "The data was successfully purged."
else:
print "There was an error removing the ingested data. There are still entries in BaseFactData " \
"corresponding to your replication, channel, and run. Please see the server logs and the database " \
"administrator for assistance."
# return response
print "Live long and prosper."
def save_run(self, scenario_id, template_id, start_date, name, description, location_ndx,
timestep_interval, model_id='', duration=0, end_date='',
note='', run_id=-1, as_object=False):
"""This is the save_run method
This will create a "run" for a specific user. It takes as its parameters everything
needed to create the run, and will return either the run id or the run object as per a
boolean flag (defaults to returning the id).
:param run_id: ID of the run if this is being edited
:param template_id: ID of the base template that this run is using
:param start_date: Start Date of the simulation in YYYY-MM-DD or a dateime.dateime obj
:param duration: Duration of the simulation in days
:param end_date: End Date of the simulation, this is used, if given instead of duration
:param name: Name of the simulation run
:param description: Description of the simulation run
:param location_ndx: Index of the location that the run is taking place
:param version: Version of the model that is being used
:param model_id: ID of the model being used (should be set in self.model_id)
:param timestep_interval: Timestep interval in days (fractions allowed)
:param note: A note to be attached to the run
:param as_object: Returns an object if true, otherwise returns the ID otherwise
return An ID of the scenario if saved (-1 if not) or the object as per the as_object flag
"""
if not self.valid_user():
raise Exception("No valid user detected, only valid users can save runs")
if duration == 0 and end_date is '':
print "You must specify duration or end_date, none were specified"
return -1
# Make sure start and end dates are taken care of
if isinstance(start_date, str):
# Makes sure that start date is a timestamp, either it can come in that way, or be made by string
start_date = datetime.datetime(
int(start_date.split('-')[0]),
int(start_date.split('-')[1]),
int(start_date.split('-')[2]))
startDate = start_date
if end_date is not '': # rgj: and isinstance(end_date, str):
if isinstance(end_date, str):
end_date = datetime.datetime(int(end_date.split('-')[0]), int(end_date.split('-')[1]), int(end_date.split('-')[2]))
else:
end_date = start_date + datetime.timedelta(days=duration)
endDate = end_date
# Add model_version to the run based on DimTemplate
my_template = DimTemplate.objects.get(id=template_id)
# Now we piece together run
run = DimRun(
baseline_key_id=scenario_id,
template_key_id=template_id,
start_date_key=startDate,
end_date_key=endDate,
name=name,
description=description,
location_key_id=location_ndx,
timestep_interval_days=timestep_interval,
models_key_id=(self.model_id if model_id is '' else model_id),
model_version=my_template.model_version
)
if run_id == -1:
run.save()
else:
run.id = run_id
run.save()
if run.id == -1:
print "An error has occurred, the run was not saved"
return -1
# Log when the run was saved and who saved it
logger.info('Run %s was saved by %s', run.id, inspect.stack()[2][1])
# Now that we have a run_id
if note is not '':
self.save_note(run.id, note)
if as_object:
return run
else:
return run.id