def create_structure(self):
"""
Creates the necessary model input data in the database to ingest against.
:return: An integer representing the replication created.
"""
myrun = DimRun()
myrun.name = "AUTO SCOTTY TEST RUN"
myrun.models_key_id = 1
myrun.save()
mychannel = DimChannel.objects.get(pk=228441)
myrun.dimchannel_set.add(mychannel)
myex = DimExecution()
myex.run_key_id = myrun.id
myex.save()
myrep = DimReplication()
myrep.execution_key_id = myex.id
myrep.seed_used = 99
myrep.series_id = 100
myrep.save()
self.model_input.append([myrun.id, myex.id, myrep.id, mychannel.id])
return myrep.id
def expand_run(cls, run, reps_per_exec):
"""
This will expand the given run into executions. An execution is a particular configuration of
a set of inputs to be used by the models. This will take the JSON Change Document (JCD) from the
run given and break it out into execution level and run level changes.
Here we assume that the run object has a jcd attribute. If not, the method fails. Eventually a
jcd field will be part of the run model definition. At time of writing this is not true.
Run level changes are changes before a factorable type is defined. The reason for this is that
changes after the first factorable type is defined must be applied in order, and therefore belong
to the execution.
This will return the number of executions, and the list of executions
:param run: DimRun object that needs to be expanded
:param reps_per_exec: Number of replications for generated executions
:returns: Number of executions, List of saved Executions
"""
if not hasattr(run, 'jcd'):
raise ValueError("Run object does not have JCD")
jcd = run.jcd
exec_list = list()
(rl_jcd, num_executions, jcd_list) = jcd.expand()
if num_executions == 1:
new_execution = DimExecution(
run_key=run,
name=run.name,
replications=reps_per_exec
)
new_execution.save()
exec_list.append(new_execution)
return num_executions, exec_list
for doc in jcd_list:
new_execution = DimExecution(
run_key=run,
name=cls.name_from_sweeps(doc.change_list),
replications=reps_per_exec,
jcd=doc
)
new_execution.save()
exec_list.append(new_execution)
return num_executions, exec_list
def handle(self, *args, **options):
print "BEGIN TEST"
# Create execution, replication, and run
myrun = DimRun()
myrun.name = "AUTO SCOTTY TEST RUN"
myrun.models_key_id = 1
myrun.save()
mychannel = DimChannel.objects.get(pk=228441)
myrun.dimchannel_set.add(mychannel)
myex = DimExecution()
myex.run_key_id = myrun.id
myex.save()
myrep = DimReplication()
myrep.execution_key_id = myex.id
myrep.seed_used = 99
myrep.series_id = 100
myrep.save()
replication_number = myrep.id
print "REPLICATION NUMBER", replication_number
print "CHANNEL NUMBER", mychannel.id
print "RUN NUMBER", myrun.id
print "EXECUTION NUMBER", myex.id
# rename the test zip_file
oldfilename = options['filename']
oldlist = oldfilename.split(".")
newfilename = oldlist[0] + "-" + str(replication_number) + "." + oldlist[1]
shutil.copyfile(oldfilename, newfilename)
# make sure this replication output data doesn't already exist
results = [i for i in BaseFactData.objects.filter(replication_key_id=replication_number,
channel_key_id=mychannel.id, run_key_id=myrun.id)]
count = len(results)
if count != 0:
print "There currently exists data in BaseFactData for the chosen replication, channel, and run." \
"Please choose a replication, channel, and run which have not previously been ingested."
# Curl the files to AutoScotty
print "Beam us up, Scotty.\nAye, Sir."
print "File: ", newfilename
f = open(newfilename, 'rb')
files = {'zip_file': f}
urlname = options['urlname']
model_type = options['modeltype']
if options['hashname']:
r = requests.post(urlname, files=files, data={'model_type': model_type, 'sync': True,
'zip_file_hash': options['hashname']})
else:
myhash = hashlib.sha1()
myhash.update(f.read())
f.seek(0)
r = requests.post(urlname, files=files, data={'model_type': model_type, 'sync': True,
'zip_file_hash': myhash.hexdigest()})
# remove the temporary copy of the file
os.remove(newfilename)
if r.status_code != 200:
print "There was an ingestion error at the server. The HTTP Response code is: ", r.status_code
print "Please check the celery and apache logs on the server to determine the source of the error."
# fetch the ingested results
results = BaseFactData.objects.filter(replication_key_id=replication_number, channel_key_id=mychannel.id,
run_key_id=myrun.id).aggregate(Count("timestep"))
count = results['timestep__count']
if count == 10950:
print "The anticipated number of entries in BaseFactData were present. The data was successfully ingested."
else:
print "The count of " + str(count) + " is not the expected number of entries. Please make sure channel " \
"228441 exists, and that you used the 'autoscottytest.zip' file."
# remove the ingested data
BaseFactData.objects.filter(replication_key_id=replication_number, channel_key_id=mychannel.id,
run_key_id=myrun.id).delete()
myrun.dimchannel_set.remove(mychannel)
myrep.delete()
myex.delete()
myrun.delete()
# make sure data was deleted
results = BaseFactData.objects.filter(replication_key_id=replication_number, channel_key_id=mychannel.id,
run_key_id=myrun.id).aggregate(Count("timestep"))
count = results['timestep__count']
if count == 0:
print "The data was successfully purged."
else:
print "There was an error removing the ingested data. There are still entries in BaseFactData " \
"corresponding to your replication, channel, and run. Please see the server logs and the database " \
"administrator for assistance."
# return response
print "Live long and prosper."
