def create_structure(self):
"""
Creates the necessary model input data in the database to ingest against.
:return: An integer representing the replication created.
"""
myrun = DimRun()
myrun.name = "AUTO SCOTTY TEST RUN"
myrun.models_key_id = 1
myrun.save()
mychannel = DimChannel.objects.get(pk=228441)
myrun.dimchannel_set.add(mychannel)
myex = DimExecution()
myex.run_key_id = myrun.id
myex.save()
myrep = DimReplication()
myrep.execution_key_id = myex.id
myrep.seed_used = 99
myrep.series_id = 100
myrep.save()
self.model_input.append([myrun.id, myex.id, myrep.id, mychannel.id])
return myrep.id
def handle(self, *args, **options):
print "BEGIN TEST"
# Create execution, replication, and run
myrun = DimRun()
myrun.name = "AUTO SCOTTY TEST RUN"
myrun.models_key_id = 1
myrun.save()
mychannel = DimChannel.objects.get(pk=228441)
myrun.dimchannel_set.add(mychannel)
myex = DimExecution()
myex.run_key_id = myrun.id
myex.save()
myrep = DimReplication()
myrep.execution_key_id = myex.id
myrep.seed_used = 99
myrep.series_id = 100
myrep.save()
replication_number = myrep.id
print "REPLICATION NUMBER", replication_number
print "CHANNEL NUMBER", mychannel.id
print "RUN NUMBER", myrun.id
print "EXECUTION NUMBER", myex.id
# rename the test zip_file
oldfilename = options['filename']
oldlist = oldfilename.split(".")
newfilename = oldlist[0] + "-" + str(replication_number) + "." + oldlist[1]
shutil.copyfile(oldfilename, newfilename)
# make sure this replication output data doesn't already exist
results = [i for i in BaseFactData.objects.filter(replication_key_id=replication_number,
channel_key_id=mychannel.id, run_key_id=myrun.id)]
count = len(results)
if count != 0:
print "There currently exists data in BaseFactData for the chosen replication, channel, and run." \
"Please choose a replication, channel, and run which have not previously been ingested."
# Curl the files to AutoScotty
print "Beam us up, Scotty.\nAye, Sir."
print "File: ", newfilename
f = open(newfilename, 'rb')
files = {'zip_file': f}
urlname = options['urlname']
model_type = options['modeltype']
if options['hashname']:
r = requests.post(urlname, files=files, data={'model_type': model_type, 'sync': True,
'zip_file_hash': options['hashname']})
else:
myhash = hashlib.sha1()
myhash.update(f.read())
f.seek(0)
r = requests.post(urlname, files=files, data={'model_type': model_type, 'sync': True,
'zip_file_hash': myhash.hexdigest()})
# remove the temporary copy of the file
os.remove(newfilename)
if r.status_code != 200:
print "There was an ingestion error at the server. The HTTP Response code is: ", r.status_code
print "Please check the celery and apache logs on the server to determine the source of the error."
# fetch the ingested results
results = BaseFactData.objects.filter(replication_key_id=replication_number, channel_key_id=mychannel.id,
run_key_id=myrun.id).aggregate(Count("timestep"))
count = results['timestep__count']
if count == 10950:
print "The anticipated number of entries in BaseFactData were present. The data was successfully ingested."
else:
print "The count of " + str(count) + " is not the expected number of entries. Please make sure channel " \
"228441 exists, and that you used the 'autoscottytest.zip' file."
# remove the ingested data
BaseFactData.objects.filter(replication_key_id=replication_number, channel_key_id=mychannel.id,
run_key_id=myrun.id).delete()
myrun.dimchannel_set.remove(mychannel)
myrep.delete()
myex.delete()
myrun.delete()
# make sure data was deleted
results = BaseFactData.objects.filter(replication_key_id=replication_number, channel_key_id=mychannel.id,
run_key_id=myrun.id).aggregate(Count("timestep"))
count = results['timestep__count']
if count == 0:
print "The data was successfully purged."
else:
print "There was an error removing the ingested data. There are still entries in BaseFactData " \
"corresponding to your replication, channel, and run. Please see the server logs and the database " \
"administrator for assistance."
# return response
print "Live long and prosper."