Esempio n. 1
0
    def create_structure(self):
        """
        Creates the necessary model input data in the database to ingest against.
        :return: An integer representing the replication created.
        """
        myrun = DimRun()
        myrun.name = "AUTO SCOTTY TEST RUN"
        myrun.models_key_id = 1
        myrun.save()

        mychannel = DimChannel.objects.get(pk=228441)
        myrun.dimchannel_set.add(mychannel)

        myex = DimExecution()
        myex.run_key_id = myrun.id
        myex.save()

        myrep = DimReplication()
        myrep.execution_key_id = myex.id
        myrep.seed_used = 99
        myrep.series_id = 100
        myrep.save()

        self.model_input.append([myrun.id, myex.id, myrep.id, mychannel.id])

        return myrep.id
Esempio n. 2
0
    def handle(self, *args, **options):
        print "BEGIN TEST"
        # Create execution, replication, and run
        myrun = DimRun()
        myrun.name = "AUTO SCOTTY TEST RUN"
        myrun.models_key_id = 1
        myrun.save()

        mychannel = DimChannel.objects.get(pk=228441)
        myrun.dimchannel_set.add(mychannel)

        myex = DimExecution()
        myex.run_key_id = myrun.id
        myex.save()

        myrep = DimReplication()
        myrep.execution_key_id = myex.id
        myrep.seed_used = 99
        myrep.series_id = 100
        myrep.save()

        replication_number = myrep.id
        print "REPLICATION NUMBER", replication_number
        print "CHANNEL NUMBER", mychannel.id
        print "RUN NUMBER", myrun.id
        print "EXECUTION NUMBER", myex.id
        # rename the test zip_file
        oldfilename = options['filename']
        oldlist = oldfilename.split(".")
        newfilename = oldlist[0] + "-" + str(replication_number) + "." + oldlist[1]
        shutil.copyfile(oldfilename, newfilename)
        # make sure this replication output data doesn't already exist
        results = [i for i in BaseFactData.objects.filter(replication_key_id=replication_number,
                                                          channel_key_id=mychannel.id, run_key_id=myrun.id)]
        count = len(results)
        if count != 0:
            print "There currently exists data in BaseFactData for the chosen replication, channel, and run." \
                  "Please choose a replication, channel, and run which have not previously been ingested."

        # Curl the files to AutoScotty
        print "Beam us up, Scotty.\nAye, Sir."
        print "File: ", newfilename
        f = open(newfilename, 'rb')
        files = {'zip_file': f}
        urlname = options['urlname']
        model_type = options['modeltype']
        if options['hashname']:
            r = requests.post(urlname, files=files, data={'model_type': model_type, 'sync': True,
                                                          'zip_file_hash': options['hashname']})
        else:
            myhash = hashlib.sha1()
            myhash.update(f.read())
            f.seek(0)
            r = requests.post(urlname, files=files, data={'model_type': model_type, 'sync': True,
                                                          'zip_file_hash': myhash.hexdigest()})

        # remove the temporary copy of the file
        os.remove(newfilename)

        if r.status_code != 200:
            print "There was an ingestion error at the server. The HTTP Response code is: ", r.status_code
            print "Please check the celery and apache logs on the server to determine the source of the error."

        # fetch the ingested results
        results = BaseFactData.objects.filter(replication_key_id=replication_number, channel_key_id=mychannel.id,
                                              run_key_id=myrun.id).aggregate(Count("timestep"))
        count = results['timestep__count']

        if count == 10950:
            print "The anticipated number of entries in BaseFactData were present. The data was successfully ingested."
        else:
            print "The count of " + str(count) + " is not the expected number of entries. Please make sure channel " \
                                                 "228441 exists, and that you used the 'autoscottytest.zip' file."

        # remove the ingested data
        BaseFactData.objects.filter(replication_key_id=replication_number, channel_key_id=mychannel.id,
                                    run_key_id=myrun.id).delete()
        myrun.dimchannel_set.remove(mychannel)
        myrep.delete()
        myex.delete()
        myrun.delete()

        # make sure data was deleted
        results = BaseFactData.objects.filter(replication_key_id=replication_number, channel_key_id=mychannel.id,
                                              run_key_id=myrun.id).aggregate(Count("timestep"))
        count = results['timestep__count']
        if count == 0:
            print "The data was successfully purged."
        else:
            print "There was an error removing the ingested data.  There are still entries in BaseFactData " \
                  "corresponding to your replication, channel, and run. Please see the server logs and the database " \
                  "administrator for assistance."

        # return response
        print "Live long and prosper."
Esempio n. 3
0
    def save_run(self, scenario_id, template_id, start_date, name, description, location_ndx,
                timestep_interval, model_id='', duration=0, end_date='',
                note='', run_id=-1, as_object=False):
        """This is the save_run method
        
        This will create a "run" for a specific user.  It takes as its parameters everything
        needed to create the run, and will return either the run id or the run object as per a
        boolean flag (defaults to returning the id).

        :param run_id:          ID of the run if this is being edited
        :param template_id:     ID of the base template that this run is using
        :param start_date:      Start Date of the simulation in YYYY-MM-DD or a dateime.dateime obj
        :param duration:        Duration of the simulation in days
        :param end_date:        End Date of the simulation, this is used, if given instead of duration
        :param name:            Name of the simulation run
        :param description:     Description of the simulation run
        :param location_ndx:    Index of the location that the run is taking place
        :param version:         Version of the model that is being used
        :param model_id:        ID of the model being used (should be set in self.model_id)
        :param timestep_interval:   Timestep interval in days (fractions allowed)
        :param note:            A note to be attached to the run
        :param as_object:       Returns an object if true, otherwise returns the ID otherwise
                                return An ID of the scenario if saved (-1 if not) or the object as per the as_object flag
        """
        if not self.valid_user():
            raise Exception("No valid user detected, only valid users can save runs")
        if duration == 0 and end_date is '':
            print "You must specify duration or end_date, none were specified"
            return -1
        # Make sure start and end dates are taken care of
        if isinstance(start_date, str):
            # Makes sure that start date is a timestamp, either it can come in that way, or be made by string
            start_date = datetime.datetime(
                    int(start_date.split('-')[0]),
                    int(start_date.split('-')[1]),
                    int(start_date.split('-')[2]))
        startDate = start_date
        if end_date is not '': # rgj: and isinstance(end_date, str):
            if isinstance(end_date, str):
                end_date = datetime.datetime(int(end_date.split('-')[0]), int(end_date.split('-')[1]), int(end_date.split('-')[2]))
        else:
            end_date = start_date + datetime.timedelta(days=duration)
        endDate = end_date

        # Add model_version to the run based on DimTemplate
        my_template = DimTemplate.objects.get(id=template_id)

        # Now we piece together run
        run = DimRun(
            baseline_key_id=scenario_id,
            template_key_id=template_id,
            start_date_key=startDate,
            end_date_key=endDate,
            name=name,
            description=description,
            location_key_id=location_ndx,
            timestep_interval_days=timestep_interval,
            models_key_id=(self.model_id if model_id is '' else model_id),
            model_version=my_template.model_version
        )

        if run_id == -1:
            run.save()
        else:
            run.id = run_id
            run.save()

        if run.id == -1:
            print "An error has occurred, the run was not saved"
            return -1

        # Log when the run was saved and who saved it
        logger.info('Run %s was saved by %s', run.id, inspect.stack()[2][1])
        # Now that we have a run_id

        if note is not '':
            self.save_note(run.id, note)

        if as_object:
            return run
        else:
            return run.id