Example #1
0
    def loader(self, session):
        success = 0
        failure = 0

        os.chdir(import_path)
        targets = os.listdir('.')

        for target in targets:
            print "now serving:%s" % target

            start_time = time.time()

            command = "%s %s" % (gunzip_command, target)
            print command
            os.system(command)

            # trim trailing gz
            file_name = target[:-3]

            try:
                pickled = pickle.load(open(file_name, "rb"))
                flag = self.pickle_loader(pickled, session)
                if flag:
                    success += 1
                    stop_time = time.time()

                    loadlog = LoadLog(target)
                    loadlog.duration = stop_time - start_time
                    session.add(loadlog)
                    session.commit()
                else:
                    failure += 1
            except:
                print "failure noted"
                failure += 1

            command = "%s -rf %s" % (rm_command, file_name)
            print command
            os.system(command)

        message = "load complete success:%d failure:%d" % (success, failure)
        print message

        return success
Example #2
0
    def file_handler(self, file_name, task_id, session):
        with open(file_name, 'rb') as handle:
            pickled_band = pickle.load(handle)

            print pickled_band.installation

            installation = session.query(Installation).filter_by(uuid = pickled_band.installation).first()
            if installation == None:
                installation = Installation(pickled_band.installation)
                session.add(installation)
                session.commit()

            load_log = LoadLog(file_name)
            load_log.installation_id = installation.id
            load_log.task_id = task_id
            load_log.pickle_time = datetime.datetime.utcfromtimestamp(pickled_band.create_time)
            load_log.version = pickled_band.version
            load_log.band_ndx = pickled_band.band_ndx
            session.add(load_log)

            for observation in pickled_band.observations:
#                print observation
                obs_time = datetime.datetime.utcfromtimestamp(observation.sample_time)
                candidate = Observation(observation.frequency, observation.sample, obs_time, task_id)
                session.add(candidate)

            session.commit()
Example #3
0
    def file_handler(self, candidate, task_id, session):
        infile = open(candidate, 'r')
        raw_buffer = infile.readlines()
        infile.close()

        for raw_line in raw_buffer:
            print raw_line
            parsed_json = json.loads(raw_line)
#            print parsed_json
#            print parsed_json['networkName']

            try:
                version = parsed_json['version']
            except:
                version = 'missing_version'

            if version == 'missing_version':
                log_message = "missing version:%s" % candidate
                self.log_writer(6, log_message, task_id, session)
            elif version == 1:
                loader_v1 = LoaderVersion1()
                retval = loader_v1.execute(parsed_json, task_id, session)

                load_log = LoadLog(task_id, retval[0], 1, candidate)
                load_log.ble_pop = retval[1]
                load_log.wifi_pop = retval[2]
                load_log.cell_cdma_pop = retval[3]
                load_log.cell_gsm_pop = retval[4]
                load_log.cell_lte_pop = retval[5]
                load_log.cell_wcdma_pop = retval[6]

                session.add(load_log)
                session.commit()

                return True
        else:
                log_message = "bad version:%d:%s" % (version, candidate)
                self.log_writer(6, log_message, task_id, session)

        return False