def loader(self, session):
success = 0
failure = 0
os.chdir(import_path)
targets = os.listdir('.')
for target in targets:
print "now serving:%s" % target
start_time = time.time()
command = "%s %s" % (gunzip_command, target)
print command
os.system(command)
# trim trailing gz
file_name = target[:-3]
try:
pickled = pickle.load(open(file_name, "rb"))
flag = self.pickle_loader(pickled, session)
if flag:
success += 1
stop_time = time.time()
loadlog = LoadLog(target)
loadlog.duration = stop_time - start_time
session.add(loadlog)
session.commit()
else:
failure += 1
except:
print "failure noted"
failure += 1
command = "%s -rf %s" % (rm_command, file_name)
print command
os.system(command)
message = "load complete success:%d failure:%d" % (success, failure)
print message
return success
def file_handler(self, file_name, task_id, session):
with open(file_name, 'rb') as handle:
pickled_band = pickle.load(handle)
print pickled_band.installation
installation = session.query(Installation).filter_by(uuid = pickled_band.installation).first()
if installation == None:
installation = Installation(pickled_band.installation)
session.add(installation)
session.commit()
load_log = LoadLog(file_name)
load_log.installation_id = installation.id
load_log.task_id = task_id
load_log.pickle_time = datetime.datetime.utcfromtimestamp(pickled_band.create_time)
load_log.version = pickled_band.version
load_log.band_ndx = pickled_band.band_ndx
session.add(load_log)
for observation in pickled_band.observations:
# print observation
obs_time = datetime.datetime.utcfromtimestamp(observation.sample_time)
candidate = Observation(observation.frequency, observation.sample, obs_time, task_id)
session.add(candidate)
session.commit()
def file_handler(self, candidate, task_id, session):
infile = open(candidate, 'r')
raw_buffer = infile.readlines()
infile.close()
for raw_line in raw_buffer:
print raw_line
parsed_json = json.loads(raw_line)
# print parsed_json
# print parsed_json['networkName']
try:
version = parsed_json['version']
except:
version = 'missing_version'
if version == 'missing_version':
log_message = "missing version:%s" % candidate
self.log_writer(6, log_message, task_id, session)
elif version == 1:
loader_v1 = LoaderVersion1()
retval = loader_v1.execute(parsed_json, task_id, session)
load_log = LoadLog(task_id, retval[0], 1, candidate)
load_log.ble_pop = retval[1]
load_log.wifi_pop = retval[2]
load_log.cell_cdma_pop = retval[3]
load_log.cell_gsm_pop = retval[4]
load_log.cell_lte_pop = retval[5]
load_log.cell_wcdma_pop = retval[6]
session.add(load_log)
session.commit()
return True
else:
log_message = "bad version:%d:%s" % (version, candidate)
self.log_writer(6, log_message, task_id, session)
return False
