def handler(self, params):
try:
log.info("evtNetworkLog get")
#cocoa Log skip
if params.has_key('evtKey') == False and params.has_key('startTime') == False:
return
# 구 버전의 로그 무시
apatLogVer = map(int, params.get('ver', "1.00.0").split('.'))
if apatLogVer < [1,0,10]:
return
dataVer = map(int, params.get('ansLogVer', "0.5").split('.'))
if dataVer < [1,0] and params.has_key('evtKey') == False:
return
self.logInsert(params)
log.info("logInsert done")
except Exception, e:
log.error("processAnsLog ERROR[deviceID:%s, sID:%s, numTotalHits:%s] %s" % (params['deviceID'], params['sID'], params['numTotalHits'], e) )
if str(e).find('restart') > 0:
log.error("processAnsLog raise e")
raise e
def import_vcf(file_id):
file_data = VcfFileCollection().get_by_id(file_id)
VcfFileCollection().update_by_id(file_id, {'status': 'processing'})
sample_count = 0
try:
stream = open(os.path.join(Settings.file_store, file_id + '.vcf'), 'r')
vcf_reader = vcf.Reader(stream)
samples = next(vcf_reader).samples
sample_count = len(samples)
stream.seek(0)
vcf_reader = vcf.Reader(stream)
for i in range(0, sample_count):
stream.seek(0)
vcf_reader = vcf.Reader(stream)
variants = list()
for record in vcf_reader:
sample = record.samples[i]
#TODO - there are better ways to handle this
# Do we need to store the reference for this query
allleles = []
if sample.gt_bases is not None:
alleles = re.split(r'[\\/|]', sample.gt_bases)
# remove duplicates
alleles = set(alleles)
for allele in alleles:
chrom = record.CHROM
# remove preceeding chr if exists
if (re.match('chr', chrom, re.I)):
chrom = chrom[3:].upper()
if chrom in [
'1', '2', '3', '4', '5', '6', '7', '8', '9', '10',
'11', '12', '13', '14', '15', '16', '17', '18',
'19', '20', '21', '22', 'X', 'Y', 'M'
]:
variants.append(chrom + '_' + str(record.POS) + '_' +
allele)
VcfSampleCollection().add({
'fileid': file_id,
'variants': variants,
'caseId': file_data['caseId']
})
except:
log.exception('error importing case vcf')
VcfFileCollection().update_by_id(file_id, {
'status': 'complete',
'samples': sample_count
})
log.info('import complete')
def logInsert(self, params):
try:
if params.has_key('tTM') == False:
try:
vrdict = params.get('__validityReport', '')
if vrdict.has_key('tTM'):
params['tTM'] = float(vrdict['tTM'][0][0].replace(",", "."))
except Exception, e:
log.error("logInsert tTM convert [androidID:%s, pkgName:%s, sID:%s, tTM:%s, numTotalHits:%s] %s" % \
(params.get('deviceID', '-'), params.get('pkgName', '-'), params.get('sID', -9), params.get('tTM', -9), params.get('numTotalHits', -9), e))
return
self.rdbCur = None
self.rdbCur = worker.dbmanager.allocDictCursor('myapwave')
log.info("logInsert: before tran")
self.rdbCur.execute("START TRANSACTION")
strSQL = """INSERT IGNORE INTO appsslog (
deviceID, pkgName, sID, tTM, objid, agentAllowAns, ansLogVer, ansVer, avgTP, bssid,
cellId, duration, estTP, evtKey, evtValue, evtSyncID, startCode, exitCode, startTime, eventTime,
initWifi, model, netType, numTotalHits, NWcharac, playAppPackageName, playTime, activeDownload, activeTime, totalDownload,
linkspd, apfreq, ver, verCode, myfiOn, myfiAP, parentAppName, insTs)
VALUES (
'%s', '%s', %d, %.3f, '%s', %s, '%s', '%s', %s, '%s',
'%s', %s, %s, '%s', %s, %s, %s, %s, %s, %s,
%s, '%s', '%s', %s, '%s', '%s', %s, %s, %s, %s,
%s, %s, '%s', %s, %s, %s, '%s', %s)""" % \
(params['deviceID'], params['pkgName'], int(params['sID']), float(params['tTM']), params.get('objid', ''),
int(params.get('agentAllowAns', -1)), params.get('ansLogVer', ''), params.get('ansVer', ''),
params.get('avgTP', 'NULL'), params.get('bssid', ''),
params.get('cellId', ''), params.get('duration', 'NULL'), params.get('estTP', 'NULL'),
params.get('evtKey', ''), params.get('evtValue', 'NULL'),
params.get('evtSyncID', 'NULL'), params.get('startCode', 'NULL'), params.get('exitCode', 'NULL'),
params.get('startTime', 'NULL'), params.get('eventTime', 'NULL'),
params.get('initWifi', 'NULL'), params.get('model', ''), params.get('netType', ''),
params.get('numTotalHits', 'NULL'), params.get('NWcharac', ''),
params.get('playAppPackageName', ''), params.get('playTime', 'NULL'), params.get('activeDownload', 'NULL'),
params.get('activeTime', 'NULL'),params.get('totalDownload', 'NULL'),
params.get('lnkspd', 'NULL'), params.get('apfreq', 'NULL'), params.get('ver', ''),
params.get('verCode', 'NULL'), params.get('myfiOn', 'NULL'),
params.get('myfiAP', 'NULL'), params.get('parentAppName', ''), params.get('insTs', 'NULL')
)
ret = self.rdbCur.execute(strSQL)
strSQL = """INSERT IGNORE INTO appssdist (deviceID, isbusy, lstuptmp) VALUES ('%s', '0', unix_timestamp())""" % (params['deviceID'])
ret = self.rdbCur.execute(strSQL)
self.rdbCur.execute("COMMIT")
log.info("logInsert: commit tran")
def process(self, params):
try:
self.rdbCur = None
self.rdbCur = worker.dbmanager.allocDictCursor('myapwave')
pnm = "p%x" % (int(random.random()*100) % 15 + 1)
while True:
self.rdbCur.execute("START TRANSACTION")
strSQL = "SELECT deviceID, lstuptmp from appssdist partition (%s) where isbusy = '0' and procTmp < %d order by lstuptmp limit 1 for update" % (pnm, params['tmp'])
ret = self.rdbCur.execute(strSQL)
if ret > 0 and ret < 10000: # Sometimes Mysql returns 18446744073709551615 instead of -1
row = self.rdbCur.fetchone()
devid = row['deviceID']
lstuptmp = row['lstuptmp']
strSQL = "UPDATE appssdist SET isbusy = '1' where deviceID = '%s'" % (devid)
ret = self.rdbCur.execute(strSQL)
self.rdbCur.execute("COMMIT")
log.info("process:appssdist '1' %s" % devid)
try:
procnt = self.passLog(devid)
except Exception, e:
log.error("Appsession.process in while [%s] %s" % (devid, e))
while True:
try:
rtcnt = 0
if procnt > 0:
strSQL = "UPDATE appssdist SET isbusy = '0', procTmp = %d, lstuptmp = unix_timestamp() where deviceID = '%s'" % (params['tmp'], devid)
else:
if lstuptmp < params['tmp'] - 600:
strSQL = "DELETE FROM appssdist where deviceID = '%s'" % (devid)
else:
strSQL = "UPDATE appssdist SET isbusy = '0', procTmp = %d where deviceID = '%s'" % (params['tmp'], devid)
ret = self.rdbCur.execute(strSQL)
log.info("process: appssdist '0' %s" % devid)
break
except Exception, e:
if str(e).find('restart') > 0:
log.warn("process: appssdist[%s] set to '0' retry: %d, %s" % (devid, rtcnt, e))
if rtcnt > 10:
raise e
time.sleep(rtcnt+1)
rtcnt += 1
else:
log.error("process: appssdist[%s] set to '0' %s" % (devid, e))
raise e
else:
def http_get(url: str) -> Optional[bytes]:
"""
Retrieves the HTML from a page via HTTP(s).
If the page is present on the server (200 status code), returns the html.
If the server does not host this page (404 status code), returns None.
Any other HTTP code are considered as unexpected and raise an HTTPBadStatusCodeError.
:param url: url to the page
:return: HTML of the page or None
"""
r = session.get(url)
if r.status_code == 200:
# log.debug(f'http_get_success', status_code=r.status_code, url=url)
return r.content
if r.status_code == 404:
log.info(f'http_get_error', status_code=r.status_code, url=url)
return None
raise HTTPBadStatusCodeError(r.status_code)
def passLog(self, devid):
try:
strSQL = """SELECT
deviceID, pkgName, sID, tTM, objid, agentAllowAns, ansLogVer, ansVer, avgTP, bssid,
cellId, duration, estTP, evtKey, evtValue, evtSyncID, startCode, exitCode, startTime, eventTime,
initWifi, model, netType, numTotalHits, NWcharac, playAppPackageName, playTime, activeDownload, activeTime, totalDownload,
linkspd, apfreq, ver, verCode, myfiOn, myfiAP, parentAppName, insTs
from appsslog where deviceID = '%s'
order by deviceID, pkgName, sID, tTM""" % (devid)
ret = self.rdbCur.execute(strSQL)
log.info("passLog:appsslog ret = %d " % ret)
if ret > 0:
rows = self.rdbCur.fetchall()
cutTs = time.time() - 5
pdevid = 'NULL'
ppkgname = 'NULL'
psid = -9
for row in rows:
if pdevid == row['deviceID'] and ppkgname == row['pkgName'] and psid == row['sID']:
log.info("passLog: pass %s %s %s" % (pdevid, ppkgname, psid))
continue
if row['insTs'] > cutTs:
pdevid = row['deviceID']
ppkgname = row['pkgName']
psid = row['sID']
log.info("passLog: insTs(%d) > cutTs(%d) %s %s %s" % (row['insTs'], cutTs, pdevid, ppkgname, psid))
continue
keepkeys = filter(lambda x: row[x] <> None, row)
newrow = {k: row[k] for k in keepkeys}
# log.info("passLog: before processLog")
# log.info(newrow)
self.processLog(newrow)
strSQL = """DELETE from appsslog where deviceID = '%s' and pkgName = '%s' and sID = %d and tTM = %.3f""" % \
(newrow['deviceID'], newrow['pkgName'], newrow['sID'], newrow['tTM'])
ret = self.rdbCur.execute(strSQL)
log.info("passLog: delete appsslog %d (inTs:%.3f cutTs:%.3f) (%s, %s, %d, %.3f)" % (ret, newrow['insTs'], cutTs, newrow['deviceID'], newrow['pkgName'], newrow['sID'], newrow['tTM']))
return ret
except Exception, e:
log.error("passLog : %s" % e)
log.error(newrow)
raise e
def handler(self, params):
global appss
global lastParamTmp
try:
log.info("evtTimeInterval get %d (last:%d)" % (params['tmp'], lastParamTmp))
if params['tmp'] > lastParamTmp:
log.info("appss.process start")
appss.process(params)
lastParamTmp = params['tmp']
log.info("appss.process done")
except Exception, e:
log.error("processAnsClock ERROR %s" % (e) )
log.info("process: appssdist '0' %s" % devid)
break
except Exception, e:
if str(e).find('restart') > 0:
log.warn("process: appssdist[%s] set to '0' retry: %d, %s" % (devid, rtcnt, e))
if rtcnt > 10:
raise e
time.sleep(rtcnt+1)
rtcnt += 1
else:
log.error("process: appssdist[%s] set to '0' %s" % (devid, e))
raise e
else:
self.rdbCur.execute("COMMIT")
log.info("process:appssdist No dev return")
break
except Exception, e:
log.error("AppSession.process %s, [%s]" % (e, strSQL))
log.error(params)
finally:
if self.rdbCur <> None:
self.rdbCur.execute("COMMIT")
worker.dbmanager.freeCursor(self.rdbCur)
# 전체 device 처리
class ProcessAnsClock(object):
# static class variable