def getPackageToScore(mask=None, inOrOut=True): packToScore = {} f = SepFile(':') f.open(Qpackage.QPACKAGE_SCORE_TXT, mode='txt', flag='r') for line in f: if mask == None: packToScore[line[0]] = float(line[1]) else: if inOrOut: if line[0] in mask: packToScore[line[0]] = float(line[1]) else: if line[0] not in mask: packToScore[line[0]] = float(line[1]) f.close() return packToScore
def getQuserOpenPackage(basePath='s3://datamining.ym/dmuser/ykang/results/qUserInLast5EachDay', beginDay='2016-01-24', interval_='30', isForward='0', s3DictBasePath='s3://datamining.ym/dmuser/ykang/data/spark.ouwan.qPackageToId', isDownload=True): mconf = MissionConf().setAppName('getQuserOpenPackage') msc = MissionContext(conf=mconf) [_, appPath] = msc.getFolder() if isDownload: for theDay in getDaysGen(beginDay, int(interval_), int(isForward)): BashUtil.s3Cp(os.path.join(basePath,theDay), appPath+os.sep+theDay, recursived=True) openPackage = {} mask = {'imei=333333333333333':1, 'imei=123456789abcdef':1, 'imei=111111111111111':1, 'imei=012345678912345':1, 'imei=000000000000000':1, 'imei=00000000000000':1} for (filename, _, files) in os.walk(appPath): print filename for gzfile in files: [_, ext] = os.path.splitext(gzfile) if ext == '.gz': f = SepFile('|') f.open(filename+os.sep+gzfile, mode='gzip', flag='rb') for line in f: if line[0] not in mask: if line[1] not in openPackage: openPackage[line[1]] = int(line[2]) else: openPackage[line[1]] += int(line[2]) f.close() openTimes = [] print 'sorting' packs = openPackage.keys() for key in packs: openTimes.append(openPackage[key]) index = sorted(range(len(openTimes)), key=lambda k: openTimes[k], reverse=True) print 'sorted' writer = LineFile() writer.open(os.path.join(appPath, 'qUserOpenPackage.txt'), mode='txt', flag='w') for i in index: key = packs[i] value = openPackage[key] writer.writeLine(key + '|' + str(value)) writer.close() #可以将qUserOpenPackageToOpenTimes写入到该位置Qpackage.QPACKAGE_ID_TXT index = 0; f = LineFile().open(Qpackage.QPACKAGE_ID_TXT, mode='txt', flag='w') for qPackage in openPackage: f.writeLine(qPackage + '|' + str(index)) index += 1 f.close() BashUtil.s3Cp(Qpackage.QPACKAGE_ID_TXT, dst=os.path.join(s3DictBasePath, 'qPackageToId.txt'), recursived=False) return openPackage
def getQpackageToOpenTimes(appPath): #idf qPackageToOpenTimes = {} for (filename, _, files) in os.walk(appPath): for gzfile in files: print gzfile [_, ext] = os.path.splitext(gzfile) if ext == '.gz': reader = SepFile('|') reader.open(os.path.join(filename, gzfile), mode='gzip', flag='rb') for line in reader: qPackageToOpenTimes[line[0]] = int(line[1]) reader.close() return qPackageToOpenTimes
def getQpackageToOpenTimes(appPath): #idf qPackageToOpenTimes = {} for (filename, _, files) in os.walk(appPath): for gzfile in files: print gzfile [_, ext] = os.path.splitext(gzfile) if ext == '.gz': reader = SepFile('|') reader.open(os.path.join(filename, gzfile), mode='gzip', flag='rb') for line in reader: qPackageToOpenTimes[line[0]] = int(line[1]) reader.close() return qPackageToOpenTimes
def getPackageToScore(mask=None, inOrOut=True): packToScore = {} f = SepFile(':') f.open(Qpackage.QPACKAGE_SCORE_TXT, mode='txt', flag='r') for line in f: if mask == None: packToScore[line[0]] = float(line[1]) else: if inOrOut: if line[0] in mask: packToScore[line[0]] = float(line[1]) else: if line[0] not in mask: packToScore[line[0]] = float(line[1]) f.close() return packToScore
def getUserTotalNumber(s3Path, isDownload=True): mconf = MissionConf().setAppName('userTotalNumber') msc = MissionContext(conf=mconf) [self, appPath] = msc.getFolder() if isDownload: BashUtil.s3Cp(s3Path, appPath, recursived=True) userTotalNumber = 0 for (filename, _, files) in os.walk(appPath): for gzfile in files: [_, ext] = os.path.splitext(gzfile) if ext == '.gz': reader = SepFile('|') reader.open(os.path.join(filename, gzfile), mode='gzip', flag='rb') for line in reader: userTotalNumber += 1 reader.close() return userTotalNumber
[_, appPath] = msc.getFolder() basePath = 's3://datamining.ym/dmuser/ykang/results/qUserInLast5EachDay' for theDay in getDaysGen('2016-01-24', 30, 0): BashUtil.s3Cp(basePath+os.sep+theDay, appPath+os.sep+theDay, recursived=True) openPackage = {} mask = {'imei=333333333333333':1, 'imei=123456789abcdef':1, 'imei=111111111111111':1, 'imei=012345678912345':1, 'imei=000000000000000':1, 'imei=00000000000000':1} #mask = {} for (filename, dirs, files) in os.walk(appPath): print filename for gzfile in files: [name, ext] = os.path.splitext(gzfile) if ext == '.gz': f = SepFile('|') f.open(filename+os.sep+gzfile, mode='gzip', flag='rb') for line in f: if line[0] not in mask: if line[1] not in openPackage: openPackage[line[1]] = int(line[2]) else: openPackage[line[1]] += int(line[2]) f.close() openTimes = [] print 'sorting' packs = openPackage.keys() for key in packs: openTimes.append(openPackage[key]) index = sorted(range(len(openTimes)), key=lambda k: openTimes[k], reverse=True) print 'sorted'
def getQuserOpenPackage( basePath='s3://datamining.ym/dmuser/ykang/results/qUserInLast5EachDay', beginDay='2016-01-24', interval_='30', isForward='0', s3DictBasePath='s3://datamining.ym/dmuser/ykang/data/spark.ouwan.qUserOpenPackage', isDownload=True): mconf = MissionConf().setAppName('getQuserOpenPackage') msc = MissionContext(conf=mconf) [_, appPath] = msc.getFolder() if isDownload: for theDay in getDaysGen(beginDay, int(interval_), int(isForward)): BashUtil.s3Cp(os.path.join(basePath, theDay), appPath + os.sep + theDay, recursived=True) openPackage = {} mask = { 'imei=333333333333333': 1, 'imei=123456789abcdef': 1, 'imei=111111111111111': 1, 'imei=012345678912345': 1, 'imei=000000000000000': 1, 'imei=00000000000000': 1 } for (filename, _, files) in os.walk(appPath): print filename for gzfile in files: [_, ext] = os.path.splitext(gzfile) if ext == '.gz': f = SepFile('|') f.open(filename + os.sep + gzfile, mode='gzip', flag='rb') for line in f: if line[0] not in mask: if line[1] not in openPackage: openPackage[line[1]] = int(line[2]) else: openPackage[line[1]] += int(line[2]) f.close() openTimes = [] print 'sorting' packs = openPackage.keys() for key in packs: openTimes.append(openPackage[key]) index = sorted(range(len(openTimes)), key=lambda k: openTimes[k], reverse=True) print 'sorted' writer = LineFile() writer.open(os.path.join(appPath, 'qUserOpenPackage.txt'), mode='txt', flag='w') for i in index: key = packs[i] value = openPackage[key] writer.writeLine(key + '|' + str(value)) writer.close() BashUtil.s3Cp(os.path.join(appPath, 'qUserOpenPackage.txt'), dst=os.path.join(s3DictBasePath, 'qUserOpenPackage.txt'), recursived=False) return openPackage
mask = { 'imei=333333333333333': 1, 'imei=123456789abcdef': 1, 'imei=111111111111111': 1, 'imei=012345678912345': 1, 'imei=000000000000000': 1, 'imei=00000000000000': 1 } #mask = {} for (filename, dirs, files) in os.walk(appPath): print filename for gzfile in files: [name, ext] = os.path.splitext(gzfile) if ext == '.gz': f = SepFile('|') f.open(filename + os.sep + gzfile, mode='gzip', flag='rb') for line in f: if line[0] not in mask: if line[1] not in openPackage: openPackage[line[1]] = int(line[2]) else: openPackage[line[1]] += int(line[2]) f.close() openTimes = [] print 'sorting' packs = openPackage.keys() for key in packs: openTimes.append(openPackage[key]) index = sorted(range(len(openTimes)), key=lambda k: openTimes[k], reverse=True)