def makeexpecteddata(hour=-1): resultfile1 = open(os.path.abspath(os.path.dirname(__file__)) + '/../inputdata/DailyPeerOnlineTime.txt', 'r') orglines1 = resultfile1.readlines() resultfile1.close() tempfile = open(os.path.abspath(os.path.dirname(__file__)) + '/../outputdata/DailyPeerOnlineTimeDBWeek1.txt', 'w') for line in orglines1: tmpstr = line.split(',')[1] if float(tmpstr) > 64000: newline = line.replace(',%s' % tmpstr, ',%s' % (float(tmpstr) + float(random.randint(0, 120)))) tmpstr1 = newline.split(',')[2] newline1 = newline.replace(',%s' % tmpstr1, ',%s,%s' % ('NULL', tmpstr1)) else: newline = line.replace(',%s' % tmpstr, ',%s' % (float(tmpstr) - float(random.randint(0, 180)))) tmpstr1 = newline.split(',')[2] newline1 = newline.replace(',%s' % tmpstr1, ',%s,%s' % ('NULL', tmpstr1)) tempfile.write(newline1) tempfile.close() pipeofodps.uploaddatatoodps( 'output_daily_peer_online_time', os.path.abspath(os.path.dirname(__file__) + '/../outputdata/DailyPeerOnlineTimeDBWeek1.txt'), 3) resultfile2 = open(os.path.abspath(os.path.dirname(__file__)) + '/../outputdata/DailyPeerOnlineTimeDBWeek1.txt', 'r') orglines2 = resultfile2.readlines() resultfile2.close() expectedfile = open(os.path.abspath(os.path.dirname(__file__)) + "/../inputdata/%s.txt"%__name__.split('.')[-1],"w") countlist={} sumcount={} for line in orglines1: prefix,avatime,actcount = line.split(',') actcount = actcount.replace('\n', '') if prefix not in countlist: countlist[prefix] = countlist.setdefault(prefix,0) + float(avatime) * int(actcount) if prefix not in sumcount: sumcount[prefix] = sumcount.setdefault(prefix,0) + int(actcount) for line in orglines2: prefix,avatime,_,actcount = line.split(',') actcount = actcount.replace('\n', '') if prefix not in countlist: countlist[prefix] = countlist.setdefault(prefix,0) + float(avatime) * int(actcount) if prefix not in sumcount: sumcount[prefix] = sumcount.setdefault(prefix,0) + int(actcount) else: countlist[prefix] = countlist[prefix] + float(avatime) * int(actcount) sumcount[prefix] = sumcount[prefix] + int(actcount) for pre, avetime in countlist.items(): expectedfile.write('%s,%f,%s\n' % (pre,float(avetime/sumcount[pre]),' ')) expectedfile.close() return os.path.abspath(os.path.dirname(__file__)) + "/../inputdata/%s.txt"%__name__.split('.')[-1]
def createtabledata(classname): m=sys.modules['testdata.'+classname] # m=sys.modules['lib.platform.dataprocess.testdata.'+classname] attstr=dir(m) for str in attstr: att=getattr(m,str) if issubclass(att,Dataprovider): tmpObj = att() tmpfile = tmpObj.make_data() print tmpObj.gettablename() pipeofodps.uploaddatatoodps(tmpObj.gettablename(), tmpfile) break
def createalldata(): m = sys.modules['testdata.dataprovider'] attstr=dir(m) for str in attstr: att=getattr(m,str) if type(att)==types.ModuleType: subattstr=dir(att) for substr in subattstr: subatt = getattr(att, substr) if type(subatt)==types.TypeType and issubclass(subatt,m.Dataprovider): tmpObj = subatt() tmpfile = tmpObj.make_data() print tmpObj.gettablename() pipeofodps.uploaddatatoodps(tmpObj.gettablename() ,tmpfile) break
def makeexpecteddata(hour=-1): resultfile = open( os.path.abspath(os.path.dirname(__file__)) + '/../outputdata/TodayFilePlayCountDB.txt', 'r') orgdownlines = resultfile.readlines() resultfile.close() tempfile = open( os.path.abspath(os.path.dirname(__file__)) + '/../outputdata/TodayFilePlayCountDBtmp.txt', 'w') for line in orgdownlines: tmpstr = line.split(',')[2] newline = line.replace(',%s' % tmpstr, ',%s,%s' % (tmpstr, tmpstr)) tempfile.write(newline) tempfile.close() pipeofodps.uploaddatatoodps( 'output_daily_file_play_count', os.path.abspath( os.path.dirname(__file__) + '/../outputdata/TodayFilePlayCountDBtmp.txt'), -1) expectedfile = open( os.path.abspath(os.path.dirname(__file__)) + "/../inputdata/%s.txt" % __name__.split('.')[-1], "w") resultlist = {} for line in orgdownlines: url, username, count, _ = line.split(',') if url not in resultlist: resultlist[url] = {} resultlist[url][username] = resultlist[url].setdefault( username, 0.0) + float(float(count) * (1.0 + math.exp(-1))) for url, users in resultlist.items(): for user, count in users.items(): expectedfile.write('%s,%s,%f,%s\n' % (url, user, count, '')) expectedfile.close() return os.path.abspath(os.path.dirname( __file__)) + "/../inputdata/%s.txt" % __name__.split('.')[-1]
def makeexpecteddata(hour=-1): with open( os.path.abspath(os.path.dirname(__file__)) + '/../outputdata/DailyProvinceFilePlayCountDB.txt', 'r') as resultfile1: orglines1 = resultfile1.readlines() with open( os.path.abspath(os.path.dirname(__file__)) + '/../outputdata/DailyProvinceFilePlayCountDBWeek1.txt', 'w') as tempfile: for line in orglines1: tmpstr = line.split(',')[2] if int(tmpstr) % 2 == 0: newline = line.replace( ',%s' % tmpstr, ',%s' % (int(tmpstr) + random.randint(0, 10))) else: newline = line.replace( ',%s' % tmpstr, ',%s' % (int(tmpstr) - random.randint(0, 6))) tempfile.write(newline) pipeofodps.uploaddatatoodps( 'output_daily_province_file_play_count', os.path.abspath( os.path.dirname(__file__) + '/../outputdata/DailyProvinceFilePlayCountDBWeek1.txt'), 2) with open( os.path.abspath(os.path.dirname(__file__)) + '/../outputdata/DailyProvinceFilePlayCountDBWeek1.txt', 'r') as resultfile2: orglines2 = resultfile2.readlines() with open( os.path.abspath(os.path.dirname(__file__)) + "/../inputdata/%s.txt" % __name__.split('.')[-1], "w") as expectedfile: resultlist = {} for line in orglines1: province, username, playcount, _ = line.split(',') if province not in resultlist: resultlist[province] = {} if username not in resultlist[province]: resultlist[province][username] = int(playcount) else: resultlist[province][ username] = resultlist[province][username] + int(playcount) for line in orglines2: province, username, playcount, _ = line.split(',') if province not in resultlist: resultlist[province] = {} if username not in resultlist[province]: resultlist[province][username] = int(playcount) else: resultlist[province][ username] = resultlist[province][username] + int(playcount) for pro, value in resultlist.items(): for user, count in value.items(): expectedfile.write('%s,%s,%d\n' % (pro, user, count)) return os.path.abspath(os.path.dirname( __file__)) + "/../inputdata/%s.txt" % __name__.split('.')[-1]
def makeexpecteddata(hour=-1): with open( os.path.abspath(os.path.dirname(__file__)) + '/../testdata/PeerInfoCleaned.txt', 'r') as resultfile: orglines = resultfile.readlines() # prepare middle table data and upload to odps. middle data will be involved in making the expected data create peeridlist = [] orglinestmp = [] with open( os.path.abspath(os.path.dirname(__file__)) + '/../testdata/%stmp.txt' % __name__.split('.')[-1], "w") as tmpfile: linecount = 0 for line in orglines: _, _, peerid, _, _, _, _, public_ip, _, _, _, _, _, _, _ = line.split( ',') if peerid not in peeridlist: if linecount % 100 == 0: peerid = peerid[:-3] + 'xyz' username = datavars.name_list[peerid[:8]] ispinfo = datavars.ip2isp[public_ip].split(',')[1] proviceinfo = datavars.ip2isp[public_ip].split(',')[0] tmpstr = '%s,%s,%s,%s\n' % (peerid, ispinfo, proviceinfo, username) orglinestmp.append(tmpstr) tmpfile.write(tmpstr) linecount += 1 peeridlist.append(peerid) pipeofodps.uploaddatatoodps( 'output_all_peer_isp_province_distribute', os.path.abspath(os.path.dirname(__file__)) + '/../testdata/%stmp.txt' % __name__.split('.')[-1], -1) ispresultlist = {} ispresultlist1 = {} proviceresultlist = {} proviceresultlist1 = {} for line in orglines: _, _, peerid, _, _, _, _, public_ip, _, _, _, _, _, _, _ = line.split( ',') if peerid not in peeridlist: username = datavars.name_list[peerid[:8]] ispinfo = datavars.ip2isp[public_ip].split(',')[1] proviceinfo = datavars.ip2isp[public_ip].split(',')[0] tmpstr = '%s,%s,%s,%s\n' % (peerid, ispinfo, proviceinfo, username) orglinestmp.append(tmpstr) peeridlist.append(peerid) for tmpline in orglinestmp: peerid, ispinfo, proviceinfo, username = tmpline.split(',') username = username.replace('\n', '') if username not in ispresultlist: ispresultlist[username] = {} ispresultlist[username][ispinfo] = ispresultlist[username].setdefault( ispinfo, 0) + 1 ispresultlist1[ispinfo] = ispresultlist1.setdefault(ispinfo, 0) + 1 if username not in proviceresultlist: proviceresultlist[username] = {} proviceresultlist[username][ proviceinfo] = proviceresultlist[username].setdefault( proviceinfo, 0) + 1 proviceresultlist1[proviceinfo] = proviceresultlist1.setdefault( proviceinfo, 0) + 1 with open( os.path.abspath(os.path.dirname(__file__)) + "/../inputdata/%sISP.txt" % __name__.split('.')[-1], "w") as expectedfile: for username, ipsinfos in ispresultlist.items(): for ips, count in ipsinfos.items(): expectedfile.write('%s,%s,%d\n' % (username, ips, count)) for ips, count in ispresultlist1.items(): expectedfile.write('%s,%s,%d\n' % ('all', ips, count)) with open( os.path.abspath(os.path.dirname(__file__)) + "/../inputdata/%sProvince.txt" % __name__.split('.')[-1], "w") as expectedfile: for username, proviceinfos in proviceresultlist.items(): for provice, count in proviceinfos.items(): expectedfile.write('%s,%s,%d\n' % (username, provice, count)) for provice, count in proviceresultlist1.items(): expectedfile.write('%s,%s,%d\n' % ('all', provice, count)) return (os.path.abspath(os.path.dirname(__file__)) + "/../inputdata/%sISP.txt" % __name__.split('.')[-1], os.path.abspath(os.path.dirname(__file__)) + "/../inputdata/%sProvince.txt" % __name__.split('.')[-1])
def makeexpecteddata(hour=-1): with open( os.path.abspath(os.path.dirname(__file__)) + '/../inputdata/DailyPeerActivity.txt', 'r') as resultfile: orglines = resultfile.readlines() with open( os.path.abspath(os.path.dirname(__file__)) + '/../outputdata/DailyPeerActivityDBWeek1.txt', 'w') as tempfile1: for line in orglines: randomnum = random.randint(150, 300) tmpcount = line.split(',')[1] newline = line.replace(',%s' % tmpcount, ',%s' % str(int(tmpcount) * randomnum)) tmpstr1 = newline.split(',')[1] newline1 = newline.replace(',%s' % tmpstr1, ',%s,%s\n' % (tmpstr1, 'NULL')) tempfile1.write(newline1) with open( os.path.abspath(os.path.dirname(__file__)) + '/../outputdata/DailyPeerActivityDBWeek2.txt', 'w') as tempfile2: for line in orglines: randomnum = random.randint(150, 300) tmpcount = line.split(',')[1] newline = line.replace(',%s' % tmpcount, ',%s' % str(int(tmpcount) * randomnum)) tmpstr1 = newline.split(',')[1] newline1 = newline.replace(',%s' % tmpstr1, ',%s,%s\n' % (tmpstr1, 'NULL')) tempfile2.write(newline1) pipeofodps.uploaddatatoodps( 'output_daily_peer_activity', os.path.abspath( os.path.dirname(__file__) + '/../outputdata/DailyPeerActivityDBWeek1.txt'), 3) pipeofodps.uploaddatatoodps( 'output_daily_peer_activity', os.path.abspath( os.path.dirname(__file__) + '/../outputdata/DailyPeerActivityDBWeek2.txt'), 7) with open( os.path.abspath(os.path.dirname(__file__)) + '/../outputdata/DailyPeerActivityDBWeek1.txt', 'r') as resultfile1: orglines1 = resultfile1.readlines() with open( os.path.abspath(os.path.dirname(__file__)) + '/../outputdata/DailyPeerActivityDBWeek2.txt', 'r') as resultfile2: orglines2 = resultfile2.readlines() with open( os.path.abspath(os.path.dirname(__file__)) + "/../inputdata/%s.txt" % __name__.split('.')[-1], "w") as expectedfile: sumcount = {} for line in orglines: prefix, actcount = line.split(',') actcount = actcount.replace('\n', '') if prefix not in sumcount: sumcount[prefix] = sumcount.setdefault(prefix, 0) + int(actcount) for line in orglines1: prefix, actcount, _ = line.split(',') if prefix not in sumcount: sumcount[prefix] = sumcount.setdefault(prefix, 0) + int(actcount) else: sumcount[prefix] = sumcount[prefix] + int(actcount) for line in orglines2: prefix, actcount, _ = line.split(',') if prefix not in sumcount: sumcount[prefix] = sumcount.setdefault(prefix, 0) + int(actcount) else: sumcount[prefix] = sumcount[prefix] + int(actcount) for pre, count in sumcount.items(): if (pre != '99999999'): expectedfile.write('%s,%d\n' % (pre, int(count / 30))) return os.path.abspath(os.path.dirname( __file__)) + "/../inputdata/%s.txt" % __name__.split('.')[-1]
def makeexpecteddata(hour=-1): with open( os.path.abspath(os.path.dirname(__file__)) + '/../outputdata/DailyTotalFlowDB.txt', 'r') as resultfile: orglines = resultfile.readlines() with open( os.path.abspath(os.path.dirname(__file__)) + '/../outputdata/DailyTotalFlowDBWeek1.txt', 'w') as tempfile1: for line in orglines: tempfile1.write(line) with open( os.path.abspath(os.path.dirname(__file__)) + '/../outputdata/DailyTotalFlowDBWeek2.txt', 'w') as tempfile2: for line in orglines: tempfile2.write(line) pipeofodps.uploaddatatoodps( 'output_daily_total_flow', os.path.abspath( os.path.dirname(__file__) + '/../outputdata/DailyTotalFlowDBWeek1.txt'), -7, False) pipeofodps.uploaddatatoodps( 'output_daily_total_flow', os.path.abspath( os.path.dirname(__file__) + '/../outputdata/DailyTotalFlowDBWeek2.txt'), 7, False) with open( os.path.abspath(os.path.dirname(__file__)) + '/../outputdata/DailyTotalFlowDBWeek1.txt', 'r') as resultfile1: orglines1 = resultfile1.readlines() with open( os.path.abspath(os.path.dirname(__file__)) + '/../outputdata/DailyTotalFlowDBWeek2.txt', 'r') as resultfile2: orglines2 = resultfile2.readlines() with open( os.path.abspath(os.path.dirname(__file__)) + "/../inputdata/%s.txt" % __name__.split('.')[-1], "w") as expectedfile: resultlist = {} for line in orglines: username, cdndown, p2pdown, totaldown, upload, typ = line.split( ',') typ = typ.replace('\n', '') if username not in resultlist: resultlist[username] = {} if typ not in resultlist[username]: resultlist[username][typ] = {} resultlist[username][typ][ 'cdndown'] = resultlist[username][typ].setdefault( 'cdndown', 0) + int(cdndown) resultlist[username][typ][ 'p2pdown'] = resultlist[username][typ].setdefault( 'p2pdown', 0) + int(p2pdown) resultlist[username][typ][ 'totaldown'] = resultlist[username][typ].setdefault( 'totaldown', 0) + int(totaldown) resultlist[username][ typ]['upload'] = resultlist[username][typ].setdefault( 'upload', 0) + int(upload) for line in orglines1: username, cdndown, p2pdown, totaldown, upload, typ = line.split( ',') typ = typ.replace('\n', '') if username not in resultlist: resultlist[username] = {} if typ not in resultlist[username]: resultlist[username][typ] = {} resultlist[username][typ]['cdndown'] = resultlist[username][typ][ 'cdndown'] + int(cdndown) resultlist[username][typ]['p2pdown'] = resultlist[username][typ][ 'p2pdown'] + int(p2pdown) resultlist[username][typ]['totaldown'] = resultlist[username][typ][ 'totaldown'] + int(totaldown) resultlist[username][typ][ 'upload'] = resultlist[username][typ]['upload'] + int(upload) for line in orglines2: username, cdndown, p2pdown, totaldown, upload, typ = line.split( ',') typ = typ.replace('\n', '') if username not in resultlist: resultlist[username] = {} if typ not in resultlist[username]: resultlist[username][typ] = {} resultlist[username][typ]['cdndown'] = resultlist[username][typ][ 'cdndown'] + int(cdndown) resultlist[username][typ]['p2pdown'] = resultlist[username][typ][ 'p2pdown'] + int(p2pdown) resultlist[username][typ]['totaldown'] = resultlist[username][typ][ 'totaldown'] + int(totaldown) resultlist[username][typ][ 'upload'] = resultlist[username][typ]['upload'] + int(upload) for user, value1 in resultlist.items(): for typeloc, value2 in value1.items(): expectedfile.write( '%s,%d,%d,%d,%d,%s\n' % (user, resultlist[user][typeloc]['cdndown'], resultlist[user][typeloc]['p2pdown'], resultlist[user][typeloc]['totaldown'], resultlist[user][typeloc]['upload'], typeloc)) return os.path.abspath(os.path.dirname( __file__)) + "/../inputdata/%s.txt" % __name__.split('.')[-1]
def makeexpecteddata(hour=-1): resultfile = open( os.path.abspath(os.path.dirname(__file__)) + '/../outputdata/DailyStartTimeCountDB.txt', 'r') orglines = resultfile.readlines() resultfile.close() tempfile1 = open( os.path.abspath(os.path.dirname(__file__)) + '/../outputdata/DailyStartTimeCountDBWeek1.txt', 'w') for line in orglines: tmpstr = line.split(',')[1] tmpcount = line.split(',')[2] if int(tmpstr) > 3000: newline = line.replace( ',%s' % tmpcount, ',%s' % (int(tmpcount) + random.randint(0, 10))) else: newline = line.replace( ',%s' % tmpcount, ',%s' % (int(tmpcount) - random.randint(0, 6))) tempfile1.write(newline) tempfile1.close() tempfile2 = open( os.path.abspath(os.path.dirname(__file__)) + '/../outputdata/DailyStartTimeCountDBWeek2.txt', 'w') for line in orglines: tmpstr = line.split(',')[1] tmpcount = line.split(',')[2] if int(tmpstr) < 3000: newline = line.replace( ',%s' % tmpcount, ',%s' % (int(tmpcount) + random.randint(0, 10))) else: newline = line.replace( ',%s' % tmpcount, ',%s' % (int(tmpcount) - random.randint(0, 6))) tempfile2.write(newline) tempfile2.close() pipeofodps.uploaddatatoodps( 'output_daily_start_time_count', os.path.abspath( os.path.dirname(__file__) + '/../outputdata/DailyStartTimeCountDBWeek1.txt'), -7) pipeofodps.uploaddatatoodps( 'output_daily_start_time_count', os.path.abspath( os.path.dirname(__file__) + '/../outputdata/DailyStartTimeCountDBWeek2.txt'), -13) resultfile1 = open( os.path.abspath(os.path.dirname(__file__)) + '/../outputdata/DailyStartTimeCountDBWeek1.txt', 'r') orglines1 = resultfile1.readlines() resultfile1.close() resultfile2 = open( os.path.abspath(os.path.dirname(__file__)) + '/../outputdata/DailyStartTimeCountDBWeek2.txt', 'r') orglines2 = resultfile2.readlines() resultfile2.close() expectedfile = open( os.path.abspath(os.path.dirname(__file__)) + "/../inputdata/%s.txt" % __name__.split('.')[-1], "w") resultlist = {} for line in orglines: username, period, count, _ = line.split(',') if username not in resultlist: resultlist[username] = {} if period not in resultlist[username]: resultlist[username][period] = int(count) else: resultlist[username][period] = resultlist[username][period] + int( count) for line in orglines1: username, period, count, _ = line.split(',') if username not in resultlist: resultlist[username] = {} if period not in resultlist[username]: resultlist[username][period] = int(count) else: resultlist[username][period] = resultlist[username][period] + int( count) for line in orglines2: username, period, count, _ = line.split(',') if username not in resultlist: resultlist[username] = {} if period not in resultlist[username]: resultlist[username][period] = int(count) else: resultlist[username][period] = resultlist[username][period] + int( count) for username, value in resultlist.items(): for period, count in value.items(): expectedfile.write('%s,%s,%d,%s\n' % (username, period, count, '')) expectedfile.close() return os.path.abspath(os.path.dirname( __file__)) + "/../inputdata/%s.txt" % __name__.split('.')[-1]