def makedata(self): data_format = '%s,%d,%s,CloutropySDK,%s,%d,%d,1000,%s,%d\n' with open( os.path.abspath(os.path.dirname(__file__)) + '/DownloadFlowCleaned.txt', 'w') as filedemanddata: for i in range(24): for j in range(60): for k in range(60): peerid = datavars.peeid_range[random.randint(0, 9)] timestamp = get_timestamp_by_time( datavars.time_format_with_second % (i, j, k)) url = datavars.url_range[random.randint(0, 4)] type = datavars.type_range[random.randint(0, 5)] p2pdown = random.randint(10000, 1000000) cdndown = random.randint(10000, 1000000) id = datavars.id_range[random.randint(0, 14)] if k % 9 == 0: p2pdown = 0 cdndown = 0 filedemanddata.write( data_format % (peerid, int(timestamp), url, type, p2pdown, cdndown, id, int(timestamp) + random.randint(10, 1000))) return os.path.abspath( os.path.dirname(__file__)) + '/DownloadFlowCleaned.txt'
def makeexpecteddata(hour=-1): resultfile = open(os.path.abspath(os.path.dirname(__file__)) + '/../testdata/FileOnDemandCleaned.txt', 'r') orglines = resultfile.readlines() resultfile.close() time_format = constvars.recorddate + '000001' timestamp = long(get_timestamp_by_time(time_format)[:-3]) expectedfile = open(os.path.abspath(os.path.dirname(__file__)) + "/../inputdata/%s.txt" % __name__.split('.')[-1],"w") resultlist = {} for line in orglines: _,peerid,timesp,_,_,_,_,_,_= line.split(',') for i in range(24): if (timestamp + 3600*(i+1)) > long(timesp[:-3]) >= (timestamp + 3600*i): if i not in resultlist: resultlist[i] = {} resultlist[i][peerid[:8]] = resultlist[i].setdefault(peerid[:8], 0) + 1 break if hour > -1: for peerfix, count in resultlist[hour].items(): expectedfile.write('%s,%s,%d\n' % (peerfix, hour, count)) else: for time, values in resultlist.items(): for peerfix, count in values.items(): expectedfile.write('%s,%s,%d,%s\n' % (peerfix, time, count,'')) expectedfile.close() return os.path.abspath(os.path.dirname(__file__)) + "/../inputdata/%s.txt" % __name__.split('.')[-1]
def makeexpecteddata(hour=-1): with open(os.path.abspath(os.path.dirname(__file__)) + '/../testdata/FileOnDemandCleaned.txt', 'r') as resultfile: orglines = resultfile.readlines() time_format = constvars.recorddate + '000001' timestamp = long(get_timestamp_by_time(time_format)[:-3]) with open(os.path.abspath(os.path.dirname(__file__)) + "/../inputdata/%s.txt"%__name__.split('.')[-1],"w") as expectedfile: resultlist = {} for line in orglines: _,peerid,timesp,_,_,_,_,_,type = line.split(',') type=type.replace('\n','') for i in range(24): if (timestamp + 3600*(i+1)) > long(timesp[:-3]) >= (timestamp + 3600*i): if i not in resultlist: resultlist[i]={} if peerid not in resultlist[i]: resultlist[i][peerid] = 1 else: resultlist[i][peerid] = resultlist[i][peerid] + 1 break for hour, values in resultlist.items(): for peeid, count in values.items(): expectedfile.write('%s,%s,%d\n' % (peeid,hour,count)) return os.path.abspath(os.path.dirname(__file__)) + "/../inputdata/%s.txt"%__name__.split('.')[-1]
def makedata(self): data_format = '%s,%d,1000,%s,%s,0,0,%s,%s,cloudtropy,%s,%d,0,0,%d,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,,%d,%d\n' with open( os.path.abspath(os.path.dirname(__file__)) + '/VodPerCleaned.txt', 'w') as filedemanddata: for i in range(24): for j in range(60): id = datavars.id_range[random.randint(0, 14)] timestamp = get_timestamp_by_time(datavars.time_format % (11, 11)) peerid = datavars.peeid_range[random.randint(0, 9)] fileid = datavars.file_range[random.randint(0, 9)] url = datavars.url_range[random.randint(0, 4)] username = datavars.url_username.values()[random.randint( 0, 4)] type = datavars.peertype[random.randint(0, 1)] start_delay = datavars.delay_range[random.randint(0, 14)] seek_delay = datavars.delay_range[random.randint(0, 14)] filedemanddata.write( data_format % (id, int(timestamp), peerid, fileid, url, username, type, int(start_delay), int(seek_delay), int(timestamp) + random.randint(10, 1000), int(timestamp) + random.randint(1001, 10000))) return os.path.abspath( os.path.dirname(__file__)) + '/VodPerCleaned.txt'
def makedata(self): data_format = '%s,%d,%s,%s,%s,android,%d,%s,80,192.168.1.10,80,%s,Intel,%d,%d\n' with open( os.path.abspath(os.path.dirname(__file__)) + '/RawPeerInfo.txt', 'w') as filedemanddata: for i in range(24): for j in range(60): id = datavars.id_range[random.randint(0, 14)] timestamp = get_timestamp_by_time(datavars.time_format % (i, j)) peerid = create_random_unique_peerid() os_type = datavars.os_versions[random.randint( 0, 1)].split(',')[0] os_version = datavars.os_versions[random.randint( 0, 1)].split(',')[1] nat_type = datavars.nattype[random.randint(0, 2)] publicip = datavars.publicip_range[random.randint(0, 4)] macs = datavars.macs[random.randint(0, 1)] filedemanddata.write( data_format % (id, int(timestamp), peerid, os_type, os_version, int(nat_type), publicip, macs, int(timestamp) + random.randint(10, 1000), int(timestamp) + random.randint(1001, 10000))) return os.path.abspath(os.path.dirname(__file__)) + '/RawPeerInfo.txt'
def makedata(self): data_format = '%s,%d,%s,%s,%s,Intel,i7,%d,%s,80,192.168.1.10,80,%s,%s,%s,error,%d,%d,%s\n' with open( os.path.abspath(os.path.dirname(__file__)) + '/RawClientException.txt', 'w') as filedemanddata: for i in range(24): for j in range(60): id = datavars.id_range[random.randint(0, 14)] timestamp = get_timestamp_by_time(datavars.time_format % (11, 11)) peerid = datavars.peeid_range[random.randint(0, 9)] os_type = datavars.os_versions[random.randint( 0, 1)].split(',')[0] os_version = datavars.os_versions[random.randint( 0, 1)].split(',')[1] nat_type = datavars.nattype[random.randint(0, 2)] publicip = datavars.publicip_range[random.randint(0, 4)] macs = datavars.macs[random.randint(0, 1)] op = datavars.op_and_error[random.randint(0, 8)].split(',')[0] errorcode = datavars.op_and_error[random.randint( 0, 8)].split(',')[1] type = datavars.type_range[random.randint(0, 3)] filedemanddata.write( data_format % (id, int(timestamp), peerid, os_type, os_version, int(nat_type), publicip, macs, op, errorcode, int(timestamp) + random.randint(10, 1000), int(timestamp) + random.randint(1001, 10000), type)) return os.path.abspath( os.path.dirname(__file__)) + '/RawClientException.txt'
def makeexpecteddata(hour=-1): with open(os.path.abspath(os.path.dirname(__file__)) + '/../testdata/DownloadFlowCleaned.txt', 'r') as resultfile: orgdownlines = resultfile.readlines() time_format = constvars.recorddate + '000001' timestamp = long(get_timestamp_by_time(time_format)[:-3]) with open(os.path.abspath(os.path.dirname(__file__)) + "/../inputdata/%s.txt" % __name__.split('.')[-1], "w") as expectedfile: resultlist = {} totallist = {} for line in orgdownlines: peerid, timesp, _, _, playtype, p2pdown, cdndown, _, _, _ = line.split(',') for i in range(24): if (timestamp + 3600*(i+1)) > long(timesp[:-3]) >= (timestamp + 3600*i): if i not in resultlist: resultlist[i] = {} if i not in totallist: totallist[i] = {} tmpusername = datavars.name_list[peerid[:8]] if tmpusername not in resultlist[i]: resultlist[i][tmpusername] = {} if playtype not in resultlist[i][tmpusername]: resultlist[i][tmpusername][playtype] = {} if playtype not in totallist[i]: totallist[i][playtype] = {} resultlist[i][tmpusername][playtype]['p2pdown'] = resultlist[i][tmpusername][playtype].setdefault( 'p2pdown', 0L) + long(p2pdown) resultlist[i][tmpusername][playtype]['totaldown'] = resultlist[i][tmpusername][playtype].setdefault( 'totaldown', 0L) + long(cdndown) + long(p2pdown) totallist[i][playtype]['p2pdown'] = totallist[i][playtype].setdefault('p2pdown', 0L) + long(p2pdown) totallist[i][playtype]['totaldown'] = totallist[i][playtype].setdefault('totaldown', 0L) + long(cdndown) + long(p2pdown) break if hour > -1: for username, ptypes in resultlist[hour].items(): for ptype, values in ptypes.items(): expectedfile.write('%s,%d,%f,%s\n' % (username, hour, float(values.setdefault('p2pdown', 0))/float( values.setdefault('totaldown', 0)), ptype)) for ptype, values in totallist[hour].items(): expectedfile.write('%s,%d,%f,%s\n' % ('all', hour, float(values.setdefault('p2pdown', 0))/float( values.setdefault('totaldown', 0)), ptype)) else: for time, users in resultlist.items(): for username, ptypes in users.items(): for ptype, values in ptypes.items(): expectedfile.write('%s,%d,%f,%s\n' % (username, hour, float(values.setdefault('p2pdown', 0))/float( values.setdefault('totaldown', 0)), ptype)) for time, ptypes in totallist.items(): for ptype, values in ptypes.items(): expectedfile.write('%s,%d,%f,%s\n' % ('all', hour, float(values.setdefault('p2pdown', 0))/float( values.setdefault('totaldown', 0)), ptype)) return os.path.abspath(os.path.dirname(__file__)) + "/../inputdata/%s.txt" % __name__.split('.')[-1]
def makedata(self): data_format ='%s,%d,%s,10000,1000,%d,%d,%s\n' with open(os.path.abspath(os.path.dirname(__file__)) + '/UploadFlowCleaned.txt', 'w') as filedemanddata: for i in range(24): for j in range(60): id = datavars.id_range[random.randint(0, 14)] timestamp = get_timestamp_by_time(datavars.time_format % (i, j)) peerid = datavars.peeid_range[random.randint(0, 9)] type=datavars.type_range[random.randint(0, 3)] filedemanddata.write(data_format % ( id, int(timestamp),peerid,int(timestamp)+random.randint(10, 1000),int(timestamp)+random.randint(1001, 10000),type)) return os.path.abspath(os.path.dirname(__file__)) + '/UploadFlowCleaned.txt'
def makedata(self): data_format = '%s,%d,%s,%d,%d,%d,%d,%s\n' with open(os.path.abspath(os.path.dirname(__file__)) + '/RawPeerOnlineTime.txt', 'w') as filedemanddata: for i in range(24): for j in range(60): id = datavars.id_range[random.randint(0, 14)] timestamp = get_timestamp_by_time(datavars.time_format % (i, j)) peerid = datavars.peeid_range[random.randint(0, 9)] quarter=random.randint(0,95) online=random.randint(0,1) type= datavars.type_range[random.randint(0, 3)] filedemanddata.write(data_format % ( id, int(timestamp),peerid,quarter,online,int(timestamp)+random.randint(10, 1000),int(timestamp)+random.randint(1001, 10000),type)) return os.path.abspath(os.path.dirname(__file__)) + '/RawPeerOnlineTime.txt'
def makedata(self): data_format = '%s,%d,%s,%s,%s,%s,%s,%d,%d,%s\n' with open(os.path.abspath(os.path.dirname(__file__)) + '/RawFileOnDemand.txt', 'w') as filedemanddata: for i in range(1000): id = datavars.id_range[random.randint(0, 14)] timestamp = get_timestamp_by_time(datavars.time_format % (11, 11)) peerid = datavars.peeid_range[random.randint(0, 9)] url = datavars.url_range[random.randint(0, 4)] fodtype = datavars.type_range[random.randint(0, 3)] fileid = datavars.file_range[random.randint(0, 9)] publicip = datavars.publicip_range[random.randint(0, 4)] type = datavars.type_range[random.randint(0, 3)] filedemanddata.write(data_format % ( id,int(timestamp),peerid,url,fodtype,fileid,publicip,int(timestamp)+random.randint(1, 100),int(timestamp)+random.randint(100, 10000),type)) return os.path.abspath(os.path.dirname(__file__)) + '/RawFileOnDemand.txt'
def makedata(self): data_format = '%s,%d,%s,%s,itsavvidstring,%s,1111,222,%d,%d\n' with open(os.path.abspath(os.path.dirname(__file__)) + '/RawQosBuffering.txt', 'w') as filedemanddata: for i in range(24): for j in [2, 6, 15, 26]: id = datavars.id_range[random.randint(0,14)] timestamp = get_timestamp_by_time(datavars.time_format% (i, j)) peerid = datavars.peeid_range[random.randint(0,9)] url = datavars.url_range[random.randint(0,4)] type = datavars.type_range[random.randint(0, 3)] line = data_format % ( id, int(timestamp), peerid, url, type, int(timestamp)+random.randint(1,100), int(timestamp) + random.randint(100,10000)) filedemanddata.write(line) return os.path.abspath(os.path.dirname(__file__)) + '/RawQosBuffering.txt'
def makeexpecteddata(hour=-1): resultfile = open( os.path.abspath(os.path.dirname(__file__)) + '/../testdata/FileOnDemandCleaned.txt', 'r') orgdownlines = resultfile.readlines() resultfile.close() time_format = constvars.recorddate + '000001' timestamp = long(get_timestamp_by_time(time_format)[:-3]) expectedfile = open( os.path.abspath(os.path.dirname(__file__)) + "/../inputdata/%s.txt" % __name__.split('.')[-1], "w") resultlist = {} for line in orgdownlines: _, _, timesp, url, _, _, fileid, _, play_type = line.split(',') play_type = play_type.replace('\n', '') for i in range(24): if (timestamp + 3600 * (i + 1)) > long(timesp[:-3]) >= (timestamp + 3600 * i): if i not in resultlist: resultlist[i] = {} if url not in resultlist[i]: resultlist[i][url] = {} resultlist[i][url][play_type] = resultlist[i][url].setdefault( play_type, 0) + 1 break if hour > -1: for hindex in range(hour + 1): for url, play_types in resultlist[hindex].items(): for play_type, count in play_types.items(): expectedfile.write('%s,%s,%d,%d,%s\n' % (url, datavars.url_username[url], count, hindex, play_type)) else: for time, urls in resultlist.items(): for url, play_types in urls.items(): for play_type, count in play_types.items(): expectedfile.write('%s,%s,%d,%d,%s\n' % (url, datavars.url_username[url], count, time, play_type)) expectedfile.close() return os.path.abspath(os.path.dirname( __file__)) + "/../inputdata/%s.txt" % __name__.split('.')[-1]
def makeexpecteddata(hour=-1): resultfile = open(os.path.abspath(os.path.dirname(__file__)) + '/../testdata/RawLiveDelay.txt', 'r') orgdownlines = resultfile.readlines() resultfile.close() time_format = constvars.recorddate + '000001' timestamp = long(get_timestamp_by_time(time_format)[:-3]) expectedfile = open(os.path.abspath(os.path.dirname(__file__)) + "/../inputdata/%s.txt" % __name__.split('.')[-1], "w") resultlist = {} for line in orgdownlines: _, timesp, peerid, _, _, _, _, delaytime, _, _ = line.split(',') for i in range(288): if (timestamp + 300*(i+1)) > long(timesp[:-3]) >= (timestamp + 300*i): if i not in resultlist: resultlist[i] = {} tmpusername = datavars.name_list[peerid[:8]] if tmpusername not in resultlist[i]: resultlist[i][tmpusername] = [] resultlist[i][tmpusername].append(int(delaytime)) break if hour > -1: for index in (range(0, (hour+1)*12)): # for index in (range(hour*12, (hour+1)*12)): for username, values in resultlist[index].items(): if len(values) != 0: average_delay = sum(values)/len(values) else: average_delay = 0 expectedfile.write('%s,%d,%d\n' % (username, index, average_delay)) else: for index, users in resultlist.items(): for username, values in users.items(): if len(values) != 0: average_delay = sum(values)/len(values) else: average_delay = 0 expectedfile.write('%s,%d,%d\n' % (username, index, average_delay)) expectedfile.close() return os.path.abspath(os.path.dirname(__file__)) + "/../inputdata/%s.txt" % __name__.split('.')[-1]
def makedata(self): data_format = '%s,%d,%s,%s,CloutropySDK,%s,%d,%d,1000,%d,%d\n' with open( os.path.abspath(os.path.dirname(__file__)) + '/RawDownloadFlow.txt', 'w') as filedemanddata: for i in range(1000): peerid = datavars.peeid_range[random.randint(0, 9)] timestamp = get_timestamp_by_time(datavars.time_format % (11, 11)) url = datavars.url_range[random.randint(0, 4)] type = datavars.peertype[random.randint(0, 1)] p2pdown = random.randint(10000, 1000000) cdndown = random.randint(10000, 1000000) id = datavars.id_range[random.randint(0, 14)] if i % 5 == 0: p2pdown = 0 cdndown = 0 filedemanddata.write( data_format % (id, int(timestamp), peerid, url, type, p2pdown, cdndown, int(timestamp) + random.randint(1, 100), int(timestamp) + random.randint(100, 10000))) return os.path.abspath( os.path.dirname(__file__)) + '/RawDownloadFlow.txt'
def makedata(self): data_format = '%s,%d,%s,%s,%s,%s,%d,%d,%d,%d\n' with open( os.path.abspath(os.path.dirname(__file__)) + '/RawLiveDelay.txt', 'w') as filedemanddata: for i in range(24): for j in range(60): id = datavars.id_range[random.randint(0, 14)] timestamp = get_timestamp_by_time(datavars.time_format % (i, j)) peerid = datavars.peeid_range[random.randint(0, 9)] fileid = datavars.file_range[random.randint(0, 9)] url = datavars.url_range[random.randint(0, 4)] sourcetype = datavars.peertype[random.randint(0, 1)] offset = int(timestamp) - random.randint(50, 5000) delay = random.randint(100, 10000) filedemanddata.write( data_format % (id, int(timestamp), peerid, fileid, url, sourcetype, offset, delay, int(timestamp) + random.randint(1, 100), int(timestamp) + random.randint(100, 1000))) return os.path.abspath(os.path.dirname(__file__)) + '/RawLiveDelay.txt'
def makeexpecteddata(hour=-1): resultfile = open( os.path.abspath(os.path.dirname(__file__)) + '/../testdata/DownloadFlowCleaned.txt', 'r') orgdownlines = resultfile.readlines() resultfile.close() resultfile = open( os.path.abspath(os.path.dirname(__file__)) + '/../testdata/UploadFlowCleaned.txt', 'r') orguplines = resultfile.readlines() resultfile.close() time_format = constvars.recorddate + '000001' timestamp = long(get_timestamp_by_time(time_format)[:-3]) expectedfile = open( os.path.abspath(os.path.dirname(__file__)) + "/../inputdata/%s.txt" % __name__.split('.')[-1], "w") resultlist = {} totallist = {} for line in orgdownlines: peerid, timesp, _, _, playtype, p2pdown, cdndown, _, _, _ = line.split( ',') tmpusername = datavars.name_list[peerid[:8]] if tmpusername not in resultlist: resultlist[tmpusername] = {} if playtype not in resultlist[tmpusername]: resultlist[tmpusername][playtype] = {} if playtype not in totallist: totallist[playtype] = {} resultlist[tmpusername][playtype][ 'p2pdown'] = resultlist[tmpusername][playtype].setdefault( 'p2pdown', 0L) + long(p2pdown) resultlist[tmpusername][playtype][ 'cdndown'] = resultlist[tmpusername][playtype].setdefault( 'cdndown', 0L) + long(cdndown) resultlist[tmpusername][playtype][ 'totaldown'] = resultlist[tmpusername][playtype].setdefault( 'totaldown', 0L) + long(cdndown) + long(p2pdown) totallist[playtype]['p2pdown'] = totallist[playtype].setdefault( 'p2pdown', 0L) + long(p2pdown) totallist[playtype]['cdndown'] = totallist[playtype].setdefault( 'cdndown', 0L) + long(cdndown) totallist[playtype]['totaldown'] = totallist[playtype].setdefault( 'totaldown', 0L) + long(cdndown) + long(p2pdown) for line in orguplines: _, timesp, peerid, upload, _, _, _, playtype = line.split(',') playtype = playtype.replace('\n', '') tmpusername = datavars.name_list[peerid[:8]] if tmpusername not in resultlist: resultlist[tmpusername] = {} if playtype not in resultlist[tmpusername]: resultlist[tmpusername][playtype] = {} if playtype not in totallist: totallist[playtype] = {} resultlist[tmpusername][playtype][ 'upload'] = resultlist[tmpusername][playtype].setdefault( 'upload', 0L) + long(upload) totallist[playtype]['upload'] = totallist[playtype].setdefault( 'upload', 0L) + long(upload) for username, ptypes in resultlist.items(): for ptype, values in ptypes.items(): expectedfile.write('%s,%d,%d,%d,%d,%s\n' % (username, values.setdefault('cdndown', 0), values.setdefault('p2pdown', 0), values.setdefault('totaldown', 0), values.setdefault('upload', 0), ptype)) for ptype, values in totallist.items(): expectedfile.write( 'all,%d,%d,%d,%d,%s\n' % (values.setdefault('cdndown', 0), values.setdefault( 'p2pdown', 0), values.setdefault( 'totaldown', 0), values.setdefault('upload', 0), ptype)) expectedfile.close() return os.path.abspath(os.path.dirname( __file__)) + "/../inputdata/%s.txt" % __name__.split('.')[-1]
import sys import os import constvars import testdata.datavars as datavars sys.path.append(os.path.abspath(os.path.dirname(__file__) + '/' + '..')) from commonfunc import get_timestamp_by_time time_format = constvars.recorddate + '000001' timestamp = long(get_timestamp_by_time(time_format)[:-3]) def makeexpecteddata(hour=-1): with open( os.path.abspath(os.path.dirname(__file__)) + '/../testdata/FileOnDemandCleaned.txt', 'r') as resultfile: orglines = resultfile.readlines() resultlist = {} peeridlist = {} with open( os.path.abspath(os.path.dirname(__file__)) + "/../inputdata/%s.txt" % __name__.split('.')[-1], "w") as expectedfile: for line in orglines: _, peerid, timesp, _, _, publicip, _, _, _ = line.split(',') for i in range(24): if (timestamp + 3600 * (i + 1)) > long(timesp[:-3]) >= (timestamp + 3600 * i):
def makeexpecteddata(hour=-1): with open( os.path.abspath(os.path.dirname(__file__)) + '/../testdata/DownloadFlowCleaned.txt', 'r') as resultfile: orgdownlines = resultfile.readlines() time_format = constvars.recorddate + '000000' # timestamp = long(get_timestamp_by_time(time_format)[:-3]) timestamp = long(get_timestamp_by_time(time_format)[:-3] + '000') with open( os.path.abspath(os.path.dirname(__file__)) + "/../inputdata/%s.txt" % __name__.split('.')[-1], "w") as expectedfile: resultlist = {} totallist = {} typelist = {} for line in orgdownlines: peerid, timesp, _, _, playtype, p2pdown, cdndown, _, _, _ = line.split( ',') if playtype == '': playtype = 'vod' for i in range(288): if (timestamp + 300 * 1000 * (i + 1)) > long(timesp) >= (timestamp + 300 * 1000 * i): if i not in resultlist: resultlist[i] = {} if i not in totallist: totallist[i] = {} totallist[i]['all'] = {} totallist[i]['all']['live'] = {} totallist[i]['all']['vods'] = {} totallist[i]['all']['all'] = {} if i not in typelist: typelist[i] = {} tmpusername = datavars.name_list[peerid[:8]] if tmpusername not in resultlist[i]: resultlist[i][tmpusername] = {} if tmpusername not in typelist[i]: typelist[i][tmpusername] = {} typelist[i][tmpusername]['live'] = {} typelist[i][tmpusername]['vods'] = {} typelist[i][tmpusername]['all'] = {} if playtype not in resultlist[i][tmpusername]: resultlist[i][tmpusername][playtype] = {} if playtype not in totallist[i]['all']: totallist[i]['all'][playtype] = {} typelist[i][tmpusername]['all']['totaldown'] = typelist[i][ tmpusername]['all'].setdefault( 'totaldown', 0) + long(cdndown) + long(p2pdown) typelist[i][tmpusername]['all']['p2pdown'] = typelist[i][ tmpusername]['all'].setdefault('p2pdown', 0) + long(p2pdown) typelist[i][tmpusername]['all']['cdndown'] = typelist[i][ tmpusername]['all'].setdefault('cdndown', 0) + long(cdndown) if playtype.startswith('live'): typelist[i][tmpusername]['live']['totaldown'] = typelist[i][tmpusername]['live'].setdefault('totaldown', 0) + \ long(cdndown) + long(p2pdown) typelist[i][tmpusername]['live']['p2pdown'] = typelist[ i][tmpusername]['live'].setdefault( 'p2pdown', 0) + long(p2pdown) typelist[i][tmpusername]['live']['cdndown'] = typelist[ i][tmpusername]['live'].setdefault( 'cdndown', 0) + long(cdndown) else: typelist[i][tmpusername]['vods']['totaldown'] = typelist[i][tmpusername]['vods'].setdefault('totaldown', 0) + \ long(cdndown) + long(p2pdown) typelist[i][tmpusername]['vods']['p2pdown'] = typelist[ i][tmpusername]['vods'].setdefault( 'p2pdown', 0) + long(p2pdown) typelist[i][tmpusername]['vods']['cdndown'] = typelist[ i][tmpusername]['vods'].setdefault( 'cdndown', 0) + long(cdndown) resultlist[i][tmpusername][playtype][ 'totaldown'] = resultlist[ i][tmpusername][playtype].setdefault( 'totaldown', 0) + long(cdndown) + long(p2pdown) resultlist[i][tmpusername][playtype][ 'p2pdown'] = resultlist[i][tmpusername][ playtype].setdefault('p2pdown', 0) + long(p2pdown) resultlist[i][tmpusername][playtype][ 'cdndown'] = resultlist[i][tmpusername][ playtype].setdefault('cdndown', 0) + long(cdndown) totallist[i]['all'][playtype]['totaldown'] = totallist[i][ 'all'][playtype].setdefault( 'totaldown', 0) + long(cdndown) + long(p2pdown) totallist[i]['all'][playtype][ 'p2pdown'] = totallist[i]['all'][playtype].setdefault( 'p2pdown', 0) + long(p2pdown) totallist[i]['all'][playtype][ 'cdndown'] = totallist[i]['all'][playtype].setdefault( 'cdndown', 0) + long(cdndown) totallist[i]['all']['all'][ 'totaldown'] = totallist[i]['all']['all'].setdefault( 'totaldown', 0) + long(cdndown) + long(p2pdown) totallist[i]['all']['all'][ 'p2pdown'] = totallist[i]['all']['all'].setdefault( 'p2pdown', 0) + long(p2pdown) totallist[i]['all']['all'][ 'cdndown'] = totallist[i]['all']['all'].setdefault( 'cdndown', 0) + long(cdndown) if playtype.startswith('live'): totallist[i]['all']['live']['totaldown'] = totallist[ i]['all']['live'].setdefault( 'totaldown', 0) + long(cdndown) + long(p2pdown) totallist[i]['all']['live']['p2pdown'] = totallist[i][ 'all']['live'].setdefault('p2pdown', 0) + long(p2pdown) totallist[i]['all']['live']['cdndown'] = totallist[i][ 'all']['live'].setdefault('cdndown', 0) + long(cdndown) else: totallist[i]['all']['vods']['totaldown'] = totallist[ i]['all']['vods'].setdefault( 'totaldown', 0) + long(cdndown) + long(p2pdown) totallist[i]['all']['vods']['p2pdown'] = totallist[i][ 'all']['vods'].setdefault('p2pdown', 0) + long(p2pdown) totallist[i]['all']['vods']['cdndown'] = totallist[i][ 'all']['vods'].setdefault('cdndown', 0) + long(cdndown) break if hour > -1: for index in (range(hour * 12, (hour + 1) * 12)): for username, ptypes in resultlist[index].items(): for ptype, values in ptypes.items(): expectedfile.write( '%d,%s,%f,%f,%f,%s\n' % (timestamp + 300 * 1000 * index, username, float( float(values.setdefault('totaldown', 0)) * 8 / 1024 / 1024) / float(300), float( float(values.setdefault('p2pdown', 0)) * 8 / 1024 / 1024) / float(300), float( float(values.setdefault('cdndown', 0)) * 8 / 1024 / 1024) / float(300), ptype)) for username, ptypes in typelist[index].items(): for ptype, values in ptypes.items(): expectedfile.write( '%d,%s,%f,%f,%f,%s\n' % (timestamp + 300 * 1000 * index, username, float( float(values.setdefault('totaldown', 0)) * 8 / 1024 / 1024) / float(300), float( float(values.setdefault('p2pdown', 0)) * 8 / 1024 / 1024) / float(300), float( float(values.setdefault('cdndown', 0)) * 8 / 1024 / 1024) / float(300), ptype)) for username, ptypes in totallist[index].items(): for ptype, values in ptypes.items(): expectedfile.write( '%d,%s,%f,%f,%f,%s\n' % (timestamp + 300 * 1000 * index, username, float( float(values.setdefault('totaldown', 0)) * 8 / 1024 / 1024) / float(300), float( float(values.setdefault('p2pdown', 0)) * 8 / 1024 / 1024) / float(300), float( float(values.setdefault('cdndown', 0)) * 8 / 1024 / 1024) / float(300), ptype)) else: for time, users in resultlist.items(): for username, ptypes in users.items(): for ptype, values in ptypes.items(): expectedfile.write( '%d,%s,%f,%f,%f,%s\n' % (timestamp + 300 * 1000 * time, username, float( float(values.setdefault('totaldown', 0)) * 8 / 1024 / 1024) / float(300), float( float(values.setdefault('p2pdown', 0)) * 8 / 1024 / 1024) / float(300), float( float(values.setdefault('cdndown', 0)) * 8 / 1024 / 1024) / float(300), ptype)) for time, users in typelist.items(): for username, ptypes in users.items(): for ptype, values in ptypes.items(): expectedfile.write( '%d,%s,%f,%f,%f,%s\n' % (timestamp + 300 * 1000 * time, username, float( float(values.setdefault('totaldown', 0)) * 8 / 1024 / 1024) / float(300), float( float(values.setdefault('p2pdown', 0)) * 8 / 1024 / 1024) / float(300), float( float(values.setdefault('cdndown', 0)) * 8 / 1024 / 1024) / float(300), ptype)) for time, users in totallist.items(): for username, ptypes in users.items(): for ptype, values in ptypes.items(): expectedfile.write( '%d,%s,%f,%f,%f,%s\n' % (timestamp + 300 * 1000 * time, username, float( float(values.setdefault('totaldown', 0)) * 8 / 1024 / 1024) / float(300), float( float(values.setdefault('p2pdown', 0)) * 8 / 1024 / 1024) / float(300), float( float(values.setdefault('cdndown', 0)) * 8 / 1024 / 1024) / float(300), ptype)) return os.path.abspath(os.path.dirname( __file__)) + "/../inputdata/%s.txt" % __name__.split('.')[-1]
def makeexpecteddata(hour=-1): resultfile = open(os.path.abspath(os.path.dirname(__file__)) + '/../testdata/DownloadFlowCleaned.txt', 'r') orgdownlines = resultfile.readlines() resultfile.close() time_format = constvars.recorddate + '000001' timestamp = long(get_timestamp_by_time(time_format)[:-3]) expectedfile = open(os.path.abspath(os.path.dirname(__file__)) + "/../inputdata/%s.txt" % __name__.split('.')[-1], "w") resultlist = {} totallist = {} typelist = {} for line in orgdownlines: peerid, timesp, _, _, playtype, p2pdown, cdndown, _, _, _ = line.split(',') if playtype == '': playtype = 'vod' for i in range(288): if (timestamp + 300*(i+1)) > long(timesp[:-3]) >= (timestamp + 300*i): if i not in resultlist: resultlist[i] = {} if i not in totallist: totallist[i] = {} if i not in typelist: typelist[i] = {} tmpusername = datavars.name_list[peerid[:8]] if tmpusername not in resultlist[i]: resultlist[i][tmpusername] = {} if tmpusername not in typelist[i]: typelist[i][tmpusername] = {} typelist[i][tmpusername]['live'] = {} typelist[i][tmpusername]['vods'] = {} typelist[i][tmpusername]['all'] = {} if playtype not in resultlist[i][tmpusername]: resultlist[i][tmpusername][playtype] = {} if playtype not in totallist[i]: totallist[i][playtype] = {} typelist[i][tmpusername]['all']['totaldown'] = typelist[i][tmpusername]['all'].setdefault('totaldown', 0) + int(cdndown) + int(p2pdown) typelist[i][tmpusername]['all']['p2pdown'] = typelist[i][tmpusername]['all'].setdefault('p2pdown', 0) + int(p2pdown) typelist[i][tmpusername]['all']['cdndown'] = typelist[i][tmpusername]['all'].setdefault('cdndown', 0) + int(cdndown) if playtype.startswith('live'): typelist[i][tmpusername]['live']['totaldown'] = typelist[i][tmpusername]['live'].setdefault('totaldown', 0) + \ int(cdndown) + int(p2pdown) typelist[i][tmpusername]['live']['p2pdown'] = typelist[i][tmpusername]['live'].setdefault('p2pdown', 0) + int(p2pdown) typelist[i][tmpusername]['live']['cdndown'] = typelist[i][tmpusername]['live'].setdefault('cdndown', 0) + int(cdndown) else: typelist[i][tmpusername]['vods']['totaldown'] = typelist[i][tmpusername]['vods'].setdefault('totaldown', 0) + \ int(cdndown) + int(p2pdown) typelist[i][tmpusername]['vods']['p2pdown'] = typelist[i][tmpusername]['vods'].setdefault('p2pdown', 0) + int(p2pdown) typelist[i][tmpusername]['vods']['cdndown'] = typelist[i][tmpusername]['vods'].setdefault('cdndown', 0) + int(cdndown) resultlist[i][tmpusername][playtype]['totaldown'] = resultlist[i][tmpusername][playtype].setdefault( 'totaldown', 0) + long(cdndown) + long(p2pdown) resultlist[i][tmpusername][playtype]['cdndown'] = resultlist[i][tmpusername][playtype].setdefault( 'cdndown', 0) + long(cdndown) resultlist[i][tmpusername][playtype]['p2pdown'] = resultlist[i][tmpusername][playtype].setdefault( 'p2pdown', 0) + long(p2pdown) totallist[i][playtype]['totaldown'] = totallist[i][playtype].setdefault('totaldown', 0) + long(cdndown) + long(p2pdown) totallist[i][playtype]['p2pdown'] = totallist[i][playtype].setdefault('p2pdown', 0) + int(p2pdown) totallist[i][playtype]['cdndown'] = totallist[i][playtype].setdefault('cdndown', 0) + int(cdndown) break if hour > -1: outputlist = {} outputtotallist = {} outputtypelist = {} for index in (range(hour*12, (hour+1)*12)): for username, ptypes in resultlist[index].items(): for ptype, values in ptypes.items(): if username not in outputlist: outputlist[username] = {} if ptype not in outputlist[username]: outputlist[username][ptype] = {} tmpvalue = float(float(values.setdefault('totaldown', 0))*8/1024/1024)/float(300) tmpvalue2 = float(float(values.setdefault('cdndown', 0))*8/1024/1024)/float(300) tmpvalue3 = float(float(values.setdefault('p2pdown', 0))*8/1024/1024)/float(300) if tmpvalue > outputlist[username][ptype].setdefault('peak', 0.0): outputlist[username][ptype]['peak'] = tmpvalue if tmpvalue2 > outputlist[username][ptype].setdefault('cdnpeak', 0.0): outputlist[username][ptype]['cdnpeak'] = tmpvalue2 if tmpvalue3 > outputlist[username][ptype].setdefault('p2ppeak', 0.0): outputlist[username][ptype]['p2ppeak'] = tmpvalue3 for username, ptypes in typelist[index].items(): for ptype, values in ptypes.items(): if username not in outputtypelist: outputtypelist[username] = {} if ptype not in outputtypelist[username]: outputtypelist[username][ptype] = {} tmpvalue = float(float(values.setdefault('totaldown', 0))*8/1024/1024)/float(300) tmpvalue2 = float(float(values.setdefault('cdndown', 0))*8/1024/1024)/float(300) tmpvalue3 = float(float(values.setdefault('p2pdown', 0))*8/1024/1024)/float(300) if tmpvalue > outputtypelist[username][ptype].setdefault('peak', 0.0): outputtypelist[username][ptype]['peak'] = tmpvalue if tmpvalue2 > outputtypelist[username][ptype].setdefault('cdnpeak', 0.0): outputtypelist[username][ptype]['cdnpeak'] = tmpvalue2 if tmpvalue3 > outputtypelist[username][ptype].setdefault('p2ppeak', 0.0): outputtypelist[username][ptype]['p2ppeak'] = tmpvalue3 for ptype, values in totallist[index].items(): if ptype not in outputtotallist: outputtotallist[ptype] = {} tmpvalue = float(float(values.setdefault('totaldown', 0))*8/1024/1024)/float(300) tmpvalue2 = float(float(values.setdefault('cdndown', 0))*8/1024/1024)/float(300) tmpvalue3 = float(float(values.setdefault('p2pdown', 0))*8/1024/1024)/float(300) if tmpvalue > outputtotallist[ptype].setdefault('peak', 0.0): outputtotallist[ptype]['peak'] = tmpvalue if tmpvalue2 > outputtotallist[ptype].setdefault('cdnpeak', 0.0): outputtotallist[ptype]['cdnpeak'] = tmpvalue2 if tmpvalue3 > outputtotallist[ptype].setdefault('p2ppeak', 0.0): outputtotallist[ptype]['p2ppeak'] = tmpvalue3 outputtotallist['vods'] = {} outputtotallist['live'] = {} outputtotallist['all'] = {} outputtotallist['vods']['peak'] = outputtotallist['vod']['peak'] + outputtotallist['hls']['peak'] outputtotallist['vods']['cdnpeak'] = outputtotallist['vod']['cdnpeak'] + outputtotallist['hls']['cdnpeak'] outputtotallist['vods']['p2ppeak'] = outputtotallist['vod']['p2ppeak'] + outputtotallist['hls']['p2ppeak'] outputtotallist['live']['peak'] = outputtotallist['live_flv']['peak'] + outputtotallist['live_ts']['peak'] + outputtotallist['live_m3u8']['peak'] outputtotallist['live']['cdnpeak'] = outputtotallist['live_flv']['cdnpeak'] + outputtotallist['live_ts']['cdnpeak'] + outputtotallist['live_m3u8']['cdnpeak'] outputtotallist['live']['p2ppeak'] = outputtotallist['live_flv']['p2ppeak'] + outputtotallist['live_ts']['p2ppeak'] + outputtotallist['live_m3u8']['p2ppeak'] outputtotallist['all']['peak'] = outputtotallist['vods']['peak'] + outputtotallist['live']['peak'] outputtotallist['all']['cdnpeak'] = outputtotallist['vods']['cdnpeak'] + outputtotallist['live']['cdnpeak'] outputtotallist['all']['p2ppeak'] = outputtotallist['vods']['p2ppeak'] + outputtotallist['live']['p2ppeak'] for username, ptypes in outputlist.items(): for ptype, values in ptypes.items(): expectedfile.write('%d,%s,%f,%f,%f,%s\n' % (hour, username, values.setdefault('peak', 0.0), values.setdefault('cdnpeak', 0.0), values.setdefault('p2ppeak', 0.0), ptype)) for username, ptypes in outputtypelist.items(): for ptype, values in ptypes.items(): expectedfile.write('%d,%s,%f,%f,%f,%s\n' % (hour, username, values.setdefault('peak', 0.0), values.setdefault('cdnpeak', 0.0), values.setdefault('p2ppeak', 0.0), ptype)) for ptype, values in outputtotallist.items(): expectedfile.write('%d,%s,%f,%f,%f,%s\n' % (hour, 'all', values.setdefault('peak', 0.0), values.setdefault('cdnpeak', 0.0), values.setdefault('p2ppeak', 0.0), ptype)) else: raise expectedfile.close() return os.path.abspath(os.path.dirname(__file__)) + "/../inputdata/%s.txt" % __name__.split('.')[-1]
def makeexpecteddata(hour=-1): resultfile = open( os.path.abspath(os.path.dirname(__file__)) + '/../testdata/LiveProgressCleaned.txt', 'r') liveprogresslines = resultfile.readlines() resultfile.close() resultfile = open( os.path.abspath(os.path.dirname(__file__)) + '/../testdata/QosBufferingCleaned.txt', 'r') QosBufferinglines = resultfile.readlines() resultfile.close() time_format = constvars.recorddate + '000001' timestamp = long(get_timestamp_by_time(time_format)[:-3]) expectedfile = open( os.path.abspath(os.path.dirname(__file__)) + "/../inputdata/%s.txt" % __name__.split('.')[-1], "w") allpeerinfo = {} totalpeerinfo = {} bufferpeerinfo = {} totalbuffering = {} for line in liveprogresslines: _, timesp, peerid, _, _, _, _, _ = line.split(',') for i in range(288): if (timestamp + 300 * (i + 1)) > long(timesp[:-3]) >= (timestamp + 300 * i): if i not in allpeerinfo: allpeerinfo[i] = {} if i not in totalpeerinfo: totalpeerinfo[i] = set() tmpusernum = peerid[:8] if tmpusernum not in allpeerinfo[i]: allpeerinfo[i][tmpusernum] = set() allpeerinfo[i][tmpusernum].add(peerid) totalpeerinfo[i].add(peerid) for line in QosBufferinglines: _, timesp, peerid, _, _, _, _, _, _, _ = line.split(',') for i in range(288): if (timestamp + 300 * (i + 1)) > long(timesp[:-3]) >= (timestamp + 300 * i): if i not in bufferpeerinfo: bufferpeerinfo[i] = {} if i not in totalbuffering: totalbuffering[i] = set() tmpusernum = peerid[:8] if tmpusernum not in bufferpeerinfo[i]: bufferpeerinfo[i][tmpusernum] = set() bufferpeerinfo[i][tmpusernum].add(peerid) totalbuffering[i].add(peerid) if hour > -1: for index in range(hour * 12, (hour + 1) * 12): for username, sets in allpeerinfo[index].items(): if index not in bufferpeerinfo: tmpnum = 0 else: if username not in bufferpeerinfo[index]: tmpnum = 0 else: tmpnum = len(bufferpeerinfo[index][username]) tmproughness = float(tmpnum) / float(len(sets)) expectedfile.write('%s,%f,%d,%s\n' % (username, tmproughness, index, ' ')) if index not in totalbuffering: tmpnum = 0 else: tmpnum = len(totalbuffering[index]) expectedfile.write('%s,%f,%d,%s\n' % ('99999999', float(tmpnum) / float(len(totalpeerinfo[index])), index, ' ')) else: for index in range(288): for username, sets in allpeerinfo[index].items(): if index not in bufferpeerinfo: tmpnum = 0 else: if username not in bufferpeerinfo[index]: tmpnum = 0 else: tmpnum = len(bufferpeerinfo[index][username]) tmproughness = float(tmpnum) / float(len(sets)) expectedfile.write('%s,%f,%d,%s\n' % (username, tmproughness, index, ' ')) if index not in totalbuffering: tmpnum = 0 else: tmpnum = len(totalbuffering[index]) expectedfile.write('%s,%f,%d,%s\n' % ('99999999', float(tmpnum) / float(len(totalpeerinfo[index])), index, ' ')) expectedfile.close() return os.path.abspath(os.path.dirname( __file__)) + "/../inputdata/%s.txt" % __name__.split('.')[-1]
def makeexpecteddata(hour=-1): resultfile = open( os.path.abspath(os.path.dirname(__file__)) + '/../testdata/DownloadFlowCleaned.txt', 'r') orgdownlines = resultfile.readlines() resultfile.close() time_format = constvars.recorddate + '000001' timestamp = long(get_timestamp_by_time(time_format)[:-3]) expectedfile = open( os.path.abspath(os.path.dirname(__file__)) + "/../inputdata/%s.txt" % __name__.split('.')[-1], "w") resultlist = {} totallist = {} for line in orgdownlines: peerid, timesp, url, _, playtype, p2pdown, cdndown, _, _, _ = line.split( ',') for i in range(24): if (timestamp + 3600 * (i + 1)) > long(timesp[:-3]) >= (timestamp + 3600 * i): if i not in resultlist: resultlist[i] = {} if i not in totallist: totallist[i] = {} tmpusername = datavars.name_list[peerid[:8]] if tmpusername not in resultlist[i]: resultlist[i][tmpusername] = {} if playtype not in resultlist[i][tmpusername]: resultlist[i][tmpusername][playtype] = {} if url not in resultlist[i][tmpusername][playtype]: resultlist[i][tmpusername][playtype][url] = {} if playtype not in totallist[i]: totallist[i][playtype] = {} if url not in totallist[i][playtype]: totallist[i][playtype][url] = {} resultlist[i][tmpusername][playtype][url][ 'p2pdown'] = resultlist[i][tmpusername][playtype][ url].setdefault('p2pdown', 0L) + long(p2pdown) resultlist[i][tmpusername][playtype][url][ 'totaldown'] = resultlist[i][ tmpusername][playtype][url].setdefault( 'totaldown', 0L) + long(cdndown) + long(p2pdown) totallist[i][playtype][url][ 'p2pdown'] = totallist[i][playtype][url].setdefault( 'p2pdown', 0L) + long(p2pdown) totallist[i][playtype][url][ 'totaldown'] = totallist[i][playtype][url].setdefault( 'totaldown', 0L) + long(cdndown) + long(p2pdown) break if hour > -1: currentresultlist = {} currenttotallist = {} for index in range(hour + 1): for username, ptypes in resultlist[index].items(): for ptype, urls in ptypes.items(): for url, values in urls.items(): if username not in currentresultlist: currentresultlist[username] = {} if ptype not in currentresultlist[username]: currentresultlist[username][ptype] = {} if url not in currentresultlist[username][ptype]: currentresultlist[username][ptype][url] = {} if ptype not in currenttotallist: currenttotallist[ptype] = {} if url not in currenttotallist[ptype]: currenttotallist[ptype][url] = {} currentresultlist[username][ptype][url][ 'p2pdown'] = currentresultlist[ username][ptype][url].setdefault( 'p2pdown', 0L) + values.setdefault( 'p2pdown', 0L) currentresultlist[username][ptype][url][ 'totaldown'] = currentresultlist[ username][ptype][url].setdefault( 'totaldown', 0L) + values.setdefault( 'totaldown', 0L) currenttotallist[ptype][url][ 'p2pdown'] = currenttotallist[ptype][ url].setdefault('p2pdown', 0L) + values.setdefault( 'p2pdown', 0L) currenttotallist[ptype][url][ 'totaldown'] = currenttotallist[ptype][ url].setdefault('totaldown', 0L) + values.setdefault( 'totaldown', 0L) for username, ptypes in currentresultlist.items(): for ptype, urls in ptypes.items(): for url, values in urls.items(): if values.setdefault('totaldown', 0) != 0: expectedfile.write( '%s,%s,%f,%s\n' % (username, url, float(values.setdefault('p2pdown', 0)) / float(values.setdefault('totaldown', 0)), ptype)) for ptype, urls in currenttotallist.items(): for url, values in currenttotallist.items(): if values.setdefault('totaldown', 0) != 0: expectedfile.write( '%s,%s,%f,%s\n' % ('all', url, float(values.setdefault('p2pdown', 0)) / float(values.setdefault('totaldown', 0)), ptype)) else: pass # print('it should be an hour job not for all day!') # return expectedfile.close() return os.path.abspath(os.path.dirname( __file__)) + "/../inputdata/%s.txt" % __name__.split('.')[-1]