def main(argv): global experiments global current_experiment global processed_batches global mobile_connections global desktop_connections json_data=open('cdn_regex.json').read() data = json.loads(json_data) for entry in data["cdns"]: cdn_regex.append(re.compile(entry["pattern"], re.I)) for file in sys.argv: if not file == sys.argv[0]: # if I got a penny for every dirty hack I ever used... exp = Experiment() current_experiment = exp print file sql = "select * from desktopMeasurement" read_batch_from_sql(file, sql, exp._processed_desktop_batches) sql = "select * from mobileMeasurement" read_batch_from_sql(file, sql, exp._processed_mobile_batches) sql = "select * from desktopConnections" read_connection_from_sql(file,sql, exp._desktop_connections) sql = "select * from mobileConnections" read_connection_from_sql(file,sql, exp._mobile_connections) experiments.append(exp) #assign the connections to the corresponding batches for conn in exp._mobile_connections: exp._processed_mobile_batches[conn._parentBatchID - 1]._active_connections.append(conn) for conn in exp._desktop_connections: exp._processed_desktop_batches[conn._parentBatchID - 1]._active_connections.append(conn) create_table("gold") ######################HARDCODED STUFF exp1 = experiments[0] exp2 = experiments[1] exp3 = experiments[2] for x,y,z in zip(exp1._processed_mobile_batches, exp2._processed_mobile_batches, exp3._processed_mobile_batches): batch = RequestBatch() batch._getCount = (x._getCount + y._getCount + z._getCount ) / 3.0 batch._dnsCount = (x._dnsCount + y._dnsCount + z._dnsCount ) / 3.0 batch._downstreamVolumeBytes = (x._downstreamVolumeBytes + y._downstreamVolumeBytes + z._downstreamVolumeBytes ) / 3.0 batch._nr_of_host_contacts = (x._nr_of_host_contacts + y._nr_of_host_contacts + z._nr_of_host_contacts ) / 3.0 batch._upstreamVolumeBytes = (x._upstreamVolumeBytes + y._upstreamVolumeBytes + z._upstreamVolumeBytes ) / 3.0 batch._connectionCount = (x._connectionCount + y._connectionCount + z._connectionCount ) / 3.0 batch._nr_of_webbgus = (x._nr_of_webbgus + y._nr_of_webbgus + z._nr_of_webbgus ) / 3.0 batch._requestURL = x._requestURL x_cdn_volume = 0 x_normal_volume = 0 for connection in x._active_connections: if connection._is_CDN_connection: x_cdn_volume += connection._current_volume else: x_normal_volume += connection._current_volume y_cdn_volume = 0 y_normal_volume = 0 for connection in y._active_connections: if connection._is_CDN_connection: y_cdn_volume += connection._current_volume else: y_normal_volume += connection._current_volume z_cdn_volume = 0 z_normal_volume = 0 for connection in z._active_connections: if connection._is_CDN_connection: z_cdn_volume += connection._current_volume else: z_normal_volume += connection._current_volume cdn_avg = float(z_cdn_volume + y_cdn_volume + x_cdn_volume) / 3.0 normal_avg = float(x_normal_volume + y_normal_volume + z_normal_volume) / 3.0 avg_normal_conn = Connection() avg_cdn_conn = Connection() avg_normal_conn._is_CDN_connection = False avg_normal_conn._current_volume = normal_avg avg_normal_conn._DNS = "dirtyhack" avg_cdn_conn._is_CDN_connection = True avg_cdn_conn._current_volume = cdn_avg avg_cdn_conn._DNS = "akamai" #dirty hack aswell batch._active_connections.append(avg_cdn_conn) batch._active_connections.append(avg_normal_conn) processed_batches.append(batch) for x,y,z in zip(exp1._processed_desktop_batches, exp2._processed_desktop_batches, exp3._processed_desktop_batches): batch = RequestBatch() batch._getCount = (x._getCount + y._getCount + z._getCount ) / 3.0 batch._dnsCount = (x._dnsCount + y._dnsCount + z._dnsCount ) / 3.0 batch._downstreamVolumeBytes = (x._downstreamVolumeBytes + y._downstreamVolumeBytes + z._downstreamVolumeBytes ) / 3.0 batch._nr_of_host_contacts = (x._nr_of_host_contacts + y._nr_of_host_contacts + z._nr_of_host_contacts ) / 3.0 batch._upstreamVolumeBytes = (x._upstreamVolumeBytes + y._upstreamVolumeBytes + z._upstreamVolumeBytes ) / 3.0 batch._connectionCount = (x._connectionCount + y._connectionCount + z._connectionCount ) / 3.0 batch._nr_of_webbgus = (x._nr_of_webbgus + y._nr_of_webbgus + z._nr_of_webbgus ) / 3.0 batch._requestURL = x._requestURL x_cdn_volume = 0 x_normal_volume = 0 for connection in x._active_connections: if connection._is_CDN_connection: x_cdn_volume += connection._current_volume else: x_normal_volume += connection._current_volume y_cdn_volume = 0 y_normal_volume = 0 for connection in y._active_connections: if connection._is_CDN_connection: y_cdn_volume += connection._current_volume else: y_normal_volume += connection._current_volume z_cdn_volume = 0 z_normal_volume = 0 for connection in z._active_connections: if connection._is_CDN_connection: z_cdn_volume += connection._current_volume else: z_normal_volume += connection._current_volume cdn_avg = float(z_cdn_volume + y_cdn_volume + x_cdn_volume) / 3.0 normal_avg = float(x_normal_volume + y_normal_volume + z_normal_volume) / 3.0 avg_normal_conn = Connection() avg_cdn_conn = Connection() avg_normal_conn._is_CDN_connection = False avg_normal_conn._current_volume = normal_avg avg_normal_conn._DNS = "dirtyhack" avg_cdn_conn._is_CDN_connection = True avg_cdn_conn._current_volume = cdn_avg avg_cdn_conn._DNS = "akamai" #dirty hack aswell batch._active_connections.append(avg_cdn_conn) batch._active_connections.append(avg_normal_conn) processed_batches.append(batch) print len(processed_batches) # for x,y,z in zip(exp1._processed_mobile_batches, exp2._processed_mobile_batches, exp3._processed_mobile_batches) # result_mobile_http_gets.append((x+y+z)/3.0) add_to_database("gold")