def install_bench(): """install bench""" logging.info('Install bench') for n in [el for s in topology['fog'] for el in s['client']] + topology['client']: execo.Process('scp /home/bconfais/ipfs+dns/bench_logtime.py ' + str(n['ip_5k']) + ':/tmp/bench_s1.py').run().stdout execo.Process('scp /home/bconfais/ipfs+dns/bench_permutation.py ' + str(n['ip_5k']) + ':/tmp/bench_s2.py').run().stdout
def install_ipfs(): """install the modified ipfs""" logging.info('Install IPFS.') commands = ['umount /tmp'] ## exec_commands(commands, [el for s in topology for el in s['storaged']]); for n in [el for s in topology for el in s['storaged']]: execo.Process('scp /home/bconfais/ipfs ' + str(n['ip2']) + ':/tmp/ipfs').run().stdout execo.Process('scp /home/bconfais/ipfs.service ' + str(n['ip2']) + ':/etc/systemd/system/ipfs.service').run().stdout commands = [ 'mkdir -p ' + str(config['rozofs']['mount_dir']), ] exec_commands(commands, [el for s in topology for el in s['storaged']])
def exec_commands_thread(node, command, stdout): cmd = execo.Process('ssh ' + str(node['ip_5k']) + ' \'' + str(command) + '\'').run() logging.info('[' + str(node) + '] ' + str(command)) if stdout: logging.info('[' + str(node) + '] ' + str(cmd.stdout)) return
def install_nuttcp(): """install nuttcp""" logging.info('Install nuttcp') for n in [ el for s in topology['fog'] for el in s['storaged'] + s['client'] ] + [el for el in topology['dns']] + [el for el in topology['client']]: execo.Process('scp /home/bconfais/nuttcp-8.1.4.x86_64 ' + str(n['ip_5k']) + ':/root/nuttcp').run().stdout
def collect_metric(startTime, endTime, metric, result_dir, site, resultFile, hostname): std = resultFile + '_std.txt' ste = resultFile + '_err.txt' stdCSV = resultFile + '.csv' if not os.path.exists(result_dir): os.mkdir(result_dir) host = hostname.split('.')[0] cmd_template = ( 'curl -kn "https://api.grid5000.fr/stable/sites/{site}/metrics/{metric}/timeseries?resolution=15&only={hostname}&from={startTime}&to={endTime}"' ) cmd = cmd_template.format(site=site, metric=metric, hostname=host, startTime=startTime, endTime=endTime) print(cmd) collector = execo.Process(cmd=cmd) collector.run() with open(os.path.join(result_dir, std), "w") as stdout, open(os.path.join(result_dir, ste), "w") as stderr: stdout.write(collector.stdout) stderr.write(collector.stderr) with open(os.path.join(result_dir, std), "r") as fp, open(os.path.join(result_dir, stdCSV), "w") as out: data = json.load(fp, object_hook=JSONObject) if metric == "power" or metric == "network_in" or metric == "network_out": item_index = 0 for item in data.items: href = item.links[0].href host_name = os.path.basename(href) item_index += 1 for index in range(len(item.values)): out.write("%s,%f,%d\n" % (host_name, item.timestamps[index], item.values[index])) else: item_index = 0 for item in data.items: timestamp = [ i for i in range(int(startTime), int(startTime) + len(item.values)) ] href = item.links[0].href host_name = os.path.basename(href) item_index += 1 for index in range(len(item.values)): out.write( "%s,%f,%s\n" % (host_name, timestamp[index], item.values[index]))
def install_ipfs(): """install the modified ipfs""" logging.info('Install IPFS.') commands = [ 'umount /tmp' ] for n in [el for s in topology['fog'] for el in s['storaged']]: # execo.Process('scp /home/bconfais/ipfs+dns/ipfs_dht_logtime '+str(n['ip_5k'])+':/tmp/ipfs').run().stdout execo.Process('scp /home/bconfais/ipfs+dns2/ipfs_dht_k1_profiler2 '+str(n['ip_5k'])+':/tmp/ipfs').run().stdout execo.Process('scp /home/bconfais/ipfs+dns/ipfs_dht.service '+str(n['ip_5k'])+':/etc/systemd/system/ipfs.service').run().stdout # execo.Process('scp /home/bconfais/ipfs+dns/ipfs_dht '+str(n['ip_5k'])+':/tmp/ipfs').run().stdout # execo.Process('scp /home/bconfais/ipfs+dns/ipfs_dht_routing3 '+str(n['ip_5k'])+':/tmp/ipfs').run().stdout # execo.Process('scp /home/bconfais/ipfs+dns/ipfs_dht_routing2 '+str(n['ip_5k'])+':/tmp/ipfs').run().stdout # execo.Process('scp /home/bconfais/ipfs+dns/ipfs_dht.service '+str(n['ip_5k'])+':/etc/systemd/system/ipfs.service').run().stdout commands = [ 'mkdir -p '+str(config['rozofs']['mount_dir']), ] exec_commands(commands, [el for s in topology['fog'] for el in s['storaged']]);
def install_rozofs(): """install rozofs on each site""" logging.info('Install rozofs.') for n in [ el for s in topology for el in s['storaged'] + s['exportd'] + s['client'] ]: execo.Process('scp -r /home/bconfais/rozodeb ' + str(n['ip2']) + ':/tmp/rozodeb').run().stdout # execo.Process('scp -r /home/bconfais/rozobuild '+str(n['ip'])+':/tmp/rozofs').run().stdout commands = [ 'dpkg -i /tmp/rozodeb/*.deb', # 'cd /tmp/rozofs/build; make install', 'pkill rozo', 'pkill storaged', 'pkill exportd', ] exec_commands(commands, [ el for s in topology for el in s['storaged'] + s['exportd'] + s['client'] ]) commands = [ 'umount /home', 'umount /grid5000', 'mkdir -p ' + str(config['rozofs']['data_dir']), 'mount -t tmpfs tmpfs ' + str(config['rozofs']['data_dir']), # 'umount /tmp', # 'mount -o data=writeback,noatime,barrier=0 /dev/sda5 /tmp', 'mkdir -p ' + str(config['rozofs']['config_dir']), ] exec_commands( commands, [el for s in topology for el in s['storaged'] + s['exportd']]) commands = [ 'mkdir -p ' + str(config['rozofs']['data_dir']) + '/exports/export-1' ] exec_commands(commands, [el for s in topology for el in s['exportd']]) commands = [ 'rm -fr ' + str(config['rozofs']['data_dir']), 'mkdir -p ' + str(config['rozofs']['data_dir']) + '/storaged/storage-1/0', 'mkdir -p ' + str(config['rozofs']['data_dir']) + '/storaged/storage-2/0', 'mkdir -p ' + str(config['rozofs']['data_dir']) + '/storaged/storage-1/1', 'mkdir -p ' + str(config['rozofs']['data_dir']) + '/storaged/storage-2/1', 'mkdir -p ' + str(config['rozofs']['mount_dir']) ] exec_commands(commands, [el for s in topology for el in s['storaged']]) commands = [ 'umount /home', 'umount /grid5000', 'mkdir -p ' + str(config['rozofs']['mount_dir']) ] exec_commands(commands, [el for s in topology for el in s['client']])
def install_named(): """install Bind""" logging.info('Install bind.') for n in [el for s in topology['fog'] for el in s['dns']] + [el for el in topology['dns']]: execo.Process('scp -r /home/bconfais/bind ' + str(n['ip_5k']) + ':/tmp/bind').run().stdout commands = ['dpkg -i /tmp/bind/*.deb', '/etc/init.d/bind9 stop'] exec_commands(commands, [el for s in topology['fog'] for el in s['dns']] + [el for el in topology['dns']])
def install_ipfs(): """install the modified ipfs""" logging.info('Install IPFS.') commands = ['umount /tmp'] ## exec_commands(commands, [el for s in topology for el in s['storaged']]); for n in [el for s in topology for el in s['storaged']]: execo.Process( 'scp /home/bconfais/ipfs_common_backend_v1a_big_workers_and_no_ratelimiter ' + str(n['ip']) + ':/tmp/ipfs').run().stdout # execo.Process('scp /home/bconfais/ipfs_k1_normal '+str(n['ip'])+':/tmp/ipfs').run().stdout execo.Process('scp /home/bconfais/ipfs.service ' + str(n['ip']) + ':/etc/systemd/system/ipfs.service').run().stdout # execo.Process('scp /home/bconfais/ipfs_common_backend_v1a '+str(n['ip'])+':/tmp/ipfs').run().stdout # execo.Process('scp /home/bconfais/ipfs2 '+str(n['ip'])+':/tmp/ipfs').run().stdout # execo.Process('scp /home/bconfais/common_ipfs_try '+str(n['ip'])+':/tmp/ipfs').run().stdout commands = [ 'mkdir -p ' + str(config['rozofs']['mount_dir']), ## 'mount -o data=writeback,noatime,barrier=0 /dev/sda5 '+str(config['rozofs']['mount_dir']) ] exec_commands(commands, [el for s in topology for el in s['storaged']])
def get_data(host, user, password, debug=False): """ use execo as a better subprocess substitute """ host = host + '-mngt' user = user password = password cmd = "ipmitool -I lanplus -U {} -P {} -H {} sensor".format( user, password, host) process = execo.Process(cmd, shell=False) process.run() # split process.stdout line by line if debug: pprint(process.stderr) return process.stdout.split('\n')
def install_ycsb(): """install ycsb on clients""" logging.info('Install YCSB.') for n in [el for s in topology for el in s['client']]: # execo.Process('scp /home/bconfais/YCSB_ipfs_random_4nodes.tar.gz '+str(n['ip'])+':/tmp/YCSB_ipfs_random.tar.gz').run().stdout # execo.Process('scp /home/bconfais/YCSB_ipfs_random_4nodes_dummyfile.tar.gz '+str(n['ip'])+':/tmp/YCSB_ipfs_random.tar.gz').run().stdout execo.Process( 'scp /home/bconfais/YCSB_ipfs_random_4nodes_cache.tar.gz ' + str(n['ip2']) + ':/tmp/YCSB_ipfs_random.tar.gz').run().stdout commands = [ 'apt-get update; apt-get --yes --force-yes install openjdk-8-jre-headless', 'cd /tmp; tar zxvf YCSB_ipfs_random.tar.gz', # 'cd /tmp; mv YCSB_ipfs_random_4nodes YCSB', # 'cd /tmp; mv YCSB_ipfs_random_4nodes_dummyfile YCSB', 'cd /tmp; mv YCSB_ipfs_random_4nodes_cache YCSB', ] exec_commands(commands, [el for s in topology for el in s['client']])
def deploy_named(): logging.info('Deploy named') # create config file config_ = "" \ "options {\n" \ " directory \"/tmp/dns/\";\n" \ " pid-file \"/run/named/named.pid\";\n" \ " allow-recursion { 0.0.0.0/0; };\n" \ " allow-transfer { none; };\n" \ " allow-update { none; };\n" \ " version none;\n" \ " hostname none;\n" \ " server-id none;\n" \ " minimal-responses yes;\n" \ " tcp-clients 10000;\n" \ " cleaning-interval 0;\n" \ " rate-limit {\n" \ " exempt-clients { 0.0.0.0/0; };\n" \ " };\n" \ "};\n\n" \ "zone \"example.com\" IN {\n" \ " type master;\n" \ " file \"example.com.zone\";\n" \ " allow-update {0.0.0.0/0;};\n" \ "};\n" with open('named', 'w') as f: f.write(config_) for n in [el for s in topology['fog'] for el in s['dns']] + [el for el in topology['dns']]: execo.Process('scp named ' + str(n['ip_5k']) + ':/etc/bind/named.conf').run().stdout config_ = "" \ "# run resolvconf?\n" \ "RESOLVCONF=no\n" \ "# startup options for the server\n" \ "OPTIONS=\"-u bind -n 10000\"\n" with open('named', 'w') as f: f.write(config_) for n in [el for s in topology['fog'] for el in s['dns']] + [el for el in topology['dns']]: execo.Process('scp named ' + str(n['ip_5k']) + ':/etc/default/bind9').run().stdout exec_commands([ 'rm -rf /tmp/dns/example.com.*', 'mkdir -p /tmp/dns', 'mount -t tmpfs tmpfs /tmp/dns' ], [el for s in topology['fog'] for el in s['dns']] + [el for el in topology['dns']]) # create root zone # c = "" # for isite, site in enumerate(topology['fog']): # c = c + "\nsite%d IN A %s" % ((isite+1), site['dns'][0]['ip_vlan']) # c = c + "\n*.site%d IN A %s" % ((isite+1), site['dns'][0]['ip_vlan']) config_ = "" \ "$TTL 3000\n" \ "example.com. IN SOA example.com. example.com. (2017052314 3600 1800 7257600 86400)\n" \ "@ IN NS example.com.\n" \ "@ IN A %s\n" \ "" % (topology['fog'][3]['dns'][0]['ip_vlan']) with open('named', 'w') as f: f.write(config_) for n in range(len(topology['fog'])): execo.Process('scp named ' + str(topology['fog'][n]['dns'][0]['ip_5k']) + ':/tmp/dns/example.com.zone').run().stdout config_ = "" \ "$TTL 3000\n" \ "example.com. IN SOA example.com. example.com. (2017052314 3600 1800 7257600 86400)\n" \ "@ IN NS example.com.\n" \ "@ IN A %s\n" \ "site1 IN TXT \"%s,%s\"\n" \ "site2 IN TXT \"%s,%s\"\n" \ "site3 IN TXT \"%s,%s\"\n" \ "site4 IN TXT \"%s\"\n" \ "site5 IN TXT \"%s,%s\"\n" \ "" % (topology['fog'][3]['dns'][0]['ip_vlan'], topology['fog'][3]['dns'][0]['ip_vlan'], topology['fog'][0]['dns'][0]['ip_vlan'], topology['fog'][3]['dns'][0]['ip_vlan'], topology['fog'][1]['dns'][0]['ip_vlan'], topology['fog'][3]['dns'][0]['ip_vlan'],topology['fog'][2]['dns'][0]['ip_vlan'], topology['fog'][3]['dns'][0]['ip_vlan'], topology['fog'][3]['dns'][0]['ip_vlan'],topology['fog'][4]['dns'][0]['ip_vlan']) with open('named', 'w') as f: f.write(config_) execo.Process('scp named ' + str(topology['fog'][3]['dns'][0]['ip_5k']) + ':/tmp/dns/example.com.zone').run().stdout exec_commands([ 'chown bind:bind /tmp/dns/example.com.zone', 'systemctl restart bind9' ], [el for s in topology['fog'] for el in s['dns']] + [el for el in topology['dns']])
def s2(): for i_object, object in enumerate(config['objects']): big_empty() sleep = 5 if 1 == object['number']: sleep = 2 for trial in range(config['trials']): logging.info('Iteration %d' % (trial)) threads_write = [] threads_first_read = [] threads_second_read = [] ycsb_args = '-p requestdistribution=sequential -p fieldcount=1 -p fieldlength=%d -p recordcount=%d -p operationcount=%d -target %d -threads %d' % ( object['size'] * 1000 * 1000, object['number'], object['number'], object['number'], object['number']) for i_site, site in [(0, topology[0])]: for i_client, c in enumerate(site['client']): threads_write.append( Thread( target=ycsb_thread, # args=(('cd /tmp/YCSB; ./bin/ycsb load ipfs -p ipfs.host1=%s -p ipfs.host2=%s -p ipfs.host3=%s -p ipfs.host4=%s %s -P workloads/workloadc' % (str(topology[i_site]['storaged'][0]['ip']), str(topology[i_site]['storaged'][1]['ip']), str(topology[i_site]['storaged'][2]['ip']), str(topology[i_site]['storaged'][3]['ip']), ycsb_args)), topology[i_site]['client'][i_client], ('object%d_trial%d_site%d_client%d_write.txt' % (i_object, trial, i_site+1, i_client+1)),) args=( ('cd /tmp/; python2 bench.py write %d %d %s %s %s %s' % ( object['number'], object['size'] * 1000 * 1000, str(topology[i_site]['storaged'][0] ['ip']), str(topology[i_site]['storaged'][1] ['ip']), str(topology[i_site]['storaged'][2] ['ip']), str(topology[i_site]['storaged'][3] ['ip']), )), topology[i_site]['client'][i_client], ('object%d_trial%d_site%d_client%d_write.txt' % (i_object, trial, i_site + 1, i_client + 1)), ))) for i_site, site in [(1, topology[1])]: for i_client, c in enumerate(site['client']): threads_first_read.append( Thread( target=ycsb_thread, # args=(('cd /tmp/YCSB; ./bin/ycsb run ipfs -p ipfs.host1=%s -p ipfs.host2=%s -p ipfs.host3=%s -p ipfs.host4=%s %s -P workloads/workloadc' % (str(topology[i_site]['storaged'][0]['ip']), str(topology[i_site]['storaged'][1]['ip']), str(topology[i_site]['storaged'][2]['ip']), str(topology[i_site]['storaged'][3]['ip']), ycsb_args)), topology[i_site]['client'][i_client], ('object%d_trial%d_site%d_client%d_read.txt' % (i_object, trial, i_site+1, i_client+1)),) args=( ('cd /tmp/; python2 bench.py read %d %d %s %s %s %s' % ( object['number'], object['size'] * 1000 * 1000, str(topology[i_site]['storaged'][0] ['ip']), str(topology[i_site]['storaged'][1] ['ip']), str(topology[i_site]['storaged'][2] ['ip']), str(topology[i_site]['storaged'][3] ['ip']), )), topology[i_site]['client'][i_client], ('object%d_trial%d_site%d_client%d_read.txt' % (i_object, trial, i_site + 1, i_client + 1)), ))) for i_site, site in [(1, topology[1])]: for i_client, c in enumerate(site['client']): threads_second_read.append( Thread( target=ycsb_thread, # args=(('cd /tmp/YCSB; ./bin/ycsb run ipfs -p ipfs.host1=%s -p ipfs.host2=%s -p ipfs.host3=%s -p ipfs.host4=%s %s -P workloads/workloadc' % (str(topology[i_site]['storaged'][0]['ip']), str(topology[i_site]['storaged'][1]['ip']), str(topology[i_site]['storaged'][2]['ip']), str(topology[i_site]['storaged'][3]['ip']), ycsb_args)), topology[i_site]['client'][i_client], ('object%d_trial%d_site%d_client%d_secondread.txt' % (i_object, trial, i_site+1, i_client+1)),) args=( ('cd /tmp/; python2 bench.py read %d %d %s %s %s %s' % ( object['number'], object['size'] * 1000 * 1000, str(topology[i_site]['storaged'][0] ['ip']), str(topology[i_site]['storaged'][1] ['ip']), str(topology[i_site]['storaged'][2] ['ip']), str(topology[i_site]['storaged'][3] ['ip']), )), topology[i_site]['client'][i_client], ('object%d_trial%d_site%d_client%d_secondread.txt' % (i_object, trial, i_site + 1, i_client + 1)), ))) logging.info('Clean /tmp/ids') empty() logging.info( 'Write %d objects of %dMB with %d clients' % (object['number'], object['size'], len(threads_write))) # write on all the sites sync() for t in threads_write: t.start() for t in threads_write: t.join() iptables_collect_end('ipfs_trial%d_object%d' % (trial, i_object), 'postwrite') logging.info('scp') execo.Process('scp ' + str(topology[0]['client'][0]['ip2']) + ':/tmp/ids ids').run().stdout execo.Process('scp ids ' + str(topology[1]['client'][0]['ip2']) + ':/tmp/ids').run().stdout logging.info( 'Read %d objects of %dMB with %d clients' % (object['number'], object['size'], len(threads_first_read))) # read on second site sync() for t in threads_first_read: t.start() for t in threads_first_read: t.join() time.sleep(sleep) iptables_collect_end('ipfs_trial%d_object%d' % (trial, i_object), 'postread') logging.info( 'Read %d objects of %dMB with %d clients' % (object['number'], object['size'], len(threads_second_read))) # read on the second site sync() for t in threads_second_read: t.start() for t in threads_second_read: t.join() time.sleep(sleep) iptables_collect_end('ipfs_trial%d_object%d' % (trial, i_object), 'postsecondread')
def deploy_rozofs(): """ """ logging.info('Deploy rozofs.') for site in topology: logging.info("Create exportd config") sids = [] for i, s in enumerate(site['storaged']): sids.append("\t\t\t\t\t{sid = %d, host = \"%s\";}\n" % ((i + 1), s['ip'])) config_ = "" \ "layout = %d;\n" \ "volumes = \n" \ "( \n" \ "\t{\n" \ "\t\tvid = 1;\n" \ "\t\tcids=\n" \ "\t\t(\n" \ "\t\t\t{\n" \ "\t\t\t\tcid = 1;\n" \ "\t\t\t\tsids=\n" \ "\t\t\t\t(\n" \ "%s" \ "\t\t\t\t);\n" \ "\t\t\t}\n" \ "\t\t);\n" \ "\t}\n" \ "); \n" \ "exports = (\n" \ "\t{eid = 1; bsize=\"4K\"; root = \"%s\"; md5=\"\"; squota=\"\"; hquota=\"\"; vid=1;}\n" \ ");\n" % (config['rozofs']['layout'], ",".join(sids), str(config['rozofs']['data_dir'])+'/exports/export-1') with open('export', 'w') as f: f.write(config_) for e in site['exportd']: execo.Process('scp export ' + str(e['ip2']) + ':' + str(config['rozofs']['config_dir']) + '/export.conf').run().stdout logging.info("Launch exportd") commands = [ 'exportd -c ' + str(config['rozofs']['config_dir']) + '/export.conf', 'rozo agent start' ] exec_commands(commands, site['exportd']) for i, s in enumerate(site['storaged']): logging.info("Create storaged config") config_ = "" \ "crc32c_check = True; \n" \ "crc32c_generate = True; \n" \ "listen = (\n" \ "\t{\n" \ "\t\taddr = \"*\";\n" \ "\t\tport = 41001;\n" \ "\t}\n" \ ");\n" \ "storages = (\n" \ "\t{cid = 1; sid = %d; root = \"%s\"; device-total = 1; device-mapper = 1; device-redundancy = 1;}\n" \ ");\n" % ((i+1), config['rozofs']['data_dir']+'/storaged/storage-1') with open('storage', 'w') as f: f.write(config_) execo.Process('scp storage ' + str(s['ip2']) + ':' + str(config['rozofs']['config_dir']) + '/storage.conf').run().stdout config_ = "" \ "nb_disk_thread = 250; // 2:32" \ "\n" with open('storage', 'w') as f: f.write(config_) execo.Process('scp storage ' + str(s['ip2']) + ':' + str(config['rozofs']['config_dir']) + '/rozofs.conf').run().stdout logging.info("Launch storaged") commands = ['rozo agent start'] exec_commands(commands, site['storaged']) commands = ['rozo node start -E 127.0.0.1'] time.sleep(5) exec_commands(commands, site['exportd'])