def install_bench():
"""install bench"""
logging.info('Install bench')
for n in [el for s in topology['fog']
for el in s['client']] + topology['client']:
execo.Process('scp /home/bconfais/ipfs+dns/bench_logtime.py ' +
str(n['ip_5k']) + ':/tmp/bench_s1.py').run().stdout
execo.Process('scp /home/bconfais/ipfs+dns/bench_permutation.py ' +
str(n['ip_5k']) + ':/tmp/bench_s2.py').run().stdout
def install_ipfs():
"""install the modified ipfs"""
logging.info('Install IPFS.')
commands = ['umount /tmp']
## exec_commands(commands, [el for s in topology for el in s['storaged']]);
for n in [el for s in topology for el in s['storaged']]:
execo.Process('scp /home/bconfais/ipfs ' + str(n['ip2']) +
':/tmp/ipfs').run().stdout
execo.Process('scp /home/bconfais/ipfs.service ' + str(n['ip2']) +
':/etc/systemd/system/ipfs.service').run().stdout
commands = [
'mkdir -p ' + str(config['rozofs']['mount_dir']),
]
exec_commands(commands, [el for s in topology for el in s['storaged']])
def exec_commands_thread(node, command, stdout):
cmd = execo.Process('ssh ' + str(node['ip_5k']) + ' \'' + str(command) +
'\'').run()
logging.info('[' + str(node) + '] ' + str(command))
if stdout:
logging.info('[' + str(node) + '] ' + str(cmd.stdout))
return
def install_nuttcp():
"""install nuttcp"""
logging.info('Install nuttcp')
for n in [
el for s in topology['fog'] for el in s['storaged'] + s['client']
] + [el for el in topology['dns']] + [el for el in topology['client']]:
execo.Process('scp /home/bconfais/nuttcp-8.1.4.x86_64 ' +
str(n['ip_5k']) + ':/root/nuttcp').run().stdout
def collect_metric(startTime, endTime, metric, result_dir, site, resultFile,
hostname):
std = resultFile + '_std.txt'
ste = resultFile + '_err.txt'
stdCSV = resultFile + '.csv'
if not os.path.exists(result_dir):
os.mkdir(result_dir)
host = hostname.split('.')[0]
cmd_template = (
'curl -kn "https://api.grid5000.fr/stable/sites/{site}/metrics/{metric}/timeseries?resolution=15&only={hostname}&from={startTime}&to={endTime}"'
)
cmd = cmd_template.format(site=site,
metric=metric,
hostname=host,
startTime=startTime,
endTime=endTime)
print(cmd)
collector = execo.Process(cmd=cmd)
collector.run()
with open(os.path.join(result_dir, std),
"w") as stdout, open(os.path.join(result_dir, ste),
"w") as stderr:
stdout.write(collector.stdout)
stderr.write(collector.stderr)
with open(os.path.join(result_dir, std),
"r") as fp, open(os.path.join(result_dir, stdCSV), "w") as out:
data = json.load(fp, object_hook=JSONObject)
if metric == "power" or metric == "network_in" or metric == "network_out":
item_index = 0
for item in data.items:
href = item.links[0].href
host_name = os.path.basename(href)
item_index += 1
for index in range(len(item.values)):
out.write("%s,%f,%d\n" %
(host_name, item.timestamps[index],
item.values[index]))
else:
item_index = 0
for item in data.items:
timestamp = [
i for i in range(int(startTime),
int(startTime) + len(item.values))
]
href = item.links[0].href
host_name = os.path.basename(href)
item_index += 1
for index in range(len(item.values)):
out.write(
"%s,%f,%s\n" %
(host_name, timestamp[index], item.values[index]))
def install_ipfs():
"""install the modified ipfs"""
logging.info('Install IPFS.')
commands = [
'umount /tmp'
]
for n in [el for s in topology['fog'] for el in s['storaged']]:
# execo.Process('scp /home/bconfais/ipfs+dns/ipfs_dht_logtime '+str(n['ip_5k'])+':/tmp/ipfs').run().stdout
execo.Process('scp /home/bconfais/ipfs+dns2/ipfs_dht_k1_profiler2 '+str(n['ip_5k'])+':/tmp/ipfs').run().stdout
execo.Process('scp /home/bconfais/ipfs+dns/ipfs_dht.service '+str(n['ip_5k'])+':/etc/systemd/system/ipfs.service').run().stdout
# execo.Process('scp /home/bconfais/ipfs+dns/ipfs_dht '+str(n['ip_5k'])+':/tmp/ipfs').run().stdout
# execo.Process('scp /home/bconfais/ipfs+dns/ipfs_dht_routing3 '+str(n['ip_5k'])+':/tmp/ipfs').run().stdout
# execo.Process('scp /home/bconfais/ipfs+dns/ipfs_dht_routing2 '+str(n['ip_5k'])+':/tmp/ipfs').run().stdout
# execo.Process('scp /home/bconfais/ipfs+dns/ipfs_dht.service '+str(n['ip_5k'])+':/etc/systemd/system/ipfs.service').run().stdout
commands = [
'mkdir -p '+str(config['rozofs']['mount_dir']),
]
exec_commands(commands, [el for s in topology['fog'] for el in s['storaged']]);
def install_rozofs():
"""install rozofs on each site"""
logging.info('Install rozofs.')
for n in [
el for s in topology
for el in s['storaged'] + s['exportd'] + s['client']
]:
execo.Process('scp -r /home/bconfais/rozodeb ' + str(n['ip2']) +
':/tmp/rozodeb').run().stdout
# execo.Process('scp -r /home/bconfais/rozobuild '+str(n['ip'])+':/tmp/rozofs').run().stdout
commands = [
'dpkg -i /tmp/rozodeb/*.deb',
# 'cd /tmp/rozofs/build; make install',
'pkill rozo',
'pkill storaged',
'pkill exportd',
]
exec_commands(commands, [
el for s in topology
for el in s['storaged'] + s['exportd'] + s['client']
])
commands = [
'umount /home',
'umount /grid5000',
'mkdir -p ' + str(config['rozofs']['data_dir']),
'mount -t tmpfs tmpfs ' + str(config['rozofs']['data_dir']),
# 'umount /tmp',
# 'mount -o data=writeback,noatime,barrier=0 /dev/sda5 /tmp',
'mkdir -p ' + str(config['rozofs']['config_dir']),
]
exec_commands(
commands,
[el for s in topology for el in s['storaged'] + s['exportd']])
commands = [
'mkdir -p ' + str(config['rozofs']['data_dir']) + '/exports/export-1'
]
exec_commands(commands, [el for s in topology for el in s['exportd']])
commands = [
'rm -fr ' + str(config['rozofs']['data_dir']), 'mkdir -p ' +
str(config['rozofs']['data_dir']) + '/storaged/storage-1/0',
'mkdir -p ' + str(config['rozofs']['data_dir']) +
'/storaged/storage-2/0', 'mkdir -p ' +
str(config['rozofs']['data_dir']) + '/storaged/storage-1/1',
'mkdir -p ' + str(config['rozofs']['data_dir']) +
'/storaged/storage-2/1',
'mkdir -p ' + str(config['rozofs']['mount_dir'])
]
exec_commands(commands, [el for s in topology for el in s['storaged']])
commands = [
'umount /home', 'umount /grid5000',
'mkdir -p ' + str(config['rozofs']['mount_dir'])
]
exec_commands(commands, [el for s in topology for el in s['client']])
def install_named():
"""install Bind"""
logging.info('Install bind.')
for n in [el for s in topology['fog']
for el in s['dns']] + [el for el in topology['dns']]:
execo.Process('scp -r /home/bconfais/bind ' + str(n['ip_5k']) +
':/tmp/bind').run().stdout
commands = ['dpkg -i /tmp/bind/*.deb', '/etc/init.d/bind9 stop']
exec_commands(commands,
[el for s in topology['fog']
for el in s['dns']] + [el for el in topology['dns']])
def install_ipfs():
"""install the modified ipfs"""
logging.info('Install IPFS.')
commands = ['umount /tmp']
## exec_commands(commands, [el for s in topology for el in s['storaged']]);
for n in [el for s in topology for el in s['storaged']]:
execo.Process(
'scp /home/bconfais/ipfs_common_backend_v1a_big_workers_and_no_ratelimiter '
+ str(n['ip']) + ':/tmp/ipfs').run().stdout
# execo.Process('scp /home/bconfais/ipfs_k1_normal '+str(n['ip'])+':/tmp/ipfs').run().stdout
execo.Process('scp /home/bconfais/ipfs.service ' + str(n['ip']) +
':/etc/systemd/system/ipfs.service').run().stdout
# execo.Process('scp /home/bconfais/ipfs_common_backend_v1a '+str(n['ip'])+':/tmp/ipfs').run().stdout
# execo.Process('scp /home/bconfais/ipfs2 '+str(n['ip'])+':/tmp/ipfs').run().stdout
# execo.Process('scp /home/bconfais/common_ipfs_try '+str(n['ip'])+':/tmp/ipfs').run().stdout
commands = [
'mkdir -p ' + str(config['rozofs']['mount_dir']),
## 'mount -o data=writeback,noatime,barrier=0 /dev/sda5 '+str(config['rozofs']['mount_dir'])
]
exec_commands(commands, [el for s in topology for el in s['storaged']])
def get_data(host, user, password, debug=False):
""" use execo as a better subprocess substitute """
host = host + '-mngt'
user = user
password = password
cmd = "ipmitool -I lanplus -U {} -P {} -H {} sensor".format(
user, password, host)
process = execo.Process(cmd, shell=False)
process.run()
# split process.stdout line by line
if debug:
pprint(process.stderr)
return process.stdout.split('\n')
def install_ycsb():
"""install ycsb on clients"""
logging.info('Install YCSB.')
for n in [el for s in topology for el in s['client']]:
# execo.Process('scp /home/bconfais/YCSB_ipfs_random_4nodes.tar.gz '+str(n['ip'])+':/tmp/YCSB_ipfs_random.tar.gz').run().stdout
# execo.Process('scp /home/bconfais/YCSB_ipfs_random_4nodes_dummyfile.tar.gz '+str(n['ip'])+':/tmp/YCSB_ipfs_random.tar.gz').run().stdout
execo.Process(
'scp /home/bconfais/YCSB_ipfs_random_4nodes_cache.tar.gz ' +
str(n['ip2']) + ':/tmp/YCSB_ipfs_random.tar.gz').run().stdout
commands = [
'apt-get update; apt-get --yes --force-yes install openjdk-8-jre-headless',
'cd /tmp; tar zxvf YCSB_ipfs_random.tar.gz',
# 'cd /tmp; mv YCSB_ipfs_random_4nodes YCSB',
# 'cd /tmp; mv YCSB_ipfs_random_4nodes_dummyfile YCSB',
'cd /tmp; mv YCSB_ipfs_random_4nodes_cache YCSB',
]
exec_commands(commands, [el for s in topology for el in s['client']])
def deploy_named():
logging.info('Deploy named')
# create config file
config_ = "" \
"options {\n" \
" directory \"/tmp/dns/\";\n" \
" pid-file \"/run/named/named.pid\";\n" \
" allow-recursion { 0.0.0.0/0; };\n" \
" allow-transfer { none; };\n" \
" allow-update { none; };\n" \
" version none;\n" \
" hostname none;\n" \
" server-id none;\n" \
" minimal-responses yes;\n" \
" tcp-clients 10000;\n" \
" cleaning-interval 0;\n" \
" rate-limit {\n" \
" exempt-clients { 0.0.0.0/0; };\n" \
" };\n" \
"};\n\n" \
"zone \"example.com\" IN {\n" \
" type master;\n" \
" file \"example.com.zone\";\n" \
" allow-update {0.0.0.0/0;};\n" \
"};\n"
with open('named', 'w') as f:
f.write(config_)
for n in [el for s in topology['fog']
for el in s['dns']] + [el for el in topology['dns']]:
execo.Process('scp named ' + str(n['ip_5k']) +
':/etc/bind/named.conf').run().stdout
config_ = "" \
"# run resolvconf?\n" \
"RESOLVCONF=no\n" \
"# startup options for the server\n" \
"OPTIONS=\"-u bind -n 10000\"\n"
with open('named', 'w') as f:
f.write(config_)
for n in [el for s in topology['fog']
for el in s['dns']] + [el for el in topology['dns']]:
execo.Process('scp named ' + str(n['ip_5k']) +
':/etc/default/bind9').run().stdout
exec_commands([
'rm -rf /tmp/dns/example.com.*', 'mkdir -p /tmp/dns',
'mount -t tmpfs tmpfs /tmp/dns'
], [el for s in topology['fog']
for el in s['dns']] + [el for el in topology['dns']])
# create root zone
# c = ""
# for isite, site in enumerate(topology['fog']):
# c = c + "\nsite%d IN A %s" % ((isite+1), site['dns'][0]['ip_vlan'])
# c = c + "\n*.site%d IN A %s" % ((isite+1), site['dns'][0]['ip_vlan'])
config_ = "" \
"$TTL 3000\n" \
"example.com. IN SOA example.com. example.com. (2017052314 3600 1800 7257600 86400)\n" \
"@ IN NS example.com.\n" \
"@ IN A %s\n" \
"" % (topology['fog'][3]['dns'][0]['ip_vlan'])
with open('named', 'w') as f:
f.write(config_)
for n in range(len(topology['fog'])):
execo.Process('scp named ' +
str(topology['fog'][n]['dns'][0]['ip_5k']) +
':/tmp/dns/example.com.zone').run().stdout
config_ = "" \
"$TTL 3000\n" \
"example.com. IN SOA example.com. example.com. (2017052314 3600 1800 7257600 86400)\n" \
"@ IN NS example.com.\n" \
"@ IN A %s\n" \
"site1 IN TXT \"%s,%s\"\n" \
"site2 IN TXT \"%s,%s\"\n" \
"site3 IN TXT \"%s,%s\"\n" \
"site4 IN TXT \"%s\"\n" \
"site5 IN TXT \"%s,%s\"\n" \
"" % (topology['fog'][3]['dns'][0]['ip_vlan'], topology['fog'][3]['dns'][0]['ip_vlan'], topology['fog'][0]['dns'][0]['ip_vlan'], topology['fog'][3]['dns'][0]['ip_vlan'], topology['fog'][1]['dns'][0]['ip_vlan'], topology['fog'][3]['dns'][0]['ip_vlan'],topology['fog'][2]['dns'][0]['ip_vlan'], topology['fog'][3]['dns'][0]['ip_vlan'], topology['fog'][3]['dns'][0]['ip_vlan'],topology['fog'][4]['dns'][0]['ip_vlan'])
with open('named', 'w') as f:
f.write(config_)
execo.Process('scp named ' + str(topology['fog'][3]['dns'][0]['ip_5k']) +
':/tmp/dns/example.com.zone').run().stdout
exec_commands([
'chown bind:bind /tmp/dns/example.com.zone', 'systemctl restart bind9'
], [el for s in topology['fog']
for el in s['dns']] + [el for el in topology['dns']])
def s2():
for i_object, object in enumerate(config['objects']):
big_empty()
sleep = 5
if 1 == object['number']:
sleep = 2
for trial in range(config['trials']):
logging.info('Iteration %d' % (trial))
threads_write = []
threads_first_read = []
threads_second_read = []
ycsb_args = '-p requestdistribution=sequential -p fieldcount=1 -p fieldlength=%d -p recordcount=%d -p operationcount=%d -target %d -threads %d' % (
object['size'] * 1000 * 1000, object['number'],
object['number'], object['number'], object['number'])
for i_site, site in [(0, topology[0])]:
for i_client, c in enumerate(site['client']):
threads_write.append(
Thread(
target=ycsb_thread,
# args=(('cd /tmp/YCSB; ./bin/ycsb load ipfs -p ipfs.host1=%s -p ipfs.host2=%s -p ipfs.host3=%s -p ipfs.host4=%s %s -P workloads/workloadc' % (str(topology[i_site]['storaged'][0]['ip']), str(topology[i_site]['storaged'][1]['ip']), str(topology[i_site]['storaged'][2]['ip']), str(topology[i_site]['storaged'][3]['ip']), ycsb_args)), topology[i_site]['client'][i_client], ('object%d_trial%d_site%d_client%d_write.txt' % (i_object, trial, i_site+1, i_client+1)),)
args=(
('cd /tmp/; python2 bench.py write %d %d %s %s %s %s'
% (
object['number'],
object['size'] * 1000 * 1000,
str(topology[i_site]['storaged'][0]
['ip']),
str(topology[i_site]['storaged'][1]
['ip']),
str(topology[i_site]['storaged'][2]
['ip']),
str(topology[i_site]['storaged'][3]
['ip']),
)),
topology[i_site]['client'][i_client],
('object%d_trial%d_site%d_client%d_write.txt' %
(i_object, trial, i_site + 1, i_client + 1)),
)))
for i_site, site in [(1, topology[1])]:
for i_client, c in enumerate(site['client']):
threads_first_read.append(
Thread(
target=ycsb_thread,
# args=(('cd /tmp/YCSB; ./bin/ycsb run ipfs -p ipfs.host1=%s -p ipfs.host2=%s -p ipfs.host3=%s -p ipfs.host4=%s %s -P workloads/workloadc' % (str(topology[i_site]['storaged'][0]['ip']), str(topology[i_site]['storaged'][1]['ip']), str(topology[i_site]['storaged'][2]['ip']), str(topology[i_site]['storaged'][3]['ip']), ycsb_args)), topology[i_site]['client'][i_client], ('object%d_trial%d_site%d_client%d_read.txt' % (i_object, trial, i_site+1, i_client+1)),)
args=(
('cd /tmp/; python2 bench.py read %d %d %s %s %s %s'
% (
object['number'],
object['size'] * 1000 * 1000,
str(topology[i_site]['storaged'][0]
['ip']),
str(topology[i_site]['storaged'][1]
['ip']),
str(topology[i_site]['storaged'][2]
['ip']),
str(topology[i_site]['storaged'][3]
['ip']),
)),
topology[i_site]['client'][i_client],
('object%d_trial%d_site%d_client%d_read.txt' %
(i_object, trial, i_site + 1, i_client + 1)),
)))
for i_site, site in [(1, topology[1])]:
for i_client, c in enumerate(site['client']):
threads_second_read.append(
Thread(
target=ycsb_thread,
# args=(('cd /tmp/YCSB; ./bin/ycsb run ipfs -p ipfs.host1=%s -p ipfs.host2=%s -p ipfs.host3=%s -p ipfs.host4=%s %s -P workloads/workloadc' % (str(topology[i_site]['storaged'][0]['ip']), str(topology[i_site]['storaged'][1]['ip']), str(topology[i_site]['storaged'][2]['ip']), str(topology[i_site]['storaged'][3]['ip']), ycsb_args)), topology[i_site]['client'][i_client], ('object%d_trial%d_site%d_client%d_secondread.txt' % (i_object, trial, i_site+1, i_client+1)),)
args=(
('cd /tmp/; python2 bench.py read %d %d %s %s %s %s'
% (
object['number'],
object['size'] * 1000 * 1000,
str(topology[i_site]['storaged'][0]
['ip']),
str(topology[i_site]['storaged'][1]
['ip']),
str(topology[i_site]['storaged'][2]
['ip']),
str(topology[i_site]['storaged'][3]
['ip']),
)),
topology[i_site]['client'][i_client],
('object%d_trial%d_site%d_client%d_secondread.txt'
%
(i_object, trial, i_site + 1, i_client + 1)),
)))
logging.info('Clean /tmp/ids')
empty()
logging.info(
'Write %d objects of %dMB with %d clients' %
(object['number'], object['size'], len(threads_write)))
# write on all the sites
sync()
for t in threads_write:
t.start()
for t in threads_write:
t.join()
iptables_collect_end('ipfs_trial%d_object%d' % (trial, i_object),
'postwrite')
logging.info('scp')
execo.Process('scp ' + str(topology[0]['client'][0]['ip2']) +
':/tmp/ids ids').run().stdout
execo.Process('scp ids ' + str(topology[1]['client'][0]['ip2']) +
':/tmp/ids').run().stdout
logging.info(
'Read %d objects of %dMB with %d clients' %
(object['number'], object['size'], len(threads_first_read)))
# read on second site
sync()
for t in threads_first_read:
t.start()
for t in threads_first_read:
t.join()
time.sleep(sleep)
iptables_collect_end('ipfs_trial%d_object%d' % (trial, i_object),
'postread')
logging.info(
'Read %d objects of %dMB with %d clients' %
(object['number'], object['size'], len(threads_second_read)))
# read on the second site
sync()
for t in threads_second_read:
t.start()
for t in threads_second_read:
t.join()
time.sleep(sleep)
iptables_collect_end('ipfs_trial%d_object%d' % (trial, i_object),
'postsecondread')
def deploy_rozofs():
""" """
logging.info('Deploy rozofs.')
for site in topology:
logging.info("Create exportd config")
sids = []
for i, s in enumerate(site['storaged']):
sids.append("\t\t\t\t\t{sid = %d, host = \"%s\";}\n" %
((i + 1), s['ip']))
config_ = "" \
"layout = %d;\n" \
"volumes = \n" \
"( \n" \
"\t{\n" \
"\t\tvid = 1;\n" \
"\t\tcids=\n" \
"\t\t(\n" \
"\t\t\t{\n" \
"\t\t\t\tcid = 1;\n" \
"\t\t\t\tsids=\n" \
"\t\t\t\t(\n" \
"%s" \
"\t\t\t\t);\n" \
"\t\t\t}\n" \
"\t\t);\n" \
"\t}\n" \
"); \n" \
"exports = (\n" \
"\t{eid = 1; bsize=\"4K\"; root = \"%s\"; md5=\"\"; squota=\"\"; hquota=\"\"; vid=1;}\n" \
");\n" % (config['rozofs']['layout'], ",".join(sids), str(config['rozofs']['data_dir'])+'/exports/export-1')
with open('export', 'w') as f:
f.write(config_)
for e in site['exportd']:
execo.Process('scp export ' + str(e['ip2']) + ':' +
str(config['rozofs']['config_dir']) +
'/export.conf').run().stdout
logging.info("Launch exportd")
commands = [
'exportd -c ' + str(config['rozofs']['config_dir']) +
'/export.conf', 'rozo agent start'
]
exec_commands(commands, site['exportd'])
for i, s in enumerate(site['storaged']):
logging.info("Create storaged config")
config_ = "" \
"crc32c_check = True; \n" \
"crc32c_generate = True; \n" \
"listen = (\n" \
"\t{\n" \
"\t\taddr = \"*\";\n" \
"\t\tport = 41001;\n" \
"\t}\n" \
");\n" \
"storages = (\n" \
"\t{cid = 1; sid = %d; root = \"%s\"; device-total = 1; device-mapper = 1; device-redundancy = 1;}\n" \
");\n" % ((i+1), config['rozofs']['data_dir']+'/storaged/storage-1')
with open('storage', 'w') as f:
f.write(config_)
execo.Process('scp storage ' + str(s['ip2']) + ':' +
str(config['rozofs']['config_dir']) +
'/storage.conf').run().stdout
config_ = "" \
"nb_disk_thread = 250; // 2:32" \
"\n"
with open('storage', 'w') as f:
f.write(config_)
execo.Process('scp storage ' + str(s['ip2']) + ':' +
str(config['rozofs']['config_dir']) +
'/rozofs.conf').run().stdout
logging.info("Launch storaged")
commands = ['rozo agent start']
exec_commands(commands, site['storaged'])
commands = ['rozo node start -E 127.0.0.1']
time.sleep(5)
exec_commands(commands, site['exportd'])
