def compute_qos_metrics(driver, url_to_fetch, qos_metrics):
"""
Having retrieved the web page, this method computes QoS metrics by executing their associated javascript scripts.
Args:
driver(WebDriver): an initialized Selenium WebDriver.
url_to_fetch(str): the url address to retrieve prior to compute the different metrics.
qos_metrics(dict(str,str)): a dictionary where keys are metric names and values are javascript methods.
Returns:
results(dict(str,object)): a dictionary containing the different metrics/values.
"""
results = dict()
try:
driver.get(url_to_fetch)
for key, value in qos_metrics.items():
results[key] = driver.execute_script(value)
for request in driver.requests:
if request.url in (url_to_fetch, url_to_fetch + '/'):
results['status_code'] = request.response.status_code
if results['status_code'] == 404:
message = 'Warning : Fetched url {} returned response code 404 (Not Found)'.format(
url_to_fetch)
collect_agent.send_log(syslog.LOG_WARNING, message)
print(message)
break
except WebDriverException as ErrorMessage:
message = 'ERROR when getting url: {}'.format(ErrorMessage)
print(message)
collect_agent.send_log(syslog.LOG_ERR, message)
driver.quit()
exit(message)
return results
def launch_thread(collect_agent, url, config, qos_metrics, stop_compression,
proxy_add, proxy_port):
binary_path = config['driver']['binary_path']
binary_type = config['driver']['binary_type']
try:
my_driver = init_driver(binary_path, binary_type, stop_compression,
proxy_add, proxy_port)
except Exception as ex:
message = 'ERROR when initializing the web driver: {}'.format(ex)
collect_agent.send_log(syslog.LOG_ERR, message)
exit(message)
if my_driver is not None:
timestamp = int(time.time() * 1000)
statistics = compute_qos_metrics(my_driver, url, qos_metrics)
my_driver.quit()
statistics['compression_savings'] = 1 - (
statistics['encoded_body_size'] / statistics['decoded_body_size'])
statistics['overhead'] = statistics['transfer_size'] - statistics[
'encoded_body_size']
s = '# Report for web page ' + url + ' #'
print('\n' + s)
print_metrics(statistics, config)
collect_agent.send_stat(timestamp, **statistics, suffix=url)
else:
message = 'Sorry, specified driver is not available. For now, only Firefox driver is supported'
collect_agent.send_log(syslog.LOG_ERR, message)
exit(message)
def main(sampling_interval,
chain_name,
source_ip=None,
destination_ip=None,
protocol=None,
in_interface=None,
out_interface=None,
dport=None,
sport=None):
table = iptc.Table(iptc.Table.FILTER)
chains = [chain for chain in table.chains if chain.name == chain_name]
try:
chain, = chains
except ValueError:
message = 'ERROR: {} does not exist in FILTER table'.format(chain_name)
collect_agent.send_log(syslog.LOG_ERR, message)
exit(message)
# Creation of the Rule
rule = iptc.Rule(chain=chain)
signal.signal(signal.SIGTERM, partial(signal_term_handler, chain, rule))
# Add Matchs
if source_ip is not None:
rule.src = source_ip
if destination_ip is not None:
rule.dst = destination_ip
if protocol is not None:
rule.protocol = protocol
if in_interface is not None:
rule.in_interface = in_interface
if out_interface is not None:
rule.out_interface = out_interface
if sport is not None:
match = iptc.Match(rule, protocol)
match.sport = sport
rule.add_match(match)
if dport is not None:
match = iptc.Match(rule, protocol)
match.dport = dport
rule.add_match(match)
# Add the Target
rule.create_target('')
chain.insert_rule(rule)
collect_agent.send_log(syslog.LOG_DEBUG,
"Added iptables rule for monitoring")
# Save the first stats for computing the rate
mutex = threading.Lock()
previous = [int(time.perf_counter() * 1000), rule.get_counters()[1]]
# Monitoring
sched = BlockingScheduler()
sched.add_job(monitor,
'interval',
seconds=sampling_interval,
args=(chain, mutex, previous))
sched.start()
def send_stats(filename):
print(filename)
with open(filename) as statistics:
try:
# Parse the first line independently
# so we can update os.ENVIRON
statistic = json.loads(next(statistics))
except StopIteration:
return # File was empty
# Setup os.ENVIRON for register_collect to work properly
metadata = statistic.pop('_metadata')
timestamp = metadata['time']
suffix = metadata.get('suffix')
for name in ENVIRON_METADATA:
# This way rstats will be aware and will not locally store the
# stats again
if name == 'job_name':
metadata[name] = 'send_stats-' + str(metadata[name])
os.environ[name.upper()] = str(metadata[name])
# Recreate connection with rstats
success = collect_agent.register_collect(CONF_FILE, new=True)
if not success:
message = 'Cannot communicate with rstats'
collect_agent.send_log(syslog.LOG_ERR, message)
raise ConnectionError(message)
collect_agent.send_stat(timestamp, suffix=suffix, **statistic)
for line in statistics:
statistic = json.loads(line)
metadata = statistic.pop('_metadata')
timestamp = metadata['time']
suffix = metadata.get('suffix')
collect_agent.send_stat(timestamp, suffix=suffix, **statistic)
def check_timeout(start_time, timeout):
elapsed = perf_counter() - start_time
if elapsed >= timeout:
message = 'Timeout: the agent is taking too long to synchronize '
'or the offset is too demanding. Try increasing the timeout or '
'the offset for this job.'
collect_agent.send_log(syslog.LOG_ERR, message)
sys.exit(message)
def run_command(cmd):
p = subprocess.run(cmd, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
if p.returncode:
message = "Error when executing command '{}': '{}'".format(
' '.join(cmd), p.stderr.decode())
collect_agent.send_log(syslog.LOG_ERR, message)
sys.exit(message)
return p.returncode, p.stdout.decode()
def run_process(cmd):
try:
p = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
except Exception as ex:
message = 'Error running {} : {}'.format(cmd, ex)
collect_agent.send_log(syslog.LOG_ERR, message)
sys.exit(message)
return p
def run_command(cmd, mode='run'):
try:
if mode == 'Popen':
p = subprocess.Popen(cmd, stderr=subprocess.PIPE)
else:
p = subprocess.run(cmd, stderr=subprocess.PIPE)
except Exception as ex:
message = 'Error running {} : {}'.format(cmd, ex)
collect_agent.send_log(syslog.LOG_ERR, message)
sys.exit(message)
return p
def main(log_buffer_size, signal_port=9000):
if log_buffer_size:
cmd = ['ITGRecv', '-Sp', str(signal_port), '-q', str(log_buffer_size)]
else:
cmd = ['ITGRecv', '-Sp', str(signal_port)]
try:
subprocess.run(cmd, stderr=subprocess.PIPE)
except Exception as ex:
message = 'Error running {} : {}'.format(cmd, ex)
collect_agent.send_log(syslog.LOG_ERR, message)
sys.exit(message)
def main(operation, iface, address_mask=None):
command = ['ip', 'address', operation]
if address_mask is not None:
command.append(str(address_mask))
command.extend(['dev', str(iface)])
try:
p = subprocess.run(command, stderr=subprocess.PIPE)
except subprocess.CalledProcessError as ex:
message = 'ERROR: {}'.format(ex)
collect_agent.send_log(syslog.LOG_ERR, message)
sys.exit(message)
if p.returncode:
error = p.stderr.decode()
if any(err in error for err in {'File exists', 'No such process'}):
message = 'WARNING: {} exited with non-zero return value ({}): {}'.format(
command, p.returncode, error)
collect_agent.send_log(syslog.LOG_WARNING, message)
sys.exit(0)
else:
message = 'ERROR: {} exited with non-zero return value ({})'.format(
command, p.returncode)
collect_agent.send_log(syslog.LOG_ERR, message)
sys.exit(message)
else:
collect_agent.send_log(
syslog.LOG_DEBUG,
'{} address {} to iface {}'.format(operation, address_mask, iface))
def run_command(command):
try:
p = subprocess.run(command,
stderr=subprocess.PIPE,
stdout=subprocess.PIPE)
except subprocess.CalledProcessError as ex:
message = 'ERROR: {}'.format(ex)
collect_agent.send_log(syslog.LOG_ERR, message)
sys.exit(message)
if p.returncode:
error = p.stderr.decode()
if 'No such process' in error:
message = 'WARNING: {} exited with non-zero return value ({}): {}'.format(
command, p.returncode, error)
collect_agent.send_log(syslog.LOG_WARNING, message)
sys.exit(0)
else:
message = 'ERROR: {} exited with non-zero return value ({}): {}'.format(
command, p.returncode, error)
collect_agent.send_log(syslog.LOG_ERR, message)
sys.exit(message)
else:
collect_agent.send_log(syslog.LOG_DEBUG,
'Applied successfully : ' + ' '.join(command))
return p.stdout.decode()
def run_command(rule):
""" Run a command, return error """
list_rule = rule.split()
cmd = ["iptables"] + list_rule
try:
p = subprocess.run(cmd, stderr=subprocess.PIPE)
except subprocess.CalledProcessError as ex:
collect_agent.send_log(
syslog.LOG_ERR,
"Error when executing command {}: {}".format(cmd, ex))
if p.returncode:
message = 'WARNING: {} exited with non-zero return value ({}): {}'.format(
cmd, p.returncode, p.stderr.decode())
collect_agent.send_log(syslog.LOG_WARNING, message)
sys.exit(0)
def send_logs(filename, send_log):
with open(os.path.join(LOGS_DIR, filename)) as log:
for line in log:
message = (
'<{line[pri]}>{line[timestamp]} '
'{line[hostname]} {line[programname]}'
'[{line[procid]}]: {line[msg]}'
.format(line=json.loads(line))
)
try:
send_log(message.encode())
except socket.error as error:
collect_agent.send_log(
syslog.LOG_NOTICE,
'Error code: {}, Message {}'.format(*error))
raise
def main(synchro_offset, timeout, retries=None, sleep_time=None):
start_time = perf_counter()
while abs(get_ntp_offset(retries, sleep_time)) > synchro_offset:
check_timeout(start_time, timeout)
try:
subprocess.run(['systemctl', 'stop', 'ntp.service'], check=True)
subprocess.run(['ntpd', '-gq'], check=True)
except subprocess.CalledProcessError as e:
message = 'Error when interacting with the NTP daemon: {}'.format(e)
collect_agent.send_log(syslog.LOG_ERR, message)
sys.exit(message)
else:
check_timeout(start_time, timeout)
finally:
subprocess.run(['systemctl', 'start', 'ntp.service'])
def stop(signalNumber, frame):
"""
Stop apache2
Args:
Returns:
NoneType
"""
cmd = ["systemctl", "stop", "apache2"]
global STOP_JOB
try:
p = subprocess.Popen(cmd, stderr=subprocess.PIPE)
STOP_JOB = True
except Exception as ex:
message = "Error when stopping apache2: {}".format(ex)
collect_agent.send_log(syslog.LOG_ERR, message)
sys.exit(message)
def client(metrics_interval,
port,
num_flows,
server_ip,
window_size,
tos,
time_duration,
transmitted_size,
protocol,
reverse,
bandwidth=None,
cong_control=None,
mss=None,
udp_size=None):
cmd = ['stdbuf', '-oL', 'iperf3', '-c', server_ip, '-f', 'k']
cmd.extend(_command_build_helper('-i', metrics_interval))
cmd.extend(_command_build_helper('-w', window_size))
cmd.extend(_command_build_helper('-p', port))
if reverse:
cmd.append('-R')
if protocol == "udp":
cmd.append('-u')
cmd.extend(_command_build_helper('-b', bandwidth))
cmd.extend(_command_build_helper('--length', udp_size))
else:
cmd.extend(_command_build_helper('-C', cong_control))
cmd.extend(_command_build_helper('-M', mss))
cmd.extend(_command_build_helper('-t', time_duration))
if time_duration is None:
if transmitted_size is not None and _parse_to_bytes(
transmitted_size) < 1024 * 1024:
message = 'Error : the number of bytes to transmit is too low.'
collect_agent.send_log(syslog.LOG_ERR, message)
sys.exit(message)
cmd.extend(_command_build_helper('-n', transmitted_size))
cmd.extend(_command_build_helper('-P', num_flows))
cmd.extend(_command_build_helper('-S', tos))
if reverse:
receiver(cmd)
else:
sender(cmd)
def main(destination_ip, count, interval, interface, packetsize, ttl, n_mean):
cmd = ['fping', destination_ip, '-e', '-D']
if count == 0:
cmd += ['-l']
else:
cmd += ['-c', str(count)]
cmd.extend(command_line_flag_for_argument(interval, '-p'))
cmd.extend(command_line_flag_for_argument(interface, '-I'))
cmd.extend(command_line_flag_for_argument(packetsize, '-b'))
cmd.extend(command_line_flag_for_argument(ttl, '-t'))
pattern = re.compile(
r'\[(\d+\.\d+)\] {} : \[\d+\], \d+ bytes, (\d+\.?\d*) '.format(
destination_ip))
measurements = []
# launch command
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
while True:
# read output
output = p.stdout.readline().decode().rstrip()
if not output:
# this should be the blank line before summary
if p.poll() is not None:
# Process ended gracefully
break
continue
match = re.match(pattern, output)
if match is None:
message = 'Unrecognised fping output: {}'.format(output)
collect_agent.send_log(syslog.LOG_WARNING, message)
continue
try:
timestamp, rtt_data = map(float, match.groups())
except ValueError as exception:
message = 'ERROR on line \'{}\': {}'.format(output, exception)
collect_agent.send_log(syslog.LOG_ERR, message)
else:
measurements.append(rtt_data)
if len(measurements) == n_mean:
collect_agent.send_stat(int(timestamp * 1000),
rtt=mean(measurements))
measurements.clear()
def restore_route(old_route, destination, operation, signal, frame):
if operation == Operations.ADD.value:
# Delete added route
cmd = ['ip', 'route', 'del', str(destination)]
elif operation == Operations.DELETE.value:
# Add deleted route
cmd = ['ip', 'route', 'add'] + old_route
else:
# Restore previous route
if old_route:
cmd = ['ip', 'route', operation] + old_route
else:
cmd = ['ip', 'route', 'del', str(destination)]
run_command(cmd)
message = 'Stopped job ip_route. Previous route has been restored.'
collect_agent.send_log(syslog.LOG_DEBUG, message)
sys.exit(message)
def multiplier(unit, base):
if unit == base:
return 1
if unit.startswith('GBytes'):
return 1024 * 1024 * 1024
if unit.startswith('MBytes'):
return 1024 * 1024
if unit.startswith('KBytes'):
return 1024
if unit.startswith('m'):
return 0.001
if unit.startswith('Gbits'):
return 1000 * 1000 * 1000
if unit.startswith('Mbits'):
return 1000 * 1000
if unit.startswith('Kbits'):
return 1000
collect_agent.send_log(syslog.LOG_ERR,
'Units of iperf metrics are not available/correct')
return 1
def main(nb_runs, max_threads, stop_compression, proxy_address, proxy_port,
urls):
# Set signal handler
signal_handler_partial = partial(kill_all, os.getpid())
signal.signal(signal.SIGTERM, signal_handler_partial)
signal.signal(signal.SIGINT, signal_handler_partial)
# Init local variables
qos_metrics_lists = dict()
qos_metrics = dict()
# Load config from config.yaml
config_filepath = os.path.join(os.path.abspath(os.path.dirname(__file__)),
'config.yml')
with open(config_filepath) as config_file:
config = yaml.safe_load(config_file)
if not urls:
urls = config['web_pages_to_fetch']
for metric in config['qos_metrics']:
qos_metrics[metric] = config['qos_metrics'][metric]['js']
# Compute qos metrics for each url 'nb_runs' times
thread_list = []
try:
for i in range(1, args.nb_runs + 1, 1):
for url in urls:
# condition of "max_threads + 1" because seleniumwire uses 1 additional inner thread to monitor requests
while threading.active_count() > max_threads + 1:
time.sleep(1)
t = threading.Thread(target=launch_thread,
args=(collect_agent, url, config,
qos_metrics, stop_compression,
proxy_address, proxy_port))
thread_list.append(t)
t.start()
time.sleep(2)
except Exception as ex:
message = 'An unexpected error occured: {}'.format(ex)
collect_agent.send_log(syslog.LOG_ERR, message)
exit(message)
finally:
for t in thread_list:
t.join()
kill_children(os.getpid())
def main(src_ip, dst_ip, src_port, dst_port, proto, interface, capture_file,
duration):
"""Capture packets on a live network interface. Only consider packets matching the specified fields."""
capture_filter = build_capture_filter(src_ip, dst_ip, src_port, dst_port,
proto)
copy = False
if capture_file == "":
capture_file = "/tmp/tcpdump_capture.pcap"
copy = True
signal_handler_partial = partial(save_pcap, capture_file, copy,
os.getpid())
original_sigint_handler = signal.getsignal(signal.SIGINT)
original_sigterm_handler = signal.getsignal(signal.SIGTERM)
signal.signal(signal.SIGTERM, signal_handler_partial)
signal.signal(signal.SIGINT, signal_handler_partial)
try:
parent = pathlib.Path(capture_file).parent
pathlib.Path(parent).mkdir(parents=True, exist_ok=True)
subprocess.run(["rm", capture_file])
cmd = [
'tcpdump', '-i', interface, capture_filter, '-w', capture_file,
'-Z', 'root'
]
if duration:
cmd += ['-G', str(duration), '-W', str(1)]
p = subprocess.run(cmd)
if p.returncode != 0:
message = 'ERROR when lauching tcpdump: {}'.format(p.stderr)
collect_agent.send_log(syslog.LOG_ERR, message)
sys.exit(message)
except Exception as ex:
message = 'ERROR when capturing: {}'.format(ex)
collect_agent.send_log(syslog.LOG_ERR, message)
sys.exit(message)
collect_agent.store_files(int(time.time() * 1000),
pcap_file=capture_file,
copy=copy)
signal.signal(signal.SIGTERM, original_sigint_handler)
signal.signal(signal.SIGINT, original_sigterm_handler)
def use_configuration(filepath):
success = collect_agent.register_collect(filepath)
if not success:
message = 'ERROR connecting to collect-agent'
collect_agent.send_log(syslog.LOG_ERR, message)
sys.exit(message)
collect_agent.send_log(syslog.LOG_DEBUG, 'Starting job ' + os.environ.get('JOB_NAME', '!'))
try:
yield
except Exception:
message = traceback.format_exc()
collect_agent.send_log(syslog.LOG_CRIT, message)
raise
except SystemExit as e:
if e.code != 0:
collect_agent.send_log(syslog.LOG_CRIT, 'Abrupt program termination: ' + str(e.code))
raise
def build_socket_sender():
try:
collector = get_collector_infos()
except yaml.YAMLError:
collect_agent.send_log(
syslog.LOG_NOTICE,
'Collector configuration file is malformed')
raise
except FileNotFoundError:
collect_agent.send_log(
syslog.LOG_NOTICE,
'Collector configuration file not found')
raise
try:
address = collector['address']
port = collector['logs']['port']
sock_type = {
'tcp': socket.SOCK_STREAM,
'udp': socket.SOCK_DGRAM,
}[collector['logstash']['mode']]
except KeyError:
collect_agent.send_log(
syslog.LOG_NOTICE,
'Collector configuration file is malformed')
raise
logstash = (address, int(port))
try:
sock = socket.socket(socket.AF_INET, sock_type)
except socket.error:
collect_agent.send_log(syslog.LOG_NOTICE, 'Failed to create socket')
raise
if sock_type == socket.SOCK_STREAM:
sock.connect(logstash)
sender = sock.send
else:
def sender(message):
sock.sendto(message, logstash)
return sock, sender
def start():
"""
Start apache2 which will listen http/1.1 requests on port 8081 and http2 on port 8082
Args:
Returns:
NoneType
"""
cmd = ["systemctl", "is-active", "apache2"]
try:
p = subprocess.Popen(cmd, stdout=subprocess.PIPE)
line = p.stdout.readline().decode()
if "inactive" in line:
cmd = ["systemctl", "start", "apache2"]
p = subprocess.Popen(cmd, stderr=subprocess.PIPE)
else:
message = "An Apache instance is already running, stopping this instance"
collect_agent.send_log(syslog.LOG_ERR, message)
return
except Exception as ex:
message = "Error when starting apache2: {}".format(ex)
collect_agent.send_log(syslog.LOG_ERR, message)
sys.exit(message)
# Set signal handler
signal.signal(signal.SIGTERM, stop)
signal.signal(signal.SIGINT, stop)
# Wait for status to change to active
cmd = ['systemctl', 'show', '-p', 'SubState', '-p', 'ActiveState', 'apache2']
global STOP_JOB
while True:
time.sleep(0.5)
status = subprocess.run(cmd, stdout=subprocess.PIPE)
output = status.stdout.decode()
if not 'SubState=running' in output:
if not STOP_JOB:
message = "Apache2 stopped abnormally : {}".format(output)
collect_agent.send_log(syslog.LOG_ERR, message)
sys.exit(message)
else:
return
def one_file(capture_file, src_ip, dst_ip, src_port, dst_port, proto,
metrics_interval):
"""Analyze packets from pcap file located at capture_file and comptute statistics.
Only consider packets matching the specified fields.
"""
display_filter = build_display_filter(src_ip, dst_ip, src_port, dst_port,
proto)
To = now()
try:
with closing(
pyshark.FileCapture(
capture_file, display_filter=display_filter)) as cap_file:
flow_id_funct = lambda pkt: (pkt.ip.src, pkt[
pkt.transport_layer].srcport, pkt.ip.dst, pkt[
pkt.transport_layer].dstport, pkt.transport_layer)
packets = [
packet for packet in cap_file if 'IP' in str(packet.layers)
and packet.transport_layer is not None
]
key_funct = lambda pkt: pkt.sniff_timestamp
if (packets):
grouped_packets = sort_and_group(packets, key=flow_id_funct)
flow_id_to_flow = dict((flow_id, sorted(flow, key=key_funct))
for flow_id, flow in grouped_packets)
all_flows = list()
to = float(packets[0].sniff_timestamp) * 1000
samples_count = 1
while to < float(packets[-1].sniff_timestamp) * 1000:
time = to + metrics_interval
ids_of_new_flows_at_time = [
x[0] for x in list(
filter(
lambda item:
(item[0] not in all_flows and float(item[1][
0].sniff_timestamp) * 1000 < time),
flow_id_to_flow.items()))
]
all_flows.extend(ids_of_new_flows_at_time)
flows_count = 0
total_flows_count = 0
flow_number = 1
total_flow_duration = 0
for flow_id in all_flows:
flow = flow_id_to_flow[flow_id]
stat_time = To + samples_count * metrics_interval
# If the flow always exists
if (float(flow[-1].sniff_timestamp) * 1000 > to):
compute_and_send_statistics(
flow, to, metrics_interval,
suffix(flow_number), stat_time)
flow_duration = 1000 * (
float(flow[-1].sniff_timestamp) -
float(flow[0].sniff_timestamp))
flows_count += 1
flow_number += 1
total_flow_duration += 1000 * (
float(flow[-1].sniff_timestamp) -
float(flow[0].sniff_timestamp))
total_flows_count += 1
statistics = {'flows_count': flows_count}
# Check if it the last sample
if total_flows_count > 0 and float(
packets[-1].sniff_timestamp) * 1000 <= time:
statistics.update({
'avg_flow_duration':
int(total_flow_duration / total_flows_count)
})
statistics.update({
'total_packets':
len(packets),
'total_bytes':
packets_length(packets)
})
collect_agent.send_stat(stat_time, **statistics)
samples_count += 1
to = time
except Exception as ex:
message = 'ERROR when analyzing: {}'.format(ex)
collect_agent.send_log(syslog.LOG_ERR, message)
sys.exit(message)
sys.exit(0) # Explicitly exit properly. This is required by pyshark module
def gilbert_elliot(capture_file, second_capture_file, src_ip, dst_ip, src_port,
dst_port, proto):
display_filter = build_display_filter(src_ip, dst_ip, src_port, dst_port,
proto)
try:
with closing(
pyshark.FileCapture(
capture_file,
display_filter=display_filter)) as cap_file_sent:
cap_file_sent_iter = iter(cap_file_sent)
goods = []
bads = []
with closing(
pyshark.FileCapture(
second_capture_file,
display_filter=display_filter)) as cap_file_received:
try:
current_packet_sent = get_next_packet(cap_file_sent_iter)
total_good = 0
total_bad = 0
for packet in cap_file_received:
if not ('IP' in str(packet.layers)
and packet.transport_layer is not None):
continue
if packet.ip.id == current_packet_sent:
total_good += 1
if total_bad:
bads.append(total_bad)
total_bad = 0
while packet.ip.id != current_packet_sent:
if total_good:
goods.append(total_good)
total_good = 0
total_bad += 1
current_packet_sent = get_next_packet(
cap_file_sent_iter)
current_packet_sent = get_next_packet(
cap_file_sent_iter)
except StopIteration:
pass
if total_good:
goods.append(total_good)
if total_bad:
bads.append(total_bad)
statistics = {
'gilbert_elliot_sent': sum(goods) + sum(bads),
'gilbert_elliot_received': sum(goods)
}
if goods or bads:
statistics['gilbert_elliot_lost_rate'] = sum(bads) / (sum(goods) +
sum(bads))
if goods:
g = sum(goods) / len(
goods) # average number of steps when we stay in good state
statistics['gilbert_elliot_p'] = 1 / g
else:
collect_agent.send_log(
syslog.LOG_WARNING,
"Cannot compute p parameter. Maybe the capture files are too short."
)
if bads:
b = sum(bads) / len(
bads) # average number of steps when we stay in bad state
statistics['gilbert_elliot_r'] = 1 / b
else:
collect_agent.send_log(
syslog.LOG_WARNING,
"Cannot compute r parameter. Maybe the capture files are too short."
)
collect_agent.send_stat(now(), **statistics)
except Exception as ex:
message = 'ERROR when analyzing: {}'.format(ex)
collect_agent.send_log(syslog.LOG_ERR, message)
sys.exit(message)
sys.exit(0) # Explicitly exit properly. This is required by pyshark module
def main(rule):
collect_agent.send_log(
syslog.LOG_DEBUG,
'Starting iptables job with rule: iptables {}'.format(rule))
run_command(rule)
def apply_conf(interfaces,
mode,
delay=None,
jitter=None,
delay_distribution=None,
bandwidth=None,
loss_model=None,
loss_model_params=None,
buffer_size=None):
collect_agent.send_log(
syslog.LOG_DEBUG,
'Starting tc_configure_link job (apply {})'.format(mode))
if mode == 'egress' or mode == 'all':
for interface in interfaces.split(','):
handle = ['root', 'handle', '1:']
# Delete existing qdisc
# Check if a root qdisc exists, if so remove it on interface
cmd = ['tc', 'qdisc', 'show', 'dev', interface]
_, output = run_command(cmd)
output = re.findall('qdisc netem', output)
if output:
delete_qdisc(interface, 'root')
# Add qlen
run_command(
['ip', 'link', 'set', interface, 'qlen',
str(buffer_size)])
# Add bandwidth if relevant
if bandwidth:
if not re.findall(r'^[0-9]+[KM]$', bandwidth):
collect_agent.send_log(
syslog.LOG_ERR,
"Invalid format for bandwidth: expecting "
"'{}', found '{}'".format('{VALUE}{M|K}', bandwidth))
sys.exit(1)
add_qdisc_bandwidth(interface, bandwidth)
handle = ['parent', '1:11', 'handle', '10:']
# Add delay
add_qdisc_delay(interface, delay, jitter, delay_distribution,
loss_model, loss_model_params, handle, buffer_size)
if mode == 'ingress' or mode == 'all':
# Ingress configuration
clear_ingress(interfaces)
run_command(['modprobe', '-r', 'ifb'])
run_command([
'modprobe', 'ifb',
'numifbs={}'.format(str(len(interfaces.split(','))))
])
for index, interface in enumerate(interfaces.split(',')):
handle = ['root', 'handle', '1:']
# Clear ingress configuration and add a new one
add_qdisc_ingress(interface, IFB.format(str(index)), buffer_size)
# Add bandwidth if relevant
if bandwidth:
if not re.findall(r'^[0-9]+[KM]$', bandwidth):
collect_agent.send_log(
syslog.LOG_ERR,
"Invalid format for bandwidth: expecting "
"'{}', found '{}'".format('{VALUE}{M|K}', bandwidth))
sys.exit(1)
add_qdisc_bandwidth(IFB.format(str(index)), bandwidth)
handle = ['parent', '1:11', 'handle', '10:']
# Add delay
add_qdisc_delay(IFB.format(str(index)), delay, jitter,
delay_distribution, loss_model, loss_model_params,
handle, buffer_size)
def main(target_address,
log_address,
dest_path,
granularity,
traffic_type='UDP',
port=8999,
signal_port=9000,
packet_size=512,
packet_rate=1000,
bandwidth='0',
duration=10,
data_size='0',
meter='owdm'):
# Clean previous log and set up the D-ITG LogServer*
if os.path.isfile('/tmp/ITGRecv.log'):
os.remove('/tmp/ITGRecv.log')
if os.path.isfile('/tmp/ITGSend.log'):
os.remove('/tmp/ITGSend.log')
proc_log = run_command('ITGLog', 'Popen')
# Get the reference time for changing the stats generated by D-ITG
time_ref = int(round(time.time() * 1000))
#Set packet_rate depending on bandwidth parameter
if bandwidth != '0':
if 'K' == 'K' in bandwidth:
packet_rate = 1000 * int(bandwidth.replace('K', ''))
elif 'M' == 'M' in bandwidth:
packet_rate = 1000 * 1000 * int(bandwidth.replace('M', ''))
elif 'G' == 'G' in bandwidth:
packet_rate = 1000 * 1000 * 1000 * int(bandwidth.replace('G', ''))
else:
packet_rate = int(bandwidth)
packet_rate = packet_rate / (8 * packet_size)
# Build and launch the D-ITGSend command
cmd_send = [
'ITGSend', '-a', target_address, '-L', log_address, '-X', log_address,
'-T', traffic_type, '-c',
str(packet_size), '-C',
str(packet_rate), '-t',
str(duration * 1000), '-m', meter, '-Sdp',
str(signal_port), '-Ssp',
str(signal_port), '-rp',
str(port)
]
#Set number of KBytes to generate
if data_size != '0':
if 'M' == 'M' in data_size:
data_size_f = 1024 * float(data_size.replace('M', ''))
elif 'G' == 'G' in data_size:
data_size_f = 1024 * 1024 * float(data_size.replace('G', ''))
else:
data_size_f = float(data_size.replace('K', ''))
cmd_send.extend(['-k', str(data_size_f)])
run_command(cmd_send)
# Terminate the process of the D-ITG LogServer
proc_log.terminate()
# Clear potential old stats
if os.path.isfile(os.path.join(dest_path, 'RCV')):
os.remove(os.path.join(dest_path, 'RCV'))
if os.path.isfile(os.path.join(dest_path, 'SND')):
os.remove(os.path.join(dest_path, 'SND'))
# Get the stats from the logs
cmd_rcv_cb = [
'ITGDec', '/tmp/ITGRecv.log', '-c',
str(granularity),
os.path.join(dest_path, 'RCV')
]
run_command(cmd_rcv_cb)
cmd_snd_cb = [
'ITGDec', '/tmp/ITGSend.log', '-c',
str(granularity),
os.path.join(dest_path, 'SND')
]
run_command(cmd_snd_cb)
# Send the stats of the receiver to the collector
path_RCV = os.path.join(dest_path, 'RCV')
try:
stats = open(path_RCV, "r")
except Exception as ex:
message = 'Error opening file {} : {}'.format(path_RCV, ex)
collect_agent.send_log(syslog.LOG_ERR, message)
sys.exit(message)
owd_r = []
with stats:
for line in stats:
txt = line.strip()
txt = txt.split(' ')
# Get the timestamp (in ms)
timestamp = txt[0].replace('.', '')
timestamp = int(timestamp[:-3])
timestamp = timestamp + time_ref
# Get the bitrate (in bps)
bitrate = txt[1]
bitrate = float(bitrate) * 1024
statistics = {'bitrate_receiver': bitrate}
collect_agent.send_stat(timestamp, **statistics)
# Get the delay (in ms)
delay = txt[2]
delay = float(delay) * 1000
owd_r.append(delay)
statistics = {'owd_receiver': delay}
collect_agent.send_stat(timestamp, **statistics)
# Get the jitter (in ms)
jitter = txt[3]
jitter = float(jitter) * 1000
statistics = {'jitter_receiver': jitter}
collect_agent.send_stat(timestamp, **statistics)
# Get the packetloss
pck_loss = txt[4]
pck_loss = float(pck_loss)
statistics = {'packetloss_receiver': pck_loss}
collect_agent.send_stat(timestamp, **statistics)
# Calculate packet_loss_rate
pck_loss_per_sec = pck_loss * 1000 / granularity
plr = (pck_loss_per_sec / packet_rate) * 100
statistics = {'packetloss_rate_receiver': plr}
collect_agent.send_stat(timestamp, **statistics)
# Send the stats of the sender to the collector
path_SND = os.path.join(dest_path, 'SND')
try:
stats = open(path_SND, "r")
except Exception as ex:
message = 'Error opening file {} : {}'.format(path_SND, ex)
collect_agent.send_log(syslog.LOG_ERR, message)
sys.exit(message)
owd_s = []
timetab = []
with stats:
for line in stats:
txt = line.strip()
txt = txt.split(' ')
# Get the timestamp (in ms)
timestamp = txt[0].replace('.', '')
timestamp = int(timestamp[:-3])
timestamp = timestamp + time_ref
# Get the bitrate (in bps)
bitrate = txt[1]
bitrate = float(bitrate) * 1024
statistics = {'bitrate_sender': bitrate}
collect_agent.send_stat(timestamp, **statistics)
if meter.upper() == "RTTM":
# Get the delay (in ms)
delay = txt[2]
delay = float(delay) * 1000
owd_s.append(delay)
timetab.append(timestamp)
statistics = {'rtt_sender': delay}
collect_agent.send_stat(timestamp, **statistics)
# Get the jitter (in ms)
jitter = txt[3]
jitter = float(jitter) * 1000
statistics = {'jitter_sender': jitter}
collect_agent.send_stat(timestamp, **statistics)
# Get the packetloss
pck_loss = txt[4]
pck_loss = float(pck_loss)
statistics = {'packetloss_sender': pck_loss}
collect_agent.send_stat(timestamp, **statistics)
# Calculate packet_loss_rate
pck_loss_per_sec = pck_loss * 1000 / granularity
plr = (pck_loss_per_sec / packet_rate) * 100
statistics = {'packetloss_rate_sender': plr}
collect_agent.send_stat(timestamp, **statistics)
if meter.upper() == 'RTTM':
for time_tab, owdr, owds in zip(timetab, owd_r, owd_s):
owd_return = owds - owdr
statistics = {'owd_return': owd_return}
collect_agent.send_stat(time_tab, **statistics)
def main(job_instance_ids, statistics_names, aggregations_periods, bins_sizes,
offset, maximum, stats_with_suffixes, axis_labels, figures_titles,
legends_titles, use_legend, add_global, pickle):
file_ext = 'pickle' if pickle else 'png'
statistics = Statistics.from_default_collector()
statistics.origin = 0
with tempfile.TemporaryDirectory(
prefix='openbach-temporal-binning-histogram-') as root:
for job, fields, aggregations, bin_sizes, labels, titles, legend_titles in itertools.zip_longest(
job_instance_ids,
statistics_names,
aggregations_periods,
bins_sizes,
axis_labels,
figures_titles,
legends_titles,
fillvalue=[]):
data_collection = statistics.fetch(
job_instances=job,
suffix=None if stats_with_suffixes else '',
fields=fields)
# Drop multi-index columns to easily concatenate dataframes from their statistic names
df = pd.concat([
plot.dataframe.set_axis(
plot.dataframe.columns.get_level_values('statistic'),
axis=1,
inplace=False) for plot in data_collection
])
# Recreate a multi-indexed columns so the plot can function properly
df.columns = pd.MultiIndex.from_tuples(
[('', '', '', '', stat) for stat in df.columns],
names=['job', 'scenario', 'agent', 'suffix', 'statistic'])
plot = _Plot(df)
if not fields:
fields = list(df.columns.get_level_values('statistic'))
metadata = itertools.zip_longest(fields, labels, bin_sizes,
aggregations, legend_titles,
titles)
for field, label, bin_size, aggregation, legend, title in metadata:
if field not in df.columns.get_level_values('statistic'):
message = 'job instances {} did not produce the statistic {}'.format(
job, field)
collect_agent.send_log(syslog.LOG_WARNING, message)
print(message)
continue
if label is None:
collect_agent.send_log(
syslog.LOG_WARNING,
'no y-axis label provided for the {} statistic of job '
'instances {}: using the empty string instead'.format(
field, job))
label = ''
if aggregation is None:
collect_agent.send_log(
syslog.LOG_WARNING,
'invalid aggregation value of {} for the {} '
'statistic of job instances {}: choose from {}, using '
'"hour" instead'.format(aggregation, field, job,
TIME_OPTIONS))
aggregation = 'hour'
if legend is None and use_legend:
collect_agent.send_log(
syslog.LOG_WARNING,
'no legend title provided for the {} statistic of job '
'instances {}: using the empty string instead'.format(
field, job))
legend = ''
if bin_size is None:
collect_agent.send_log(
syslog.LOG_WARNING,
'no bin size provided for the {} statistic of job '
'instances {}: using the default value 100 instead'.
format(field, job))
bin_size = 100
figure, axis = plt.subplots()
axis = plot.plot_temporal_binning_histogram(
axis, label, field, None, bin_size, offset, maximum,
aggregation, add_global, use_legend, legend)
if title is not None:
axis.set_title(title)
filepath = os.path.join(
root,
'temporal_binning_histogram_{}.{}'.format(field, file_ext))
save(figure, filepath, pickle, False)
collect_agent.store_files(now(), figure=filepath)
