def run_curl_middlebox(job, interface, interface_event, expconfig,
monroe_exporter, meta_info):
cmd = [
'curl', '-s', expconfig['middlebox_detection_target'] + "?" +
expconfig['nodeid'] + "_" + interface + "_middlebox_test", '/dev/null'
]
timeout = False
time_begin = int(time.time())
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
try:
stdout, stderr = p.communicate(timeout=None)
except subprocess.TimeoutExpired:
timeout = True
p.kill()
stdout, stderr = p.communicate()
time_end = int(time.time())
# To use monroe_exporter the following fields must be present
# "DataId"
# "DataVersion"
# "NodeId"
# "SequenceNumber"
result_object = {
'NodeId':
"curl_middlebox_" + expconfig['middlebox_detection_target'] + "_" +
expconfig['nodeid'],
'DataId':
interface,
'DataVersion':
0,
'Timestamp':
time.time(),
'SequenceNumber':
-1,
'IPAddress':
meta_info['modem']['extra_ipaddress'],
'ImsiMccMnc':
meta_info['modem']['extra_imsi'],
'Location':
meta_info['modem']['extra_location'],
'target':
job['target'],
'interface':
interface,
'measurement_begin':
time_begin,
'measurement_end':
time_end,
'timeout':
timeout,
'cmd':
cmd,
'stdout':
stdout,
'stderr':
stderr,
}
monroe_exporter.save_output(result_object)
# According to specification all messages that ends with .UPDATE in the
# topic are rebrodcasts so we skip these.
if topic.endswith(".UPDATE"):
if topic in UPDATECACHE:
continue
else:
UPDATECACHE.add(topic)
# If not correct JSON, skip and wait for next message
try:
if not DEBUG:
msg = json.loads(msgdata)
# Some zmq messages do not have nodeid information so I set it here
msg['NodeId'] = str(CONFIG['nodeid'])
#Check so we have all mandatory fields
str(msg['DataId'])
int(msg['DataVersion'])
float(msg['Timestamp'])
int(msg['SequenceNumber'])
else:
msg = msgdata
except:
if CONFIG['verbosity'] > 0:
print("Error: Recived invalid JSON msg with topic {} from "
"metadata-publisher : {}").format(topic, msgdata)
continue
if CONFIG['verbosity'] > 2:
print(msg)
if not DEBUG:
monroe_exporter.save_output(msg, CONFIG['resultdir'])
def executeAction(action, selectedInterface, expconfig, log_list,
dynamicSelection):
meta_info = interfacesMap[selectedInterface]['meta_info']
rssis = interfacesMap[selectedInterface]['rssis']
rtts = interfacesMap[selectedInterface]['rtts']
ifname = meta_info[expconfig["modeminterfacename"]]
interval = float(expconfig['interval'] / 1000.0)
cmd = [
"curl",
"-o",
"/dev/null", # to not output filecontents on stdout
"--fail", # to get the curl exit code 22 for http failures
"--insecure", # to allow selfsigned certificates
"--raw",
"--silent",
"--write-out",
"{}".format(CURL_METRICS),
"--interface",
"{}".format(ifname),
"--max-time",
"{}".format(expconfig['time']),
# "--range", "0-{}".format(expconfig['size'] - 1),
"{}".format(action['Url'])
]
# Safeguard to always have a defined output variable
output = None
# Iterate over the number of repetitions for the current action
for i in range(0, action['Repetitions']):
err_code = 0
try:
start_curl = time.time()
try:
output = check_output(cmd)
except CalledProcessError as e:
err_code = e.returncode # AEL get the error code here
output = e.output
# if e.returncode == 28: # time-limit exceeded
# if expconfig['verbosity'] > 2:
# print ("Exceding timelimit {}, "
# "saving what we have").format(expconfig['time'])
# output = e.output
# else:
# raise e
# Clean away leading and trailing whitespace
output = output.strip(' \t\r\n\0')
# Convert to JSON
msg = json.loads(output)
msg.update({
"ErrorCode": err_code,
"Guid": expconfig['guid'],
"DataId": expconfig['dataidCurl'],
"DataVersion": expconfig['dataversion'],
"NodeId": expconfig['nodeid'],
"Timestamp": start_curl,
"Iccid": meta_info["ICCID"],
"Operator": meta_info["Operator"],
"DownloadTime": msg["TotalTime"] - msg["SetupTime"],
"SequenceNumber": i
})
if dynamicSelection:
msg.update({"DynamicSelection": True})
else:
msg.update({"DynamicSelection": False})
if expconfig['verbosity'] > 2:
print msg
if not DEBUG:
monroe_exporter.save_output(msg, expconfig['resultdir'])
log_list.append(unicode(json.dumps(msg) + '\n'))
except Exception as e:
if expconfig['verbosity'] > 0:
print("Execution or parsing failed for "
"command : {}, "
"output : {}, "
"error: {}").format(cmd, output, e)
def run_ping_exp(my_interface_map, expconfig, log_list):
global FPING_PROCESS
meta_info = my_interface_map['meta_info']
rtts = my_interface_map['rtts']
rssis = my_interface_map['rssis']
# Set some variables for saving data every export_interval
monroe_exporter.initalize(expconfig['export_interval'],
expconfig['resultdir'])
ifname = meta_info[expconfig["modeminterfacename"]]
interval = float(expconfig['interval'] / 1000.0)
pingTarget = expconfig['pingTarget']
cmd = ["fping", "-I", ifname, "-D", "-c", "1", pingTarget]
# Regexp to parse fping ouput from command
r = re.compile(
r'^\[(?P<ts>[0-9]+\.[0-9]+)\] (?P<host>[^ ]+) : \[(?P<seq>[0-9]+)\], (?P<bytes>\d+) bytes, (?P<rtt>[0-9]+(?:\.[0-9]+)?) ms \(.*\)$'
)
# This is the inner loop where we wait for output from fping
# This will run until we get a interface hickup, where the process will be
# killed from parent process.
seq = 0
while True:
popen = subprocess.Popen(cmd, stdout=subprocess.PIPE, bufsize=1)
output = popen.stdout.readline()
m = r.match(output)
if m is not None: # We could send and got a reply
# keys are defined in regexp compilation. Nice!
exp_result = m.groupdict()
# Add RTT and RSSI values to the relevant lists (aka windows)
add_value(rtts, float(exp_result['rtt']))
add_value(rssis, meta_info["RSSI"])
msg = {
'Bytes': int(exp_result['bytes']),
'Host': exp_result['host'],
'Rtt': float(exp_result['rtt']),
'AvgRtt': calc_avg(rtts),
'SequenceNumber': int(seq),
'Timestamp': float(exp_result['ts']),
"Guid": expconfig['guid'],
"DataId": expconfig['dataid'],
"Interface": ifname,
"DataVersion": expconfig['dataversion'],
"NodeId": expconfig['nodeid'],
"Iccid": meta_info["ICCID"],
"Operator": meta_info["Operator"],
"Rssi": meta_info["RSSI"],
"AvgRssi": calc_avg(rssis),
}
else: # We lost the interface or did not get a reply
msg = {
'Host': pingTarget,
'SequenceNumber': int(seq),
'Timestamp': time.time(),
"Guid": expconfig['guid'],
"DataId": expconfig['dataid'],
"Interface": ifname,
"DataVersion": expconfig['dataversion'],
"NodeId": expconfig['nodeid'],
"Iccid": meta_info["ICCID"],
"Operator": meta_info["Operator"]
}
if expconfig['verbosity'] > 2:
print msg
if not DEBUG:
# We have already initalized the exporter with the export dir
monroe_exporter.save_output(msg)
log_list.append(unicode(json.dumps(msg) + '\n'))
# with io.open(expconfig['resultfile'], 'a') as f:
# f.write(unicode(json.dumps(msg) + '\n'))
seq += 1
# Sleep for the predefined interval and re-execute loop
time.sleep(interval)
# Cleanup
if expconfig['verbosity'] > 1:
print "Cleaning up fping process"
popen.stdout.close()
popen.terminate()
popen.kill()
while True:
# If not correct msg, skip and wait for next message
try:
(topic, msgdata) = socket.recv().split(' ', 1)
except:
if CONFIG['verbosity'] > 0:
print ("Error: Invalid zmq msg")
continue
# According to specification all messages that ends with .UPDATE in the
# topic are rebrodcasts so we skip these.
if topic.endswith(".UPDATE"):
if topic in UPDATECACHE:
continue
else:
UPDATECACHE.add(topic)
# If not correct JSON, skip and wait for next message
try:
msg = json.loads(msgdata)
# Some zmq messages do not have nodeid information so I set it here
msg['NodeId'] = CONFIG['nodeid']
except:
if CONFIG['verbosity'] > 0:
print ("Error: Recived invalid JSON msg with topic {} from "
"metadata-multicaster : {}").format(topic, msg)
continue
if CONFIG['verbosity'] > 2:
print msg
monroe_exporter.save_output(msg, CONFIG['resultdir'])
log_str = "Starting Experiment Run : {}".format(cmd)
print(log_str) if (debug) else syslog.syslog(log_str)
try:
msg = run_exp(cmd)
# Should be replaced with real value from "scheduler"/initscript
GUID = "{}.{}.{}.{}".format("experiment_id",
"scheduling_id",
"node_id",
"repetition")
msg.update({
"Guid": GUID,
"TimeStamp": time.time(),
"InterfaceName": interface,
"DownloadTime": msg["TotalTime"] - msg["SetupTime"]
})
print (msg) if (debug) else monroe_exporter.save_output(msg)
time.sleep(5)
except CalledProcessError as e:
log_str = "Execution failed: {}".format(e)
print (msg) if (debug) else syslog.syslog(log_str)
elapsed = time.time() - start_time
# Wait if interval > 0 else break loop
if (interval > 0):
wait = interval - elapsed if (interval - elapsed > 0) else 0
log_str = "Now waiting {} s before next run".format(wait)
print (log_str) if (debug) else syslog.syslog(log_str)
time.sleep(wait)
else:
log_str = "Finsished after {}".format(elapsed)
print (log_str) if (debug) else syslog.syslog(log_str)
def run_exp(meta_info, expconfig):
global FPING_PROCESS
# Set some variables for saving data every export_interval
monroe_exporter.initalize(expconfig['export_interval'],
expconfig['resultdir'])
# Register signla handlers
signal.signal(signal.SIGTERM, handle_signal)
signal.signal(signal.SIGINT, handle_signal)
ifname = meta_info['InterfaceName']
interval = str(expconfig['interval'])
server = expconfig['server']
cmd = ["fping",
"-I", ifname,
"-D",
"-p", interval,
"-l", server]
# Regexp to parse fping ouput from command
r = re.compile(r'^\[(?P<ts>[0-9]+\.[0-9]+)\] (?P<host>[^ ]+) : \[(?P<seq>[0-9]+)\], (?P<bytes>\d+) bytes, (?P<rtt>[0-9]+(?:\.[0-9]+)?) ms \(.*\)$')
popen = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
bufsize=1)
FPING_PROCESS = popen
stdout_lines = iter(popen.stdout.readline, "")
# This is the inner loop where we wait for output from fping
# This will run until we get a interface hickup
for line in stdout_lines:
m = r.match(line)
if m is None:
if expconfig['verbosity'] > 1:
print "Could not match regexp, exiting"
sys.exit(1)
# keys are defined in regexp compilation. Nice!
exp_result = m.groupdict()
# Experiment outputinfo["ICCID"] = "local"
msg = {
'Bytes': int(exp_result['bytes']),
'Host': exp_result['host'],
'Rtt': float(exp_result['rtt']),
'SequenceNumber': int(exp_result['seq']),
'Timestamp': float(exp_result['ts']),
"Guid": expconfig['guid'],
"DataId": expconfig['dataid'],
"DataVersion": expconfig['dataversion'],
"NodeId": expconfig['nodeid'],
"Iccid": meta_info["ICCID"],
"Operator": meta_info["Operator"]
}
if expconfig['verbosity'] > 2:
print msg
if not DEBUG:
# We have already initalized the exporter with the export dir
monroe_exporter.save_output(msg)
# Cleanup
if expconfig['verbosity'] > 1:
print "Cleaning up fping process"
popen.stdout.close()
popen.terminate()
popen.kill()
def run_exp(meta_info, expconfig):
global FPING_PROCESS
# Set some variables for saving data every export_interval
monroe_exporter.initalize(expconfig['export_interval'],
expconfig['resultdir'])
ifname = meta_info[expconfig["modeminterfacename"]]
interval = float(expconfig['interval'] / 1000.0)
server = expconfig['server']
cmd = ["fping", "-I", ifname, "-D", "-c", "1", server]
# Regexp to parse fping ouput from command
r = re.compile(
r'^\[(?P<ts>[0-9]+\.[0-9]+)\] (?P<host>[^ ]+) : \[(?P<seq>[0-9]+)\], (?P<bytes>\d+) bytes, (?P<rtt>[0-9]+(?:\.[0-9]+)?) ms \(.*\)$'
)
# This is the inner loop where we wait for output from fping
# This will run until we get a interface hickup, where the process will be
# killed from parent process.
seq = 0
while True:
popen = subprocess.Popen(cmd, stdout=subprocess.PIPE, bufsize=1)
output = popen.stdout.readline()
m = r.match(output)
if m is not None: # We could send and got a reply
# keys are defined in regexp compilation. Nice!
exp_result = m.groupdict()
msg = {
'Bytes': int(exp_result['bytes']),
'Host': exp_result['host'],
'Rtt': float(exp_result['rtt']),
'SequenceNumber': int(seq),
'Timestamp': float(exp_result['ts']),
"Guid": expconfig['guid'],
"DataId": expconfig['dataid'],
"DataVersion": expconfig['dataversion'],
"NodeId": expconfig['nodeid'],
"Iccid": meta_info["ICCID"],
"Operator": meta_info["Operator"]
}
else: # We lost the interface or did not get a reply
msg = {
'Host': server,
'SequenceNumber': int(seq),
'Timestamp': time.time(),
"Guid": expconfig['guid'],
"DataId": expconfig['dataid'],
"DataVersion": expconfig['dataversion'],
"NodeId": expconfig['nodeid'],
"Iccid": meta_info["ICCID"],
"Operator": meta_info["Operator"]
}
if expconfig['verbosity'] > 2:
print msg
if not DEBUG:
# We have already initalized the exporter with the export dir
monroe_exporter.save_output(msg)
seq += 1
time.sleep(interval)
# Cleanup
if expconfig['verbosity'] > 1:
print "Cleaning up fping process"
popen.stdout.close()
popen.terminate()
popen.kill()