def execute_workflow(wf):
""" this is the main workflow for executing a given workflow """
try:
# check that the definition is correct
info("######################################", Fore.CYAN)
info("> Workflow | " + str(wf.name), Fore.CYAN)
# check variables
b = wf.change_provider
c = wf.primary_data_provider
d = wf.execution_strategy
except KeyError as e:
error("definition.py is missing value " + str(e))
exit(1)
# initialize the test environment
init_pre_processors(wf)
init_change_provider(wf)
init_data_providers(wf)
# here we also execute the strategy
run_execution_strategy(wf)
# we are done, now we clean up
kill_pre_processors(wf)
info("> Finished workflow")
print("\n")
info(">start comparison of methods now")
from compare_methods import regressor_compare_methods, classifier_compare_methods
classifier_compare_methods()
regressor_compare_methods()
def __init__(self, wf, p):
try:
self.submit_mode = p["submit_mode"]
self.job_file = p["job_file"]
self.job_class = p["job_class"]
info(
"> PreProcessor | Spark | Mode: " + str(self.submit_mode) +
" | Args: " + str(self.job_class), Fore.CYAN)
except KeyError as e:
error("configuration.spark was incomplete: " + str(e))
exit(1)
spark_home = os.environ.get("SPARK_HOME")
spark_bin = "/bin/spark-submit"
# now we start the spark to run the job in
# http://stackoverflow.com/questions/13243807/popen-waiting-for-child-process-even-when-the-immediate-child-has-terminated/13256908#13256908
# set system/version dependent "start_new_session" analogs
kwargs = {}
if platform.system() == 'Windows':
CREATE_NEW_PROCESS_GROUP = 0x00000200 # note: could get it from subprocess
DETACHED_PROCESS = 0x00000008 # 0x8 | 0x200 == 0x208
kwargs.update(creationflags=DETACHED_PROCESS
| CREATE_NEW_PROCESS_GROUP)
elif sys.version_info < (3, 2): # assume posix
kwargs.update(preexec_fn=os.setsid)
else: # Python 3.2+ and Unix
kwargs.update(start_new_session=True)
# starting a subprocess to allow termination of spark after we are done
self.process = subprocess.Popen(spark_home + spark_bin + ' --class ' + self.job_class + \
' ./' + wf.folder + '/' + self.job_file, stdout=subprocess.PIPE, shell=True,
**kwargs)
# register a shutdown callback on this thread
atexit.register(self.shutdown)
# wait for some time to get spark time to boot up
time.sleep(10)
def run_execution_strategy(wf):
""" we run the correct execution strategy """
applyInitKnobs(wf)
try:
# start the right execution strategy
if wf.execution_strategy["type"] == "sequential":
log_results(wf.folder, wf.execution_strategy["knobs"][0].keys() + ["result"], append=False)
start_sequential_strategy(wf)
elif wf.execution_strategy["type"] == "self_optimizer":
log_results(wf.folder, wf.execution_strategy["knobs"].keys() + ["result"], append=False)
start_self_optimizer_strategy(wf)
elif wf.execution_strategy["type"] == "uncorrelated_self_optimizer":
log_results(wf.folder, wf.execution_strategy["knobs"].keys() + ["result"], append=False)
start_uncorrelated_self_optimizer_strategy(wf)
elif wf.execution_strategy["type"] == "step_explorer":
log_results(wf.folder, wf.execution_strategy["knobs"].keys() + ["result"], append=False)
start_step_strategy(wf)
elif wf.execution_strategy["type"] == "forever":
start_forever_strategy(wf)
except RuntimeError:
error("Stopped the whole workflow as requested by a RuntimeError")
# finished
info(">")
applyDefaultKnobs(wf)
def __init__(self, wf, cp):
# load config
try:
info("> EWSChangePro | ", Fore.CYAN)
except KeyError:
error("EWSChangePro was incomplete")
exit(1)
def returnDataListNonBlocking(self):
""" by logic this can not be non-blocking, so it is implemented as returnData """
try:
r = requests.get(self.url)
return [self.serialize_function(r.content)]
except:
error("HTTP Connection Problems")
return None
def returnData(self):
""" does a http GET request and returns the result value """
try:
r = requests.get(self.url)
return self.serialize_function(r.content)
except:
error("HTTP Connection Problems")
return None
def applyDefaultKnobs(wf):
""" we are done, so revert to default if given """
if "post_workflow_knobs" in wf.execution_strategy:
try:
info("> Applied the post_workflow_knobs")
wf.change_provider["instance"] \
.applyChange(wf.change_event_creator(wf.execution_strategy["post_workflow_knobs"]))
except:
error("apply changes did not work")
def run_execution_strategy(wf):
""" we run the correct execution strategy """
applyInitKnobs(wf)
print(wf.execution_strategy["type"])
try:
# start the right execution strategy
if wf.execution_strategy["type"] == "sequential":
log_results(wf.folder, list(wf.execution_strategy["knobs"][0].keys()) + ["result"], append=False)
start_sequential_strategy(wf)
elif wf.execution_strategy["type"] == "self_optimizer":
log_results(wf.folder, list(wf.execution_strategy["knobs"].keys()) + ["result"], append=False)
start_self_optimizer_strategy(wf)
# elif wf.execution_strategy["type"] == "discrete_optimizer":
# log_results(wf.folder, wf.execution_strategy["knobs"] + ["result"], append=False)
# start_discrete_optimizer_strategy(wf)
elif wf.execution_strategy["type"] == "uncorrelated_self_optimizer":
log_results(wf.folder, list(wf.execution_strategy["knobs"].keys()) + ["result"], append=False)
start_uncorrelated_self_optimizer_strategy(wf)
elif wf.execution_strategy["type"] == "step_explorer":
log_results(wf.folder, list(wf.execution_strategy["knobs"].keys()) + ["result"], append=False)
start_step_strategy(wf)
elif wf.execution_strategy["type"] == "forever":
start_forever_strategy(wf)
elif wf.execution_strategy["type"] == "simple_am":
start_simple_am(wf)
# elif wf.execution_strategy["type"] == "sequential_runtime":
# print("got here")
# log_results(wf.folder, wf.execution_strategy["knobs"] + ["result"], append=False)
# start_seq_runtime_stategy(wf)
# elif wf.execution_strategy["type"] == "mabandit_ucb1":
# print(type(wf.execution_strategy["knobs"]))
# log_results(wf.folder, wf.execution_strategy["knobs"] + ["result"], append=False)
# start_mab_ucb1_strategy(wf)
# elif wf.execution_strategy["type"] == "discount_ucb":
# log_results(wf.folder, wf.execution_strategy["knobs"] + ["result"], append=False)
# start_mab_discount_ucb_strategy(wf)
# elif wf.execution_strategy["type"] == "sliding_ucb":
# log_results(wf.folder, wf.execution_strategy["knobs"] + ["result"], append=False)
# start_mab_sw_ucb_strategy(wf)
except RuntimeError:
error("Stopped the whole workflow as requested by a RuntimeError")
# finished
info(">")
applyDefaultKnobs(wf)
def __init__(self, wf, cp):
self.timer = 0
# load config
try:
self.seconds = cp["seconds"]
info("> Interval | Seconds: " + str(self.seconds), Fore.CYAN)
except KeyError as e:
error("IntervalDataProvider definition was incomplete: " + str(e))
exit(1)
def __init__(self, wf, dp):
self.callBackFunction = None
# load config
try:
self.chosen_metric = dp["chosen_metric"]
info(">EWSDataPro | Metric: " + self.chosen_metric, Fore.CYAN)
except KeyError as e:
error("HTTPDataPro definition was incomplete: " + str(e))
exit(1)
def __init__(self, wf, cp):
# load config
self.host = wf.host
self.port = wf.port
self.send_message = wf.send_message
try:
info("> SWIMChangePro | ", Fore.CYAN)
except KeyError:
error("SWIMChangePro was incomplete")
exit(1)
def reset(self):
""" creates a new consumer to get to the current position of the queue """
try:
self.consumer = KafkaConsumer(
bootstrap_servers=self.kafka_uri,
value_deserializer=self.serialize_function,
group_id=None,
consumer_timeout_ms=3000)
self.consumer.subscribe([self.topic])
except RuntimeError as e:
error("connection to kafka failed: " + str(e))
exit(1)
def init_change_provider(wf):
""" loads the specified change provider into the workflow """
cp = wf.change_provider
if cp["type"] == "kafka_producer":
cp["instance"] = KafkaProducerChangeProvider(wf, cp)
elif cp["type"] == "mqtt_publisher":
cp["instance"] = MQTTPublisherChangeProvider(wf, cp)
elif cp["type"] == "http_request":
cp["instance"] = HTTPRequestChangeProvider(wf, cp)
elif cp["type"] == "dummy":
cp["instance"] = DummyChangeChangeProvider(wf, cp)
else:
error("Not a valid changeProvider")
exit(1)
def plot(wf):
""" here we try to generate automatic plotting of the experiments results """
info("######################################", Fore.CYAN)
info("> Reporting on | " + str(wf.name), Fore.CYAN)
plot_file_dir = './' + str(wf.folder)
# try to access the results.csv values
try:
with open('./' + str(plot_file_dir) + '/results.csv', 'r') as csv_file:
reader = csv.reader(csv_file, dialect='excel')
header = next(reader)
except IOError:
error('Please first generate a "' + str(plot_file_dir) +
'/results.csv" file by running the start command')
return
results_data_frame = pd.read_csv(str(plot_file_dir) + '/results.csv')
# 1 input -> 1 output variable case
if len(header) == 2:
info("> Found 1 knob, creating scatter plot...", Fore.CYAN)
plot_file = plot_file_dir + '/scatter_plot.png'
ax = sns.regplot(x=header[0],
y=header[1],
data=results_data_frame,
fit_reg=False)
fig = ax.get_figure()
fig.savefig(plot_file)
info("> Plot saved at " + plot_file, Fore.CYAN)
# 2 input -> 1 output variables case
elif len(header) == 3:
info("> Found 2 knobs, creating heat map...", Fore.CYAN)
plot_file = plot_file_dir + '/heatmap.png'
results = results_data_frame.pivot(*header)
try:
ax = sns.heatmap(results, annot=True, fmt=".1f", linewidths=.5)
fig = ax.get_figure()
fig.savefig(plot_file)
fig.show()
info("> Plot saved at " + plot_file, Fore.CYAN)
except ValueError:
info("> This strategy does not support heatmap generation")
else:
info(
"> Cannot plot these results (RTX can only plot experiments of one of two variables for now)",
Fore.CYAN)
def __init__(self, wf, cp):
# load config
try:
self.url = cp["url"]
self.serializer = cp["serializer"]
info("> HTTPChangePro | " + self.serializer + " | URL: " + self.url, Fore.CYAN)
except KeyError:
error("HTTPChangePro was incomplete")
exit(1)
# look at the serializer
if self.serializer == "JSON":
self.serialize_function = lambda v: json.dumps(v).encode('utf-8')
else:
error("serializer not implemented")
exit(1)
def __init__(self, wf, dp):
self.callBackFunction = None
# load config
try:
self.host = wf.host
self.port = wf.port
self.metricslist = dp["server_metrics"]
self.delays = wf.require_delays
self.last_action = wf.last_action
self.send_message = wf.send_message
info(
">SWIMDataPro | Metric: " + self.host + ":" +
str(self.port), Fore.CYAN)
except KeyError as e:
error("SWIMDataPro definition was incomplete: " + str(e))
exit(1)
def __init__(self, wf, cp):
self.callBackFunction = None
# load config
try:
self.url = cp["url"]
self.serializer = cp["serializer"]
info(
"> HTTPDataPro | " + self.serializer + " | URL: " +
self.url, Fore.CYAN)
except KeyError as e:
error("HTTPDataPro definition was incomplete: " + str(e))
exit(1)
if self.serializer == "JSON":
self.serialize_function = lambda m: json.loads(m.decode('utf-8'))
else:
error("serializer not implemented")
exit(1)
def __init__(self, wf, cp):
# load config
try:
self.queue = []
self.host = cp["host"]
self.port = cp["port"]
self.topic = cp["topic"]
self.serializer = cp["serializer"]
info("> MQTTPublisher | " + self.serializer + " | URI: " + self.host + ":" + self.port + " | Topic: " +
self.topic, Fore.CYAN)
except KeyError:
error("mqttPublisher definition was incomplete")
exit(1)
# look at the serializer
if self.serializer == "JSON":
self.serialize_function = lambda v: json.dumps(v).encode('utf-8')
else:
error("serializer not implemented")
exit(1)
def __init__(self, wf, cp):
# load config
try:
self.kafka_uri = cp["kafka_uri"]
self.topic = cp["topic"]
self.serializer = cp["serializer"]
info("> KafkaProducer | " + self.serializer + " | URI: " + self.kafka_uri + " | Topic: " +
self.topic, Fore.CYAN)
except KeyError:
error("configuration.kafka_producer was incomplete")
exit(1)
# look at the serializer
if self.serializer == "JSON":
self.serialize_function = lambda v: json.dumps(v).encode('utf-8')
else:
error("serializer not implemented")
exit(1)
# try to connect
try:
# stop annoying logging
logging.getLogger("kafka.coordinator.consumer").setLevel("ERROR")
logging.getLogger("kafka.conn").setLevel("ERROR")
self.producer = KafkaProducer(bootstrap_servers=self.kafka_uri,
value_serializer=self.serialize_function,
request_timeout_ms=5000)
except:
error("connection to kafka failed")
exit(1)
def retrieve_true_knobs(wf):
knobs = wf.execution_strategy["knobs"][0]
if "source" in knobs:
wf.execution_strategy["knobs"].clear()
#SOURCE SHOULD STILL BE LOADED IN (replace the IP used somehow)
config_model = ConfigurationModel()
list_rel_w_alt = config_model.relations_with_alternatives()
for rel_w_alt in list_rel_w_alt:
rel_alts = config_model.relation_alternative(rel_w_alt)
rel_alts.append(rel_w_alt)
values = []
for alt in rel_alts:
values.append(alt.child_comp)
variable = {rel_w_alt.get_name() : values}
wf.execution_strategy["knobs"].append(variable)
print(wf.execution_strategy["knobs"])
#global_vars.IP = "wrong" #knobs["source"] #this doesn't work
#configs = eRI.get_all_configs()
#for config_obj in configs:
# wf.execution_strategy["knobs"].append({"config":config_obj.original_json})
else:
error("source of configurations not included")
exit(1)
def __init__(self, wf, cp):
self.callBackFunction = None
# load the configuration
try:
self.queue = []
self.host = cp["host"]
self.port = cp["port"]
self.topic = cp["topic"]
self.serializer = cp["serializer"]
info(
"> MQTTListener | " + self.serializer + " | URI: " +
self.host + ":" + self.port + " | Topic: " + self.topic,
Fore.CYAN)
except KeyError as e:
error("mqttListener definition was incomplete: " + str(e))
exit(1)
# create serializer
if self.serializer == "JSON":
self.serialize_function = lambda m: json.loads(m.decode('utf-8'))
else:
error("serializer not implemented")
exit(1)
try:
# create mqtt client and connect
self.mqtt = mqtt.Client()
self.mqtt.connect(self.host, port=self.port)
# register callback
self.mqtt.on_message = self.on_message
# subscribe and start listing on second thread
self.mqtt.subscribe(self.topic, 0)
self.mqtt.loop_start()
except RuntimeError as e:
error("connection to mqtt failed: " + str(e))
exit(1)
def loadDefinition(folder):
""" opens the given folder and searches for a definition.py file and checks if it looks valid"""
if len(sys.argv) != 3:
error("missing experiment folder")
exit(1)
try:
wf = imp.load_source('wf', './' + folder + '/definition.py')
wf.folder = sys.argv[2]
testName = wf.name
return wf
except IOError:
error("Folder is not a valid experiment folder (does not contain definition.py)")
exit(1)
except AttributeError:
error("Workflow did not had a name attribute")
exit(1)
except ImportError as e:
error("Import failed: " + str(e))
exit(1)
def __init__(self, wf, cp):
self.callBackFunction = None
# load config
try:
self.kafka_uri = cp["kafka_uri"]
self.topic = cp["topic"]
self.serializer = cp["serializer"]
info(
"> KafkaConsumer | " + self.serializer + " | URI: " +
self.kafka_uri + " | Topic: " + self.topic, Fore.CYAN)
except KeyError as e:
error("system.kafkaConsumer was incomplete: " + str(e))
exit(1)
# look at the serializer
if self.serializer == "JSON":
self.serialize_function = lambda m: json.loads(m.decode('utf-8'))
else:
error("serializer not implemented")
exit(1)
# try to connect
try:
# disable annoying logging
logging.getLogger("kafka.coordinator.consumer").setLevel("ERROR")
logging.getLogger("kafka.conn").setLevel("ERROR")
# connect to kafka
self.consumer = KafkaConsumer(
bootstrap_servers=self.kafka_uri,
value_deserializer=self.serialize_function,
enable_auto_commit=False,
group_id=None,
consumer_timeout_ms=3000)
# subscribe to the requested topic
self.consumer.subscribe([self.topic])
except RuntimeError as e:
error("connection to kafka failed: " + str(e))
exit(1)
def experimentFunction(wf, exp):
""" executes a given experiment """
start_time = current_milli_time()
# remove all old data from the queues
wf.primary_data_provider["instance"].reset()
# load change event creator or use a default
if hasattr(wf, "change_event_creator"):
change_creator = wf.change_event_creator
else:
change_creator = _defaultChangeProvider
# start
info(">")
info("> KnobValues | " + str(exp["knobs"]))
# create new state
exp["state"] = wf.state_initializer(dict(),wf)
# apply changes to system
try:
wf.change_provider["instance"].applyChange(change_creator(exp["knobs"],wf))
except:
error("apply changes did not work")
# ignore the first data sets
to_ignore = exp["ignore_first_n_results"]
if to_ignore > 0:
i = 0
while i < to_ignore:
new_data = wf.primary_data_provider["instance"].returnData()
if new_data is not None:
i += 1
process("IgnoreSamples | ", i, to_ignore)
print("")
# start collecting data
sample_size = exp["sample_size"]
i = 0
try:
while i < sample_size:
# we start with the primary data provider using blocking returnData
new_data = wf.primary_data_provider["instance"].returnData()
if new_data is not None:
try:
# print(new_data)
exp["state"] = wf.primary_data_provider["data_reducer"](exp["state"], new_data,wf)
except StopIteration:
raise StopIteration() # just fwd
except RuntimeError:
raise RuntimeError() # just fwd
except:
error("could not reducing data set: " + str(new_data))
i += 1
process("CollectSamples | ", i, sample_size)
# now we use returnDataListNonBlocking on all secondary data providers
if hasattr(wf, "secondary_data_providers"):
for cp in wf.secondary_data_providers:
new_data = cp["instance"].returnDataListNonBlocking()
for nd in new_data:
try:
exp["state"] = cp["data_reducer"](exp["state"], nd,wf)
except StopIteration:
raise StopIteration() # just
except RuntimeError:
raise RuntimeError() # just fwd
except:
error("could not reducing data set: " + str(nd))
print("")
except StopIteration:
# this iteration should stop asap
error("This experiment got stopped as requested by a StopIteration exception")
try:
result = wf.evaluator(exp["state"],wf)
except:
result = 0
error("evaluator failed")
# we store the counter of this experiment in the workflow
if hasattr(wf, "experimentCounter"):
wf.experimentCounter += 1
else:
wf.experimentCounter = 1
# print the results
duration = current_milli_time() - start_time
# do not show stats for forever strategy
if wf.totalExperiments > 0:
info("> Statistics | " + str(wf.experimentCounter) + "/" + str(wf.totalExperiments)
+ " took " + str(duration) + "ms" + " - remaining ~" + str(
(wf.totalExperiments - wf.experimentCounter) * duration / 1000) + "sec")
info("> FullState | " + str(exp["state"]))
info("> ResultValue | " + str(result))
# log the result values into a csv file
log_results(wf.folder, exp["knobs"].values() + [result])
# return the result value of the evaluator
return result
def returnDataListNonBlocking(self):
""" does wait for x seconds and then return the timer counter value """
error(
"IntervalDataProvider is not able to work as a secondary data provider"
)
exit(1)
