def _setup(self):
'''
Clear and configure redis conf, log files in the ros home
directories under a subdirectory styled by the name of this hub.
'''
if os.path.isdir(self._home_dir):
shutil.rmtree(self._home_dir)
os.makedirs(self._home_dir)
rospack = rospkg.RosPack()
redis_conf_template = utils.read_template(
os.path.join(rospack.get_path('rocon_hub'), 'redis', 'redis.conf'))
redis_conf_template = instantiate_redis_conf_template(
redis_conf_template, self._files['redis_conf_local'])
redis_local_template = utils.read_template(
os.path.join(rospack.get_path('rocon_hub'), 'redis',
'redis.conf.local'))
redis_local_template = instantiate_local_conf_template(
redis_local_template, self._parameters['port'],
self._parameters['max_memory'], self._files['redis_server_log'])
try:
f = open(self._files['redis_conf'], 'w')
f.write(redis_conf_template.encode('utf-8'))
finally:
f.close()
try:
f = open(self._files['redis_conf_local'], 'w')
f.write(redis_local_template.encode('utf-8'))
finally:
f.close()
def get_qt_text_templates(package, type):
template_dir = os.path.join(os.path.dirname(__file__), 'templates', type)
templates = {}
templates['mainpage.dox'] = read_template(
os.path.join(template_dir, 'mainpage.dox'))
templates['CMakeLists.txt'] = read_template(
os.path.join(template_dir, 'CMakeLists.txt'))
templates['package.xml'] = read_template(
os.path.join(template_dir, 'package.xml'))
templates[os.path.join('ui', 'main_window.ui')] = read_template(
os.path.join(template_dir, 'ui', 'main_window.ui'))
templates[os.path.join('src', 'main.cpp')] = read_template(
os.path.join(template_dir, 'src', 'main.cpp'))
templates[os.path.join('src', 'main_window.cpp')] = read_template(
os.path.join(template_dir, 'src', 'main_window.cpp'))
templates[os.path.join('src', 'qnode.cpp')] = read_template(
os.path.join(template_dir, 'src', 'qnode.cpp'))
templates[os.path.join('resources', 'images.qrc')] = read_template(
os.path.join(template_dir, 'resources', 'images.qrc'))
templates[os.path.join('include', package,
'main_window.hpp')] = read_template(
os.path.join(template_dir, 'include',
'PACKAGE_NAME', 'main_window.hpp'))
templates[os.path.join('include', package, 'qnode.hpp')] = read_template(
os.path.join(template_dir, 'include', 'PACKAGE_NAME', 'qnode.hpp'))
return templates
def get_common_templates(package):
template_dir = os.path.join(os.path.dirname(__file__), 'templates',
'common')
templates = {}
templates['CMakeLists.txt'] = utils.read_template(
os.path.join(template_dir, 'CMakeLists.txt'))
templates['package.xml'] = utils.read_template(
os.path.join(template_dir, 'package.xml'))
return templates
def get_ros_text_templates(type):
template_common_dir = os.path.join(os.path.dirname(__file__),'templates','common')
template_dir = os.path.join(os.path.dirname(__file__),'templates',type)
templates = {}
templates['mainpage.dox'] = utils.read_template(os.path.join(template_common_dir,'mainpage.dox'))
templates['manifest.xml'] = utils.read_template(os.path.join(template_common_dir,'manifest.xml'))
templates['CMakeLists.txt'] = utils.read_template(os.path.join(template_dir,'CMakeLists.txt'))
if type == 'ros-legacy':
templates['Makefile'] = utils.read_template(os.path.join(template_common_dir,'Makefile'))
return templates
def get_text_templates(package, type):
template_common_dir = os.path.join(os.path.dirname(__file__),'templates','common')
template_dir = os.path.join(os.path.dirname(__file__),'templates',type)
templates = {}
templates['mainpage.dox'] = utils.read_template(os.path.join(template_common_dir,'mainpage.dox'))
templates['CMakeLists.txt'] = utils.read_template(os.path.join(template_dir,'CMakeLists.txt'))
templates['manifest.xml'] = utils.read_template(os.path.join(template_dir,'manifest.xml'))
templates[os.path.join('msg','Dude.msg')] = utils.read_template(os.path.join(template_dir,'msg','Dude.msg'))
templates[os.path.join('srv','HelloDude.srv')] = utils.read_template(os.path.join(template_dir,'srv','HelloDude.srv'))
return templates
def get_qt_text_templates(package, type):
template_dir = os.path.join(os.path.dirname(__file__),'templates',type)
templates = {}
templates['mainpage.dox'] = read_template(os.path.join(template_dir,'mainpage.dox'))
templates['CMakeLists.txt'] = read_template(os.path.join(template_dir,'CMakeLists.txt'))
templates['package.xml'] = read_template(os.path.join(template_dir,'package.xml'))
templates[os.path.join('ui','main_window.ui')] = read_template(os.path.join(template_dir,'ui','main_window.ui'))
templates[os.path.join('src','main.cpp')] = read_template(os.path.join(template_dir,'src','main.cpp'))
templates[os.path.join('src','main_window.cpp')] = read_template(os.path.join(template_dir,'src','main_window.cpp'))
templates[os.path.join('src','qnode.cpp')] = read_template(os.path.join(template_dir,'src','qnode.cpp'))
templates[os.path.join('resources','images.qrc')] = read_template(os.path.join(template_dir,'resources','images.qrc'))
templates[os.path.join('include',package,'main_window.hpp')] = read_template(os.path.join(template_dir,'include','PACKAGE_NAME','main_window.hpp'))
templates[os.path.join('include',package,'qnode.hpp')] = read_template(os.path.join(template_dir,'include','PACKAGE_NAME','qnode.hpp'))
return templates
def get_templates(package):
template_dir = os.path.join(os.path.dirname(__file__), 'templates',
'comms')
templates = {}
templates['mainpage.dox'] = utils.read_template(
os.path.join(template_dir, 'mainpage.dox'))
templates['CMakeLists.txt'] = utils.read_template(
os.path.join(template_dir, 'CMakeLists.txt'))
templates['package.xml'] = utils.read_template(
os.path.join(template_dir, 'package.xml'))
templates[os.path.join('msg', 'Dude.msg')] = utils.read_template(
os.path.join(template_dir, 'msg', 'Dude.msg'))
templates[os.path.join('srv', 'HelloDude.srv')] = utils.read_template(
os.path.join(template_dir, 'srv', 'HelloDude.srv'))
return templates
def generate_static_pktclassifier_h(fields):
skeleton = read_template(TEMPLATES_DIR + "/static_pktclassifier.h.template")
skeleton_attrs = copy.copy(fields[0])
header = fields[0]['header']
skeleton_attrs['headers'] = "#include <rofl/common/protocols/f%sframe.h>\n" % header
skeleton_attrs['header_access'] = "\tvirtual rofl::f%sframe* %s (int idx = 0) const;\n" % (header, header)
skeleton_attrs['pop_operations'] = ""
for field in fields:
if field.get('action') == 'pop':
skeleton_attrs['pop_operations'] += "\tvirtual void pop_%s(uint16_t ether_type);\n" % header
skeleton_attrs['push_operations'] = ""
for field in fields:
if field.get('action') == 'push':
skeleton_attrs['push_operations'] += "\tvirtual rofl::f%sframe* push_%s(uint16_t ether_type);\n" % (header, header)
skeleton_attrs['header_type'] = "\t\tHEADER_TYPE_%s,\n" % header.upper()
skeleton_attrs['max_occurances'] = "\tstatic const unsigned int MAX_%s_FRAMES = 1;\n" % header.upper()
skeleton_attrs['max_occurances_values'] = "\t\t\t\t\t\t\tMAX_%s_FRAMES +\n" % header.upper()
skeleton_attrs['relative_positions'] = "static const unsigned int FIRST_%s_FRAME_POS = FIRST_GTP_FRAME_POS+MAX_GTP_FRAMES;\n" % header.upper()
skeleton_attrs['header_parse'] = "void parse_%s(uint8_t *data, size_t datalen);\n" % header
return skeleton % skeleton_attrs
def get_py_templates(package):
template_dir = os.path.join(os.path.dirname(__file__), 'templates', 'py')
templates = {}
templates['CMakeLists.txt'] = utils.read_template(
os.path.join(template_dir, 'CMakeLists.txt'))
templates['package.xml'] = utils.read_template(
os.path.join(template_dir, 'package.xml'))
templates['setup.py'] = utils.read_template(
os.path.join(template_dir, 'setup.py'))
templates[os.path.join('scripts', package)] = utils.read_template(
os.path.join(template_dir, 'scripts', 'PACKAGE_NAME'))
templates[os.path.join('src', package,
'__init__.py')] = utils.read_template(
os.path.join(template_dir, 'src',
'PACKAGE_NAME', '__init__.py'))
return templates
def generate_static_pktclassifier_c(fields):
skeleton = read_template(TEMPLATES_DIR + "/static_pktclassifier.c.template")
skeleton_attrs = copy.copy(fields[0])
header = fields[0]['header']
skeleton_attrs['initializers'] = INIT_SKELETON % skeleton_attrs
skeleton_attrs['getters'] = GETTER_SKELETON % skeleton_attrs
skeleton_attrs['parsers'] = PARSER_SKELETON % skeleton_attrs
skeleton_attrs['pop_operations'] = ""
for field in fields:
if field.get('action') == 'pop':
skeleton_attrs['pop_operations'] += "\tvirtual void pop_%s(uint16_t ether_type);\n" % header
skeleton_attrs['push_operations'] = ""
for field in fields:
if field.get('action') == 'push':
skeleton_attrs['push_operations'] += "\tvirtual rofl::f%sframe* push_%s(uint16_t ether_type);\n" % (header, header)
skeleton_attrs['header_type'] = "\t\tHEADER_TYPE_%s,\n" % header.upper()
skeleton_attrs['max_occurances'] = "\tstatic const unsigned int MAX_%s_FRAMES = 1;\n" % header.upper()
skeleton_attrs['max_occurances_values'] = "\t\t\t\t\t\t\tMAX_%s_FRAMES +\n" % header.upper()
skeleton_attrs['relative_positions'] = "static const unsigned int FIRST_%s_FRAME_POS = FIRST_GTP_FRAME_POS+MAX_GTP_FRAMES;\n" % header.upper()
skeleton_attrs['header_parse'] = "void parse_%s(uint8_t *data, size_t datalen);\n" % header
skeleton_attrs['pop_operations_body'] = POP_SKELETON % skeleton_attrs
skeleton_attrs['push_operations_body'] = PUSH_SKELETON % skeleton_attrs
skeleton_attrs['ether_parse'] = ETHER_PARSE_SKELETON % skeleton_attrs
return skeleton % skeleton_attrs
def generate_translation_utils_c(fields):
skeleton = read_template(TEMPLATES_DIR + "/of12_translation_utils.c.template")
header = fields[0]['header']
code0 = EXPERIMENTAL_INCLUDES_SKELETON % (header, header)
code1 = ""
for field in fields_iterator(fields):
code1 += MATCH_TRANSLATION_SKELETON % field
code2 = ""
for field in fields_iterator(fields):
code2 += MATCH_SET_SKELETON % field
code3 = ""
for field in fields:
if field.get('action') in ('push', 'pop'):
code3 += ACTION_SET_SKELETON % field
code4 = ""
for field in fields_iterator(fields):
code4 += REVERSE_MATCH_SKELETON % field
code5 = ""
for field in fields:
if field.get('action') in ('push', 'pop'):
code5 += REVERSE_ACTION_SKELETON % field
code6 = ""
for field in fields_iterator(fields):
code6+= MAP_REVERSE_PACKET_MATCH_SKELETON % field
return skeleton % (code0, code1, code2, code3, code4, code5, code6)
def generate_datapath_pipeline_platform_actions_h(fields):
skeleton = read_template(TEMPLATES_DIR + "/packet_actions.h.template")
code = ""
for field in fields:
if 'field' not in field:
code += "void platform_packet_%(action)s_%(header)s(datapacket_t* pkt, uint16_t ether_type);\n" % field # TODO
return skeleton % code
def get_templates(package):
template_common_dir = os.path.join(os.path.dirname(__file__),'templates','common')
template_dir = os.path.join(os.path.dirname(__file__),'templates','cpp-ros')
templates = {}
templates['mainpage.dox'] = utils.read_template(os.path.join(template_common_dir,'mainpage.dox'))
templates['manifest.xml'] = utils.read_template(os.path.join(template_common_dir,'manifest.xml'))
templates['CMakeLists.txt'] = utils.read_template(os.path.join(template_dir,'CMakeLists.txt'))
templates[os.path.join('src','main.cpp')] = utils.read_template(os.path.join(template_dir,'src','main.cpp'))
templates[os.path.join('src','CMakeLists.txt')] = utils.read_template(os.path.join(template_dir,'src','CMakeLists.txt'))
templates[os.path.join('src','lib',package+'.cpp')] = utils.read_template(os.path.join(template_dir,'src','lib','package_name.cpp'))
templates[os.path.join('src','lib','CMakeLists.txt')] = utils.read_template(os.path.join(template_dir,'src','lib','CMakeLists.txt'))
templates[os.path.join('include',package,package+'.hpp')] = utils.read_template(os.path.join(template_dir,'include','PACKAGE_NAME','package_name.hpp'))
templates[os.path.join('launch','test.launch')] = utils.read_template(os.path.join(template_dir,'launch','test.launch'))
return templates
def get_cpp_templates(package):
template_dir = os.path.join(os.path.dirname(__file__), 'templates', 'cpp')
templates = {}
templates['CMakeLists.txt'] = utils.read_template(
os.path.join(template_dir, 'CMakeLists.txt'))
templates['package.xml'] = utils.read_template(
os.path.join(template_dir, 'package.xml'))
templates[os.path.join('include', package,
package + '.hpp')] = utils.read_template(
os.path.join(template_dir, 'include',
'PACKAGE_NAME',
'package_name.hpp'))
templates[os.path.join('src', 'lib',
package + '.cpp')] = utils.read_template(
os.path.join(template_dir, 'src', 'lib',
'package_name.cpp'))
templates[os.path.join('src', 'main.cpp')] = utils.read_template(
os.path.join(template_dir, 'src', 'main.cpp'))
return templates
def __init__(self, run):
self.run = run
self.collections = ['test']
self.data_dicts = self.get_data_dicts()
self.question_templates, self.level_dict = utils.read_template(run)
self.relations = [rel for rel_list in self.level_dict.values()
for rel in rel_list]
self.prop_info = utils.read_property_info()
self.relation_examples_dict = utils.read_examples(self.relations)
self.questions = self.get_questions()
def __init__(self, run, restrict = True):
self.run = run
self.collections = ['perceptual', 'activities', 'complex', 'parts']
self.data_dicts = self.get_data_dicts()
self.question_templates, self.level_dict = utils.read_template(run)
self.relations = [rel for rel_list in self.level_dict.values()\
for rel in rel_list]
self.prop_info = utils.read_property_info()
self.relation_examples_dict = utils.read_examples(self.relations)
self.restrict = restrict
self.questions = self.get_questions()
def generate(N_bodies=500):
id_base = 0
xml = read_template("body-templates/ant.template.xml")
conf = read_yaml("body-templates/ant.yaml")
data = conf["classA"]
write_xml(f"{utils.folder}/envs/{id_base}.mean.xml", data, xml)
for i in range(N_bodies):
data_clone = data.copy()
for key in data_clone:
data_clone[key] += np.random.normal(0, 0.1)
write_xml(f"{utils.folder}/envs/{i+id_base}.xml", data_clone, xml)
def get_qt_text_templates(package, type):
template_dir = os.path.join(os.path.dirname(__file__), "templates", type)
templates = {}
templates["mainpage.dox"] = read_template(os.path.join(template_dir, "mainpage.dox"))
templates["CMakeLists.txt"] = read_template(os.path.join(template_dir, "CMakeLists.txt"))
templates["package.xml"] = read_template(os.path.join(template_dir, "package.xml"))
templates[os.path.join("ui", "main_window.ui")] = read_template(os.path.join(template_dir, "ui", "main_window.ui"))
templates[os.path.join("src", "main.cpp")] = read_template(os.path.join(template_dir, "src", "main.cpp"))
templates[os.path.join("src", "main_window.cpp")] = read_template(
os.path.join(template_dir, "src", "main_window.cpp")
)
templates[os.path.join("src", "qnode.cpp")] = read_template(os.path.join(template_dir, "src", "qnode.cpp"))
templates[os.path.join("resources", "images.qrc")] = read_template(
os.path.join(template_dir, "resources", "images.qrc")
)
templates[os.path.join("include", package, "main_window.hpp")] = read_template(
os.path.join(template_dir, "include", "PACKAGE_NAME", "main_window.hpp")
)
templates[os.path.join("include", package, "qnode.hpp")] = read_template(
os.path.join(template_dir, "include", "PACKAGE_NAME", "qnode.hpp")
)
return templates
def new_user(email, password):
message = MIMEMultipart()
message['From'] = '*****@*****.**'
message['To'] = email
message['Subject'] = 'Welcome to GrapevinePD'
new_user_template = read_template('email_templates/new_user.html')
message_body = new_user_template.substitute(EMAIL=email, PASSWORD=password)
message.attach(MIMEText(message_body, 'html'))
smtp = smtplib.SMTP(host=SMTP_HOST, port=SMTP_PORT)
smtp.starttls()
smtp.login(SMTP_EMAIL, SMTP_PASSWORD)
smtp.send_message(message)
def generate_openflow_pipeline_action_h(fields):
skeleton = read_template(TEMPLATES_DIR + "/of1x_action.h.template")
code1 = ""
for field in fields:
if 'field' not in field:
code1 += "\tOF1X_AT_%(action_upper)s_%(header_upper)s,\n" % field
elif field['field'] != '__skip__':
code1 += "\tOF1X_AT_%(action_upper)s_FIELD_%(header_upper)s_%(field_upper)s,\n" % field
code2 = ""
for field in fields:
if 'field' not in field:
code2 += "\tOF12PAT_%(action_upper)s_%(header_upper)s,\n" % field
return skeleton % (code1, code2)
def get_prophet_daemon_resource_request(cfg):
ret = {"cpu-resource": 0, "mem-resource": 0}
if "qos-switch" not in cfg or cfg["qos-switch"] == "false":
logger.info(
"Ignore calculate prophet daemon resource usage since qos-switch set to false"
)
return ret
prophet_daemon_services = ["node-exporter", "job-exporter", "log-manager"]
prophet_source_path = os.path.join(
os.path.dirname(os.path.realpath(__file__)), "../../../src")
# {%- if cluster_cfg['cluster']['common']['qos-switch'] == "true" %}
start_match = r"{%-?\s*if\s*cluster_cfg\['cluster'\]\['common'\]\['qos-switch'\][^}]+%}"
end_match = r"{%-?\s*endif\s*%}" # {%- end %}
str_match = "{}(.*?){}".format(start_match, end_match)
regex = re.compile(str_match, flags=re.DOTALL)
for prophet_daemon in prophet_daemon_services:
deploy_template_path = os.path.join(
prophet_source_path,
"{0}/deploy/{0}.yaml.template".format(prophet_daemon))
if os.path.exists(deploy_template_path):
template = read_template(deploy_template_path)
match = regex.search(template)
if not match:
logger.warning(
"Could not find resource request for service %s",
prophet_daemon)
continue
resources = yaml.load(match.group(1), yaml.SafeLoader)["resources"]
if "requests" in resources:
ret["cpu-resource"] += parse_quantity(
resources["requests"].get("cpu", 0))
ret["mem-resource"] += parse_quantity(
resources["requests"].get("memory", 0)) / 1024 / 1024
elif "limits" in resources:
ret["cpu-resource"] += parse_quantity(resources["limits"].get(
"cpu", 0))
ret["mem-resource"] += parse_quantity(resources["limits"].get(
"memory", 0)) / 1024 / 1024
else:
logger.warning("Could not find resource request for PAI daemon %s",
prophet_daemon)
return ret
def generate_packet_c(fields):
skeleton = read_template(TEMPLATES_DIR + "/packet.c.template")
skeleton_attrs = copy.copy(fields[0])
header = fields[0]['header']
code1 = "#include <rofl/common/protocols/f%sframe.h>" % header
code2 = ""
for field in fields:
if 'field' in field and field['field'] != '__skip__':
code2 += GET_SKELETON % field
code2 += SET_SKELETON % field
if field.get('action') == 'push':
code2 += PUSH2_SKELETON % field
if field.get('action') == 'pop':
code2 += POP2_SKELETON % field
return skeleton % (code1, code2)
def generate_common_protocol_frame_h(fields):
skeleton = read_template(TEMPLATES_DIR + "/frame.h.template")
skeleton_attrs = copy.copy(fields[0])
skeleton_attrs['fields'] = ""
for field in fields:
skeleton_attrs['fields'] += "\t\tuint%(length)s_t %(field)s;\n" % field
skeleton_attrs['fields_getters'] = ""
for field in fields_iterator(fields):
skeleton_attrs['fields_getters'] += "\tuint%(length)s_t get_%(field)s();\n" % field
skeleton_attrs['fields_setters'] = ""
for field in fields_iterator(fields):
skeleton_attrs['fields_setters'] += "\tvoid set_%(field)s(uint%(length)s_t %(field)s);\n" % field
return skeleton % skeleton_attrs
def __copy_modified_files():
generate_file(ROFL_DIR + "/common/endianess_other.h", read_template(MODIFIED_DIR + "/endianess_other.h"))
generate_file(ROFL_DIR + "/common/Makefile.am", read_template(MODIFIED_DIR + "/rofl_common_Makefile.am"))
generate_file(ROFL_DIR + "/common/openflow/cofaction.h", read_template(MODIFIED_DIR + "/cofaction.h"))
generate_file(ROFL_DIR + "/common/openflow/cofaction.cc", read_template(MODIFIED_DIR + "/cofaction.cc"))
generate_file(ROFL_DIR + "/datapath/pipeline/openflow/openflow1x/pipeline/Makefile.am", read_template(MODIFIED_DIR + "/rofl_datapath_pipeline_Makefile.am"))
generate_file(ROFL_DIR + "/datapath/pipeline/openflow/openflow1x/pipeline/of1x_match.h", read_template(MODIFIED_DIR + "/of1x_match.h"))
generate_file(ROFL_DIR + "/datapath/pipeline/openflow/openflow1x/pipeline/of1x_match.c", read_template(MODIFIED_DIR + "/of1x_match.c"))
generate_file(ROFL_DIR + "/datapath/pipeline/platform/Makefile.am", read_template(MODIFIED_DIR + "/rofl_datapath_platform_Makefile.am"))
generate_file(ROFL_DIR + "/datapath/pipeline/platform/packet.h", read_template(MODIFIED_DIR + "/packet.h"))
def generate_packet_classifier_h(fields):
skeleton = read_template(TEMPLATES_DIR + "/packetclassifier.h.template")
skeleton_attrs = copy.copy(fields[0])
header = fields[0]['header']
skeleton_attrs['class'] = "\tclass f%sframe;\n" % header
skeleton_attrs['header_access'] = "\tvirtual rofl::f%sframe* %s (int idx = 0) const=0;\n" % (header, header)
skeleton_attrs['pop_operations'] = ""
for field in fields:
if field.get('action') == 'pop':
skeleton_attrs['pop_operations'] += "\tvirtual void pop_%s(uint16_t ether_type)=0;\n" % header
skeleton_attrs['push_operations'] = ""
for field in fields:
if field.get('action')== 'push':
skeleton_attrs['push_operations'] += "\tvirtual rofl::f%sframe* push_%s(uint16_t ether_type)=0;\n" % (header, header)
return skeleton % skeleton_attrs
def generate(class_type="classA", id_base=0):
os.makedirs(f"{utils.folder}/envs", exist_ok=True)
os.makedirs(f"{utils.folder}/params", exist_ok=True)
xml = read_template("body-templates/ant.template.xml")
conf = read_yaml("body-templates/ant.yaml")
data = conf[class_type]
write_xml(f"{utils.folder}/envs/{id_base}.mean.xml", data, xml)
for i in range(5):
data_clone = data.copy()
for key in data_clone:
data_clone[key] += np.random.normal(0, 0.01)
data_clone["_initial_z"] = data_clone["size_torso"] + 0.5
write_xml(f"{utils.folder}/envs/{i+id_base}.xml", data_clone, xml)
params = {
"size_torso": data_clone["size_torso"]
}
write_yaml(f"{utils.folder}/params/{i+id_base}.yml", params)
def create_scripts(mode="single"):
if False: # dynamically adjust the memory allocation
memory_needed = {
10: "2G",
20: "4G",
50: "5G",
100: "6G",
}
template_filename = f"script-templates/submit_{mode}.sh"
script_template = read_template(template_filename)
filename = f"scripts/{g_env_id}_submit_{mode}.sh"
data = {
"cwd": os.path.abspath(os.getcwd()),
"partition": args.partition,
"memory": args.memory,
"dataset": f"dataset/{g_env_id}",
}
write_script(filename, data, script_template)
return filename
def get_relations(run):
dicts = []
filepath = f'../templates/relation_overview_run{run}.csv'
collection_relation_question_dict, level_relation_dict = read_template(run)
for l, rels, in level_relation_dict.items():
l = int(l)
if l == 1:
l_name = 'all'
elif l == 2:
l_name = 'some'
elif l == 3:
l_name = 'few'
for r in rels:
d = dict()
if r == 'creative':
d['level'] = 'creative'
else:
d['level'] = l_name
d['relation'] = r
dicts.append(d)
to_csv(filepath, dicts, header = True)
def generate_common_protocol_frame_c(fields):
skeleton = read_template(TEMPLATES_DIR + "/frame.c.template")
skeleton_attrs = copy.copy(fields[0])
skeleton_attrs['field_printed_list'] = ""
for field in fields_iterator(fields):
skeleton_attrs['field_printed_list'] += "%(field)s[%%d] " % field
skeleton_attrs['fields_to_be_printed'] = ""
for field in fields_iterator(fields):
skeleton_attrs['fields_to_be_printed'] += "\t\t\tbe%(length)stoh(%(header)s_hdr->%(field)s),\n" % field
skeleton_attrs['fields_getters'] = ""
for field in fields_iterator(fields):
skeleton_attrs['fields_getters'] += FIELD_GETTER % field
skeleton_attrs['fields_setters'] = ""
for field in fields_iterator(fields):
skeleton_attrs['fields_setters'] += FIELD_SETTER % field
return skeleton % skeleton_attrs
def generate_openflow_pipeline_action_c(fields):
skeleton = read_template(TEMPLATES_DIR + "/of1x_action.c.template")
code1 = ""
for field in fields:
if 'field' not in field:
code1 += "\t\tcase OF1X_AT_%(action_upper)s_%(header_upper)s:\n" % field
elif field['field'] != '__skip__':
code1 += "\t\tcase OF1X_AT_%(action_upper)s_FIELD_%(header_upper)s_%(field_upper)s:\n" % field
code1 += "\t\t\taction->field.u%(length)s = field.u%(length)s&OF1X_%(masking)s_MASK;\n" % field
code1 += "\t\t\taction->ver_req.min_ver = OF_VERSION_12;\n"
code1 += "\t\t\tbreak;\n"
code2 = ""
for field in fields:
if 'field' not in field:
code2 += "\t\tcase OF1X_AT_%(action_upper)s_%(header_upper)s:\n" % field
code2 += "\t\t\tplatform_packet_%(action)s_%(header)s(pkt, action->field.u%(length)s);\n" % field
code2 += "\t\t\tpkt_matches->eth_type= platform_packet_get_eth_type(pkt);\n" % field
code2 += "\t\t\tpkt_matches->pkt_size_bytes = platform_packet_get_size_bytes(pkt);\n" % field
elif field['field'] != '__skip__':
code2 += "\t\tcase OF1X_AT_%(action_upper)s_FIELD_%(header_upper)s_%(field_upper)s:\n" % field
code2 += "\t\t\tplatform_packet_set_%(header)s_%(field)s(pkt, action->field.u%(length)s);\n" % field
code2 += "\t\t\tpkt_matches->%(header)s_%(field)s = action->field.u%(length)s;\n" % field
code2 += "\t\t\tbreak;\n"
code3 = ""
for field in fields:
if 'field' not in field:
code3 += """\t\tcase OF1X_AT_%(action_upper)s_%(header_upper)s: ROFL_PIPELINE_DEBUG_NO_PREFIX("%(action_upper)s_%(header_upper)s"); \n""" % field
elif field['field'] != '__skip__':
code3 += """\t\tcase OF1X_AT_%(action_upper)s_FIELD_%(header_upper)s_%(field_upper)s: \n""" % field
code3 += """\t\t\tROFL_PIPELINE_DEBUG_NO_PREFIX("%(action_upper)s_%(header_upper)s_%(field_upper)s: 0x%%x",action.field.u%(length)s); \n""" % field
code3 += "\t\t\tbreak;\n"
return skeleton % (code1, code2, code3)
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from utils import get_contacts, read_template, setup_smtp
from config import configs
names, emails = get_contacts('contacts.txt') # read contacts
content_template = read_template(configs['message_file'])
for name, email in zip(names, emails):
msg = MIMEMultipart() # create a message
# add in the actual person name to the message template
content = content_template.substitute(PERSON_NAME=name.title())
# setup the parameters of the message
msg['From'] = configs['address']
msg['To'] = email
msg['Subject'] = "This is TEST"
# add in the message body
msg.attach(MIMEText(content, 'plain'))
# send the message via the server set up earlier.
session = setup_smtp(host=configs['host'],
port=configs['port'],
address=configs['address'],
password=configs['password'])
session.send_message(msg)
def get_text_templates():
template_dir = os.path.join(os.path.dirname(__file__),'templates','stack')
templates = {}
templates['CMakeLists.txt'] = utils.read_template(os.path.join(template_dir,'CMakeLists.txt'))
templates['stack.xml'] = utils.read_template(os.path.join(template_dir,'stack.xml'))
return templates
def swift_variants(data: dict) -> str:
return template_variants(data)
def swift_text_styles(data: dict) -> str:
return template_text(data)
def swift_layer_styles(data: dict) -> str:
return template_layer(data)
compiler = pybars.Compiler()
template_assets = compiler.compile(
read_template(ROOT_DIR + '/../resources/templates/enums/assets.mustache'))
template_elements = compiler.compile(
read_template(ROOT_DIR +
'/../resources/templates/enums/elements.mustache'))
template_sections = compiler.compile(
read_template(ROOT_DIR +
'/../resources/templates/enums/sections.mustache'))
template_variants = compiler.compile(
read_template(ROOT_DIR +
'/../resources/templates/enums/variants.mustache'))
template_text = compiler.compile(
read_template(ROOT_DIR + '/../resources/templates/enums/text.mustache'))
template_layer = compiler.compile(
read_template(ROOT_DIR + '/../resources/templates/enums/layer.mustache'))
def process_enums(input_folder: str, app_dir: str, output_folder: str,
keep_files: str):
enum_src: [str] = []
enum_path: [str] = [] # so they can be removed later
keep_files = (keep_files == "1")
for dir_path, _, files in os.walk(input_folder):
for file in (file for file in files
if file.startswith("enums_") and file.endswith(".yaml")):
file_path = os.path.join(dir_path, file)
try:
src = read_template(file_path)
enum_src.append(src)
enum_path.append(file_path)
print("✅ found enum '" + file + "'")
except IOError:
print("🚨 found '" + file_path +
"' but it couldn't be loaded")
if not enum_src:
print("🚨 found no enums_*.yaml files, so nothing to process ðŸ˜")
data = parse(enum_src)
data["app_name"] = "App"
enums_assets = swift_assets(data)
enums_elements = swift_elements(data)
enums_sections = swift_sections(data)
enums_variants = swift_variants(data)
enums_text_styles = swift_text_styles(data)
enums_layer_styles = swift_layer_styles(data)
output_assets = os.path.join(output_folder,
"{0}Assets.swift".format(data["app_name"]))
output_elements = os.path.join(
output_folder, "{0}Elements.swift".format(data["app_name"]))
output_sections = os.path.join(
output_folder, "{0}Sections.swift".format(data["app_name"]))
output_variants = os.path.join(
output_folder, "{0}Variants.swift".format(data["app_name"]))
output_text = os.path.join(output_folder,
"{0}TextStyles.swift".format(data["app_name"]))
output_layer = os.path.join(
output_folder, "{0}LayerStyles.swift".format(data["app_name"]))
try:
write_file(output_assets, enums_assets)
except IOError:
print('Could not write enums assets file ' + output_assets)
try:
write_file(output_elements, enums_elements)
except IOError:
print('Could not write enums elements file ' + output_elements)
try:
write_file(output_sections, enums_sections)
except IOError:
print('Could not write enums sections file ' + output_sections)
try:
write_file(output_variants, enums_variants)
except IOError:
print('Could not write enums variants file ' + output_variants)
try:
write_file(output_text, enums_text_styles)
except IOError:
print('Could not write enums text identifier file ' + output_text)
try:
write_file(output_layer, enums_layer_styles)
except IOError:
print('Could not write enums layer identifier file ' + output_layer)
if not keep_files:
[os.remove(source_file) for source_file in enum_path]
def generate_common_protocol_makefile(fields):
skeleton = read_template(TEMPLATES_DIR + "/protocols_makefile.am.template")
header = fields[0]['header']
return skeleton % ("f%sframe.cc " % header, "f%sframe.h " % header)
def get_qt_text_templates(package, type):
template_common_dir = os.path.join(os.path.dirname(__file__),'templates','common')
template_dir = os.path.join(os.path.dirname(__file__),'templates',type)
templates = {}
templates['mainpage.dox'] = utils.read_template(os.path.join(template_common_dir,'mainpage.dox'))
if type == 'qt-ros-legacy':
templates['Makefile'] = utils.read_template(os.path.join(template_common_dir,'Makefile'))
templates['CMakeLists.txt'] = utils.read_template(os.path.join(template_dir,'CMakeLists.txt'))
templates['manifest.xml'] = utils.read_template(os.path.join(template_dir,'manifest.xml'))
templates[os.path.join('ui','main_window.ui')] = utils.read_template(os.path.join(template_dir,'ui','main_window.ui'))
templates[os.path.join('src','main.cpp')] = utils.read_template(os.path.join(template_dir,'src','main.cpp'))
templates[os.path.join('src','main_window.cpp')] = utils.read_template(os.path.join(template_dir,'src','main_window.cpp'))
templates[os.path.join('src','qnode.cpp')] = utils.read_template(os.path.join(template_dir,'src','qnode.cpp'))
templates[os.path.join('resources','images.qrc')] = utils.read_template(os.path.join(template_dir,'resources','images.qrc'))
templates[os.path.join('include',package,'main_window.hpp')] = utils.read_template(os.path.join(template_dir,'include','PACKAGE_NAME','main_window.hpp'))
templates[os.path.join('include',package,'qnode.hpp')] = utils.read_template(os.path.join(template_dir,'include','PACKAGE_NAME','qnode.hpp'))
return templates
def generate_experimental_action_h(fields):
skeleton = read_template(TEMPLATES_DIR + "/rofl_experimental_actions.h.template")
fields[0]['experimental_id'] = 3
return skeleton % fields[0]
def generate_bodies():
assert args.seed_bodies<100, "The way we combining real seeds only allow seed_bodies to be smaller than 100."
assert args.body_variation_range%10==0, "The way we combining real seeds only allow body_variation_range to be multiplication of 10, e.g. 10, 20, ... 90."
assert args.num_bodies%10==0, "The way we combining real seeds only allow num_bodies to be multiplication of 10, e.g. 10, 20, 30 ..."
real_seed = args.num_bodies*1000 + args.body_variation_range * 100 + args.seed_bodies # if I only apply seed_bodies, the first 20 of walker2d_30_10-v0 will be the same of walker2d_20_10-v0.
random.seed(real_seed)
output(f"Start generating bodies {args.num_bodies} with seed {real_seed}", 1)
# 1. Check templates
template_files = check_templates()
# 2. Create Folders
dataset_path = create_folder()
# 3. Generate variations and write body, param files
body_xml = read_template(template_files[0])
body_yaml = read_yaml(template_files[1])
file_list, param_list = [], []
for i in range(args.num_bodies):
data = {}
for key in body_yaml['variable']:
data[key] = body_yaml['variable'][key] * ((random.random() * 2 - 1) * args.body_variation_range / 100 + 1.0)
data[key] = significant_digits(data[key], 4)
for key in body_yaml['fixed']:
data[key] = body_yaml['fixed'][key]
for key in body_yaml['combination']:
data[key] = 0
for key1 in body_yaml['combination'][key]:
data[key] += data[key1]
data[key] = significant_digits(data[key], 4)
# Volume calculation
for part in body_yaml['part']:
data[f"volume_{part}"] = data[f"length_{part}"] * 3.14 * data[f"weight_{part}"] * data[f"weight_{part}"]
write_xml(f"{dataset_path}/bodies/{i}.xml", data, body_xml)
write_yaml(f"{dataset_path}/params/{i}.yaml", data)
file_list.append(f"bodies/{i}.xml")
param_list.append(f"params/{i}.yaml")
# 4. Write config file
env_id = f"{args.template_body}_{args.num_bodies}_{args.body_variation_range}-v{args.seed_bodies}"
config_yaml = {
"dataset_name": args.template_body,
"real_seed": real_seed,
"bodies": {
"total": len(file_list),
"files": file_list,
"params": param_list,
},
"gym_env": {
"env_id": env_id,
"filename": f"{args.template_body}.py",
"class": f"{args.template_body.capitalize()}Env",
}
}
write_yaml(f"{dataset_path}/config.yaml", config_yaml)
# 5. Copy over Gym Env Python file
shutil.copyfile(template_files[2], f"{dataset_path}/{args.template_body}.py")
return env_id, dataset_path
buckets["atoms"] = styles.unique_entries_dict(simplified, "atom")
buckets["elements"] = styles.unique_entries_dict(simplified, "element")
buckets["sections"] = styles.unique_entries_dict(simplified, "section")
buckets["variants"] = styles.unique_entries_dict(original, "variant")
data = {
"objects": simplified,
"grouped": buckets
}
return data
def swift_code(data):
return template_stylist(data, partials={'text_style': partial_text_style, 'color': color_partial})
def enum_report(data: dict) -> str:
enums = {
"atom_text": data["grouped"]["atoms"],
"element": data["grouped"]["elements"],
"section": data["grouped"]["sections"],
"variant": data["grouped"]["variants"]
}
return yaml.dump(enums)
root = ROOT_DIR + '/../resources/'
compiler = pybars.Compiler()
template_stylist = compiler.compile(read_template(root + 'text_swift_stylist.mustache'))
partial_text_style = compiler.compile(read_template(root + 'text_swift_stylist_textstyle.mustache'))
color_partial = compiler.compile(read_template(root + 'layer_swift_stylist_color.mustache'))
def generate_experimental_action_c(fields):
skeleton = read_template(TEMPLATES_DIR + "/rofl_experimental_actions.c.template")
return skeleton % fields[0]