def abstractToMatlab(event_list, output_file): UI().objectUI.showMessage("Starting to write mat file", "w") file_data = {} c = Config() if c.matlab: struct_type = c.config_data["matlab"]["default"] else: struct_type = UI().objectUI.chooseWindow( "Which struct type do you prefer? ", cte.MATLAB_TYPES_ADMITTED) arrays = [[], [], [], []] for ev in event_list: arrays[0].append(double(ev.x)) arrays[1].append(double(ev.y)) arrays[2].append(double(ev.pol)) arrays[3].append(double(secsToNsecs(ev.ts))) # 1 struct if struct_type == cte.MATLAB_TYPES_ADMITTED[0]: if c.matlab: struct_name = c.config_data["matlab"]["1 struct"]["struct_name"] data_names = c.config_data["matlab"]["1 struct"]["names"] else: struct_name = UI().objectUI.simpleInput( "What is the name of the struct?: ") data_names = UI().objectUI.multiInputsWindow( "How are the names of these parameters", cte.MATLAB_STRUCT_NAMES) file_data[struct_name] = {} for arr, name in zip(arrays, data_names): file_data[struct_name][name] = arr # Matrix nx4 elif struct_type == cte.MATLAB_TYPES_ADMITTED[1]: if c.matlab: struct_name = c.config_data["matlab"]["Matrix nx4"]["struct_name"] else: struct_name = UI().objectUI.simpleInput( "What is the name of the struct?: ") file_data[struct_name] = column_stack( (arrays[0], arrays[1], arrays[2], arrays[3])) # 4 structs (one for each event's parameter) elif struct_type == cte.MATLAB_TYPES_ADMITTED[2]: if c.matlab: struct_names = c.config_data["matlab"]["4 structs"]["names"] else: struct_names = UI().objectUI.multiInputsWindow( "How are the names of these structs", cte.MATLAB_STRUCT_NAMES) for arr, name in zip(arrays, struct_names): file_data[name] = arr UI().objectUI.showMessage( "Starting to save the data (no progress bar available)", "w") sio.savemat(output_file, file_data, oned_as="column") UI().objectUI.showMessage("Finishing writing the mat file", "c")
def abstractToRosbag(event_list, output_file): UI().objectUI.showMessage("Starting to write bag file", "w") bag = rosbag.Bag(output_file, "w") c = Config() if c.rosbag: topic = c.config_data["rosbag"]["topic"] else: topic = UI().objectUI.simpleInput( "Introduce the name of the topic where the events are going to be write: " ) num_progress = getNumProgress(len(event_list)) for i, event in enumerate(event_list): if i % num_progress == 0: UI().objectUI.sumProgress() e = _Event() e.x = event.x e.y = event.y e.polarity = event.pol e.ts = rospy.Time.from_sec(event.ts) bag.write(topic, e) UI().objectUI.sumProgress(True) bag.close() UI().objectUI.showMessage("Finishing writing the bag file", "c")
def __init__(self): self._config = Config() self._db_server = self._config.get_db_server() self._db_port = self._config.get_db_port() self._db_database = self._config.get_db_database() self._client = MongoClient(self._db_server, self._db_port) self._date_helper = DateHelper()
def create_two_layer_dense_model(): a = Config(config_dict=None, standard_key_list=TwoLayerDenseModel.standard_key_list) a.load_config(path=CONFIG_PATH + '/model/testTwoLayerDenseAttackConfig.json') model = TwoLayerDenseModel(config=a) return model
def __init__(self): logger.info("Initializing Trello crawler") factory_controller = FactoryController() self._config = Config() self._request_helper = RequestHelper() self._date_helper = DateHelper() self._trello_card_controller = factory_controller.get_trello_card()
def main(): UI("terminal") Config("src/config/config.json") # testFileAndType("data/aedat/aedat4/Cars_sequence.aedat4", "aedat") testTypes("aedat", "aedat")
def create_data(): conf = Config(standard_key_list=FISData.standard_key_list, config_dict=None) config = utils.load_json(file_path=CONFIG_PATH + '/data/testFisAttackDataConfig.json') config['FILE_PATH'] = DATASET1_PATH + '/Attack.csv' conf.load_config(path=None, json_dict=config) data = FISData(config=conf) data.load_data() return data
def __init__(self, config, data=None): super(Model, self).__init__(config) self.config = config if not self.config: self.config = Config(standard_key_list=[], config_dict={}) self.data = data self.input = None self.delta_state_output = None self.snapshot_var = [] self.save_snapshot_op = [] self.load_snapshot_op = []
def abstractToAedat(event_list, output_file): c = Config() if c.aedat: version = c.config_data["aedat"]["version"] else: version = UI().objectUI.chooseWindow("Choose a version: ", cte.AEDAT_ACCEPTED_VERSIONS) if version == cte.AEDAT_ACCEPTED_VERSIONS[0]: abstractToAedat2(event_list, output_file) elif version == cte.AEDAT_ACCEPTED_VERSIONS[1]: abstractToAedat3(event_list, output_file) elif version == cte.AEDAT_ACCEPTED_VERSIONS[2]: abstractToAedat4(event_list, output_file)
def __init__(self, db_name: str = 'data_main'): """Constructor for Mongo() :param db_name: default database overwrite, defaults to 'dat_main' :type db_name: str, optional """ config = Config() client = pymongo.MongoClient(config.MONGO_URI) logger.info(f"Connected to MongoDB with {config.MONGO_URI}") db = client.get_database(db_name) logger.info(f"Connected to database {db_name}") self._client = client self._db = db
def __init__(self, cost_fn=None, config=None): super().__init__(config=config) if self.config is None: self.config = Config(standard_key_list=['REAL_ENVIRONMENT_STATUS', 'CYBER_ENVIRONMENT_STATUS', 'TEST_ENVIRONMENT_STATUS']) self.config.config_dict['REAL_ENVIRONMENT_STATUS'] = 1 self.config.config_dict['CYBER_ENVIRONMENT_STATUS'] = 0 self.config.config_dict['TEST_ENVIRONMENT_STATUS'] = 2 self._test_data = SamplerData() self._cyber_data = SamplerData() self._real_data = SamplerData() self.cost_fn = cost_fn self.data = None self._env_status = None self.env_status = 1
def main(): # Args parse. input_file, output_file, input_type, output_type, use_config, config_path, ui_type = parseArguments( ) # Init config features. if use_config: Config(config_path) # Create UI. if ui_type == "graphic": UI("graphic") elif ui_type == "terminal": UI("terminal") # Init UI. try: UI().objectUI.initialWindow(convert, input_file, output_file, input_type, output_type, use_config, config_path) except Exception as e: UI().objectUI.errorWindow(e)
def rosbagToAbstract(input_file): UI().objectUI.showMessage("Starting to read bag file", "w") bag = rosbag.Bag(input_file) c = Config() if c.rosbag: topic = c.config_data["rosbag"]["topic"] else: topics = bag.get_type_and_topic_info().topics topic = UI().objectUI.chooseWindow( "Which is the topic that contains the events?: ", topics) event_list = [] num_progress = getNumProgress(bag.get_message_count(topic)) i = 0 for topic, msg, t in bag.read_messages(topics=topic): if i % num_progress == 0: UI().objectUI.sumProgress() i += 1 aux_list = [] if "EventArray" in str(type(msg)): # msg._type aux_list = msg.events else: aux_list.append(msg) for event in aux_list: event_list.append( Event(event.x, event.y, event.polarity, combine(event.ts.secs, event.ts.nsecs))) bag.close() UI().objectUI.sumProgress(True) UI().objectUI.showMessage("Finishing reading the bag file", "c") return event_list
def initialWindow(self, convert, input_file="", output_file=cte.OUTPUT_DEFAULT, input_type="", output_type="", use_config=False, config_path=cte.CONFIG_PATH): input_type_aux = input_type if input_type != "" else getExtension( input_file) output_type_aux = output_type if output_type != "" else getExtension( output_file) input_layout = [[ sg.Text("Input path", size=(12, 1), text_color="white", background_color=grey_green), sg.In(size=(35, 1), key="INPUT FILE", enable_events=True, default_text=input_file), sg.FileBrowse() ], [ sg.Text("Detected input type", size=(12, 1), text_color="white", background_color=grey_green), sg.Combo(cte.ADMITTED_TYPES, key="INPUT TYPE", default_value=input_type_aux, size=(10, 1)) ]] output_layout = [[ sg.Text("Output path", size=(12, 1), text_color="white", background_color=grey_green), sg.In(size=(35, 1), key="OUTPUT FILE", enable_events=True, default_text=output_file), sg.FileBrowse() ], [ sg.Text("Output type", size=(12, 1), text_color="white", background_color=grey_green), sg.Combo(cte.ADMITTED_TYPES, key="OUTPUT TYPE", default_value=output_type_aux, size=(10, 1), enable_events=True) ]] config_layout = [[ sg.Text("Config file path", size=(12, 1), text_color="white", background_color=grey_green), sg.In(size=(35, 1), key="CONFIG PATH", enable_events=True, default_text=config_path), sg.FileBrowse(), ], [ sg.Radio('Use config file', 'CONFIG RADIO', key="USE CONFIG", default=use_config, background_color=grey_green), sg.Radio('Do not use config file', 'CONFIG RADIO', key="NOT USE CONFIG", default=not use_config, background_color=grey_green) ]] buttons_layout = [[sg.Button("RESET"), sg.Button("CONVERT")]] layout = [[ input_layout, [[sg.HSeparator()]], output_layout, [[sg.HSeparator()]], config_layout, [[sg.HSeparator()]], buttons_layout ]] self.window = sg.Window("Event Converter", layout, size=(1000, 500), margins=(30, 35), resizable=True, font=(font_family, font_size), background_color=grey_green, button_color=dark_green) while True: event, values = self.window.read() if event == "Exit" or event == sg.WIN_CLOSED: break elif event == "RESET": self.window.Element('INPUT FILE').Update(value="") self.window.Element('INPUT TYPE').Update(value="") self.window.Element('OUTPUT FILE').Update(value="") self.window.Element('OUTPUT TYPE').Update(value="") self.window.Element('CONFIG PATH').Update(value="") self.window.Element('NOT USE CONFIG').Update(value=True) elif event == "INPUT FILE": ext = getExtension(values["INPUT FILE"]) if ext in cte.ADMITTED_TYPES: self.window.Element('INPUT TYPE').Update(value=ext) elif event == "OUTPUT FILE": ext = getExtension(values["OUTPUT FILE"]) if ext in cte.ADMITTED_TYPES: self.window.Element('OUTPUT TYPE').Update(value=ext) elif event == "OUTPUT TYPE": ext = values["OUTPUT TYPE"] file = values["OUTPUT FILE"] tam = len(getExtension(file)) if tam > 0: file = file[:-tam] if file[-1] == ".": file += ext else: file += "." + ext self.window.Element('OUTPUT FILE').Update(value=file) elif event == "CONVERT": i_f = values["INPUT FILE"] i_t = values["INPUT TYPE"] o_f = values["OUTPUT FILE"] o_t = values["OUTPUT TYPE"] u_c = values["USE CONFIG"] c_p = values["CONFIG PATH"] if not os.path.isfile(i_f): sg.popup_error("The input file does not exists") elif u_c and not os.path.isfile(c_p): sg.popup_error("The config file does not exists") elif i_t not in cte.ADMITTED_TYPES: sg.popup_error("The input type is not supported") elif o_t not in cte.ADMITTED_TYPES: sg.popup_error("The output type is not supported") else: if u_c: Config(c_p) convert(i_f, o_f, i_t, o_t) sg.popup('CONVERSION FINISHED!!!') self.window.close()
'/fixedOutputModelKey.json') def __init__(self, config): super(FixedOutputModel, self).__init__(config) def predict(self, sess=None, state=None): action = [0 for _ in range(self.config.config_dict['ACTION_SPACE'][0])] action[0] = self.config.config_dict['F1'] action[1] = self.config.config_dict['PROB_SAMPLE_ON_REAL'] action[2] = self.config.config_dict['PROB_TRAIN_ON_REAL'] return np.array(action) def reset(self): pass def update(self, *args, **kwargs): pass def print_log_queue(self, status): pass if __name__ == '__main__': from conf import CONFIG conf = Config(standard_key_list=FixedOutputModel.key_list) conf.load_config(path=CONFIG + '/baselineTrainerModelTestConfig.json') a = FixedOutputModel(config=conf) print(a.predict(state=1))
return loss, optimizer def update(self, sess, q_label, state, action): loss, _, grad = sess.run( fetches=[self.loss, self.optimize_loss, self.gradients], feed_dict={ self.q_label: q_label, self.state: state, self.action: action, self.is_training: True }) return loss, grad def eval_tensor(self, ): raise NotImplementedError pass if __name__ == '__main__': from src.config.config import Config from configuration import CONFIG_PATH a = Config(standard_key_list=DenseCritic.standard_key_list) a.load_config(path=CONFIG_PATH + '/testCriticConfig.json') critic = DenseCritic(config=a) with tf.Session() as sess: with sess.as_default(): tl.layers.initialize_global_variables(sess) critic.net.print_params() pass
sess = tf.get_default_session() sess.run(self.variables_initializer) self.ddpg_model.actor_optimizer.sync() self.ddpg_model.critic_optimizer.sync() sess.run(self.ddpg_model.target_init_updates) self.env_status = self.config.config_dict['REAL_ENVIRONMENT_STATUS'] super().init() def store_one_sample(self, state, next_state, action, reward, done, *arg, **kwargs): self.memory.append(obs0=state, obs1=next_state, action=action, reward=reward, terminal1=done) if __name__ == '__main__': from config import CONFIG con = Config(standard_key_list=DDPGModel.key_list) con.load_config(path=CONFIG + '/targetModelTestConfig.json') a = DDPGModel(config=con) sess = tf.Session() with sess.as_default(): a.init() a.load_snapshot() a.save_snapshot() a.save_model(path='/home/linsen/.tmp/ddpg-model.ckpt', global_step=1) a.load_model(file='/home/linsen/.tmp/ddpg-model.ckpt-1')
def __init__(self, *args, **kwargs): self.config = Config(os.environ['HOME'] + '/tower.yml') super().__init__(*args, **kwargs)
if env_status == self.config.config_dict['REAL_ENVIRONMENT_STATUS']: memory = self.real_data_memory elif env_status == self.config.config_dict['CYBER_ENVIRONMENT_STATUS']: memory = self.simulation_data_memory else: raise ValueError('Wrong Environment status') length = memory.nb_entries return length >= sample_count if __name__ == '__main__': from src.config.config import Config con = Config(standard_key_list=TrpoModel.key_list) con.load_config( path='/home/dls/CAP/intelligenttrainerframework/conf/modelNetworkConfig/targetModelTestConfig.json') a = TrpoModel(config=con, action_bound=([-1], [1]), obs_bound=([-1], [1])) import tensorflow as tf import os os.environ["CUDA_VISIBLE_DEVICES"] = '0' tf_config = tf.ConfigProto() tf_config.gpu_options.allow_growth = True sess = tf.Session(config=tf_config) with sess.as_default(): a.init() for i in range(500): a.env_status = a.config.config_dict['CYBER_ENVIRONMENT_STATUS']
n_units=self.config.config_dict['LSTM_DENSE_LAYER_2_UNIT'], act=tf.nn.tanh, name=name_prefix + 'LSTM_DENSE_LAYER_2') net = tl.layers.DenseLayer( layer=lstm_fc2, n_units=self.config.config_dict['MERGED_LAYER_1_UNIT'], act=tf.nn.relu, name=name_prefix + 'MERGED_DENSE_LAYER_1') net = tl.layers.DenseLayer( layer=net, n_units=self.config.config_dict['MERGED_LAYER_2_UNIT'], act=tf.nn.relu, name=name_prefix + 'MERGED_DENSE_LAYER_2') return net if __name__ == '__main__': from src.config.config import Config from configuration import CONFIG_PATH a = Config(config_dict=None, standard_key_list=LSTMCritic.standard_key_list) a.load_config(path=CONFIG_PATH + '/testLSTMCriticconfig.json') critic = LSTMCritic(config=a) with tf.Session() as sess: with sess.as_default(): tl.layers.initialize_global_variables(sess) critic.net.print_params() pass
def load_config(key_list, config_path): conf = Config(standard_key_list=key_list) conf.load_config(path=config_path) return conf
def __init__(self): logger.info("Initialize JWTHelper") config = Config() self._secret = config.get_jwt_secret() self._algorithm = config.get_jwt_algorithm() self._date_helper = DateHelper()
def load_config(key_list, config_path, update_dict): conf = Config(standard_key_list=key_list) conf.load_config(path=config_path) update_config_dict(config=conf, dict=update_dict) return conf
if attachments: if not isinstance(attachments, list): attachments = [attachments] self.logger.info("Found {} attachment. Processing".format(len(attachments))) for attachment in attachments: self.logger.info("Attaching \"{}\"".format(attachment)) self.add_attachment(msg, attachment) self.logger.debug("Attached \"{}\"".format(attachment)) return msg def send(self, msg): try: self.mailserver.send_message(msg) self.logger.info("Mail sent to the {} recipients".format(len(self.C["mail.recipients"]))) except SMTPServerDisconnected: self.logger.warning("Mail server disconnected. Reconnecting.") self.connect() self.send(msg) if __name__ == '__main__': from src.config.config import Config c = Config("mail.yaml") m = MailClient(config=c) mail = m.compose_mail("test mail", "this is a test mail. \n Please ignore the content", attachments=["attachments/1.txt", "attachments/2.txt"]) m.send(mail)
self.data_list = temp_data self.state_list = [] self.output_list = [] for sample in self.data_list: self.state_list.append(sample['STATE']) self.output_list.append(sample['OUTPUT']) pass self.state_list = np.array(self.state_list) self.output_list = np.array(self.output_list) def return_batch_data(self, index, size): return self.state_list[index * size:index * size + size], self.output_list[index * size:index * size + size] pass if __name__ == '__main__': from src.config.config import Config from src.configuration.standard_key_list import CONFIG_STANDARD_KEY_LIST from src.configuration import CONFIG_PATH conf = Config(standard_key_list=FISData.standard_key_list, config_dict=None) config = utils.load_json(file_path=CONFIG_PATH + '/testFisAttackDataConfig.json') config['FILE_PATH'] = DATASET1_PATH + '/Attack.csv' conf.load_config(path=None, json_dict=config) data = FISData(config=conf) data.load_data() data.shuffle_data()
self.torcs_vision_list.put(ob.img) img = self.return_latest_state_from_vision_list() return img, reward, done def reset(self, relauch=False): ob = self.torcs_env.reset(relaunch=relauch) self.torcs_vision_list.clear() while len(self.torcs_vision_list ) < self.config.config_dict['VISION_DATA_LENGTH']: self.torcs_vision_list.put(ob.img) img = self.return_latest_state_from_vision_list() return img def end(self): self.torcs_env.end() self.torcs_vision_list.clear() def return_latest_state_from_vision_list(self): right = len(self.torcs_vision_list) - 1 left = right - self.config.config_dict['VISION_DATA_LENGTH'] img = slice_queue(q=self.torcs_vision_list, left=left, right=right) return img if __name__ == '__main__': from src.config.config import Config from configuration import CONFIG_PATH a = Config(standard_key_list=TorcsEnvironment.standard_key_list) a.load_config(path=CONFIG_PATH + '/testTorcsEnvironmentConfig.json') env = TorcsEnvironment(config=a)
import logging from flask.json import jsonify from src.config.config import Config config = Config() DEFAULT_DATE_FORMAT = "%Y-%m-%d" DEFAULT_DATETIME_FORMAT = "%Y-%m-%d %H:%M.%S" def get_logger(name: str, file_name=None) -> logging.Logger: """Get Logger with proper config :param name: The name to use, usually __name__ :type name: str :return: The Logger :rtype: Logger """ # if config.ENV.lower() == 'local': # # If local, do not write to file # logging.basicConfig(level=config.LOGGING_LEVEL) # else: # # If not local, write to file # logging.basicConfig(filename='main.log', level=config.LOGGING_LEVEL) if file_name: logging.basicConfig(level=config.LOGGING_LEVEL, filename=file_name,
# TODO HOW TO ELEGANT CHANGE THIS if self.model and hasattr(self.model, 'print_log_queue') and callable( self.model.print_log_queue): self.model.print_log_queue(status=status) def reset(self): super().reset() self.model.reset() def get_trpo_step_count(self): from src.model.trpoModel.trpoModel import TrpoModel if isinstance(self.model, TrpoModel) is True: return self.model.step_count else: return None if __name__ == '__main__': from conf import CONFIG from src.model.ddpgModel.ddpgModel import DDPGModel conf = Config(standard_key_list=TargetAgent.key_list) conf.load_config(path=CONFIG + '/ddpgAgentTestConfig.json') ddog_con = Config(standard_key_list=DDPGModel.key_list) ddog_con.load_config(path=CONFIG + '/targetModelTestConfig.json') ddpg = DDPGModel(config=ddog_con) a = TargetAgent(config=conf, real_env=2, cyber_env=1, model=ddpg)
n_units=self.config.config_dict['DENSE_LAYER_2_UNIT'], act=tf.nn.leaky_relu, name=name_prefix + 'DENSE_LAYER_2') net = tl.layers.DenseLayer( layer=net, n_units=self.config.config_dict['OUTPUT_DIM'], act=tf.nn.softmax, name=name_prefix + 'OUTPUT_LAYER') return net def create_training_method(self): # weight_decay = tf.add_n([self.config.config_dict['L2'] * tf.nn.l2_loss(var) for var in self.var_list]) # loss = tf.reduce_mean(tf.square(self.label - self.net.outputs)) + weight_decay loss = tf.reduce_mean(tf.square(self.label - self.net.outputs)) optimizer = tf.train.AdamOptimizer( self.config.config_dict['LEARNING_RATE']) return loss, optimizer if __name__ == '__main__': from src.config.config import Config from src.configuration import CONFIG_PATH a = Config(config_dict=None, standard_key_list=DenseModel.standard_key_list) a.load_config(path=CONFIG_PATH + '/testDenseConfig.json') actor = DenseModel(config=a) pass
self.action_means: self.action_scalar.means, self.output_means: self.delta_scalar.means, self.output_vars: np.sqrt(self.delta_scalar.vars) }) return utl.squeeze_array(res, dim=1 + len(self.config.config_dict['STATE_SPACE'])) def init(self): sess = tf.get_default_session() sess.run(self.variables_initializer) super().init() if __name__ == '__main__': from conf import CONFIG conf = Config(standard_key_list=DynamicsEnvMlpModel.key_list) conf.load_config(path=CONFIG + '/dynamicsEnvMlpModelTestConfig.json') a = DynamicsEnvMlpModel(config=conf) state_input = np.zeros(shape=[10, 20]) action_input = np.zeros(shape=[10, 6]) sess = tf.Session() with sess.as_default(): a.init() a.load_snapshot() a.save_snapshot() a.save_model(path='/home/linsen/.tmp/model.ckpt', global_step=1) a.load_model(file='/home/linsen/.tmp/model.ckpt-1')