class NeutralNet(object): def __init__(self, layer_num): # 初始化一个全联接网络,所以不需要指定connections,layer_num是一个list,元素代表每层node数 self.layers = [] layer_count = len(layer_num) for i in range(layer_count): self.layers.append(Layer(i, layer_num[i])) # 初始化layer,给各layer填充节点 self.connections = Connections() for i in range(layer_count - 1): # 初始化connections,全连接网络 connections = [Connection(up_node, down_node) for up_node in self.layers[i].nodes for down_node in self.layers[i + 1].nodes] for conn in connections: self.connections.add_conn(conn) conn.up_node.add_downconns(conn) conn.down_node.add_upconns(conn) def train(self, train_data, labels, rate, N): for i in range(N): for x, y in zip(train_data, labels): self.train_one_sample(x, y, rate) def train_one_sample(self, x, y, rate): self.predict(x) self._cal_delta(y) self._update_weight(rate) def predict(self, x): self.layers[0].set_output(data_set=x) # 根据样本设置输入层 for layer in self.layers[1:]: # 计算各层输出 layer.cal_output() return [node.output for node in self.layers[-1].nodes] # 返回输出层的结果 def _cal_delta(self, y): for node, label in zip(self.layers[-1].nodes, y): node.cal_output_delta(label) for layer in self.layers[-2::-1]: # 反向传播误差 for node in layer.nodes: node.cal_hidden_delta() def _update_weight(self, rate): for layer in self.layers[1:]: for node in layer.nodes: node.update_bias(rate) # 更新节点bias for conn in node.down_conns: conn.update_weight(rate) # 更新连接weight def _cal_gradient(self): # 计算网络每层的梯度,留待检查梯度计算的正确性 for layer in self.layers[1:]: for node in layer.nodes: for conn in node.up_conns: conn.cal_gradient() def get_gradient(self, x, y): self.predict(x) self._cal_delta(y) self._cal_gradient() def dump(self): pass
def __init__(self) -> None: """Initialization""" self.return_code = -1 self.state = self.State(self.State.START.value + 1) self.authentication_thread = None self.metadata_thread = None self.realtime_situation_thread = None self.total_node_count = 0 self.loaded_node_count = 0 self.settings = get_settings() self.logger = setup_log("ComponentsInformationExample", self.settings.log_level) self.client = Connections( hostname=self.settings.hostname, logger=self.logger, authentication_on_open=self.authentication_on_open, authentication_on_message=self.authentication_on_message, authentication_on_error=self.authentication_on_error, authentication_on_close=self.authentication_on_close, metadata_on_open=self.metadata_on_open, metadata_on_message=self.metadata_on_message, metadata_on_error=self.metadata_on_error, metadata_on_close=self.metadata_on_close, realtime_situation_on_open=self.realtime_situation_on_open, realtime_situation_on_message=self.realtime_situation_on_message, realtime_situation_on_error=self.realtime_situation_on_error, realtime_situation_on_close=self.realtime_situation_on_close, ) self.messages = Messages(self.logger, self.settings.protocol_version)
def __init__(self): builder = Gtk.Builder() builder.add_objects_from_file('app.glade', ('winConnections', )) self.window = builder.get_object('winConnections') self.connection_list = builder.get_object('listConnections') self.ent_host = builder.get_object('entHost') self.ent_port = builder.get_object('entPort') self.ent_user = builder.get_object('entUser') self.ent_password = builder.get_object('entPassword') self.ent_database = builder.get_object('entDatabase') self.ent_remote_host = builder.get_object('entRemoteHost') self.ent_remote_user = builder.get_object('entRemoteUser') self.file_remote_key = builder.get_object('fileRemoteKey') self.connections = Connections() for connection in self.connections.get_connections(): self.connection_list.add(ListBoxRowWithData(connection)) if self.connections.count() == 0: self.on_add_connection(None) builder.connect_signals({ 'onAddConnection': self.on_add_connection, 'onListSelected': self.on_list_selected, 'onRemoveConnection': self.on_remove_connection, 'onSaveConnection': self.on_save_connection, 'onConnect': self.on_connect }) WindowManager.add_window(self.window) self.window.show_all()
def __init__(self): self._connections = Connections() self._connections_need_game = [] self._number_of_wait_ticks = 0 self._game_number = 0 self._games = {} self._games_to_kill = []
def __init__(self) -> None: """Initialization""" self.return_code = -1 self.state = self.State(self.State.START.value + 1) self.authentication_thread = None self.metadata_thread = None self.settings = get_settings() self.logger = setup_log("BuildingExample", self.settings.log_level) self.client = Connections( hostname=self.settings.hostname, logger=self.logger, authentication_on_open=self.authentication_on_open, authentication_on_message=self.authentication_on_message, authentication_on_error=self.authentication_on_error, authentication_on_close=self.authentication_on_close, metadata_on_open=self.metadata_on_open, metadata_on_message=self.metadata_on_message, metadata_on_error=self.metadata_on_error, metadata_on_close=self.metadata_on_close, ) self.messages = BuildingMessages(self.logger, self.settings.protocol_version)
def __init__(self) -> None: """Initialization""" self.return_code = -1 self.state = self.State(self.State.START.value + 1) self.new_user = dict( username="******", password="******", full_name="John Doe", role=AuthenticationMessages.Role.OPERATOR.value, updated_full_name="John J. Doe", updated_password="******", updated_role=AuthenticationMessages.Role.ADMIN.value, ) self.settings = get_settings() self.logger = setup_log("AuthenticationExample", self.settings.log_level) self.client = Connections( hostname=self.settings.hostname, logger=self.logger, authentication_on_open=self.authentication_on_open, authentication_on_message=self.authentication_on_message, authentication_on_error=self.authentication_on_error, authentication_on_close=self.authentication_on_close, ) self.authentication = AuthenticationMessages( self.logger, self.settings.protocol_version)
def tearDownClass(cls) -> None: db = Connections(table_name=None) with db.connect() as conn: company_email = '*****@*****.**' email = '*****@*****.**' conn.execute( f"delete from provider where email = '{company_email}'") conn.execute(f"delete from client where email = '{email}'")
def __init__(self) -> None: """Initialization""" self.return_code = -1 self.state = self.State(self.State.START.value + 1) # Network id needs to point to a valid network with at least one sink # online self.network_id = 777555 self.sink_ids = None # Uncomment the line below for setting appconfig for specific sinks only # self.sink_ids = [1, 101] # When running more than once for the same network, the diagnostics interval # or application data needs to changed as WNT server will not try to set the # application configuration if it is already the same. self.diagnostics_interval = 60 self.application_data = "00112233445566778899AABBCCDDEEFF" self.is_override_on = False self.authentication_thread = None self.metadata_thread = None self.realtime_situation_thread = None self.total_node_count = 0 self.loaded_node_count = 0 self.settings = get_settings() self.logger = setup_log( "ApplicationConfigurationExample", self.settings.log_level ) self.client = Connections( hostname=self.settings.hostname, logger=self.logger, authentication_on_open=self.authentication_on_open, authentication_on_message=self.authentication_on_message, authentication_on_error=self.authentication_on_error, authentication_on_close=self.authentication_on_close, metadata_on_open=self.metadata_on_open, metadata_on_message=self.metadata_on_message, metadata_on_error=self.metadata_on_error, metadata_on_close=self.metadata_on_close, realtime_situation_on_open=self.realtime_situation_on_open, realtime_situation_on_message=self.realtime_situation_on_message, realtime_situation_on_error=self.realtime_situation_on_error, realtime_situation_on_close=self.realtime_situation_on_close, ) self.messages = Messages(self.logger, self.settings.protocol_version)
def __init__(self): """Return a Customer object whose name is *name*.""" self.exchangeRate = ExchangeRate() self.weather = Weather() self.myconnection = Connections() self.myppointments = Appointments() #place holder for other option if (SHOW_DATA_AT == "spread_sheet"): self.data_visualizer = SreadSheet() self.reset()
def __init__(self, layer_num): # 初始化一个全联接网络,所以不需要指定connections,layer_num是一个list,元素代表每层node数 self.layers = [] layer_count = len(layer_num) for i in range(layer_count): self.layers.append(Layer(i, layer_num[i])) # 初始化layer,给各layer填充节点 self.connections = Connections() for i in range(layer_count - 1): # 初始化connections,全连接网络 connections = [Connection(up_node, down_node) for up_node in self.layers[i].nodes for down_node in self.layers[i + 1].nodes] for conn in connections: self.connections.add_conn(conn) conn.up_node.add_downconns(conn) conn.down_node.add_upconns(conn)
def __init__(self) -> None: """Initialization""" self.floor_plan_image_id = None self.floor_plan_image_thumbnail_id = None self.return_code = -1 self.state = self.State(self.State.START.value + 1) self.authentication_thread = None self.metadata_thread = None self.settings = get_settings() self.logger = setup_log("FloorPlanAreaExample", self.settings.log_level) self.client = Connections( hostname=self.settings.hostname, logger=self.logger, authentication_on_open=self.authentication_on_open, authentication_on_message=self.authentication_on_message, authentication_on_error=self.authentication_on_error, authentication_on_close=self.authentication_on_close, metadata_on_open=self.metadata_on_open, metadata_on_message=self.metadata_on_message, metadata_on_error=self.metadata_on_error, metadata_on_close=self.metadata_on_close, ) self.messages = Messages(self.logger, self.settings.protocol_version) script_path = os.path.dirname(os.path.realpath(__file__)) self.floor_plan_image_file_path = os.path.join( script_path, "assets/floor_plan.png" ) self.floor_plan_image_thumbnail_file_path = os.path.join( script_path, "assets/floor_plan_thumbnail.png" ) self.floor_plan_image_width = 8989 self.floor_plan_image_height = 4432 self.temp_floor_plan_image_file_path = ( self.floor_plan_image_file_path + ".tmp.png" ) self.temp_floor_plan_image_thumbnail_file_path = ( self.floor_plan_image_thumbnail_file_path + ".tmp.png" )
def lambda_handler(event, context, **kwargs): print( 'Starting function\n-------------------------------------------------') start_time = time() print('Loading account credentials') accounts = Connections() print('Gathering Usage and Limit Data') comparisons = {} for count, account in enumerate(accounts.env_names): comparisons[account] = AccountUsage(accounts.env_names[count], accounts.connection_list[count]) comparisons[account].get_limits() comparisons[account].compare() print('Generationg Email') email = EmailGenerator(comparisons) email.create_html email.send_email print('Creating Report') report = ReportGenerator(email.utilization) json = report.write_json() csv = report.write_csv print('Uplaoding Report to S3') push_json = report.push_to_s3(report.metadata) push_csv = report.push_to_s3(report.report) end_time = time() - start_time print('Time required: {0:.2f} s'.format(end_time))
def __init__(self, layers): self.connections = Connections() self.layers = [] layer_count = len(layers) node_count = 0 for i in range(layer_count): self.layer.append(Layer(i, layers[i])) for layer in range(layer_count - 1): connections = [ Connection(upstream_node, downstream_node) for upstream_node in self.layers[layer].nodex for downstream_node in self.layers[layer + 1].nodes[:-1] ] for conn in connections: self.connections.add_connection(conn) conn.downstream_node.append_upstream_connection(conn) conn.upstream_node.append_downstream_connection(conn)
def get_perimeter_sec_ids(perimeter_id): ''' Returns the list of security for a perimeter ''' ids = Connections.exec_sql("QUANT", ''' SELECT security_id FROM %s..perimeter_domain pd JOIN %s..domain_security ds on ds.domain_id = pd.domain_id WHERE pd.perimeter_id = %d ''' % (bq, bq, perimeter_id)) return [id[0] for id in ids] if ids is not None else None
def get_se_estimator_by_run_id(run_id, estimator_id, context): estimator = Connections.exec_sql('QUANT', ''' SELECT pd.parameter_name, pv.value FROM %s..param_desc pd, %s..param_value pv WHERE pv.parameter_id = pd.parameter_id AND pd.estimator_id = %d AND pv.run_id = %d ''' % (bq, bq, estimator_id, run_id) ) return pyData('init', date=[x[0] for x in estimator], value={context:[x[1] for x in estimator]})
def get_se_estimator(estimator, security_id, trading_destinations, day, context): estimator_id = get_estimator_id(estimator) job_id = get_job_id(estimator_id, context, security_id, trading_destinations) if job_id == None: print "no data for (security_id: %d, trading_destination: %s)" % (security_id, trading_destinations) job_id = get_job_id(estimator_id, context, security_id, 'NULL') if job_id == None: print "no data for (security_id: %d, trading_destination: NULL)" % (security_id) job_id = get_job_id(estimator_id, context, 'NULL', trading_destinations) if job_id != None: date_se = Connections.exec_sql('QUANT', ''' SELECT convert(varchar(8),max(er.stamp_date),112) FROM %s..estimator_runs er WHERE er.job_id = %d AND er.stamp_date <= '%s' AND estimator_id = %d ''' % (bq, job_id, day, estimator_id) ) if date_se != None and date_se[0][0] is not None: data_se = Connections.exec_sql('QUANT', ''' SELECT pd.parameter_name, pv.value FROM %s..estimator_runs er, %s..param_desc pd, %s..param_value pv WHERE er.estimator_id = %d AND pv.parameter_id = pd.parameter_id AND pd.estimator_id = er.estimator_id AND pv.run_id = er.run_id AND er.run_id = (SELECT max(run_id) FROM %s..estimator_runs WHERE stamp_date='%s' AND job_id=%d and estimator_id=%d) ''' % (bq, bq, bq, estimator_id, bq, date_se[0][0], job_id, estimator_id) ) data_se.append(('run_date', date_se[0][0])) return data_se else: raise ValueError("No SE entries for this date.") else: raise ValueError ("No job_id for estimator %s" % estimator)
def get_runs_by_perimeters_for_date(perimeter_ids, estimator_id, context_id, date): ids = Connections.exec_sql("QUANT", ''' SELECT ds.security_id, er.stamp_date, er.run_id, run_quality FROM %s..perimeter_domain pd JOIN %s..domain_security ds on ds.domain_id=pd.domain_id JOIN %s..association a on a.security_id=ds.security_id AND a.estimator_id= %d AND a.context_id= %d AND isnull(ds.trading_destination_id,0) = isnull(a.trading_destination_id,0) JOIN %s..estimator_runs er on er.job_id=a.job_id and er.context_id = a.context_id and er.estimator_id = a.estimator_id WHERE pd.perimeter_id in (%s) and stamp_date < '%s' GROUP by ds.security_id having er.run_id = max(er.run_id) ''' % (bq, bq, bq, estimator_id, context_id, bq, ','.join(map(str,perimeter_ids)), date) ) return ids
def connectionsFactory(*args, **kwargs): """ Create a :class:`~nupic.algorithms.connections.Connections` instance. :class:`TemporalMemory` subclasses may override this method to choose a different :class:`~nupic.algorithms.connections.Connections` implementation, or to augment the instance otherwise returned by the default :class:`~nupic.algorithms.connections.Connections` implementation. See :class:`~nupic.algorithms.connections.Connections` for constructor signature and usage. :returns: :class:`~nupic.algorithms.connections.Connections` instance """ return Connections(*args, **kwargs)
def _approximations(df, relations=[]): t1 = ut.out('approximating relational with mean, max, median...') df = df.copy() con_obj = Connections() g, sgs = con_obj.find_subgraphs(df, relations, verbose=False) approx_dict = {} sg_list = [] for i, sg in enumerate(sgs): if sg[3] > 0: # num edges > 0 sg_list.extend([(x, i) for x in sg[0]]) # give sg_id if len(sg_list) == 0: return approx_dict sg_df = pd.DataFrame(sg_list, columns=['com_id', 'sg_id']) df = df.merge(sg_df, how='left') df['sg_id'] = df['sg_id'].fillna(-1).apply(int) sg_mean = df.groupby('sg_id')['ind_pred'].mean().reset_index()\ .rename(columns={'ind_pred': 'sg_mean_pred'}) sg_median = df.groupby('sg_id')['ind_pred'].median().reset_index()\ .rename(columns={'ind_pred': 'sg_median_pred'}) sg_max = df.groupby('sg_id')['ind_pred'].max().reset_index()\ .rename(columns={'ind_pred': 'sg_max_pred'}) df = df.merge(sg_mean).merge(sg_median).merge(sg_max) filler = lambda x, c: x['ind_pred'] if x['sg_id'] == -1 else x[c] for col in ['sg_mean_pred', 'sg_median_pred', 'sg_max_pred']: cols = ['ind_pred', col, 'sg_id'] df[col] = df[cols].apply(filler, axis=1, args=(col,)) ut.time(t1) return df
def get_context_runs_by_perimeter(perimeter_ids, estimator_id, context_ids): ids = Connections.exec_sql('QUANT', ''' select ds.security_id, er.stamp_date, lr.last_run_id, lr.context_id from quant..perimeter_domain pd join quant..domain_security ds on ds.domain_id = pd.domain_id join quant..job j on ds.domain_id = j.domain_id and j.estimator_id = %d and ds.domain_id = j.domain_id join quant..perimeter_job pj on pj.job_id = j.job_id and pj.is_active = 1 join quant..last_run lr on lr.job_id = j.job_id and lr.context_id in (%s) join quant..estimator_runs er on er.run_id = lr.last_run_id where ds.trading_destination_id is null and pd.perimeter_id in (%s) ''' % (estimator_id, ','.join(map(str,context_ids)), ','.join(map(str,perimeter_ids))) ) return ids
def __init__(self, layers): self.connections = Connections() self.layers = [] layer_count = len(layers) node_count = 0 for i in range(layer_count): self.layers.append(Layer(i, layers[i])) for layer in range(layer_count - 1): connections = [ Connection(upstream_node, downstream_node) for upstream_node in self.layers[layer].node for downstream_node in self.layer[layer + 1].nodes[:-1]] for conn in connections: self.connections.add_connection(conn) conn.downstream_node.append_upstream_conneciont(conn) conn.upstream_node.append_downstream_connection(conn)
def get_job_id(estimator, context, s_security_id, s_trading_destination_id): if isinstance(estimator, int): estimator_id = estimator else: estimator_id = get_estimator_id(estimator) job_id = Connections.exec_sql("QUANT", "select job_id from %s..association WHERE \ estimator_id=%d \ AND context_id=(SELECT context_id FROM %s..context WHERE context_name = \'%s\' AND estimator_id=%d)\ AND security_id=%s\ AND trading_destination_id=%s" % (bq, estimator_id, bq, context, estimator_id, s_security_id, s_trading_destination_id) ) return job_id[0][0] if len(job_id) > 0 else None
class Games(): def __init__(self): self._connections = Connections() self._connections_need_game = [] self._number_of_wait_ticks = 0 self._game_number = 0 self._games = {} self._games_to_kill = [] def tick(self): self._kill_games() self._get_connections_looking() self._do_matchmaking() self._tick_games() def _kill_games(self): while self._games_to_kill: game = self._games_to_kill.pop() game_number = game.get_game_number() if game_number in self._games: game.end_game() del self._games[game_number] def _get_connections_looking(self): while True: connection = self._connections.get_connection_looking_and_remove() if connection == None: return if connection in self._connections_need_game: return self._connections_need_game.append(connection) def _do_matchmaking(self): if not self._connections_need_game: return for i in range(len(self._connections_need_game) - 1, -1, -1): if not self._connections_need_game[i].connected: self._connections_need_game.pop(i) if (self._number_of_wait_ticks > Settings.MATCHMAKER_WAIT_TICKS or len(self._connections_need_game) >= Settings.MATCHMAKER_MAXIMUM_PLAYERS): self._create_game() self._number_of_wait_ticks = 0 return self._number_of_wait_ticks += 1 def _create_game(self): game_number = self._get_next_game_number() game = Game(self, game_number) self._games[game_number] = game count = 0 while self._connections_need_game: game.add_connection(self._connections_need_game.pop()) count += 1 if count >= Settings.MATCHMAKER_MAXIMUM_PLAYERS: break game.start_game() def _tick_games(self): if not self._games: return for game in self._games.itervalues(): if game.get_game_over(): self._games_to_kill.append(game) else: game.tick() def _get_next_game_number(self): self._game_number += 1 while self._game_number in self._games: self._game_number += 1 if self._game_number > 999999999: self._game_number = 1 return self._game_number
class ComponentsInformationExample(object): """Main example class which is run""" class State(Enum): """State enumeration class""" START = auto() LOGIN = auto() # Started on authentication_on_open REALTIME_SITUATION_LOGIN = auto() WAIT_FOR_STARTUP_SITUATION = auto() QUERY_COMPONENTS_INFORMATION = auto() WAIT_FOREVER = auto() END = auto() def __init__(self) -> None: """Initialization""" self.return_code = -1 self.state = self.State(self.State.START.value + 1) self.authentication_thread = None self.metadata_thread = None self.realtime_situation_thread = None self.total_node_count = 0 self.loaded_node_count = 0 self.settings = get_settings() self.logger = setup_log("ComponentsInformationExample", self.settings.log_level) self.client = Connections( hostname=self.settings.hostname, logger=self.logger, authentication_on_open=self.authentication_on_open, authentication_on_message=self.authentication_on_message, authentication_on_error=self.authentication_on_error, authentication_on_close=self.authentication_on_close, metadata_on_open=self.metadata_on_open, metadata_on_message=self.metadata_on_message, metadata_on_error=self.metadata_on_error, metadata_on_close=self.metadata_on_close, realtime_situation_on_open=self.realtime_situation_on_open, realtime_situation_on_message=self.realtime_situation_on_message, realtime_situation_on_error=self.realtime_situation_on_error, realtime_situation_on_close=self.realtime_situation_on_close, ) self.messages = Messages(self.logger, self.settings.protocol_version) def send_request(self) -> None: """Send request""" if self.state == self.State.LOGIN: self.authentication_thread.socket.send( json.dumps( self.messages.message_login(self.settings.username, self.settings.password))) elif self.state == self.State.REALTIME_SITUATION_LOGIN: self.realtime_situation_thread.socket.send( json.dumps( self.messages.message_realtime_situation_login( self.messages.session_id))) elif self.state == self.State.QUERY_COMPONENTS_INFORMATION: self.metadata_thread.socket.send( json.dumps( self.messages.message_query_components_information())) def parse_response(self, message: str) -> bool: """Parse response Args: message (str): received message Returns: bool: True if response's request succeeded """ if self.state == self.State.LOGIN: return self.messages.parse_login(json.loads(message)) elif self.state == self.state.QUERY_COMPONENTS_INFORMATION: return self.messages.parse_query_components_information( json.loads(message)) def authentication_on_open(self, _websocket) -> None: """Websocket callback when the authentication websocket has been opened Args: websocket (Websocket): communication socket """ self.logger.info("Authentication socket open") self.send_request() def authentication_on_message(self, websocket, message: str) -> None: """Websocket callback when a new authentication message arrives Args: websocket (Websocket): communication socket message (str): received message """ self.on_message(websocket, message) def authentication_on_error(self, websocket, error: str) -> None: """Websocket callback when an authentication socket error occurs Args: websocket (Websocket): communication socket error (str): error message """ if websocket.keep_running: self.logger.error("Authentication socket error: {0}".format(error)) def authentication_on_close(self, _websocket) -> None: """Websocket callback when the authentication connection closes Args: _websocket (Websocket): communication socket """ self.logger.info("Authentication socket close") def metadata_on_open(self, _websocket) -> None: """Websocket callback when the metadata websocket has been opened Args: websocket (Websocket): communication socket """ self.logger.info("Metadata socket open") def metadata_on_message(self, websocket, message: str) -> None: """Websocket callback when a new metadata message arrives Args: websocket (Websocket): communication socket message (str): received message """ self.on_message(websocket, message) def metadata_on_error(self, websocket, error: str) -> None: """Websocket callback when a metadata socket error occurs Args: websocket (Websocket): communication socket error (str): error message """ if websocket.keep_running: self.logger.error("Metadata socket error: {0}".format(error)) def metadata_on_close(self, _websocket) -> None: """Websocket callback when the metadata connection closes Args: _websocket (Websocket): communication socket """ self.logger.warning("Metadata socket close") def realtime_situation_on_open(self, _websocket) -> None: """Websocket callback when the realtime situation websocket has been opened Args: websocket (Websocket): communication socket """ self.logger.info("Realtime situation socket open") def realtime_situation_on_message(self, _websocket, message: str) -> None: """Websocket callback when a new realtime situation message arrives Args: websocket (Websocket): communication socket message (str): received message """ if self.state == self.State.REALTIME_SITUATION_LOGIN: if not self.messages.parse_realtime_situation_login( json.loads(message)): self.logger.error("Test run failed. Exiting.") self.stop_connection_threads() else: self.state = self.State(self.state.value + 1) self.send_request() return elif self.state == self.state.WAIT_FOR_STARTUP_SITUATION: wnt_message = wnt_proto.Message() wnt_message.ParseFromString(message) if wnt_message.HasField("rtsituation_metadata"): self.total_node_count += wnt_message.rtsituation_metadata.node_count if wnt_message.HasField("source_address") and wnt_message.HasField( "network_id"): # Here it would be good to count distinct node count self.loaded_node_count += 1 if self.loaded_node_count == self.total_node_count: # Initial nodes' data loaded self.state = self.State(self.state.value + 1) self.send_request() else: wnt_message = wnt_proto.Message() wnt_message.ParseFromString(message) if wnt_message.HasField("gateway_info"): self.logger.info("gateway_info:\n{}".format( wnt_message.gateway_info)) if wnt_message.HasField("backend_component_info"): self.logger.info("backend_component_info:\n{}".format( wnt_message.backend_component_info)) def realtime_situation_on_error(self, websocket, error: str) -> None: """Websocket callback when realtime situation socket error occurs Args: websocket (Websocket): communication socket error (str): error message """ if websocket.keep_running: self.logger.error( "Realtime situation socket error: {0}".format(error)) def realtime_situation_on_close(self, _websocket) -> None: """Websocket callback when the realtime situation connection closes Args: _websocket (Websocket): communication socket """ self.logger.warning("Realtime situation socket close") def on_message(self, _websocket, message: str) -> None: """Called when authentication or metadata message is received Handles the state machine and closing of the communication threads Args: websocket (Websocket): communication socket message (str): received message """ if not self.parse_response(message): self.logger.error("Test run failed. Exiting.") self.stop_connection_threads() elif (not self.state == self.State.WAIT_FOR_STARTUP_SITUATION and not self.state == self.State.WAIT_FOREVER): self.state = self.State(self.state.value + 1) if self.state != self.State.END: self.send_request() else: self.return_code = 0 self.stop_connection_threads() def stop_connection_threads(self) -> None: """Stop all connection threads""" self.client.stop_realtime_situation_thread() self.client.stop_metadata_thread() self.client.stop_authentication_thread() def run(self) -> int: """Run method which starts and waits the communication thread(s) Returns: int: Process return code """ try: self.realtime_situation_thread = ( self.client.start_realtime_situation_thread()) self.metadata_thread = self.client.start_metadata_thread() self.authentication_thread = self.client.start_authentication_thread( ) # Run for 10 seconds self.realtime_situation_thread.join(10) self.metadata_thread.join(0) self.authentication_thread.join(0) self.return_code = 0 except: pass return self.return_code
def get_estimator_id(estimator): estimator_id = Connections.exec_sql("QUANT", "SELECT estimator_id FROM %s..estimator WHERE estimator_name = \'%s\'" % (bq, estimator)) return int(estimator_id[0][0]) if estimator_id is not None else None
class BuildingExample(object): """Main example class which is run""" class State(Enum): """State enumeration class""" START = auto() LOGIN = auto() # Started on authentication_on_open CREATE_BUILDING = auto() GET_BUILDINGS = auto() UPDATE_BUILDING = auto() GET_BUILDINGS_2 = auto() DELETE_BUILDING = auto() END = auto() def __init__(self) -> None: """Initialization""" self.return_code = -1 self.state = self.State(self.State.START.value + 1) self.authentication_thread = None self.metadata_thread = None self.settings = get_settings() self.logger = setup_log("BuildingExample", self.settings.log_level) self.client = Connections( hostname=self.settings.hostname, logger=self.logger, authentication_on_open=self.authentication_on_open, authentication_on_message=self.authentication_on_message, authentication_on_error=self.authentication_on_error, authentication_on_close=self.authentication_on_close, metadata_on_open=self.metadata_on_open, metadata_on_message=self.metadata_on_message, metadata_on_error=self.metadata_on_error, metadata_on_close=self.metadata_on_close, ) self.messages = BuildingMessages(self.logger, self.settings.protocol_version) def send_request(self) -> None: """Send request""" if self.state == self.State.LOGIN: self.authentication_thread.socket.send( json.dumps( self.messages.message_login(self.settings.username, self.settings.password))) elif self.state == self.State.CREATE_BUILDING: self.metadata_thread.socket.send( json.dumps( self.messages.message_create_building("New building"))) elif self.state.name.startswith(self.State.GET_BUILDINGS.name): self.metadata_thread.socket.send( json.dumps(self.messages.message_get_buildings())) elif self.state == self.State.UPDATE_BUILDING: self.metadata_thread.socket.send( json.dumps( self.messages.message_update_building( self.messages.new_building_id, "Updated building"))) elif self.state == self.State.DELETE_BUILDING: self.metadata_thread.socket.send( json.dumps( self.messages.message_delete_building( self.messages.new_building_id))) def parse_response(self, message: str) -> bool: """Parse response Args: message (str): received message Returns: bool: True if response's request succeeded """ if self.state == self.State.LOGIN: return self.messages.parse_login(json.loads(message)) elif self.state == self.State.CREATE_BUILDING: return self.messages.parse_create_building(json.loads(message)) elif self.state.name.startswith(self.State.GET_BUILDINGS.name): return self.messages.parse_get_buildings(json.loads(message)) elif self.state == self.State.UPDATE_BUILDING: return self.messages.parse_update_building(json.loads(message)) elif self.state == self.State.DELETE_BUILDING: return self.messages.parse_delete_building(json.loads(message)) def authentication_on_open(self, _websocket) -> None: """Websocket callback when the authentication websocket has been opened Args: websocket (Websocket): communication socket """ self.logger.info("Authentication socket open") self.send_request() def authentication_on_message(self, websocket, message: str) -> None: """Websocket callback when a new authentication message arrives Args: websocket (Websocket): communication socket message (str): received message """ self.on_message(websocket, message) def authentication_on_error(self, websocket, error: str) -> None: """Websocket callback when an authentication socket error occurs Args: websocket (Websocket): communication socket error (str): error message """ if websocket.keep_running: self.logger.error("Authentication socket error: {0}".format(error)) def authentication_on_close(self, _websocket) -> None: """Websocket callback when the authentication connection closes Args: _websocket (Websocket): communication socket """ self.logger.info("Authentication socket close") def metadata_on_open(self, _websocket) -> None: """Websocket callback when the metadata websocket has been opened Args: websocket (Websocket): communication socket """ self.logger.info("Metadata socket open") def metadata_on_message(self, websocket, message: str) -> None: """Websocket callback when a new metadata message arrives Args: websocket (Websocket): communication socket message (str): received message """ self.on_message(websocket, message) def metadata_on_error(self, websocket, error: str) -> None: """Websocket callback when a metadata socket error occurs Args: websocket (Websocket): communication socket error (str): error message """ if websocket.keep_running: self.logger.error("Metadata socket error: {0}".format(error)) def metadata_on_close(self, _websocket) -> None: """Websocket callback when the metadata connection closes Args: _websocket (Websocket): communication socket """ self.logger.warning("Metadata socket close") def on_message(self, _websocket, message: str) -> None: """Called when authentication or metadata message is received Handles the state machine and closing of the communication threads Args: websocket (Websocket): communication socket message (str): received message """ if not self.parse_response(message): self.logger.error("Test run failed. Exiting.") self.client.stop_metadata_thread() self.client.stop_authentication_thread() else: self.state = self.State(self.state.value + 1) if self.state != self.State.END: self.send_request() else: self.return_code = 0 self.client.stop_metadata_thread() self.client.stop_authentication_thread() def run(self) -> int: """Run method which starts and waits the communication thread(s) Returns: int: Process return code """ try: self.authentication_thread = self.client.start_authentication_thread( ) self.metadata_thread = self.client.start_metadata_thread() self.metadata_thread.join() self.authentication_thread.join() except: pass return self.return_code
def tearDown(self) -> None: db = Connections(table_name='provider') with db.connect() as conn: conn.execute( db.table.update().where(db.table.c.id == self._id).values(active=None) )
class FloorPlanAreaExample(object): """Main example class which is run""" class State(Enum): """State enumeration class""" START = auto() LOGIN = auto() # Started on authentication_on_open CREATE_BUILDING = auto() CREATE_FLOOR_PLAN = auto() SET_FLOOR_PLAN_IMAGE = auto() SET_FLOOR_PLAN_IMAGE_THUMBNAIL = auto() UPDATE_FLOOR_PLAN = auto() CREATE_AREA = auto() UPDATE_AREA = auto() GET_AREAS = auto() DELETE_AREA = auto() DELETE_FLOOR_PLAN = auto() DELETE_BUILDING = auto() END = auto() def __init__(self) -> None: """Initialization""" self.floor_plan_image_id = None self.floor_plan_image_thumbnail_id = None self.return_code = -1 self.state = self.State(self.State.START.value + 1) self.authentication_thread = None self.metadata_thread = None self.settings = get_settings() self.logger = setup_log("FloorPlanAreaExample", self.settings.log_level) self.client = Connections( hostname=self.settings.hostname, logger=self.logger, authentication_on_open=self.authentication_on_open, authentication_on_message=self.authentication_on_message, authentication_on_error=self.authentication_on_error, authentication_on_close=self.authentication_on_close, metadata_on_open=self.metadata_on_open, metadata_on_message=self.metadata_on_message, metadata_on_error=self.metadata_on_error, metadata_on_close=self.metadata_on_close, ) self.messages = Messages(self.logger, self.settings.protocol_version) script_path = os.path.dirname(os.path.realpath(__file__)) self.floor_plan_image_file_path = os.path.join( script_path, "assets/floor_plan.png" ) self.floor_plan_image_thumbnail_file_path = os.path.join( script_path, "assets/floor_plan_thumbnail.png" ) self.floor_plan_image_width = 8989 self.floor_plan_image_height = 4432 self.temp_floor_plan_image_file_path = ( self.floor_plan_image_file_path + ".tmp.png" ) self.temp_floor_plan_image_thumbnail_file_path = ( self.floor_plan_image_thumbnail_file_path + ".tmp.png" ) def send_request(self) -> None: """Send request""" if self.state == self.State.LOGIN: self.authentication_thread.socket.send( json.dumps( self.messages.message_login( self.settings.username, self.settings.password ) ) ) elif self.state == self.State.CREATE_BUILDING: self.metadata_thread.socket.send( json.dumps(self.messages.message_create_building("New building")) ) elif self.state == self.State.CREATE_FLOOR_PLAN: self.metadata_thread.socket.send( json.dumps( self.messages.message_create_floor_plan( self.messages.new_building_id, "New floor plan" ) ) ) elif self.state == self.state.SET_FLOOR_PLAN_IMAGE: self.metadata_thread.socket.send( json.dumps( self.messages.message_set_image( FileHelper.read_file_content_as_base64( self.floor_plan_image_file_path ) ) ) ) elif self.state == self.state.SET_FLOOR_PLAN_IMAGE_THUMBNAIL: self.metadata_thread.socket.send( json.dumps( self.messages.message_set_image( FileHelper.read_file_content_as_base64( self.floor_plan_image_thumbnail_file_path ) ) ) ) elif self.state == self.State.UPDATE_FLOOR_PLAN: self.metadata_thread.socket.send( json.dumps( self.messages.message_update_floor_plan( self.messages.new_floor_plan_id, image_id=self.floor_plan_image_id, image_thumbnail_id=self.floor_plan_image_thumbnail_id, latitude_lefttop=61.454823, longitude_lefttop=23.884526, altitude_lefttop=0, x_normcoord_lefttop=0.0748329808357999, y_normcoord_lefttop=0.203506328386351, latitude_righttop=61.454773, longitude_righttop=23.886096, altitude_righttop=0, x_normcoord_righttop=0.903860782456575, y_normcoord_righttop=0.203571943827163, latitude_leftbottom=61.454612, longitude_leftbottom=23.884503, altitude_leftbottom=0, x_normcoord_leftbottom=0.0747559429065484, y_normcoord_leftbottom=0.780014805319742, latitude_rightbottom=61.454562, longitude_rightbottom=23.88607, altitude_rightbottom=0, x_normcoord_rightbottom=0.904069882566427, y_normcoord_rightbottom=0.78039444527477, x_distance_point1=0.450065006833406, y_distance_point1=0.203192686229106, x_distance_point2=0.449649314572983, y_distance_point2=0.780260953915855, distance_in_m=25.1, level=0, image_width=self.floor_plan_image_width, image_height=self.floor_plan_image_height, ) ) ) elif self.state == self.state.CREATE_AREA: self.metadata_thread.socket.send( json.dumps( self.messages.message_create_area( self.messages.new_floor_plan_id, "new area" ) ) ) elif self.state == self.state.UPDATE_AREA: corner_point_llas = [ dict(latitude=61.454674940345, longitude=23.8857998957377, altitude=0), dict(latitude=61.4546694661163, longitude=23.8859745817991, altitude=0), dict(latitude=61.4545636984609, longitude=23.8859602235394, altitude=0), dict(latitude=61.4545691231503, longitude=23.8857853265751, altitude=0), ] self.metadata_thread.socket.send( json.dumps( self.messages.message_update_area( self.messages.new_area_id, self.messages.new_floor_plan_id, "updated name", 80, 0, 0, 200, corner_point_llas, ) ) ) elif self.state == self.State.GET_AREAS: self.metadata_thread.socket.send( json.dumps( self.messages.message_get_floor_plan_areas( self.messages.new_floor_plan_id ) ) ) elif self.state == self.State.DELETE_AREA: self.metadata_thread.socket.send( json.dumps( self.messages.message_delete_area( self.messages.new_area_id, self.messages.new_floor_plan_id ) ) ) elif self.state == self.State.DELETE_FLOOR_PLAN: self.metadata_thread.socket.send( json.dumps( self.messages.message_delete_floor_plan( self.messages.new_floor_plan_id ) ) ) elif self.state == self.State.DELETE_BUILDING: self.metadata_thread.socket.send( json.dumps( self.messages.message_delete_building(self.messages.new_building_id) ) ) def parse_response(self, message: str) -> bool: """Parse response Args: message (str): received message Returns: bool: True if response's request succeeded """ if self.state == self.State.LOGIN: return self.messages.parse_login(json.loads(message)) elif self.state == self.State.CREATE_BUILDING: return self.messages.parse_create_building(json.loads(message)) elif self.state == self.State.CREATE_FLOOR_PLAN: return self.messages.parse_create_floor_plan(json.loads(message)) elif self.state == self.State.SET_FLOOR_PLAN_IMAGE: parse_result = self.messages.parse_set_image(json.loads(message)) if parse_result: self.floor_plan_image_id = self.messages.image_id return parse_result elif self.state == self.State.SET_FLOOR_PLAN_IMAGE_THUMBNAIL: parse_result = self.messages.parse_set_image(json.loads(message)) if parse_result: self.floor_plan_image_thumbnail_id = self.messages.image_id return parse_result elif self.state == self.State.UPDATE_FLOOR_PLAN: return self.messages.parse_update_floor_plan(json.loads(message)) elif self.state == self.State.CREATE_AREA: return self.messages.parse_create_area(json.loads(message)) elif self.state == self.State.UPDATE_AREA: return self.messages.parse_update_area(json.loads(message)) elif self.state == self.State.GET_AREAS: return self.messages.parse_get_floor_plan_areas(json.loads(message)) elif self.state == self.State.DELETE_AREA: return self.messages.parse_delete_area(json.loads(message)) elif self.state == self.State.DELETE_FLOOR_PLAN: return self.messages.parse_delete_floor_plan(json.loads(message)) elif self.state == self.State.DELETE_BUILDING: return self.messages.parse_delete_building(json.loads(message)) def authentication_on_open(self, _websocket) -> None: """Websocket callback when the authentication websocket has been opened Args: websocket (Websocket): communication socket """ self.logger.info("Authentication socket open") self.send_request() def authentication_on_message(self, websocket, message: str) -> None: """Websocket callback when a new authentication message arrives Args: websocket (Websocket): communication socket message (str): received message """ self.on_message(websocket, message) def authentication_on_error(self, websocket, error: str) -> None: """Websocket callback when an authentication socket error occurs Args: websocket (Websocket): communication socket error (str): error message """ if websocket.keep_running: self.logger.error("Authentication socket error: {0}".format(error)) def authentication_on_close(self, _websocket) -> None: """Websocket callback when the authentication connection closes Args: _websocket (Websocket): communication socket """ self.logger.info("Authentication socket close") def metadata_on_open(self, _websocket) -> None: """Websocket callback when the metadata websocket has been opened Args: websocket (Websocket): communication socket """ self.logger.info("Metadata socket open") def metadata_on_message(self, websocket, message: str) -> None: """Websocket callback when a new metadata message arrives Args: websocket (Websocket): communication socket message (str): received message """ self.on_message(websocket, message) def metadata_on_error(self, websocket, error: str) -> None: """Websocket callback when a metadata socket error occurs Args: websocket (Websocket): communication socket error (str): error message """ if websocket.keep_running: self.logger.error("Metadata socket error: {0}".format(error)) def metadata_on_close(self, _websocket) -> None: """Websocket callback when the metadata connection closes Args: _websocket (Websocket): communication socket """ self.logger.warning("Metadata socket close") def on_message(self, _websocket, message: str) -> None: """Called when authentication or metadata message is received Handles the state machine and closing of the communication threads Args: websocket (Websocket): communication socket message (str): received message """ if not self.parse_response(message): self.logger.error("Test run failed. Exiting.") self.client.stop_metadata_thread() self.client.stop_authentication_thread() else: self.state = self.State(self.state.value + 1) if self.state != self.State.END: self.send_request() else: self.return_code = 0 self.client.stop_metadata_thread() self.client.stop_authentication_thread() def run(self) -> int: """Run method which starts and waits the communication thread(s) Returns: int: Process return code """ try: self.authentication_thread = self.client.start_authentication_thread() self.metadata_thread = self.client.start_metadata_thread() self.metadata_thread.join() self.authentication_thread.join() except: pass return self.return_code
from connections import Connections __dir__ = os.path.dirname(__file__) config = yaml.load(open(os.path.join(__dir__, "../default_config.yaml"))) try: config.update(yaml.load(open(os.path.join(__dir__, "../config.yaml")))) except IOError: # is ok if we do not have config.yaml pass logging.basicConfig(filename=config['KILLER_LOG_PATH'], level=logging.INFO, format='%(asctime)s pid:%(process)d %(message)s') logging.info("Started killer process, with limit %s", config['QUERY_TIME_LIMIT']) conn = Connections(config) cur = conn.replica.cursor() try: cur.execute('SHOW PROCESSLIST') queries = cur.fetchall() logging.info("Found %s queries running", len(queries)) to_kill = [ q for q in queries if q[5] > config['QUERY_TIME_LIMIT'] and q[4] != 'Sleep' ] logging.info("Found %s queries to kill", len(to_kill)) for q in to_kill: try: cur.execute('KILL QUERY %s', q[0]) logging.info("Killed query with thread_id:%s" % q[0])
class Network(object): def __init__(self, layers): self.connections = Connections() self.layers = [] layer_count = len(layers) node_count = 0 for i in range(layer_count): self.layers.append(Layer(i, layers[i])) for layer in range(layer_count - 1): connections = [ Connection(upstream_node, downstream_node) for upstream_node in self.layers[layer].node for downstream_node in self.layer[layer + 1].nodes[:-1]] for conn in connections: self.connections.add_connection(conn) conn.downstream_node.append_upstream_conneciont(conn) conn.upstream_node.append_downstream_connection(conn) def train(self, labels, data_set, rate, iteration): for i in range(iteration): for d in range(len(data_set)): self.train_one_sample(labels[d], data_set(d), rate) def train_one_sample(self, label, sample, rate): self.predict(sample) self.calc_delta(label) self.update_weight(rate) def calc_delta(self, label): output_nodes = self.layers[-1].nodes for i in range(len(label)): output_nodes[i].calc_output_layer_delta(label[i]) for layer in self.layers[-2::-1]: for node in layer.nodes: node.calc_hidden_layer_delta() def update_weight(self, rate): for layer in self.layers[:-1]: for node in layer.nodes: for conn in node.downstream: conn.update_weight(rate) def calc_gradient(self): for layer in self.layers[:-1]: for node in layer.nodes: for conn in node.downstream: conn.calc_gradient() def get_gradient(self, label, sample): self.predict(sample) self.calc_delta(label) self.calc_gradient() def predict(self, sample): self.layers[0].set_output(sample) for i in range(1, len(self.layers)): self.layers[i].calc_output() return map( lambda node: node.output, self.layers[-1].nodes[:-1]) def dump(self): for layer in self.layers: layer.dump()
class AuthenticationExample(object): """Main example class which is run""" class State(Enum): """State enumeration class""" START = auto() LOGIN = auto() # Started on authentication_on_open QUERY_USERS = auto() CREATE_USER = auto() QUERY_USERS_2 = auto() UPDATE_USER = auto() QUERY_USERS_3 = auto() DELETE_USER = auto() QUERY_USERS_4 = auto() END = auto() def __init__(self) -> None: """Initialization""" self.return_code = -1 self.state = self.State(self.State.START.value + 1) self.new_user = dict( username="******", password="******", full_name="John Doe", role=AuthenticationMessages.Role.OPERATOR.value, updated_full_name="John J. Doe", updated_password="******", updated_role=AuthenticationMessages.Role.ADMIN.value, ) self.settings = get_settings() self.logger = setup_log("AuthenticationExample", self.settings.log_level) self.client = Connections( hostname=self.settings.hostname, logger=self.logger, authentication_on_open=self.authentication_on_open, authentication_on_message=self.authentication_on_message, authentication_on_error=self.authentication_on_error, authentication_on_close=self.authentication_on_close, ) self.authentication = AuthenticationMessages( self.logger, self.settings.protocol_version) def send_request(self, websocket) -> None: """Send request Args: websocket (Websocket): communication socket """ if self.state.name.startswith(self.State.LOGIN.name): websocket.send( json.dumps( self.authentication.message_login(self.settings.username, self.settings.password))) elif self.state.name.startswith(self.State.QUERY_USERS.name): websocket.send( json.dumps(self.authentication.message_query_users())) elif self.state.name.startswith(self.State.CREATE_USER.name): websocket.send( json.dumps( self.authentication.message_create_user( username=self.new_user["username"], password=self.new_user["password"], full_name=self.new_user["full_name"], role=self.new_user["role"], ))) elif self.state.name.startswith(self.State.UPDATE_USER.name): websocket.send( json.dumps( self.authentication.message_update_user( username=self.new_user["username"], new_password=self.new_user["updated_password"], new_full_name=self.new_user["updated_full_name"], new_role=self.new_user["updated_role"], ))) elif self.state.name.startswith(self.State.DELETE_USER.name): websocket.send( json.dumps( self.authentication.message_delete_user( username=self.new_user["username"]))) def parse_response(self, message: str) -> bool: """Parse response Args: message (str): received message Returns: bool: True if response's request succeeded """ if self.state.name.startswith(self.State.LOGIN.name): if not self.authentication.parse_login(json.loads(message)): return False elif self.state.name.startswith(self.State.QUERY_USERS.name): if not self.authentication.parse_query_users(json.loads(message)): return False elif self.state.name.startswith(self.State.CREATE_USER.name): if not self.authentication.parse_create_user(json.loads(message)): return False elif self.state.name.startswith(self.State.UPDATE_USER.name): if not self.authentication.parse_update_user(json.loads(message)): return False elif self.state.name.startswith(self.State.DELETE_USER.name): if not self.authentication.parse_delete_user(json.loads(message)): return False return True def authentication_on_open(self, websocket) -> None: """Websocket callback when the authentication websocket has been opened Args: websocket (Websocket): communication socket """ self.logger.info("Socket open") self.send_request(websocket) def authentication_on_message(self, websocket, message: str) -> None: """Websocket callback when a new authentication message arrives Args: websocket (Websocket): communication socket message (str): received message """ if not self.parse_response(message): self.logger.error("Example run failed. Exiting.") self.client.stop_authentication_thread() else: self.state = self.State(self.state.value + 1) if self.state != self.State.END: self.send_request(websocket) else: self.return_code = 0 self.client.stop_authentication_thread() def authentication_on_error(self, websocket, error: str) -> None: """Websocket callback when an authentication socket error occurs Args: _websocket (Websocket): communication socket error (str): error message """ if websocket.keep_running: self.logger.error("Socket error: {0}".format(error)) def authentication_on_close(self, _websocket) -> None: """Websocket callback when the authentication connection closes Args: _websocket (Websocket): communication socket """ self.logger.info("Socket close") def run(self) -> int: """Run method which starts and waits the communication thread(s) Returns: int: Process return code """ try: self.client.start_authentication_thread().join() except: pass return self.return_code
class Network(object): def __init__(self, layers): self.connections = Connections() self.layers = [] layer_count = len(layers) node_count = 0 for i in range(layer_count): self.layer.append(Layer(i, layers[i])) for layer in range(layer_count - 1): connections = [ Connection(upstream_node, downstream_node) for upstream_node in self.layers[layer].nodex for downstream_node in self.layers[layer + 1].nodes[:-1] ] for conn in connections: self.connections.add_connection(conn) conn.downstream_node.append_upstream_connection(conn) conn.upstream_node.append_downstream_connection(conn) def train(self, labels, data_set, rate, epoch): for i in range(epoch): for d in range(len(data_set)): self.tran_one_sample(labels[d], data_set[d], rate) def train_one_sample(self, lable, sample, rate): self.predict(sample) self.calc_dalta(lable) self.update_weight(rate) def calc_delta(self, label): output_nodes = self.layers[-1].nodes for i in range(len(label)): output_nodes[i].calc_output_layer_delta(label[i]) for layer in self.layers[-2::-1]: for node in layer.nodes: node.calc_hidden_layer_delta() def update_weight(self, rate): for layer in self.layers[:-1]: for node in layer.nodes: for conn in node.downstream: conn.update_weight(rate) def calc_gradient(self): for layer in self.layer[:-1]: for node in layer.nodes: for conn in node.downstream: conn.calc_gradient() def predict(self, sample): self.layers[0].set_output(sample) for i in range(1, len(self.layers)): self.layers[i].calc_output() return [ m for m in map(lambda node: node.output, self.layers[-1].nodes[:-1]) ] def dump(self): for layer in self.layers: layer.dump()
def init(sender, signal): global conn conn = Connections(celery.conf) celery_log.info("Initialized lazy loaded connections")
def _spread(df, col='ind_pred', relations=[]): """This'll give some post-hoc test-set analysis, when running this, keep track of the test sets that improved using relational modeling, then average those test set statistics together to compare to the test sets that did not improve.""" t1 = ut.out('computing subgraph statistics...') con_obj = Connections() gids = [r[2] for r in relations] g, sgs = con_obj.find_subgraphs(df, relations, verbose=False) spread_dict = {} sg_list = [] for i, sg in enumerate(sgs): if sg[3] > 0: # num edges > 0 sg_list.extend([(x, i) for x in sg[0]]) # give sg_id if len(sg_list) == 0: return spread_dict sg_df = pd.DataFrame(sg_list, columns=['com_id', 'sg_id']) df = df.merge(sg_df, how='left') df['sg_id'] = df['sg_id'].fillna(-1).apply(int) p, r, ts = precision_recall_curve(df['label'], df[col]) aupr = average_precision_score(df['label'], df[col]) mp = 1.0 - aupr corrects = [] step = int(len(ts) / 100) if len(ts) > 100 else 1 for i in range(0, len(ts), step): t = ts[i] df['pred'] = np.where(df[col] > t, 1, 0) correct = df['pred'] == df['label'] corrects.append(correct.apply(int)) total_corrects = [sum(x) for x in zip(*corrects)] df['correct'] = total_corrects # extract bottom x% data df = df.sort_values('correct', ascending=False) ndx = len(df) - int(len(df) * mp) qfs = df[df['label'] == 1] qfo = df[df['label'] == 0] qf1, qf2 = df[ndx:], df[:ndx] qf1s = qf1[qf1['label'] == 1] # low performers qf1o = qf1[qf1['label'] == 0] # low performers qf2s = qf2[qf2['label'] == 1] # high performers qf2o = qf2[qf2['label'] == 0] # high performers spread_dict['spam_mean'] = round(qfs['ind_pred'].mean(), 4) spread_dict['spam_median'] = round(qfs['ind_pred'].median(), 4) spread_dict['ham_mean'] = round(qfo['ind_pred'].mean(), 4) spread_dict['ham_median'] = round(qfo['ind_pred'].median(), 4) for nm, temp_df in [('bot_spam', qf1s), ('bot_ham', qf1o), ('top_spam', qf2s), ('top_ham', qf2o)]: wf = temp_df[(temp_df[gids] != -1).any(axis=1)] sg_mean = wf.groupby('sg_id')['ind_pred'].mean().reset_index()\ .rename(columns={'ind_pred': 'sg_mean'}) sg_std = wf.groupby('sg_id')['ind_pred'].std().reset_index()\ .rename(columns={'ind_pred': 'sg_std'}) sg_median = wf.groupby('sg_id')['ind_pred'].median().reset_index()\ .rename(columns={'ind_pred': 'sg_median'}) sg_min = wf.groupby('sg_id')['ind_pred'].min().reset_index()\ .rename(columns={'ind_pred': 'sg_min'}) sg_max = wf.groupby('sg_id')['ind_pred'].max().reset_index()\ .rename(columns={'ind_pred': 'sg_max'}) wf = wf.merge(sg_mean).merge(sg_std).merge(sg_median)\ .merge(sg_min).merge(sg_max) wf['sg_spread'] = wf['sg_max'] - wf['sg_min'] spread_dict[nm + '_sg_mean'] = round(np.mean(wf['sg_mean']), 4) spread_dict[nm + '_sg_std'] = round(np.mean(wf['sg_std']), 4) spread_dict[nm + '_sg_median'] = round(np.mean(wf['sg_median']), 4) spread_dict[nm + '_sg_min'] = round(np.mean(wf['sg_min']), 4) spread_dict[nm + '_sg_max'] = round(np.mean(wf['sg_max']), 4) spread_dict[nm + '_sg_spread'] = round(np.mean(wf['sg_spread']), 4) ut.time(t1) return spread_dict
__dir__ = os.path.dirname(__file__) config = yaml.load(open(os.path.join(__dir__, "../default_config.yaml"))) try: config.update(yaml.load(open(os.path.join(__dir__, "../config.yaml")))) except IOError: # is ok if we do not have config.yaml pass logging.basicConfig( filename=config['KILLER_LOG_PATH'], level=logging.INFO, format='%(asctime)s pid:%(process)d %(message)s' ) logging.info("Started killer process, with limit %s", config['QUERY_TIME_LIMIT']) conn = Connections(config) cur = conn.replica.cursor() try: cur.execute('SHOW PROCESSLIST') queries = cur.fetchall() logging.info("Found %s queries running", len(queries)) to_kill = [q for q in queries if q[5] > config['QUERY_TIME_LIMIT'] and q[4] != 'Sleep'] logging.info("Found %s queries to kill", len(to_kill)) for q in to_kill: try: cur.execute('KILL QUERY %s', q[0]) logging.info("Killed query with thread_id:%s" % q[0]) except pymysql.InternalError as e: if e.args[0] == 1094: # Error code for 'no such thread'
def __init__(self, wt): Layers.conn = Connections(wt)
class ApplicationConfigurationExample(object): """Main example class which is run""" class State(Enum): """State enumeration class""" START = auto() LOGIN = auto() # Started on authentication_on_open REALTIME_SITUATION_LOGIN = auto() WAIT_FOR_STARTUP_SITUATION = auto() SET_APPLICATION_CONFIGURATION = auto() WAIT_FOR_APPLICATION_CONFIGURATION = auto() END = auto() def __init__(self) -> None: """Initialization""" self.return_code = -1 self.state = self.State(self.State.START.value + 1) # Network id needs to point to a valid network with at least one sink # online self.network_id = "777555" # When running more than once for the same network, the diagnostics interval # or application data needs to changed as WNT server will not try to set the # application configuration if it is already the same. self.diagnostics_interval = 30 self.application_data = "00112233445566778899AABBCCDDEEFF" self.authentication_thread = None self.metadata_thread = None self.realtime_situation_thread = None self.total_node_count = 0 self.loaded_node_count = 0 self.settings = get_settings() self.logger = setup_log("ApplicationConfigurationExample", self.settings.log_level) self.client = Connections( hostname=self.settings.hostname, logger=self.logger, authentication_on_open=self.authentication_on_open, authentication_on_message=self.authentication_on_message, authentication_on_error=self.authentication_on_error, authentication_on_close=self.authentication_on_close, metadata_on_open=self.metadata_on_open, metadata_on_message=self.metadata_on_message, metadata_on_error=self.metadata_on_error, metadata_on_close=self.metadata_on_close, realtime_situation_on_open=self.realtime_situation_on_open, realtime_situation_on_message=self.realtime_situation_on_message, realtime_situation_on_error=self.realtime_situation_on_error, realtime_situation_on_close=self.realtime_situation_on_close, ) self.messages = Messages(self.logger, self.settings.protocol_version) def send_request(self) -> None: """Send request""" if self.state == self.State.LOGIN: self.authentication_thread.socket.send( json.dumps( self.messages.message_login(self.settings.username, self.settings.password))) elif self.state == self.State.REALTIME_SITUATION_LOGIN: self.realtime_situation_thread.socket.send( json.dumps( self.messages.message_realtime_situation_login( self.messages.session_id))) elif self.state == self.State.SET_APPLICATION_CONFIGURATION: self.metadata_thread.socket.send( json.dumps( self.messages.message_set_app_config( self.network_id, self.diagnostics_interval, self.application_data, ))) def parse_response(self, message: str) -> bool: """Parse response Args: message (str): received message Returns: bool: True if response's request succeeded """ if self.state == self.State.LOGIN: return self.messages.parse_login(json.loads(message)) elif self.state == self.State.SET_APPLICATION_CONFIGURATION: return self.messages.parse_set_app_config(json.loads(message)) def authentication_on_open(self, _websocket) -> None: """Websocket callback when the authentication websocket has been opened Args: websocket (Websocket): communication socket """ self.logger.info("Authentication socket open") self.send_request() def authentication_on_message(self, websocket, message: str) -> None: """Websocket callback when a new authentication message arrives Args: websocket (Websocket): communication socket message (str): received message """ self.on_message(websocket, message) def authentication_on_error(self, websocket, error: str) -> None: """Websocket callback when an authentication socket error occurs Args: websocket (Websocket): communication socket error (str): error message """ if websocket.keep_running: self.logger.error("Authentication socket error: {0}".format(error)) def authentication_on_close(self, _websocket) -> None: """Websocket callback when the authentication connection closes Args: _websocket (Websocket): communication socket """ self.logger.info("Authentication socket close") def metadata_on_open(self, _websocket) -> None: """Websocket callback when the metadata websocket has been opened Args: websocket (Websocket): communication socket """ self.logger.info("Metadata socket open") def metadata_on_message(self, websocket, message: str) -> None: """Websocket callback when a new metadata message arrives Args: websocket (Websocket): communication socket message (str): received message """ self.on_message(websocket, message) def metadata_on_error(self, websocket, error: str) -> None: """Websocket callback when a metadata socket error occurs Args: websocket (Websocket): communication socket error (str): error message """ if websocket.keep_running: self.logger.error("Metadata socket error: {0}".format(error)) def metadata_on_close(self, _websocket) -> None: """Websocket callback when the metadata connection closes Args: _websocket (Websocket): communication socket """ self.logger.warning("Metadata socket close") def realtime_situation_on_open(self, _websocket) -> None: """Websocket callback when the realtime situation websocket has been opened Args: websocket (Websocket): communication socket """ self.logger.info("Realtime situation socket open") def realtime_situation_on_message(self, _websocket, message: str) -> None: """Websocket callback when a new realtime situation message arrives Args: websocket (Websocket): communication socket message (str): received message """ if self.state == self.State.REALTIME_SITUATION_LOGIN: if not self.messages.parse_realtime_situation_login( json.loads(message)): self.logger.error("Test run failed. Exiting.") self.stop_connection_threads() else: self.state = self.State(self.state.value + 1) self.send_request() elif self.state == self.state.WAIT_FOR_STARTUP_SITUATION: wnt_message = wnt_proto.Message() wnt_message.ParseFromString(message) if wnt_message.HasField("rtsituation_metadata"): self.total_node_count += wnt_message.rtsituation_metadata.node_count if wnt_message.HasField("source_address") and wnt_message.HasField( "network_id"): # Here it would be good to count distinct node count self.loaded_node_count += 1 if self.loaded_node_count == self.total_node_count: # Initial nodes' data loaded self.state = self.State(self.state.value + 1) self.send_request() # The response might come from the realtime situation connection before # the metadata connection elif (self.state == self.state.WAIT_FOR_APPLICATION_CONFIGURATION or self.state == self.State.SET_APPLICATION_CONFIGURATION): wnt_message = wnt_proto.Message() wnt_message.ParseFromString(message) if wnt_message.HasField("app_config"): app_config_string = str( binascii.hexlify(wnt_message.app_config.app_config), "utf-8") self.logger.info( "App config message received with diagnostics interval {} and " "application data {}".format( wnt_message.app_config.interval, app_config_string)) # Exit after we get first node app config data message self.return_code = 0 self.stop_connection_threads() def realtime_situation_on_error(self, websocket, error: str) -> None: """Websocket callback when realtime situation socket error occurs Args: websocket (Websocket): communication socket error (str): error message """ if websocket.keep_running: self.logger.error( "Realtime situation socket error: {0}".format(error)) def realtime_situation_on_close(self, _websocket) -> None: """Websocket callback when the realtime situation connection closes Args: _websocket (Websocket): communication socket """ self.logger.warning("Realtime situation socket close") def on_message(self, _websocket, message: str) -> None: """Called when authentication or metadata message is received Handles the state machine and closing of the communication threads Args: websocket (Websocket): communication socket message (str): received message """ if not self.parse_response(message): self.logger.error("Test run failed. Exiting.") self.stop_connection_threads() elif not self.state == self.State.WAIT_FOR_APPLICATION_CONFIGURATION: self.state = self.State(self.state.value + 1) if self.state != self.State.END: self.send_request() else: self.return_code = 0 self.stop_connection_threads() def stop_connection_threads(self) -> None: """Stop all connection threads""" self.client.stop_realtime_situation_thread() self.client.stop_metadata_thread() self.client.stop_authentication_thread() def run(self) -> int: """Run method which starts and waits the communication thread(s) Returns: int: Process return code """ try: self.realtime_situation_thread = ( self.client.start_realtime_situation_thread()) self.metadata_thread = self.client.start_metadata_thread() self.authentication_thread = self.client.start_authentication_thread( ) # Maximum wait time 30 seconds self.realtime_situation_thread.join(30) self.metadata_thread.join(0) self.authentication_thread.join(0) except: pass return self.return_code
def get_last_run_id(job_id, estimator_id): run_id = Connections.exec_sql('QUANT', ''' SELECT last_run_id, convert(varchar(8), last_run_date, 104) last_run_date FROM %s..last_run WHERE AND job_id=%d ''' % (bq, estimator_id, job_id)) return (int(run_id[0][0]), run_id[0][1]) if run_id is not None else None
def read(cls, proto): """ Reads deserialized data from proto object. :param proto: (DynamicStructBuilder) Proto object :returns: (:class:TemporalMemory) TemporalMemory instance """ 1 / 0 # Unimplemented. tm = object.__new__(cls) # capnp fails to save a tuple, so proto.columnDimensions was forced to # serialize as a list. We prefer a tuple, however, because columnDimensions # should be regarded as immutable. tm.columnDimensions = tuple(proto.columnDimensions) tm.cellsPerColumn = int(proto.cellsPerColumn) tm.activationThreshold = int(proto.activationThreshold) tm.initialPermanence = proto.initialPermanence tm.connectedPermanence = proto.connectedPermanence tm.minThreshold = int(proto.minThreshold) tm.maxNewSynapseCount = int(proto.maxNewSynapseCount) tm.permanenceIncrement = proto.permanenceIncrement tm.permanenceDecrement = proto.permanenceDecrement tm.predictedSegmentDecrement = proto.predictedSegmentDecrement tm.maxSegmentsPerCell = int(proto.maxSegmentsPerCell) tm.maxSynapsesPerSegment = int(proto.maxSynapsesPerSegment) tm.connections = Connections.read(proto.connections) #pylint: disable=W0212 tm._random = Random() tm._random.read(proto.random) #pylint: enable=W0212 tm.activeCells = [int(x) for x in proto.activeCells] tm.winnerCells = [int(x) for x in proto.winnerCells] flatListLength = tm.connections.segmentFlatListLength() tm.numActiveConnectedSynapsesForSegment = [0] * flatListLength tm.numActivePotentialSynapsesForSegment = [0] * flatListLength tm.lastUsedIterationForSegment = [0] * flatListLength tm.activeEvents = [] tm.matchingEvents = [] for protoSegment in proto.activeEvents: tm.activeEvents.append( tm.connections.getSegment(protoSegment.cell, protoSegment.idxOnCell)) for protoSegment in proto.matchingEvents: tm.matchingEvents.append( tm.connections.getSegment(protoSegment.cell, protoSegment.idxOnCell)) for protoSegment in proto.numActivePotentialSynapsesForSegment: segment = tm.connections.getSegment(protoSegment.cell, protoSegment.idxOnCell) tm.numActivePotentialSynapsesForSegment[segment.flatIdx] = (int( protoSegment.number)) tm.iteration = long(proto.iteration) for protoSegment in proto.lastUsedIterationForSegment: segment = tm.connections.getSegment(protoSegment.cell, protoSegment.idxOnCell) tm.lastUsedIterationForSegment[segment.flatIdx] = (long( protoSegment.number)) return tm
from arguments import Arguments from connections import Connections ''' This code runs the JPL Open Source Rover. It accepts a few command line arguments for different functionality -t : Testing mode - allows you to test commands to send to the rover emulating a signals from a controller -s : Attempts to connect to a Unix socket for controlling the LED screen. The screen.py script must be running previous to this in order to work. It lives at ../led/screen.py -c : Controller flag, letting this program know what type of controller you wish to run with b : Bluetooth app (default) x : XBox controller (requires USB reciever) An example line running this script to run the LED screen and with an Xbox controller sudo python main.py -s -c x ''' args = Arguments() conn = Connections() rover = Rover() def listener(): ''' Based on command line args decides which controller and sockets to open ''' if args.socket: print "starting LED socket client" conn.unixSockConnect() elif args.connect == 'x' or args.connect == 'b': conn.connect(args.connect) else: conn.connect('b')
def socket(self): return Connections.socket_by_character_id(self.id)