class NeutralNet(object):
def __init__(self, layer_num): # 初始化一个全联接网络，所以不需要指定connections，layer_num是一个list，元素代表每层node数
self.layers = []
layer_count = len(layer_num)
for i in range(layer_count):
self.layers.append(Layer(i, layer_num[i])) # 初始化layer，给各layer填充节点
self.connections = Connections()
for i in range(layer_count - 1): # 初始化connections，全连接网络
connections = [Connection(up_node, down_node) for up_node in self.layers[i].nodes
for down_node in self.layers[i + 1].nodes]
for conn in connections:
self.connections.add_conn(conn)
conn.up_node.add_downconns(conn)
conn.down_node.add_upconns(conn)
def train(self, train_data, labels, rate, N):
for i in range(N):
for x, y in zip(train_data, labels):
self.train_one_sample(x, y, rate)
def train_one_sample(self, x, y, rate):
self.predict(x)
self._cal_delta(y)
self._update_weight(rate)
def predict(self, x):
self.layers[0].set_output(data_set=x) # 根据样本设置输入层
for layer in self.layers[1:]: # 计算各层输出
layer.cal_output()
return [node.output for node in self.layers[-1].nodes] # 返回输出层的结果
def _cal_delta(self, y):
for node, label in zip(self.layers[-1].nodes, y):
node.cal_output_delta(label)
for layer in self.layers[-2::-1]: # 反向传播误差
for node in layer.nodes:
node.cal_hidden_delta()
def _update_weight(self, rate):
for layer in self.layers[1:]:
for node in layer.nodes:
node.update_bias(rate) # 更新节点bias
for conn in node.down_conns:
conn.update_weight(rate) # 更新连接weight
def _cal_gradient(self): # 计算网络每层的梯度，留待检查梯度计算的正确性
for layer in self.layers[1:]:
for node in layer.nodes:
for conn in node.up_conns:
conn.cal_gradient()
def get_gradient(self, x, y):
self.predict(x)
self._cal_delta(y)
self._cal_gradient()
def dump(self):
pass
def __init__(self) -> None:
"""Initialization"""
self.return_code = -1
self.state = self.State(self.State.START.value + 1)
self.authentication_thread = None
self.metadata_thread = None
self.realtime_situation_thread = None
self.total_node_count = 0
self.loaded_node_count = 0
self.settings = get_settings()
self.logger = setup_log("ComponentsInformationExample",
self.settings.log_level)
self.client = Connections(
hostname=self.settings.hostname,
logger=self.logger,
authentication_on_open=self.authentication_on_open,
authentication_on_message=self.authentication_on_message,
authentication_on_error=self.authentication_on_error,
authentication_on_close=self.authentication_on_close,
metadata_on_open=self.metadata_on_open,
metadata_on_message=self.metadata_on_message,
metadata_on_error=self.metadata_on_error,
metadata_on_close=self.metadata_on_close,
realtime_situation_on_open=self.realtime_situation_on_open,
realtime_situation_on_message=self.realtime_situation_on_message,
realtime_situation_on_error=self.realtime_situation_on_error,
realtime_situation_on_close=self.realtime_situation_on_close,
)
self.messages = Messages(self.logger, self.settings.protocol_version)
def __init__(self):
builder = Gtk.Builder()
builder.add_objects_from_file('app.glade', ('winConnections', ))
self.window = builder.get_object('winConnections')
self.connection_list = builder.get_object('listConnections')
self.ent_host = builder.get_object('entHost')
self.ent_port = builder.get_object('entPort')
self.ent_user = builder.get_object('entUser')
self.ent_password = builder.get_object('entPassword')
self.ent_database = builder.get_object('entDatabase')
self.ent_remote_host = builder.get_object('entRemoteHost')
self.ent_remote_user = builder.get_object('entRemoteUser')
self.file_remote_key = builder.get_object('fileRemoteKey')
self.connections = Connections()
for connection in self.connections.get_connections():
self.connection_list.add(ListBoxRowWithData(connection))
if self.connections.count() == 0:
self.on_add_connection(None)
builder.connect_signals({
'onAddConnection': self.on_add_connection,
'onListSelected': self.on_list_selected,
'onRemoveConnection': self.on_remove_connection,
'onSaveConnection': self.on_save_connection,
'onConnect': self.on_connect
})
WindowManager.add_window(self.window)
self.window.show_all()
def __init__(self):
self._connections = Connections()
self._connections_need_game = []
self._number_of_wait_ticks = 0
self._game_number = 0
self._games = {}
self._games_to_kill = []
def __init__(self) -> None:
"""Initialization"""
self.return_code = -1
self.state = self.State(self.State.START.value + 1)
self.authentication_thread = None
self.metadata_thread = None
self.settings = get_settings()
self.logger = setup_log("BuildingExample", self.settings.log_level)
self.client = Connections(
hostname=self.settings.hostname,
logger=self.logger,
authentication_on_open=self.authentication_on_open,
authentication_on_message=self.authentication_on_message,
authentication_on_error=self.authentication_on_error,
authentication_on_close=self.authentication_on_close,
metadata_on_open=self.metadata_on_open,
metadata_on_message=self.metadata_on_message,
metadata_on_error=self.metadata_on_error,
metadata_on_close=self.metadata_on_close,
)
self.messages = BuildingMessages(self.logger,
self.settings.protocol_version)
def __init__(self) -> None:
"""Initialization"""
self.return_code = -1
self.state = self.State(self.State.START.value + 1)
self.new_user = dict(
username="******",
password="******",
full_name="John Doe",
role=AuthenticationMessages.Role.OPERATOR.value,
updated_full_name="John J. Doe",
updated_password="******",
updated_role=AuthenticationMessages.Role.ADMIN.value,
)
self.settings = get_settings()
self.logger = setup_log("AuthenticationExample",
self.settings.log_level)
self.client = Connections(
hostname=self.settings.hostname,
logger=self.logger,
authentication_on_open=self.authentication_on_open,
authentication_on_message=self.authentication_on_message,
authentication_on_error=self.authentication_on_error,
authentication_on_close=self.authentication_on_close,
)
self.authentication = AuthenticationMessages(
self.logger, self.settings.protocol_version)
def tearDownClass(cls) -> None:
db = Connections(table_name=None)
with db.connect() as conn:
company_email = '*****@*****.**'
email = '*****@*****.**'
conn.execute(
f"delete from provider where email = '{company_email}'")
conn.execute(f"delete from client where email = '{email}'")
def __init__(self) -> None:
"""Initialization"""
self.return_code = -1
self.state = self.State(self.State.START.value + 1)
# Network id needs to point to a valid network with at least one sink
# online
self.network_id = 777555
self.sink_ids = None
# Uncomment the line below for setting appconfig for specific sinks only
# self.sink_ids = [1, 101]
# When running more than once for the same network, the diagnostics interval
# or application data needs to changed as WNT server will not try to set the
# application configuration if it is already the same.
self.diagnostics_interval = 60
self.application_data = "00112233445566778899AABBCCDDEEFF"
self.is_override_on = False
self.authentication_thread = None
self.metadata_thread = None
self.realtime_situation_thread = None
self.total_node_count = 0
self.loaded_node_count = 0
self.settings = get_settings()
self.logger = setup_log(
"ApplicationConfigurationExample", self.settings.log_level
)
self.client = Connections(
hostname=self.settings.hostname,
logger=self.logger,
authentication_on_open=self.authentication_on_open,
authentication_on_message=self.authentication_on_message,
authentication_on_error=self.authentication_on_error,
authentication_on_close=self.authentication_on_close,
metadata_on_open=self.metadata_on_open,
metadata_on_message=self.metadata_on_message,
metadata_on_error=self.metadata_on_error,
metadata_on_close=self.metadata_on_close,
realtime_situation_on_open=self.realtime_situation_on_open,
realtime_situation_on_message=self.realtime_situation_on_message,
realtime_situation_on_error=self.realtime_situation_on_error,
realtime_situation_on_close=self.realtime_situation_on_close,
)
self.messages = Messages(self.logger, self.settings.protocol_version)
def __init__(self):
"""Return a Customer object whose name is *name*."""
self.exchangeRate = ExchangeRate()
self.weather = Weather()
self.myconnection = Connections()
self.myppointments = Appointments()
#place holder for other option
if (SHOW_DATA_AT == "spread_sheet"):
self.data_visualizer = SreadSheet()
self.reset()
def __init__(self, layer_num): # 初始化一个全联接网络，所以不需要指定connections，layer_num是一个list，元素代表每层node数
self.layers = []
layer_count = len(layer_num)
for i in range(layer_count):
self.layers.append(Layer(i, layer_num[i])) # 初始化layer，给各layer填充节点
self.connections = Connections()
for i in range(layer_count - 1): # 初始化connections，全连接网络
connections = [Connection(up_node, down_node) for up_node in self.layers[i].nodes
for down_node in self.layers[i + 1].nodes]
for conn in connections:
self.connections.add_conn(conn)
conn.up_node.add_downconns(conn)
conn.down_node.add_upconns(conn)
def __init__(self) -> None:
"""Initialization"""
self.floor_plan_image_id = None
self.floor_plan_image_thumbnail_id = None
self.return_code = -1
self.state = self.State(self.State.START.value + 1)
self.authentication_thread = None
self.metadata_thread = None
self.settings = get_settings()
self.logger = setup_log("FloorPlanAreaExample", self.settings.log_level)
self.client = Connections(
hostname=self.settings.hostname,
logger=self.logger,
authentication_on_open=self.authentication_on_open,
authentication_on_message=self.authentication_on_message,
authentication_on_error=self.authentication_on_error,
authentication_on_close=self.authentication_on_close,
metadata_on_open=self.metadata_on_open,
metadata_on_message=self.metadata_on_message,
metadata_on_error=self.metadata_on_error,
metadata_on_close=self.metadata_on_close,
)
self.messages = Messages(self.logger, self.settings.protocol_version)
script_path = os.path.dirname(os.path.realpath(__file__))
self.floor_plan_image_file_path = os.path.join(
script_path, "assets/floor_plan.png"
)
self.floor_plan_image_thumbnail_file_path = os.path.join(
script_path, "assets/floor_plan_thumbnail.png"
)
self.floor_plan_image_width = 8989
self.floor_plan_image_height = 4432
self.temp_floor_plan_image_file_path = (
self.floor_plan_image_file_path + ".tmp.png"
)
self.temp_floor_plan_image_thumbnail_file_path = (
self.floor_plan_image_thumbnail_file_path + ".tmp.png"
)
def lambda_handler(event, context, **kwargs):
print(
'Starting function\n-------------------------------------------------')
start_time = time()
print('Loading account credentials')
accounts = Connections()
print('Gathering Usage and Limit Data')
comparisons = {}
for count, account in enumerate(accounts.env_names):
comparisons[account] = AccountUsage(accounts.env_names[count],
accounts.connection_list[count])
comparisons[account].get_limits()
comparisons[account].compare()
print('Generationg Email')
email = EmailGenerator(comparisons)
email.create_html
email.send_email
print('Creating Report')
report = ReportGenerator(email.utilization)
json = report.write_json()
csv = report.write_csv
print('Uplaoding Report to S3')
push_json = report.push_to_s3(report.metadata)
push_csv = report.push_to_s3(report.report)
end_time = time() - start_time
print('Time required: {0:.2f} s'.format(end_time))
def __init__(self, layers):
self.connections = Connections()
self.layers = []
layer_count = len(layers)
node_count = 0
for i in range(layer_count):
self.layer.append(Layer(i, layers[i]))
for layer in range(layer_count - 1):
connections = [
Connection(upstream_node, downstream_node)
for upstream_node in self.layers[layer].nodex
for downstream_node in self.layers[layer + 1].nodes[:-1]
]
for conn in connections:
self.connections.add_connection(conn)
conn.downstream_node.append_upstream_connection(conn)
conn.upstream_node.append_downstream_connection(conn)
def get_perimeter_sec_ids(perimeter_id):
'''
Returns the list of security for a perimeter
'''
ids = Connections.exec_sql("QUANT",
''' SELECT security_id FROM %s..perimeter_domain pd
JOIN %s..domain_security ds on ds.domain_id = pd.domain_id
WHERE pd.perimeter_id = %d
''' % (bq, bq, perimeter_id))
return [id[0] for id in ids] if ids is not None else None
def get_se_estimator_by_run_id(run_id, estimator_id, context):
estimator = Connections.exec_sql('QUANT',
''' SELECT pd.parameter_name, pv.value
FROM %s..param_desc pd, %s..param_value pv
WHERE pv.parameter_id = pd.parameter_id
AND pd.estimator_id = %d
AND pv.run_id = %d
''' % (bq, bq, estimator_id, run_id)
)
return pyData('init', date=[x[0] for x in estimator], value={context:[x[1] for x in estimator]})
def get_se_estimator(estimator, security_id, trading_destinations, day, context):
estimator_id = get_estimator_id(estimator)
job_id = get_job_id(estimator_id, context, security_id, trading_destinations)
if job_id == None:
print "no data for (security_id: %d, trading_destination: %s)" % (security_id, trading_destinations)
job_id = get_job_id(estimator_id, context, security_id, 'NULL')
if job_id == None:
print "no data for (security_id: %d, trading_destination: NULL)" % (security_id)
job_id = get_job_id(estimator_id, context, 'NULL', trading_destinations)
if job_id != None:
date_se = Connections.exec_sql('QUANT',
''' SELECT convert(varchar(8),max(er.stamp_date),112)
FROM %s..estimator_runs er WHERE
er.job_id = %d AND er.stamp_date <= '%s'
AND estimator_id = %d
''' % (bq, job_id, day, estimator_id)
)
if date_se != None and date_se[0][0] is not None:
data_se = Connections.exec_sql('QUANT',
''' SELECT pd.parameter_name, pv.value FROM %s..estimator_runs er,
%s..param_desc pd, %s..param_value pv
WHERE er.estimator_id = %d
AND pv.parameter_id = pd.parameter_id
AND pd.estimator_id = er.estimator_id
AND pv.run_id = er.run_id
AND er.run_id = (SELECT max(run_id) FROM %s..estimator_runs WHERE stamp_date='%s' AND job_id=%d and estimator_id=%d)
''' % (bq, bq, bq, estimator_id, bq, date_se[0][0], job_id, estimator_id)
)
data_se.append(('run_date', date_se[0][0]))
return data_se
else:
raise ValueError("No SE entries for this date.")
else:
raise ValueError ("No job_id for estimator %s" % estimator)
def get_runs_by_perimeters_for_date(perimeter_ids, estimator_id, context_id, date):
ids = Connections.exec_sql("QUANT",
''' SELECT ds.security_id, er.stamp_date, er.run_id, run_quality FROM %s..perimeter_domain pd
JOIN %s..domain_security ds on ds.domain_id=pd.domain_id
JOIN %s..association a on a.security_id=ds.security_id AND a.estimator_id= %d
AND a.context_id= %d
AND isnull(ds.trading_destination_id,0) = isnull(a.trading_destination_id,0)
JOIN %s..estimator_runs er on er.job_id=a.job_id and er.context_id = a.context_id and er.estimator_id = a.estimator_id
WHERE pd.perimeter_id in (%s) and stamp_date < '%s'
GROUP by ds.security_id having er.run_id = max(er.run_id)
''' % (bq, bq, bq, estimator_id, context_id, bq, ','.join(map(str,perimeter_ids)), date)
)
return ids
def connectionsFactory(*args, **kwargs):
"""
Create a :class:`~nupic.algorithms.connections.Connections` instance.
:class:`TemporalMemory` subclasses may override this method to choose a
different :class:`~nupic.algorithms.connections.Connections` implementation,
or to augment the instance otherwise returned by the default
:class:`~nupic.algorithms.connections.Connections` implementation.
See :class:`~nupic.algorithms.connections.Connections` for constructor
signature and usage.
:returns: :class:`~nupic.algorithms.connections.Connections` instance
"""
return Connections(*args, **kwargs)
def _approximations(df, relations=[]):
t1 = ut.out('approximating relational with mean, max, median...')
df = df.copy()
con_obj = Connections()
g, sgs = con_obj.find_subgraphs(df, relations, verbose=False)
approx_dict = {}
sg_list = []
for i, sg in enumerate(sgs):
if sg[3] > 0: # num edges > 0
sg_list.extend([(x, i) for x in sg[0]]) # give sg_id
if len(sg_list) == 0:
return approx_dict
sg_df = pd.DataFrame(sg_list, columns=['com_id', 'sg_id'])
df = df.merge(sg_df, how='left')
df['sg_id'] = df['sg_id'].fillna(-1).apply(int)
sg_mean = df.groupby('sg_id')['ind_pred'].mean().reset_index()\
.rename(columns={'ind_pred': 'sg_mean_pred'})
sg_median = df.groupby('sg_id')['ind_pred'].median().reset_index()\
.rename(columns={'ind_pred': 'sg_median_pred'})
sg_max = df.groupby('sg_id')['ind_pred'].max().reset_index()\
.rename(columns={'ind_pred': 'sg_max_pred'})
df = df.merge(sg_mean).merge(sg_median).merge(sg_max)
filler = lambda x, c: x['ind_pred'] if x['sg_id'] == -1 else x[c]
for col in ['sg_mean_pred', 'sg_median_pred', 'sg_max_pred']:
cols = ['ind_pred', col, 'sg_id']
df[col] = df[cols].apply(filler, axis=1, args=(col,))
ut.time(t1)
return df
def get_context_runs_by_perimeter(perimeter_ids, estimator_id, context_ids):
ids = Connections.exec_sql('QUANT',
''' select ds.security_id, er.stamp_date, lr.last_run_id, lr.context_id
from quant..perimeter_domain pd
join quant..domain_security ds on ds.domain_id = pd.domain_id
join quant..job j on ds.domain_id = j.domain_id and j.estimator_id = %d and ds.domain_id = j.domain_id
join quant..perimeter_job pj on pj.job_id = j.job_id and pj.is_active = 1
join quant..last_run lr on lr.job_id = j.job_id and lr.context_id in (%s)
join quant..estimator_runs er on er.run_id = lr.last_run_id
where ds.trading_destination_id is null and pd.perimeter_id in (%s)
''' % (estimator_id, ','.join(map(str,context_ids)), ','.join(map(str,perimeter_ids)))
)
return ids
def __init__(self, layers):
self.connections = Connections()
self.layers = []
layer_count = len(layers)
node_count = 0
for i in range(layer_count):
self.layers.append(Layer(i, layers[i]))
for layer in range(layer_count - 1):
connections = [
Connection(upstream_node, downstream_node)
for upstream_node in self.layers[layer].node
for downstream_node in self.layer[layer + 1].nodes[:-1]]
for conn in connections:
self.connections.add_connection(conn)
conn.downstream_node.append_upstream_conneciont(conn)
conn.upstream_node.append_downstream_connection(conn)
def get_job_id(estimator, context, s_security_id, s_trading_destination_id):
if isinstance(estimator, int):
estimator_id = estimator
else:
estimator_id = get_estimator_id(estimator)
job_id = Connections.exec_sql("QUANT",
"select job_id from %s..association WHERE \
estimator_id=%d \
AND context_id=(SELECT context_id FROM %s..context WHERE context_name = \'%s\' AND estimator_id=%d)\
AND security_id=%s\
AND trading_destination_id=%s" %
(bq, estimator_id, bq, context, estimator_id, s_security_id, s_trading_destination_id)
)
return job_id[0][0] if len(job_id) > 0 else None
class Games():
def __init__(self):
self._connections = Connections()
self._connections_need_game = []
self._number_of_wait_ticks = 0
self._game_number = 0
self._games = {}
self._games_to_kill = []
def tick(self):
self._kill_games()
self._get_connections_looking()
self._do_matchmaking()
self._tick_games()
def _kill_games(self):
while self._games_to_kill:
game = self._games_to_kill.pop()
game_number = game.get_game_number()
if game_number in self._games:
game.end_game()
del self._games[game_number]
def _get_connections_looking(self):
while True:
connection = self._connections.get_connection_looking_and_remove()
if connection == None:
return
if connection in self._connections_need_game:
return
self._connections_need_game.append(connection)
def _do_matchmaking(self):
if not self._connections_need_game:
return
for i in range(len(self._connections_need_game) - 1, -1, -1):
if not self._connections_need_game[i].connected:
self._connections_need_game.pop(i)
if (self._number_of_wait_ticks > Settings.MATCHMAKER_WAIT_TICKS
or len(self._connections_need_game) >=
Settings.MATCHMAKER_MAXIMUM_PLAYERS):
self._create_game()
self._number_of_wait_ticks = 0
return
self._number_of_wait_ticks += 1
def _create_game(self):
game_number = self._get_next_game_number()
game = Game(self, game_number)
self._games[game_number] = game
count = 0
while self._connections_need_game:
game.add_connection(self._connections_need_game.pop())
count += 1
if count >= Settings.MATCHMAKER_MAXIMUM_PLAYERS:
break
game.start_game()
def _tick_games(self):
if not self._games:
return
for game in self._games.itervalues():
if game.get_game_over():
self._games_to_kill.append(game)
else:
game.tick()
def _get_next_game_number(self):
self._game_number += 1
while self._game_number in self._games:
self._game_number += 1
if self._game_number > 999999999:
self._game_number = 1
return self._game_number
class ComponentsInformationExample(object):
"""Main example class which is run"""
class State(Enum):
"""State enumeration class"""
START = auto()
LOGIN = auto() # Started on authentication_on_open
REALTIME_SITUATION_LOGIN = auto()
WAIT_FOR_STARTUP_SITUATION = auto()
QUERY_COMPONENTS_INFORMATION = auto()
WAIT_FOREVER = auto()
END = auto()
def __init__(self) -> None:
"""Initialization"""
self.return_code = -1
self.state = self.State(self.State.START.value + 1)
self.authentication_thread = None
self.metadata_thread = None
self.realtime_situation_thread = None
self.total_node_count = 0
self.loaded_node_count = 0
self.settings = get_settings()
self.logger = setup_log("ComponentsInformationExample",
self.settings.log_level)
self.client = Connections(
hostname=self.settings.hostname,
logger=self.logger,
authentication_on_open=self.authentication_on_open,
authentication_on_message=self.authentication_on_message,
authentication_on_error=self.authentication_on_error,
authentication_on_close=self.authentication_on_close,
metadata_on_open=self.metadata_on_open,
metadata_on_message=self.metadata_on_message,
metadata_on_error=self.metadata_on_error,
metadata_on_close=self.metadata_on_close,
realtime_situation_on_open=self.realtime_situation_on_open,
realtime_situation_on_message=self.realtime_situation_on_message,
realtime_situation_on_error=self.realtime_situation_on_error,
realtime_situation_on_close=self.realtime_situation_on_close,
)
self.messages = Messages(self.logger, self.settings.protocol_version)
def send_request(self) -> None:
"""Send request"""
if self.state == self.State.LOGIN:
self.authentication_thread.socket.send(
json.dumps(
self.messages.message_login(self.settings.username,
self.settings.password)))
elif self.state == self.State.REALTIME_SITUATION_LOGIN:
self.realtime_situation_thread.socket.send(
json.dumps(
self.messages.message_realtime_situation_login(
self.messages.session_id)))
elif self.state == self.State.QUERY_COMPONENTS_INFORMATION:
self.metadata_thread.socket.send(
json.dumps(
self.messages.message_query_components_information()))
def parse_response(self, message: str) -> bool:
"""Parse response
Args:
message (str): received message
Returns:
bool: True if response's request succeeded
"""
if self.state == self.State.LOGIN:
return self.messages.parse_login(json.loads(message))
elif self.state == self.state.QUERY_COMPONENTS_INFORMATION:
return self.messages.parse_query_components_information(
json.loads(message))
def authentication_on_open(self, _websocket) -> None:
"""Websocket callback when the authentication websocket has been opened
Args:
websocket (Websocket): communication socket
"""
self.logger.info("Authentication socket open")
self.send_request()
def authentication_on_message(self, websocket, message: str) -> None:
"""Websocket callback when a new authentication message arrives
Args:
websocket (Websocket): communication socket
message (str): received message
"""
self.on_message(websocket, message)
def authentication_on_error(self, websocket, error: str) -> None:
"""Websocket callback when an authentication socket error occurs
Args:
websocket (Websocket): communication socket
error (str): error message
"""
if websocket.keep_running:
self.logger.error("Authentication socket error: {0}".format(error))
def authentication_on_close(self, _websocket) -> None:
"""Websocket callback when the authentication connection closes
Args:
_websocket (Websocket): communication socket
"""
self.logger.info("Authentication socket close")
def metadata_on_open(self, _websocket) -> None:
"""Websocket callback when the metadata websocket has been opened
Args:
websocket (Websocket): communication socket
"""
self.logger.info("Metadata socket open")
def metadata_on_message(self, websocket, message: str) -> None:
"""Websocket callback when a new metadata message arrives
Args:
websocket (Websocket): communication socket
message (str): received message
"""
self.on_message(websocket, message)
def metadata_on_error(self, websocket, error: str) -> None:
"""Websocket callback when a metadata socket error occurs
Args:
websocket (Websocket): communication socket
error (str): error message
"""
if websocket.keep_running:
self.logger.error("Metadata socket error: {0}".format(error))
def metadata_on_close(self, _websocket) -> None:
"""Websocket callback when the metadata connection closes
Args:
_websocket (Websocket): communication socket
"""
self.logger.warning("Metadata socket close")
def realtime_situation_on_open(self, _websocket) -> None:
"""Websocket callback when the realtime situation websocket has been opened
Args:
websocket (Websocket): communication socket
"""
self.logger.info("Realtime situation socket open")
def realtime_situation_on_message(self, _websocket, message: str) -> None:
"""Websocket callback when a new realtime situation message arrives
Args:
websocket (Websocket): communication socket
message (str): received message
"""
if self.state == self.State.REALTIME_SITUATION_LOGIN:
if not self.messages.parse_realtime_situation_login(
json.loads(message)):
self.logger.error("Test run failed. Exiting.")
self.stop_connection_threads()
else:
self.state = self.State(self.state.value + 1)
self.send_request()
return
elif self.state == self.state.WAIT_FOR_STARTUP_SITUATION:
wnt_message = wnt_proto.Message()
wnt_message.ParseFromString(message)
if wnt_message.HasField("rtsituation_metadata"):
self.total_node_count += wnt_message.rtsituation_metadata.node_count
if wnt_message.HasField("source_address") and wnt_message.HasField(
"network_id"):
# Here it would be good to count distinct node count
self.loaded_node_count += 1
if self.loaded_node_count == self.total_node_count:
# Initial nodes' data loaded
self.state = self.State(self.state.value + 1)
self.send_request()
else:
wnt_message = wnt_proto.Message()
wnt_message.ParseFromString(message)
if wnt_message.HasField("gateway_info"):
self.logger.info("gateway_info:\n{}".format(
wnt_message.gateway_info))
if wnt_message.HasField("backend_component_info"):
self.logger.info("backend_component_info:\n{}".format(
wnt_message.backend_component_info))
def realtime_situation_on_error(self, websocket, error: str) -> None:
"""Websocket callback when realtime situation socket error occurs
Args:
websocket (Websocket): communication socket
error (str): error message
"""
if websocket.keep_running:
self.logger.error(
"Realtime situation socket error: {0}".format(error))
def realtime_situation_on_close(self, _websocket) -> None:
"""Websocket callback when the realtime situation connection closes
Args:
_websocket (Websocket): communication socket
"""
self.logger.warning("Realtime situation socket close")
def on_message(self, _websocket, message: str) -> None:
"""Called when authentication or metadata message is received
Handles the state machine and closing of the communication threads
Args:
websocket (Websocket): communication socket
message (str): received message
"""
if not self.parse_response(message):
self.logger.error("Test run failed. Exiting.")
self.stop_connection_threads()
elif (not self.state == self.State.WAIT_FOR_STARTUP_SITUATION
and not self.state == self.State.WAIT_FOREVER):
self.state = self.State(self.state.value + 1)
if self.state != self.State.END:
self.send_request()
else:
self.return_code = 0
self.stop_connection_threads()
def stop_connection_threads(self) -> None:
"""Stop all connection threads"""
self.client.stop_realtime_situation_thread()
self.client.stop_metadata_thread()
self.client.stop_authentication_thread()
def run(self) -> int:
"""Run method which starts and waits the communication thread(s)
Returns:
int: Process return code
"""
try:
self.realtime_situation_thread = (
self.client.start_realtime_situation_thread())
self.metadata_thread = self.client.start_metadata_thread()
self.authentication_thread = self.client.start_authentication_thread(
)
# Run for 10 seconds
self.realtime_situation_thread.join(10)
self.metadata_thread.join(0)
self.authentication_thread.join(0)
self.return_code = 0
except:
pass
return self.return_code
def get_estimator_id(estimator):
estimator_id = Connections.exec_sql("QUANT",
"SELECT estimator_id FROM %s..estimator WHERE estimator_name = \'%s\'"
% (bq, estimator))
return int(estimator_id[0][0]) if estimator_id is not None else None
class BuildingExample(object):
"""Main example class which is run"""
class State(Enum):
"""State enumeration class"""
START = auto()
LOGIN = auto() # Started on authentication_on_open
CREATE_BUILDING = auto()
GET_BUILDINGS = auto()
UPDATE_BUILDING = auto()
GET_BUILDINGS_2 = auto()
DELETE_BUILDING = auto()
END = auto()
def __init__(self) -> None:
"""Initialization"""
self.return_code = -1
self.state = self.State(self.State.START.value + 1)
self.authentication_thread = None
self.metadata_thread = None
self.settings = get_settings()
self.logger = setup_log("BuildingExample", self.settings.log_level)
self.client = Connections(
hostname=self.settings.hostname,
logger=self.logger,
authentication_on_open=self.authentication_on_open,
authentication_on_message=self.authentication_on_message,
authentication_on_error=self.authentication_on_error,
authentication_on_close=self.authentication_on_close,
metadata_on_open=self.metadata_on_open,
metadata_on_message=self.metadata_on_message,
metadata_on_error=self.metadata_on_error,
metadata_on_close=self.metadata_on_close,
)
self.messages = BuildingMessages(self.logger,
self.settings.protocol_version)
def send_request(self) -> None:
"""Send request"""
if self.state == self.State.LOGIN:
self.authentication_thread.socket.send(
json.dumps(
self.messages.message_login(self.settings.username,
self.settings.password)))
elif self.state == self.State.CREATE_BUILDING:
self.metadata_thread.socket.send(
json.dumps(
self.messages.message_create_building("New building")))
elif self.state.name.startswith(self.State.GET_BUILDINGS.name):
self.metadata_thread.socket.send(
json.dumps(self.messages.message_get_buildings()))
elif self.state == self.State.UPDATE_BUILDING:
self.metadata_thread.socket.send(
json.dumps(
self.messages.message_update_building(
self.messages.new_building_id, "Updated building")))
elif self.state == self.State.DELETE_BUILDING:
self.metadata_thread.socket.send(
json.dumps(
self.messages.message_delete_building(
self.messages.new_building_id)))
def parse_response(self, message: str) -> bool:
"""Parse response
Args:
message (str): received message
Returns:
bool: True if response's request succeeded
"""
if self.state == self.State.LOGIN:
return self.messages.parse_login(json.loads(message))
elif self.state == self.State.CREATE_BUILDING:
return self.messages.parse_create_building(json.loads(message))
elif self.state.name.startswith(self.State.GET_BUILDINGS.name):
return self.messages.parse_get_buildings(json.loads(message))
elif self.state == self.State.UPDATE_BUILDING:
return self.messages.parse_update_building(json.loads(message))
elif self.state == self.State.DELETE_BUILDING:
return self.messages.parse_delete_building(json.loads(message))
def authentication_on_open(self, _websocket) -> None:
"""Websocket callback when the authentication websocket has been opened
Args:
websocket (Websocket): communication socket
"""
self.logger.info("Authentication socket open")
self.send_request()
def authentication_on_message(self, websocket, message: str) -> None:
"""Websocket callback when a new authentication message arrives
Args:
websocket (Websocket): communication socket
message (str): received message
"""
self.on_message(websocket, message)
def authentication_on_error(self, websocket, error: str) -> None:
"""Websocket callback when an authentication socket error occurs
Args:
websocket (Websocket): communication socket
error (str): error message
"""
if websocket.keep_running:
self.logger.error("Authentication socket error: {0}".format(error))
def authentication_on_close(self, _websocket) -> None:
"""Websocket callback when the authentication connection closes
Args:
_websocket (Websocket): communication socket
"""
self.logger.info("Authentication socket close")
def metadata_on_open(self, _websocket) -> None:
"""Websocket callback when the metadata websocket has been opened
Args:
websocket (Websocket): communication socket
"""
self.logger.info("Metadata socket open")
def metadata_on_message(self, websocket, message: str) -> None:
"""Websocket callback when a new metadata message arrives
Args:
websocket (Websocket): communication socket
message (str): received message
"""
self.on_message(websocket, message)
def metadata_on_error(self, websocket, error: str) -> None:
"""Websocket callback when a metadata socket error occurs
Args:
websocket (Websocket): communication socket
error (str): error message
"""
if websocket.keep_running:
self.logger.error("Metadata socket error: {0}".format(error))
def metadata_on_close(self, _websocket) -> None:
"""Websocket callback when the metadata connection closes
Args:
_websocket (Websocket): communication socket
"""
self.logger.warning("Metadata socket close")
def on_message(self, _websocket, message: str) -> None:
"""Called when authentication or metadata message is received
Handles the state machine and closing of the communication threads
Args:
websocket (Websocket): communication socket
message (str): received message
"""
if not self.parse_response(message):
self.logger.error("Test run failed. Exiting.")
self.client.stop_metadata_thread()
self.client.stop_authentication_thread()
else:
self.state = self.State(self.state.value + 1)
if self.state != self.State.END:
self.send_request()
else:
self.return_code = 0
self.client.stop_metadata_thread()
self.client.stop_authentication_thread()
def run(self) -> int:
"""Run method which starts and waits the communication thread(s)
Returns:
int: Process return code
"""
try:
self.authentication_thread = self.client.start_authentication_thread(
)
self.metadata_thread = self.client.start_metadata_thread()
self.metadata_thread.join()
self.authentication_thread.join()
except:
pass
return self.return_code
def tearDown(self) -> None:
db = Connections(table_name='provider')
with db.connect() as conn:
conn.execute(
db.table.update().where(db.table.c.id == self._id).values(active=None)
)
class FloorPlanAreaExample(object):
"""Main example class which is run"""
class State(Enum):
"""State enumeration class"""
START = auto()
LOGIN = auto() # Started on authentication_on_open
CREATE_BUILDING = auto()
CREATE_FLOOR_PLAN = auto()
SET_FLOOR_PLAN_IMAGE = auto()
SET_FLOOR_PLAN_IMAGE_THUMBNAIL = auto()
UPDATE_FLOOR_PLAN = auto()
CREATE_AREA = auto()
UPDATE_AREA = auto()
GET_AREAS = auto()
DELETE_AREA = auto()
DELETE_FLOOR_PLAN = auto()
DELETE_BUILDING = auto()
END = auto()
def __init__(self) -> None:
"""Initialization"""
self.floor_plan_image_id = None
self.floor_plan_image_thumbnail_id = None
self.return_code = -1
self.state = self.State(self.State.START.value + 1)
self.authentication_thread = None
self.metadata_thread = None
self.settings = get_settings()
self.logger = setup_log("FloorPlanAreaExample", self.settings.log_level)
self.client = Connections(
hostname=self.settings.hostname,
logger=self.logger,
authentication_on_open=self.authentication_on_open,
authentication_on_message=self.authentication_on_message,
authentication_on_error=self.authentication_on_error,
authentication_on_close=self.authentication_on_close,
metadata_on_open=self.metadata_on_open,
metadata_on_message=self.metadata_on_message,
metadata_on_error=self.metadata_on_error,
metadata_on_close=self.metadata_on_close,
)
self.messages = Messages(self.logger, self.settings.protocol_version)
script_path = os.path.dirname(os.path.realpath(__file__))
self.floor_plan_image_file_path = os.path.join(
script_path, "assets/floor_plan.png"
)
self.floor_plan_image_thumbnail_file_path = os.path.join(
script_path, "assets/floor_plan_thumbnail.png"
)
self.floor_plan_image_width = 8989
self.floor_plan_image_height = 4432
self.temp_floor_plan_image_file_path = (
self.floor_plan_image_file_path + ".tmp.png"
)
self.temp_floor_plan_image_thumbnail_file_path = (
self.floor_plan_image_thumbnail_file_path + ".tmp.png"
)
def send_request(self) -> None:
"""Send request"""
if self.state == self.State.LOGIN:
self.authentication_thread.socket.send(
json.dumps(
self.messages.message_login(
self.settings.username, self.settings.password
)
)
)
elif self.state == self.State.CREATE_BUILDING:
self.metadata_thread.socket.send(
json.dumps(self.messages.message_create_building("New building"))
)
elif self.state == self.State.CREATE_FLOOR_PLAN:
self.metadata_thread.socket.send(
json.dumps(
self.messages.message_create_floor_plan(
self.messages.new_building_id, "New floor plan"
)
)
)
elif self.state == self.state.SET_FLOOR_PLAN_IMAGE:
self.metadata_thread.socket.send(
json.dumps(
self.messages.message_set_image(
FileHelper.read_file_content_as_base64(
self.floor_plan_image_file_path
)
)
)
)
elif self.state == self.state.SET_FLOOR_PLAN_IMAGE_THUMBNAIL:
self.metadata_thread.socket.send(
json.dumps(
self.messages.message_set_image(
FileHelper.read_file_content_as_base64(
self.floor_plan_image_thumbnail_file_path
)
)
)
)
elif self.state == self.State.UPDATE_FLOOR_PLAN:
self.metadata_thread.socket.send(
json.dumps(
self.messages.message_update_floor_plan(
self.messages.new_floor_plan_id,
image_id=self.floor_plan_image_id,
image_thumbnail_id=self.floor_plan_image_thumbnail_id,
latitude_lefttop=61.454823,
longitude_lefttop=23.884526,
altitude_lefttop=0,
x_normcoord_lefttop=0.0748329808357999,
y_normcoord_lefttop=0.203506328386351,
latitude_righttop=61.454773,
longitude_righttop=23.886096,
altitude_righttop=0,
x_normcoord_righttop=0.903860782456575,
y_normcoord_righttop=0.203571943827163,
latitude_leftbottom=61.454612,
longitude_leftbottom=23.884503,
altitude_leftbottom=0,
x_normcoord_leftbottom=0.0747559429065484,
y_normcoord_leftbottom=0.780014805319742,
latitude_rightbottom=61.454562,
longitude_rightbottom=23.88607,
altitude_rightbottom=0,
x_normcoord_rightbottom=0.904069882566427,
y_normcoord_rightbottom=0.78039444527477,
x_distance_point1=0.450065006833406,
y_distance_point1=0.203192686229106,
x_distance_point2=0.449649314572983,
y_distance_point2=0.780260953915855,
distance_in_m=25.1,
level=0,
image_width=self.floor_plan_image_width,
image_height=self.floor_plan_image_height,
)
)
)
elif self.state == self.state.CREATE_AREA:
self.metadata_thread.socket.send(
json.dumps(
self.messages.message_create_area(
self.messages.new_floor_plan_id, "new area"
)
)
)
elif self.state == self.state.UPDATE_AREA:
corner_point_llas = [
dict(latitude=61.454674940345, longitude=23.8857998957377, altitude=0),
dict(latitude=61.4546694661163, longitude=23.8859745817991, altitude=0),
dict(latitude=61.4545636984609, longitude=23.8859602235394, altitude=0),
dict(latitude=61.4545691231503, longitude=23.8857853265751, altitude=0),
]
self.metadata_thread.socket.send(
json.dumps(
self.messages.message_update_area(
self.messages.new_area_id,
self.messages.new_floor_plan_id,
"updated name",
80,
0,
0,
200,
corner_point_llas,
)
)
)
elif self.state == self.State.GET_AREAS:
self.metadata_thread.socket.send(
json.dumps(
self.messages.message_get_floor_plan_areas(
self.messages.new_floor_plan_id
)
)
)
elif self.state == self.State.DELETE_AREA:
self.metadata_thread.socket.send(
json.dumps(
self.messages.message_delete_area(
self.messages.new_area_id, self.messages.new_floor_plan_id
)
)
)
elif self.state == self.State.DELETE_FLOOR_PLAN:
self.metadata_thread.socket.send(
json.dumps(
self.messages.message_delete_floor_plan(
self.messages.new_floor_plan_id
)
)
)
elif self.state == self.State.DELETE_BUILDING:
self.metadata_thread.socket.send(
json.dumps(
self.messages.message_delete_building(self.messages.new_building_id)
)
)
def parse_response(self, message: str) -> bool:
"""Parse response
Args:
message (str): received message
Returns:
bool: True if response's request succeeded
"""
if self.state == self.State.LOGIN:
return self.messages.parse_login(json.loads(message))
elif self.state == self.State.CREATE_BUILDING:
return self.messages.parse_create_building(json.loads(message))
elif self.state == self.State.CREATE_FLOOR_PLAN:
return self.messages.parse_create_floor_plan(json.loads(message))
elif self.state == self.State.SET_FLOOR_PLAN_IMAGE:
parse_result = self.messages.parse_set_image(json.loads(message))
if parse_result:
self.floor_plan_image_id = self.messages.image_id
return parse_result
elif self.state == self.State.SET_FLOOR_PLAN_IMAGE_THUMBNAIL:
parse_result = self.messages.parse_set_image(json.loads(message))
if parse_result:
self.floor_plan_image_thumbnail_id = self.messages.image_id
return parse_result
elif self.state == self.State.UPDATE_FLOOR_PLAN:
return self.messages.parse_update_floor_plan(json.loads(message))
elif self.state == self.State.CREATE_AREA:
return self.messages.parse_create_area(json.loads(message))
elif self.state == self.State.UPDATE_AREA:
return self.messages.parse_update_area(json.loads(message))
elif self.state == self.State.GET_AREAS:
return self.messages.parse_get_floor_plan_areas(json.loads(message))
elif self.state == self.State.DELETE_AREA:
return self.messages.parse_delete_area(json.loads(message))
elif self.state == self.State.DELETE_FLOOR_PLAN:
return self.messages.parse_delete_floor_plan(json.loads(message))
elif self.state == self.State.DELETE_BUILDING:
return self.messages.parse_delete_building(json.loads(message))
def authentication_on_open(self, _websocket) -> None:
"""Websocket callback when the authentication websocket has been opened
Args:
websocket (Websocket): communication socket
"""
self.logger.info("Authentication socket open")
self.send_request()
def authentication_on_message(self, websocket, message: str) -> None:
"""Websocket callback when a new authentication message arrives
Args:
websocket (Websocket): communication socket
message (str): received message
"""
self.on_message(websocket, message)
def authentication_on_error(self, websocket, error: str) -> None:
"""Websocket callback when an authentication socket error occurs
Args:
websocket (Websocket): communication socket
error (str): error message
"""
if websocket.keep_running:
self.logger.error("Authentication socket error: {0}".format(error))
def authentication_on_close(self, _websocket) -> None:
"""Websocket callback when the authentication connection closes
Args:
_websocket (Websocket): communication socket
"""
self.logger.info("Authentication socket close")
def metadata_on_open(self, _websocket) -> None:
"""Websocket callback when the metadata websocket has been opened
Args:
websocket (Websocket): communication socket
"""
self.logger.info("Metadata socket open")
def metadata_on_message(self, websocket, message: str) -> None:
"""Websocket callback when a new metadata message arrives
Args:
websocket (Websocket): communication socket
message (str): received message
"""
self.on_message(websocket, message)
def metadata_on_error(self, websocket, error: str) -> None:
"""Websocket callback when a metadata socket error occurs
Args:
websocket (Websocket): communication socket
error (str): error message
"""
if websocket.keep_running:
self.logger.error("Metadata socket error: {0}".format(error))
def metadata_on_close(self, _websocket) -> None:
"""Websocket callback when the metadata connection closes
Args:
_websocket (Websocket): communication socket
"""
self.logger.warning("Metadata socket close")
def on_message(self, _websocket, message: str) -> None:
"""Called when authentication or metadata message is received
Handles the state machine and closing of the communication threads
Args:
websocket (Websocket): communication socket
message (str): received message
"""
if not self.parse_response(message):
self.logger.error("Test run failed. Exiting.")
self.client.stop_metadata_thread()
self.client.stop_authentication_thread()
else:
self.state = self.State(self.state.value + 1)
if self.state != self.State.END:
self.send_request()
else:
self.return_code = 0
self.client.stop_metadata_thread()
self.client.stop_authentication_thread()
def run(self) -> int:
"""Run method which starts and waits the communication thread(s)
Returns:
int: Process return code
"""
try:
self.authentication_thread = self.client.start_authentication_thread()
self.metadata_thread = self.client.start_metadata_thread()
self.metadata_thread.join()
self.authentication_thread.join()
except:
pass
return self.return_code
from connections import Connections
__dir__ = os.path.dirname(__file__)
config = yaml.load(open(os.path.join(__dir__, "../default_config.yaml")))
try:
config.update(yaml.load(open(os.path.join(__dir__, "../config.yaml"))))
except IOError:
# is ok if we do not have config.yaml
pass
logging.basicConfig(filename=config['KILLER_LOG_PATH'],
level=logging.INFO,
format='%(asctime)s pid:%(process)d %(message)s')
logging.info("Started killer process, with limit %s",
config['QUERY_TIME_LIMIT'])
conn = Connections(config)
cur = conn.replica.cursor()
try:
cur.execute('SHOW PROCESSLIST')
queries = cur.fetchall()
logging.info("Found %s queries running", len(queries))
to_kill = [
q for q in queries
if q[5] > config['QUERY_TIME_LIMIT'] and q[4] != 'Sleep'
]
logging.info("Found %s queries to kill", len(to_kill))
for q in to_kill:
try:
cur.execute('KILL QUERY %s', q[0])
logging.info("Killed query with thread_id:%s" % q[0])
class Network(object):
def __init__(self, layers):
self.connections = Connections()
self.layers = []
layer_count = len(layers)
node_count = 0
for i in range(layer_count):
self.layers.append(Layer(i, layers[i]))
for layer in range(layer_count - 1):
connections = [
Connection(upstream_node, downstream_node)
for upstream_node in self.layers[layer].node
for downstream_node in self.layer[layer + 1].nodes[:-1]]
for conn in connections:
self.connections.add_connection(conn)
conn.downstream_node.append_upstream_conneciont(conn)
conn.upstream_node.append_downstream_connection(conn)
def train(self, labels, data_set, rate, iteration):
for i in range(iteration):
for d in range(len(data_set)):
self.train_one_sample(labels[d], data_set(d), rate)
def train_one_sample(self, label, sample, rate):
self.predict(sample)
self.calc_delta(label)
self.update_weight(rate)
def calc_delta(self, label):
output_nodes = self.layers[-1].nodes
for i in range(len(label)):
output_nodes[i].calc_output_layer_delta(label[i])
for layer in self.layers[-2::-1]:
for node in layer.nodes:
node.calc_hidden_layer_delta()
def update_weight(self, rate):
for layer in self.layers[:-1]:
for node in layer.nodes:
for conn in node.downstream:
conn.update_weight(rate)
def calc_gradient(self):
for layer in self.layers[:-1]:
for node in layer.nodes:
for conn in node.downstream:
conn.calc_gradient()
def get_gradient(self, label, sample):
self.predict(sample)
self.calc_delta(label)
self.calc_gradient()
def predict(self, sample):
self.layers[0].set_output(sample)
for i in range(1, len(self.layers)):
self.layers[i].calc_output()
return map(
lambda node: node.output,
self.layers[-1].nodes[:-1])
def dump(self):
for layer in self.layers:
layer.dump()
class AuthenticationExample(object):
"""Main example class which is run"""
class State(Enum):
"""State enumeration class"""
START = auto()
LOGIN = auto() # Started on authentication_on_open
QUERY_USERS = auto()
CREATE_USER = auto()
QUERY_USERS_2 = auto()
UPDATE_USER = auto()
QUERY_USERS_3 = auto()
DELETE_USER = auto()
QUERY_USERS_4 = auto()
END = auto()
def __init__(self) -> None:
"""Initialization"""
self.return_code = -1
self.state = self.State(self.State.START.value + 1)
self.new_user = dict(
username="******",
password="******",
full_name="John Doe",
role=AuthenticationMessages.Role.OPERATOR.value,
updated_full_name="John J. Doe",
updated_password="******",
updated_role=AuthenticationMessages.Role.ADMIN.value,
)
self.settings = get_settings()
self.logger = setup_log("AuthenticationExample",
self.settings.log_level)
self.client = Connections(
hostname=self.settings.hostname,
logger=self.logger,
authentication_on_open=self.authentication_on_open,
authentication_on_message=self.authentication_on_message,
authentication_on_error=self.authentication_on_error,
authentication_on_close=self.authentication_on_close,
)
self.authentication = AuthenticationMessages(
self.logger, self.settings.protocol_version)
def send_request(self, websocket) -> None:
"""Send request
Args:
websocket (Websocket): communication socket
"""
if self.state.name.startswith(self.State.LOGIN.name):
websocket.send(
json.dumps(
self.authentication.message_login(self.settings.username,
self.settings.password)))
elif self.state.name.startswith(self.State.QUERY_USERS.name):
websocket.send(
json.dumps(self.authentication.message_query_users()))
elif self.state.name.startswith(self.State.CREATE_USER.name):
websocket.send(
json.dumps(
self.authentication.message_create_user(
username=self.new_user["username"],
password=self.new_user["password"],
full_name=self.new_user["full_name"],
role=self.new_user["role"],
)))
elif self.state.name.startswith(self.State.UPDATE_USER.name):
websocket.send(
json.dumps(
self.authentication.message_update_user(
username=self.new_user["username"],
new_password=self.new_user["updated_password"],
new_full_name=self.new_user["updated_full_name"],
new_role=self.new_user["updated_role"],
)))
elif self.state.name.startswith(self.State.DELETE_USER.name):
websocket.send(
json.dumps(
self.authentication.message_delete_user(
username=self.new_user["username"])))
def parse_response(self, message: str) -> bool:
"""Parse response
Args:
message (str): received message
Returns:
bool: True if response's request succeeded
"""
if self.state.name.startswith(self.State.LOGIN.name):
if not self.authentication.parse_login(json.loads(message)):
return False
elif self.state.name.startswith(self.State.QUERY_USERS.name):
if not self.authentication.parse_query_users(json.loads(message)):
return False
elif self.state.name.startswith(self.State.CREATE_USER.name):
if not self.authentication.parse_create_user(json.loads(message)):
return False
elif self.state.name.startswith(self.State.UPDATE_USER.name):
if not self.authentication.parse_update_user(json.loads(message)):
return False
elif self.state.name.startswith(self.State.DELETE_USER.name):
if not self.authentication.parse_delete_user(json.loads(message)):
return False
return True
def authentication_on_open(self, websocket) -> None:
"""Websocket callback when the authentication websocket has been opened
Args:
websocket (Websocket): communication socket
"""
self.logger.info("Socket open")
self.send_request(websocket)
def authentication_on_message(self, websocket, message: str) -> None:
"""Websocket callback when a new authentication message arrives
Args:
websocket (Websocket): communication socket
message (str): received message
"""
if not self.parse_response(message):
self.logger.error("Example run failed. Exiting.")
self.client.stop_authentication_thread()
else:
self.state = self.State(self.state.value + 1)
if self.state != self.State.END:
self.send_request(websocket)
else:
self.return_code = 0
self.client.stop_authentication_thread()
def authentication_on_error(self, websocket, error: str) -> None:
"""Websocket callback when an authentication socket error occurs
Args:
_websocket (Websocket): communication socket
error (str): error message
"""
if websocket.keep_running:
self.logger.error("Socket error: {0}".format(error))
def authentication_on_close(self, _websocket) -> None:
"""Websocket callback when the authentication connection closes
Args:
_websocket (Websocket): communication socket
"""
self.logger.info("Socket close")
def run(self) -> int:
"""Run method which starts and waits the communication thread(s)
Returns:
int: Process return code
"""
try:
self.client.start_authentication_thread().join()
except:
pass
return self.return_code
class Network(object):
def __init__(self, layers):
self.connections = Connections()
self.layers = []
layer_count = len(layers)
node_count = 0
for i in range(layer_count):
self.layer.append(Layer(i, layers[i]))
for layer in range(layer_count - 1):
connections = [
Connection(upstream_node, downstream_node)
for upstream_node in self.layers[layer].nodex
for downstream_node in self.layers[layer + 1].nodes[:-1]
]
for conn in connections:
self.connections.add_connection(conn)
conn.downstream_node.append_upstream_connection(conn)
conn.upstream_node.append_downstream_connection(conn)
def train(self, labels, data_set, rate, epoch):
for i in range(epoch):
for d in range(len(data_set)):
self.tran_one_sample(labels[d], data_set[d], rate)
def train_one_sample(self, lable, sample, rate):
self.predict(sample)
self.calc_dalta(lable)
self.update_weight(rate)
def calc_delta(self, label):
output_nodes = self.layers[-1].nodes
for i in range(len(label)):
output_nodes[i].calc_output_layer_delta(label[i])
for layer in self.layers[-2::-1]:
for node in layer.nodes:
node.calc_hidden_layer_delta()
def update_weight(self, rate):
for layer in self.layers[:-1]:
for node in layer.nodes:
for conn in node.downstream:
conn.update_weight(rate)
def calc_gradient(self):
for layer in self.layer[:-1]:
for node in layer.nodes:
for conn in node.downstream:
conn.calc_gradient()
def predict(self, sample):
self.layers[0].set_output(sample)
for i in range(1, len(self.layers)):
self.layers[i].calc_output()
return [
m
for m in map(lambda node: node.output, self.layers[-1].nodes[:-1])
]
def dump(self):
for layer in self.layers:
layer.dump()
def init(sender, signal):
global conn
conn = Connections(celery.conf)
celery_log.info("Initialized lazy loaded connections")
def _spread(df, col='ind_pred', relations=[]):
"""This'll give some post-hoc test-set analysis, when running this,
keep track of the test sets that improved using relational modeling,
then average those test set statistics together to compare to the test
sets that did not improve."""
t1 = ut.out('computing subgraph statistics...')
con_obj = Connections()
gids = [r[2] for r in relations]
g, sgs = con_obj.find_subgraphs(df, relations, verbose=False)
spread_dict = {}
sg_list = []
for i, sg in enumerate(sgs):
if sg[3] > 0: # num edges > 0
sg_list.extend([(x, i) for x in sg[0]]) # give sg_id
if len(sg_list) == 0:
return spread_dict
sg_df = pd.DataFrame(sg_list, columns=['com_id', 'sg_id'])
df = df.merge(sg_df, how='left')
df['sg_id'] = df['sg_id'].fillna(-1).apply(int)
p, r, ts = precision_recall_curve(df['label'], df[col])
aupr = average_precision_score(df['label'], df[col])
mp = 1.0 - aupr
corrects = []
step = int(len(ts) / 100) if len(ts) > 100 else 1
for i in range(0, len(ts), step):
t = ts[i]
df['pred'] = np.where(df[col] > t, 1, 0)
correct = df['pred'] == df['label']
corrects.append(correct.apply(int))
total_corrects = [sum(x) for x in zip(*corrects)]
df['correct'] = total_corrects
# extract bottom x% data
df = df.sort_values('correct', ascending=False)
ndx = len(df) - int(len(df) * mp)
qfs = df[df['label'] == 1]
qfo = df[df['label'] == 0]
qf1, qf2 = df[ndx:], df[:ndx]
qf1s = qf1[qf1['label'] == 1] # low performers
qf1o = qf1[qf1['label'] == 0] # low performers
qf2s = qf2[qf2['label'] == 1] # high performers
qf2o = qf2[qf2['label'] == 0] # high performers
spread_dict['spam_mean'] = round(qfs['ind_pred'].mean(), 4)
spread_dict['spam_median'] = round(qfs['ind_pred'].median(), 4)
spread_dict['ham_mean'] = round(qfo['ind_pred'].mean(), 4)
spread_dict['ham_median'] = round(qfo['ind_pred'].median(), 4)
for nm, temp_df in [('bot_spam', qf1s), ('bot_ham', qf1o),
('top_spam', qf2s), ('top_ham', qf2o)]:
wf = temp_df[(temp_df[gids] != -1).any(axis=1)]
sg_mean = wf.groupby('sg_id')['ind_pred'].mean().reset_index()\
.rename(columns={'ind_pred': 'sg_mean'})
sg_std = wf.groupby('sg_id')['ind_pred'].std().reset_index()\
.rename(columns={'ind_pred': 'sg_std'})
sg_median = wf.groupby('sg_id')['ind_pred'].median().reset_index()\
.rename(columns={'ind_pred': 'sg_median'})
sg_min = wf.groupby('sg_id')['ind_pred'].min().reset_index()\
.rename(columns={'ind_pred': 'sg_min'})
sg_max = wf.groupby('sg_id')['ind_pred'].max().reset_index()\
.rename(columns={'ind_pred': 'sg_max'})
wf = wf.merge(sg_mean).merge(sg_std).merge(sg_median)\
.merge(sg_min).merge(sg_max)
wf['sg_spread'] = wf['sg_max'] - wf['sg_min']
spread_dict[nm + '_sg_mean'] = round(np.mean(wf['sg_mean']), 4)
spread_dict[nm + '_sg_std'] = round(np.mean(wf['sg_std']), 4)
spread_dict[nm + '_sg_median'] = round(np.mean(wf['sg_median']), 4)
spread_dict[nm + '_sg_min'] = round(np.mean(wf['sg_min']), 4)
spread_dict[nm + '_sg_max'] = round(np.mean(wf['sg_max']), 4)
spread_dict[nm + '_sg_spread'] = round(np.mean(wf['sg_spread']), 4)
ut.time(t1)
return spread_dict
__dir__ = os.path.dirname(__file__)
config = yaml.load(open(os.path.join(__dir__, "../default_config.yaml")))
try:
config.update(yaml.load(open(os.path.join(__dir__, "../config.yaml"))))
except IOError:
# is ok if we do not have config.yaml
pass
logging.basicConfig(
filename=config['KILLER_LOG_PATH'],
level=logging.INFO,
format='%(asctime)s pid:%(process)d %(message)s'
)
logging.info("Started killer process, with limit %s", config['QUERY_TIME_LIMIT'])
conn = Connections(config)
cur = conn.replica.cursor()
try:
cur.execute('SHOW PROCESSLIST')
queries = cur.fetchall()
logging.info("Found %s queries running", len(queries))
to_kill = [q for q in queries
if q[5] > config['QUERY_TIME_LIMIT'] and q[4] != 'Sleep']
logging.info("Found %s queries to kill", len(to_kill))
for q in to_kill:
try:
cur.execute('KILL QUERY %s', q[0])
logging.info("Killed query with thread_id:%s" % q[0])
except pymysql.InternalError as e:
if e.args[0] == 1094: # Error code for 'no such thread'
def __init__(self, wt): Layers.conn = Connections(wt)
class ApplicationConfigurationExample(object):
"""Main example class which is run"""
class State(Enum):
"""State enumeration class"""
START = auto()
LOGIN = auto() # Started on authentication_on_open
REALTIME_SITUATION_LOGIN = auto()
WAIT_FOR_STARTUP_SITUATION = auto()
SET_APPLICATION_CONFIGURATION = auto()
WAIT_FOR_APPLICATION_CONFIGURATION = auto()
END = auto()
def __init__(self) -> None:
"""Initialization"""
self.return_code = -1
self.state = self.State(self.State.START.value + 1)
# Network id needs to point to a valid network with at least one sink
# online
self.network_id = "777555"
# When running more than once for the same network, the diagnostics interval
# or application data needs to changed as WNT server will not try to set the
# application configuration if it is already the same.
self.diagnostics_interval = 30
self.application_data = "00112233445566778899AABBCCDDEEFF"
self.authentication_thread = None
self.metadata_thread = None
self.realtime_situation_thread = None
self.total_node_count = 0
self.loaded_node_count = 0
self.settings = get_settings()
self.logger = setup_log("ApplicationConfigurationExample",
self.settings.log_level)
self.client = Connections(
hostname=self.settings.hostname,
logger=self.logger,
authentication_on_open=self.authentication_on_open,
authentication_on_message=self.authentication_on_message,
authentication_on_error=self.authentication_on_error,
authentication_on_close=self.authentication_on_close,
metadata_on_open=self.metadata_on_open,
metadata_on_message=self.metadata_on_message,
metadata_on_error=self.metadata_on_error,
metadata_on_close=self.metadata_on_close,
realtime_situation_on_open=self.realtime_situation_on_open,
realtime_situation_on_message=self.realtime_situation_on_message,
realtime_situation_on_error=self.realtime_situation_on_error,
realtime_situation_on_close=self.realtime_situation_on_close,
)
self.messages = Messages(self.logger, self.settings.protocol_version)
def send_request(self) -> None:
"""Send request"""
if self.state == self.State.LOGIN:
self.authentication_thread.socket.send(
json.dumps(
self.messages.message_login(self.settings.username,
self.settings.password)))
elif self.state == self.State.REALTIME_SITUATION_LOGIN:
self.realtime_situation_thread.socket.send(
json.dumps(
self.messages.message_realtime_situation_login(
self.messages.session_id)))
elif self.state == self.State.SET_APPLICATION_CONFIGURATION:
self.metadata_thread.socket.send(
json.dumps(
self.messages.message_set_app_config(
self.network_id,
self.diagnostics_interval,
self.application_data,
)))
def parse_response(self, message: str) -> bool:
"""Parse response
Args:
message (str): received message
Returns:
bool: True if response's request succeeded
"""
if self.state == self.State.LOGIN:
return self.messages.parse_login(json.loads(message))
elif self.state == self.State.SET_APPLICATION_CONFIGURATION:
return self.messages.parse_set_app_config(json.loads(message))
def authentication_on_open(self, _websocket) -> None:
"""Websocket callback when the authentication websocket has been opened
Args:
websocket (Websocket): communication socket
"""
self.logger.info("Authentication socket open")
self.send_request()
def authentication_on_message(self, websocket, message: str) -> None:
"""Websocket callback when a new authentication message arrives
Args:
websocket (Websocket): communication socket
message (str): received message
"""
self.on_message(websocket, message)
def authentication_on_error(self, websocket, error: str) -> None:
"""Websocket callback when an authentication socket error occurs
Args:
websocket (Websocket): communication socket
error (str): error message
"""
if websocket.keep_running:
self.logger.error("Authentication socket error: {0}".format(error))
def authentication_on_close(self, _websocket) -> None:
"""Websocket callback when the authentication connection closes
Args:
_websocket (Websocket): communication socket
"""
self.logger.info("Authentication socket close")
def metadata_on_open(self, _websocket) -> None:
"""Websocket callback when the metadata websocket has been opened
Args:
websocket (Websocket): communication socket
"""
self.logger.info("Metadata socket open")
def metadata_on_message(self, websocket, message: str) -> None:
"""Websocket callback when a new metadata message arrives
Args:
websocket (Websocket): communication socket
message (str): received message
"""
self.on_message(websocket, message)
def metadata_on_error(self, websocket, error: str) -> None:
"""Websocket callback when a metadata socket error occurs
Args:
websocket (Websocket): communication socket
error (str): error message
"""
if websocket.keep_running:
self.logger.error("Metadata socket error: {0}".format(error))
def metadata_on_close(self, _websocket) -> None:
"""Websocket callback when the metadata connection closes
Args:
_websocket (Websocket): communication socket
"""
self.logger.warning("Metadata socket close")
def realtime_situation_on_open(self, _websocket) -> None:
"""Websocket callback when the realtime situation websocket has been opened
Args:
websocket (Websocket): communication socket
"""
self.logger.info("Realtime situation socket open")
def realtime_situation_on_message(self, _websocket, message: str) -> None:
"""Websocket callback when a new realtime situation message arrives
Args:
websocket (Websocket): communication socket
message (str): received message
"""
if self.state == self.State.REALTIME_SITUATION_LOGIN:
if not self.messages.parse_realtime_situation_login(
json.loads(message)):
self.logger.error("Test run failed. Exiting.")
self.stop_connection_threads()
else:
self.state = self.State(self.state.value + 1)
self.send_request()
elif self.state == self.state.WAIT_FOR_STARTUP_SITUATION:
wnt_message = wnt_proto.Message()
wnt_message.ParseFromString(message)
if wnt_message.HasField("rtsituation_metadata"):
self.total_node_count += wnt_message.rtsituation_metadata.node_count
if wnt_message.HasField("source_address") and wnt_message.HasField(
"network_id"):
# Here it would be good to count distinct node count
self.loaded_node_count += 1
if self.loaded_node_count == self.total_node_count:
# Initial nodes' data loaded
self.state = self.State(self.state.value + 1)
self.send_request()
# The response might come from the realtime situation connection before
# the metadata connection
elif (self.state == self.state.WAIT_FOR_APPLICATION_CONFIGURATION
or self.state == self.State.SET_APPLICATION_CONFIGURATION):
wnt_message = wnt_proto.Message()
wnt_message.ParseFromString(message)
if wnt_message.HasField("app_config"):
app_config_string = str(
binascii.hexlify(wnt_message.app_config.app_config),
"utf-8")
self.logger.info(
"App config message received with diagnostics interval {} and "
"application data {}".format(
wnt_message.app_config.interval, app_config_string))
# Exit after we get first node app config data message
self.return_code = 0
self.stop_connection_threads()
def realtime_situation_on_error(self, websocket, error: str) -> None:
"""Websocket callback when realtime situation socket error occurs
Args:
websocket (Websocket): communication socket
error (str): error message
"""
if websocket.keep_running:
self.logger.error(
"Realtime situation socket error: {0}".format(error))
def realtime_situation_on_close(self, _websocket) -> None:
"""Websocket callback when the realtime situation connection closes
Args:
_websocket (Websocket): communication socket
"""
self.logger.warning("Realtime situation socket close")
def on_message(self, _websocket, message: str) -> None:
"""Called when authentication or metadata message is received
Handles the state machine and closing of the communication threads
Args:
websocket (Websocket): communication socket
message (str): received message
"""
if not self.parse_response(message):
self.logger.error("Test run failed. Exiting.")
self.stop_connection_threads()
elif not self.state == self.State.WAIT_FOR_APPLICATION_CONFIGURATION:
self.state = self.State(self.state.value + 1)
if self.state != self.State.END:
self.send_request()
else:
self.return_code = 0
self.stop_connection_threads()
def stop_connection_threads(self) -> None:
"""Stop all connection threads"""
self.client.stop_realtime_situation_thread()
self.client.stop_metadata_thread()
self.client.stop_authentication_thread()
def run(self) -> int:
"""Run method which starts and waits the communication thread(s)
Returns:
int: Process return code
"""
try:
self.realtime_situation_thread = (
self.client.start_realtime_situation_thread())
self.metadata_thread = self.client.start_metadata_thread()
self.authentication_thread = self.client.start_authentication_thread(
)
# Maximum wait time 30 seconds
self.realtime_situation_thread.join(30)
self.metadata_thread.join(0)
self.authentication_thread.join(0)
except:
pass
return self.return_code
def get_last_run_id(job_id, estimator_id):
run_id = Connections.exec_sql('QUANT',
''' SELECT last_run_id, convert(varchar(8), last_run_date, 104) last_run_date
FROM %s..last_run WHERE AND job_id=%d
''' % (bq, estimator_id, job_id))
return (int(run_id[0][0]), run_id[0][1]) if run_id is not None else None
def read(cls, proto):
"""
Reads deserialized data from proto object.
:param proto: (DynamicStructBuilder) Proto object
:returns: (:class:TemporalMemory) TemporalMemory instance
"""
1 / 0 # Unimplemented.
tm = object.__new__(cls)
# capnp fails to save a tuple, so proto.columnDimensions was forced to
# serialize as a list. We prefer a tuple, however, because columnDimensions
# should be regarded as immutable.
tm.columnDimensions = tuple(proto.columnDimensions)
tm.cellsPerColumn = int(proto.cellsPerColumn)
tm.activationThreshold = int(proto.activationThreshold)
tm.initialPermanence = proto.initialPermanence
tm.connectedPermanence = proto.connectedPermanence
tm.minThreshold = int(proto.minThreshold)
tm.maxNewSynapseCount = int(proto.maxNewSynapseCount)
tm.permanenceIncrement = proto.permanenceIncrement
tm.permanenceDecrement = proto.permanenceDecrement
tm.predictedSegmentDecrement = proto.predictedSegmentDecrement
tm.maxSegmentsPerCell = int(proto.maxSegmentsPerCell)
tm.maxSynapsesPerSegment = int(proto.maxSynapsesPerSegment)
tm.connections = Connections.read(proto.connections)
#pylint: disable=W0212
tm._random = Random()
tm._random.read(proto.random)
#pylint: enable=W0212
tm.activeCells = [int(x) for x in proto.activeCells]
tm.winnerCells = [int(x) for x in proto.winnerCells]
flatListLength = tm.connections.segmentFlatListLength()
tm.numActiveConnectedSynapsesForSegment = [0] * flatListLength
tm.numActivePotentialSynapsesForSegment = [0] * flatListLength
tm.lastUsedIterationForSegment = [0] * flatListLength
tm.activeEvents = []
tm.matchingEvents = []
for protoSegment in proto.activeEvents:
tm.activeEvents.append(
tm.connections.getSegment(protoSegment.cell,
protoSegment.idxOnCell))
for protoSegment in proto.matchingEvents:
tm.matchingEvents.append(
tm.connections.getSegment(protoSegment.cell,
protoSegment.idxOnCell))
for protoSegment in proto.numActivePotentialSynapsesForSegment:
segment = tm.connections.getSegment(protoSegment.cell,
protoSegment.idxOnCell)
tm.numActivePotentialSynapsesForSegment[segment.flatIdx] = (int(
protoSegment.number))
tm.iteration = long(proto.iteration)
for protoSegment in proto.lastUsedIterationForSegment:
segment = tm.connections.getSegment(protoSegment.cell,
protoSegment.idxOnCell)
tm.lastUsedIterationForSegment[segment.flatIdx] = (long(
protoSegment.number))
return tm
from arguments import Arguments
from connections import Connections
'''
This code runs the JPL Open Source Rover. It accepts a few command line arguments for different functionality
-t : Testing mode - allows you to test commands to send to the rover emulating a signals from a controller
-s : Attempts to connect to a Unix socket for controlling the LED screen. The screen.py script must be running
previous to this in order to work. It lives at ../led/screen.py
-c : Controller flag, letting this program know what type of controller you wish to run with
b : Bluetooth app (default)
x : XBox controller (requires USB reciever)
An example line running this script to run the LED screen and with an Xbox controller
sudo python main.py -s -c x
'''
args = Arguments()
conn = Connections()
rover = Rover()
def listener():
'''
Based on command line args decides which controller and sockets to open
'''
if args.socket:
print "starting LED socket client"
conn.unixSockConnect()
elif args.connect == 'x' or args.connect == 'b':
conn.connect(args.connect)
else:
conn.connect('b')
def socket(self):
return Connections.socket_by_character_id(self.id)
