def register_virtual_device(socketio):
"""
Register the virtual device callback to get invoked periodically.
:param socketio:
:return:
"""
tl = Timeloop()
tl._add_job(send_event, interval=timedelta(seconds=1), args={'socketio': socketio})
# @tl.job(interval=timedelta(seconds=1))
# def on_e_bike_event_received():
# return send_event(socketio)
return tl.start()
class ExtractorJob():
def __init__(self, periodicity, text_filter, source):
self.periodicity = timedelta(seconds=periodicity)
self.source = source
self.text_filter = [text_filter]
self.t1 = Timeloop()
self.setup_logger()
def test_function(self):
self.logger.info('Begin {source_name} test'.format(
source_name=self.source.__name__))
print('In test function')
def target_function(self):
print('Begin target function')
# Begin first test
self.logger.info('Begin {source_name} test'.format(
source_name=self.source.__name__))
extractor = self.source()
# Get the urls
extractor.get_news_urls(datetime.today())
# Extract text from news
extractor.extract_text_from_news()
# Filter by keywords
extractor.filter_news_by_keywords(self.text_filter)
# Close the extractor
del extractor
def setup_logger(self):
# Configure logger: oddcrawler needsd to be the top logger
self.logger = getLogger('oddcrawler')
self.logger.setLevel(DEBUG)
# create file file handler
fh = FileHandler('extractor_test.log')
fh.setLevel(DEBUG)
# create console handler
ch = StreamHandler()
ch.setLevel(ERROR)
# create formatter and add it to handlers
formatter = Formatter('%(levelname)s %(asctime)-15s %(message)s')
fh.setFormatter(formatter)
ch.setFormatter(formatter)
self.logger.addHandler(fh)
self.logger.addHandler(ch)
def run(self):
self.t1._add_job(self.target_function, interval=self.periodicity)
self.t1.start(block=True)
def sync(key_file, user, remote_out_dir, local_sync_dir, every):
if every == 0:
# Sync only one time.
sync_once(key_file, user, remote_out_dir, local_sync_dir)
else:
# Set up timer to sync regularly.
tl = Timeloop()
tl._add_job(
sync_once,
timedelta(seconds=every),
key_file,
user,
remote_out_dir,
local_sync_dir,
)
tl.start(block=True)
class PriceFeederJobBase:
def __init__(self, price_f_config, config_net, connection_net):
self.options = price_f_config
self.config_network = config_net
self.connection_network = connection_net
# connection network is the brownie connection network
# config network is our enviroment we want to connect
network_manager.connect(connection_network=self.connection_network,
config_network=self.config_network)
address_medianizer = self.options['networks'][self.config_network]['addresses']['MoCMedianizer']
address_pricefeed = self.options['networks'][self.config_network]['addresses']['PriceFeed']
log.info("Starting with MoCMedianizer: {}".format(address_medianizer))
log.info("Starting with PriceFeed: {}".format(address_pricefeed))
self.app_mode = self.options['networks'][self.config_network]['app_mode']
# simulation don't write to blockchain
self.is_simulation = False
if 'is_simulation' in self.options:
self.is_simulation = self.options['is_simulation']
# Min prices source
self.min_prices_source = 1
if 'min_prices_source' in self.options:
self.min_prices_source = self.options['min_prices_source']
# backup writes
self.backup_writes = 0
self.tl = Timeloop()
self.last_price = 0.0
self.last_price_timestamp = datetime.datetime.now() - datetime.timedelta(seconds=300)
self.price_source = PriceEngines(self.options['networks'][self.config_network]['price_engines'],
log=log,
app_mode=self.app_mode,
min_prices=self.min_prices_source)
@staticmethod
def aws_put_metric_exception(value):
""" Only for AWS cloudwatch"""
if 'AWS_ACCESS_KEY_ID' not in os.environ:
return
# Create CloudWatch client
cloudwatch = boto3.client('cloudwatch')
# Put custom metrics
cloudwatch.put_metric_data(
MetricData=[
{
'MetricName': os.environ['PRICE_FEEDER_NAME'],
'Dimensions': [
{
'Name': 'PRICEFEEDER',
'Value': 'Error'
},
],
'Unit': 'None',
'Value': value
},
],
Namespace='MOC/EXCEPTIONS'
)
def price_feed(self):
""" Post price """
return
def price_feed_backup(self):
""" Post price in backup mode """
return
def job_price_feed(self):
try:
self.price_feed()
except Exception as e:
log.error(e, exc_info=True)
self.aws_put_metric_exception(1)
def job_price_feed_backup(self):
try:
self.price_feed_backup()
except Exception as e:
log.error(e, exc_info=True)
self.aws_put_metric_exception(1)
def add_jobs(self):
# creating the alarm
self.aws_put_metric_exception(0)
backup_mode = False
if 'backup_mode' in self.options:
if self.options['backup_mode']:
backup_mode = True
if backup_mode:
log.info("Job Price feeder as BACKUP!")
self.tl._add_job(self.job_price_feed_backup, datetime.timedelta(
seconds=self.options['interval']))
else:
self.tl._add_job(self.job_price_feed, datetime.timedelta(
seconds=self.options['interval']))
def time_loop_start(self):
self.add_jobs()
self.tl.start()
while True:
try:
time.sleep(1)
except KeyboardInterrupt:
self.tl.stop()
break
class Session():
"""
Interacts with a CouchDB server's REST API for session management.
Examples:
couchDB = Session(address="https://somehost.com", port="6984", username="******", password="******", address="https://somehost.com", port="6984")
couchDB = Session(address="https://somehost.com", port="6984", auth_token="bGVlLmx1bm5AZGFsLmNhOjVERTQ4RjhCOq-IvL4mhVVUFn4k5H1bIYiggf3X")
couchDB = Session(address="https://somehost.com", port="6984", username="******", password="******", keep_alive=290)
Attributes:
:param bool admin_party Determines whether or not to attempt connections to the CouchDB using Admin Party. (Default: False)
:param str username: Username used to authenticate to the CouchDB server. (Default: None)
:param str password: Password used to authenticate to the CouchDB server. (Default: None)
:param str auth_token: AuthSession value used to authenticate to the CouchDB server.
If provided, authentication using AuthSession will be attempted. AuthSession is set/updated
with successful authentication when connecting with username and password. (Default: None)
:param str host: Address that the CouchDB server is served from. (Default: http://127.0.0.1)
:param int port: Port number that the CouchDB server is listening on. (Default: 5984)
:param int keep_alive: Determines if automatic session renewal will be attempted and at what frequency. If > 0, session renewal is performed every keep_alive seconds. (Default: 0)
:param bool auto_connect: Determines if an authentication attempt will be made during instancing of this object. (Default: False)
:param bool basic_auth: Sets authentication method to the CouchDB server to Basic. If basic authentication is used, auto_connect has no effect. (Default: False)
:param dict custom_headers: Dictionary of custom headers to add to each request to the CouchDB server. (Default: None)
"""
def __init__(self, **kwargs):
self._host = kwargs.get('host', 'http://127.0.0.1')
self._port = kwargs.get('port', 5984)
self.address = f'{self._host}:{self._port}'
self.custom_headers = kwargs.get('custom_headers',
{}) # TODO: implement
self._keep_alive = kwargs.get('keep_alive', 0)
self._keep_alive_timeloop = Timeloop()
self._keep_alive_timeloop.logger.setLevel('WARNING')
self._name = kwargs.get('username', None)
self._password = kwargs.get('password', None)
self.auth_token = kwargs.get('auth_token', None)
self._auto_connect = kwargs.get('auto_connect', False)
self._basic_auth = kwargs.get('basic_auth',
False) # TODO: implement basic auth
self._admin_party = kwargs.get('admin_party',
False) # TODO: implement admin party
self._headers = {
'Content-type': 'application/json',
'Accept': 'application/json'
}
# reference to this object is required for the CouchDBDecorators.endpoint to be able to update the auth token
self.session = self
# TODO: implement a generic Error class to hold error information that consumer can check
if (self._auto_connect is True and self._basic_auth is False):
self.authenticate(data={
'name': self._name,
'password': self._password
})
def __del__(self):
if (self._keep_alive > 0):
self._keep_alive_timeloop.stop()
def _create_basic_auth_header(self):
return requests.auth.HTTPBasicAuth(self._name, self._password)(
requests.Request()).headers
def set_auth_token_from_headers(self, headers):
# if a new auth token is issued, include it in the response, otherwise, return the original
if ('Set-Cookie' in headers):
self.auth_token = headers.get('Set-Cookie').split(
';', 2)[0].split('=')[1]
@RelaxedDecorators.endpoint('/_session',
method='post',
data_keys={
'name': str,
'password': str
})
def authenticate(self, doc):
return doc
@RelaxedDecorators.endpoint('/_session')
def get_session_info(self, doc):
return doc
@RelaxedDecorators.endpoint('/_session', method='delete')
def close(self, doc):
return doc if isinstance(doc, CouchError) else None
def authenticate_via_proxy(self, username, roles, token):
"""
Not implemented. See:
https://docs.couchdb.org/en/stable/api/server/authn.html#proxy-authentication
https://stackoverflow.com/a/40499853/3169479 (for implementation details)
"""
pass
def renew_session(self):
"""
Alias for get_session_info()
"""
return self.get_session_info()
def keep_alive(self, isEnabled=False):
"""
Enables or disables keep alive.
"""
if (isEnabled is False):
self._keep_alive_timeloop.stop()
elif (isEnabled and self._keep_alive > 0
and self.auth_token is not None):
if (len(self._keep_alive_timeloop.jobs) == 0):
self._keep_alive_timeloop._add_job(
func=self.renew_session,
interval=timedelta(seconds=self._keep_alive))
self._keep_alive_timeloop.start()
MAX_ITERATIONS,
beta=1.1,
error_on=error,
error_start=args.error_start_index,
error_duration=10,
max_retries=NODES * 4,
manager=manager,
network_id=NETWORK_ID,
use_api=API,
api_address=API_ADDRESS))
start_time = None
end_time = None
try:
tl = Timeloop()
tl._add_job(log_status, interval=timedelta(milliseconds=5000))
tl.start()
start_time = timer()
for s in servers:
s.daemon = True
s.start()
while (True):
data = list(map(lambda s: s._j.k(), servers))
if (np.min(data) >= MAX_ITERATIONS):
break
else:
time.sleep(1)
finally:
end_time = timer()
tl.stop()
class Server(object):
def __init__(self, config: Config, signals: np.ndarray, output = None, interval = 1000, new_reference_signal_each_k = 5, max_iterations = 1200, error_on = False, error_start = 0, error_duration = 10, network_id = 2):
"""
Creates a Server object using:
str:host\t host address
int:port\t port where server runs
int:id\t ID of the Server (used in numpy to locate position in matrix)
np.array: adjacency\t Adjacency matrix of the whole/sub system
int:signal\t Initial value for the node to start
dict:out_neighbors\t contains all outneighbor host addresses in key property
bool: instant_start\t whether the server should start immediately
Returns: Server(object).
"""
logger = logging.getLogger(name='Server.__init__')
manager = Manager() # used to synchronize data between processes and threads
self._host = config.host
self._port = config.port
self.__address = (config.host, config.port)
self._adjacency = config.adjacency
self._id = config.id
self._laplacian = utility.calculate_laplacian(self._adjacency)
if not utility.check_laplacian(self._laplacian):
raise BaseException("No valid laplacian")
# self._beta = 1/np.max(np.linalg.eigvals(self._laplacian)) # calculates beta, moved to utility
self._beta = utility.calculate_beta(self._laplacian)
self._server_socket = socket(AF_INET, SOCK_STREAM)
self._server_socket.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1)
self._server_socket.bind(self.__address)
self._j = J(0, signals[0], config.out_neighbors, manager)
self._neighbor_states = manager.dict()
self._interval = interval
self._interval_sec = float(interval / 1000.0)
self.__signals = signals
self._new_reference_signal_each_k = new_reference_signal_each_k
self.__output = output
self.__max_iterations = max_iterations
self.__running = False
self.__neighbor_out_connections = manager.dict()
self.__error_on = error_on
self.__error_start = error_start
self.__error_duration = error_duration
self.__API_URL = 'http://10.0.2.2:7071'
self.__NETWORK_ID = network_id
self.__API_QUEUE = manager.list()
self.__API_QUEUE_LOCK = Lock()
self.__tl = Timeloop() # allows to start recurring threads in a given time interval
if config.instant_start:
self.start()
# logger.debug(self)
logger.warn(f"Using beta {self._beta:24.20f}")
def start(self):
''' Starting server, acccept process and broadcast Threads. '''
if self.__running:
return
try:
self.__running = True
self._server_socket.listen(500)
self.accept_process = Process(target=self.accept_connections)
self.accept_process.daemon = True
self.accept_process.start()
self.__tl._add_job(self.broadcast, interval=timedelta(milliseconds=self._interval))
self.__tl._add_job(self.callApi, interval=timedelta(seconds=2))
self.__tl.start()
except BaseException as e:
logging.getLogger(name='start').error(str(e))
self.stop()
def stop(self):
''' Stopping all processes and threads. '''
if not self.__running:
return
self.__running = False
# self.accept_process.terminate()
self.accept_process.join(2)
self._server_socket.close()
def accept_connections(self):
''' Waits for new connections. Creates a handling Thread for all new connections. '''
logger = logging.getLogger(name='Server:accept_connections')
while True:
try:
logger.info("Waiting for new connection...")
client, client_address = self._server_socket.accept()
if self._j.k() < self.__max_iterations:
thread = Thread(target=self.handle_connection, args=(client,client_address))
thread.daemon = True
logger.info(f"Retrieving information from\t{client_address}\tat host {self.__address}")
thread.start()
else:
return
except IOError as e:
if e.errno == errno.EPIPE:
return
logger.error(str(e))
def handle_connection(self, client: socket, client_address:(str, int)):
''' Handles each incomming connection and stores the message. '''
logger = logging.getLogger(name='Server:handle_connection')
while True:
try:
msg = self.receive(client)
sender = msg[0]
key = msg[2]
state = msg[1]
self._neighbor_states[(sender, key)] = state # sample item created: (('127.0.0.1', 4), 451.75)
logger.info(f"[{self._host}]: Stored [{(sender, key)}]: {str(state)}")
logger.debug(f"Node {self._id} has states {self._neighbor_states}")
if key == self.__max_iterations:
break
except OSError as e:
if e.errno == errno.EPIPE:
break
else:
logger.error(e)
# @self.__tl.job(interval=timedelta(milliseconds=args.time))
def broadcast(self):
''' Gets called by the Timeloop class. Creates a Thread which handles computation of new state and broadcasting. '''
try:
if self._j.k() < self.__max_iterations:
# use the following to start the broadcast function as a new threat -> timeloop will continoue immediately and does not wait until finished
# logging.getLogger(name='Server:broadcast').info("broadcast job current time: %s with data: %s" % (time.ctime(), str(self._neighbor_states)))
# thread = Thread(target=self.broadcast_thread) #, args=(self._j, self._neighbor_states, self._adjacency)
# thread.daemon = True
# thread.start()
# use this to have only one broadcast thread at once
self.broadcast_thread()
else:
self.stop()
except OSError as e:
if e.errno == errno.EPIPE or e.errno == errno.ECONNRESET:
self.stop()
else:
raise e
def broadcast_thread(self):
''' Initially sends message to all neighbor nodes. Calculates new state each round and distributes it. '''
logger = logging.getLogger(name='Server:broadcast_thread')
relevant_states = dict(filter(
lambda elem: elem[0][1] == self._j.k(),
self._neighbor_states.items()
)) # states with same k as own state
relevant_states = dict(map(
lambda elem: (elem[0][0],float(elem[1])),
relevant_states.items()
)) # map dict to remove k from key variable (tuple): ((str, int), float) -> (str, float)
if self._j.k() == 0:
logger.info(f"Node {self._id} initially broadcasts its value.")
self.distribute_state()
x = np.zeros(np.shape(self._adjacency)[0])
if len(relevant_states) == len(self._j.neighbors):
logger.debug(f"{self._j.neighbors.items()}")
for neighbor in relevant_states.keys():
i = self._j.neighbors[neighbor]
logger.info(f"{neighbor} > {i}")
np.put(x, int(i), relevant_states[neighbor])
np.put(x, self._id, self._j.state())
# loading new reference signals
sig_nr = int(self._j.k()/self._new_reference_signal_each_k)
if self._j.k() > 0 and sig_nr < 200:
if self._j.k() % self._new_reference_signal_each_k == 0:
self._j.set_reference_signal(self.__signals[sig_nr])
else:
self._j.set_reference_signal(self._j.reference_signal())
# calculating new state
x_new = utility.calculate_iteration(self._id, self._laplacian, x, self._j.diff(), self._beta)
self._j.increment_k()
self._j.set_state(x_new)
self.distribute_state()
# add log to api queue
with self.__API_QUEUE_LOCK:
self.__API_QUEUE.append(
{
'nodeId': self._host,
'port': self._port,
'state': float(np.real(x_new)),
'neighborStates': relevant_states,
'iteration': self._j.k(),
'timestamp': datetime.utcnow().__str__(),
'networkId': self.__NETWORK_ID,
'referenceSignal': self._j.reference_signal()
}
)
logger.debug(f" UPDATED: Node {self._id}: {self._j.state()} | Others: {self._neighbor_states}")
# log to output if specified
if self.__output != None:
try:
self.__output.write(str(float(self._j.state())) + '\n')
except BaseException as e:
logger.error("Unable to wirte to output.\n" + str(e))
else:
logger.info('Rebroadcasting on node: ' + self._host)
self.distribute_state() # sends the new state to the neighbor nodes
time.sleep(self._interval_sec)
''' Removes all data from previous calculations (session_key < k). '''
states_to_remove = dict(filter(
lambda elem: elem[0][1] < self._j.k(), # elem[0] is key of dict and consists of node name and session key k
self._neighbor_states.items()
))
for state in states_to_remove.items():
try:
del self._neighbor_states[state[0]]
except KeyError:
logger.warn(f"Key '{state[0]}' not found.")
def distribute_state(self):
''' Sends the current state to all neighbor nodes. '''
if self._j.k() >= self.__max_iterations:
return
for neighbor in self._j.neighbors.keys():
try:
if neighbor not in self.__neighbor_out_connections:
if (neighbor, self.__max_iterations) not in self._neighbor_states:
client_socket = socket(AF_INET, SOCK_STREAM)
client_socket.connect((neighbor, self._port))
self.__neighbor_out_connections[neighbor] = client_socket
else:
continue
# adding error to own signal if selected
error = 0.0
if self.__error_on:
if self._j.k() >= self.__error_start and self._j.k() < self.__error_duration:
error = utility.calculate_error(self._j.k())
logging.debug(f"Adding error on node {self._id}: {error}")
self.send(self.__neighbor_out_connections[neighbor], message=(self._host, float(self._j.state()+error), self._j.k()))
except Exception as e:
logging.getLogger(name='Server:distribute_state').debug(str(e))
try:
self.__neighbor_out_connections[neighbor].close()
except Exception as e:
pass
finally:
del self.__neighbor_out_connections[neighbor]
logging.getLogger(name='Server:distribute_state').warn('Deleting sockeet connection for ' + neighbor)
def send(self, channel: socket, message: object ):
''' Sends a message to another socket using JSON. '''
try:
msg = json.dumps(message)
logging.info(f"sending data from {self._host}:\t{msg}")
channel.send(struct.pack("i", len(msg)) + bytes(msg, "utf8"))
return True
except OSError as e:
logging.error(e)
return False
def receive(self, channel: socket ) -> (str, float, int):
''' Receives a message from another socket using JSON. '''
recv = channel.recv(struct.calcsize("i"))
if len(recv) < 4:
raise OSError(errno.EPIPE, 'Empyt message size', str(len(recv)))
size = struct.unpack("i", recv)[0]
data = ""
while len(data) < size:
msg = channel.recv(size - len(data))
if not msg:
return None
data += msg.decode("utf8")
logging.info(f"receiving data at {self._host}:\t{str(data)}")
return json.loads( data.strip() )
def callApi(self):
try:
with self.__API_QUEUE_LOCK:
data = list(self.__API_QUEUE)
self.__API_QUEUE[:] = []
if len(data) > 0:
requests.post(f"{self.__API_URL}/api/log", json=data)
except ValueError as ve:
logging.getLogger(name='callApi: ').error(ve)
except ConnectionError as ce:
logging.getLogger(name='callApi: ').error(ce)
except TimeoutError as te:
logging.getLogger(name='callApi: ').error(te)
except requests.exceptions.RequestException as re:
logging.getLogger(name='callApi: ').error(re)
