def get_experiment_target(experiment_id: int) -> pd.DataFrame:
database = Database()
query = f"SELECT project_id FROM experiment WHERE id = {experiment_id}"
project_id = database.read(query=query)[0]['project_id']
registry = Registry()
return registry.get_dataset(path=f"{project_id}-{experiment_id}",
key='target')
def deal_eve_content(filecontent):
link_mongo = Database(database_name="Semitic")
link_ioc = IoC()
eve_json = []
json_result = {}
if filecontent:
for i in filecontent:
eve_json.append(json.loads(i))
link_mongo.insert("eve", eve_json)
json_result = classify_eve(eve_json)
if json_result:
for type_json in json_result:
link_mongo.insert(type_json, json_result[type_json])
try:
if type_json == "tcp":
link_mongo.insert(
"alert_ioc", link_ioc.deal_tcp(json_result[type_json]))
elif type_json == "tls":
link_mongo.insert(
"alert_ioc", link_ioc.deal_tls(json_result[type_json]))
elif type_json == "http":
link_mongo.insert(
"alert_ioc", link_ioc.deal_http(json_result[type_json]))
elif type_json == "dns":
link_mongo.insert(
"alert_ioc", link_ioc.deal_dns(json_result[type_json]))
pass
except:
pass
link_mongo.close()
link_ioc.link_mongo.close()
def __init__(self, data):
self.link = Database("Intelligence")
self.search_type = list(data.keys())[0]
self.search_value = data[self.search_type]
self.result = {}
self.result["data"] = []
super().__init__()
def get_experiment_model(experiment_id: int) -> object:
database = Database()
query = f"SELECT project_id FROM experiment WHERE id = {experiment_id}"
project_id = database.read(query=query)[0]['project_id']
registry = Registry()
return registry.get_model(path=f"{project_id}-{experiment_id}",
key='model')
def get_availability(self):
with Database() as db:
sql = 'SELECT "day", start, "type" FROM availability WHERE user_id = %s'
result = db.query(sql, self.id)
results = []
for r in result:
results.append([r[0], convert_time(r[1]), r[2]])
return results
def test_with_commit(self):
"""Ensure that an error free with statement commits when exiting
Performs an error free with block and ensures the data remains
"""
with Database(database=TEST_DATABASE) as db:
db.query('INSERT INTO test_data (variable) VALUES (1)')
result = self.connection.query('SELECT * FROM test_data')
self.assertEqual(result, [(1, )])
def by_email(cls, email):
with Database() as db:
if db.exists("user", email=email):
sql = 'SELECT id FROM "user" WHERE email = %s'
id = db.query(sql, email)[0][0]
else:
raise KeyError('User {} not found'.format(email))
return User(id)
def login(cls, email, password):
with Database() as db:
password = hashlib.sha224(bytes(password, "utf-8")).hexdigest()
sql = 'SELECT "id", password FROM "user" WHERE email = %s'
user_id, db_password = db.query(sql, email)[0]
if password == db_password:
return User(user_id)
return None
def create(cls, course, start, end, day, location):
with Database() as db:
id = uuid.uuid4().int
id = int(str(id)[:10])
sql = 'INSERT INTO "session" (id, course, start, "end", "day", location) VALUES (%s, %s, %s, %s, %s, %s)'
db.query(sql, id, course, start, end, day, location)
return Session(id, course)
def create(cls, name):
with Database() as db:
id = uuid.uuid4().int
id = int(str(id)[:10])
sql = 'INSERT INTO "course" (id, "name") VALUES (%s, %s)'
db.query(sql, id, name)
return Course(id)
def __init__(self, id):
self.id = id
self.name = "CSSE1001"
with Database() as db:
if db.exists("course", id=self.id):
sql = 'SELECT "name" FROM "course" WHERE id = %s'
self.name = db.query(sql, self.id, limit=1)[0][0]
else:
raise KeyError('Course {} not found'.format(self.id))
def test_rollback(self):
"""Ensure that the database rollback method removes changes to the data
"""
# insert test data and rollback
self.connection.query('INSERT INTO test_data (variable) VALUES (1)')
self.connection.rollback()
self.connection.close()
# reopen connection and query to ensure data was removed
self.connection = Database(database=TEST_DATABASE)
result = self.connection.query('SELECT * FROM test_data')
self.assertEqual(result, [])
def test_commit(self):
"""Ensure that the database commit method retains
the data after reconnection.
"""
# insert test data and commit
self.connection.query('INSERT INTO test_data (variable) VALUES (1)')
self.connection.commit()
self.connection.close()
# reopen connection and query to ensure data was retained
self.connection = Database(database=TEST_DATABASE)
result = self.connection.query('SELECT * FROM test_data')
self.assertEqual(result, [(1, )])
def test_with_rollback(self):
"""Ensure that an error occurring in a database with context forces the database to rollback
Raises an error in the with context and ensures that modifications were removed
"""
try:
with Database(database=TEST_DATABASE) as db:
db.query('INSERT INTO test_data (variable) VALUES (1)')
raise Exception
except:
pass
result = self.connection.query('SELECT * FROM test_data')
self.assertEqual(result, [])
def create(cls, email, password, permissions):
with Database() as db:
if db.exists("user", email=email):
return None
password = hashlib.sha224(bytes(password, "utf-8")).hexdigest()
id = uuid.uuid4().int
id = int(str(id)[:10])
sql = 'INSERT INTO "user" (id, email, permission, password) VALUES (%s, %s, %s, %s)'
db.query(sql, id, email, permissions, password)
return User(id)
def __init__(self, id, course):
self.id = id
self.course = course
self.start = datetime.now()
self.end = datetime.now()
self.day = datetime.now()
self.location = "SAD"
with Database() as db:
if db.exists("session", id=self.id):
sql = 'SELECT start, "end", "day", location FROM "session" WHERE id = %s AND course_id = %s'
self.start, self.end, self.day, self.location = db.query(
sql, self.id, self.course, limit=1)[0]
else:
raise KeyError('Session Time {} not found'.format(self.id))
def __init__(self, id):
self.id = id
self.email = "*****@*****.**"
self.permissions = 0
with Database() as db:
if db.exists("user", id=self.id):
sql = 'SELECT "email", permission FROM "user" WHERE id = %s'
self.email, self.permissions = db.query(sql, self.id,
limit=1)[0]
else:
raise KeyError('User {} not found'.format(self.id))
self.is_authenticated = True
self.is_active = True
self.is_anonymous = False
def decorated(*args, **kwargs):
"""creates the decorator"""
token = None
if 'Authorization' in request.headers:
token = request.headers['Authorization']
if not token:
return jsonify({'message': 'Token is missing!'}), 401
try:
data = jwt.decode(token, 'mysecret')
database = Database()
query = database.get_by_argument('users', 'email', data['email'])
current_user = User(query[0], query[1], query[2], query[3])
except:
return jsonify({'message': 'Token is invalid!'}), 401
return f(current_user, *args, **kwargs)
def run_experiment(project_id: int, engine: str, model: object,
target_col: str, train_data: pd.DataFrame,
test_data: pd.DataFrame, metrics: dict) -> int:
_ENGINES = ['sklearn']
if engine not in _ENGINES:
raise Exception(
f'Engine not registered, must be one of the following: {",".join(_ENGINES)}'
)
database = Database()
query = f"INSERT INTO experiment (project_id, engine) VALUES ('{project_id}', '{engine}')"
experiment_id = database.write(query=query)
registry = Registry()
registry.put_model(path=f"{project_id}-{experiment_id}",
key='pre-model',
model=model)
registry.put_metrics(path=f"{project_id}-{experiment_id}",
key='metrics',
metrics=metrics)
registry.put_dataset(path=f"{project_id}-{experiment_id}",
key='test',
dataset=test_data)
registry.put_dataset(path=f"{project_id}-{experiment_id}",
key='train',
dataset=train_data)
pipeline = Pipeline()
conf = {
'experiment_id': experiment_id,
'project_id': project_id,
'target_col': target_col
}
triggered = pipeline.trigger_dag(dag_id=f'{engine}-pipeline',
data=dict(conf=conf))
if triggered:
query = f"UPDATE experiment SET status = 'submitted' WHERE id = {experiment_id}"
database.write(query=query)
else:
query = f"UPDATE experiment SET status = 'submission-failed' WHERE id = {experiment_id}"
database.write(query=query)
raise Exception('Pipeline failed to run experiment')
return experiment_id
def test_database_init(app):
from api.database import Database
db = Database(app)
assert db.app == app
import tornado.ioloop
import tornado.web
import json
import sys
import os
from api.database import Database
from api.service import Service, UnknownInstanceError
from api.import_data import import_local_data
from api.endpoint import Endpoint
if __name__ == "__main__":
try:
# initialize database schema (SQLAlchemy)
db = Database("hivery.db")
# pre-process raw data files and load into database
import_local_data(db, "data/companies.json", "data/people.json", "data/foods.json")
# pass database to service
service = Service(db)
# construct the API endpoint
endpoint = Endpoint(service)
# start listening on the API endpoint
endpoint.run(port_num=8888)
except ImportError as err:
print("data import failed because:: " + str(err))
def get_prediction(prediction_id: int):
database = Database()
query = f"SELECT * FROM serving WHERE id = {prediction_id}"
return database.read(query=query)[0]
from api.database import Database
from api.models import LoggedInResponse, ShortURL, SuccessfulShortURLResponse
from api.utils import authenticate_token, create_short_url, create_user
load_dotenv(".env")
API_BASE = os.environ.get("API_BASE_URL", "http://*****:*****@app.on_event("startup")
async def startup() -> None:
await db.connect()
await db.initialize_tables()
@app.on_event("shutdown")
async def shutdown() -> None:
await db.disconnect()
@app.get("/discord", response_model=LoggedInResponse)
def get_projects() -> dict:
query = "SELECT * FROM project"
database = Database()
return database.read(query=query)
from flask_restful import Resource, Api
from flask import Flask, session, request, url_for, render_template, redirect, abort, escape, flash
from flask import current_app as app
from werkzeug.security import generate_password_hash, check_password_hash
from functools import wraps
from datetime import datetime
from api.database import Database
database = Database()
users = database.get_users()
def login_required(f):
@wraps(f)
def wrap(*args, **kwargs):
print(request.data)
print(request.environ)
if session.get("authenticated"):
return f(*args, **kwargs)
else:
return {"error": "You need to authenticate to use this endpoint"}, 401
return wrap
class Login(Resource):
def __init__(self):
super().__init__()
def post(self):
data = request.json
def create_project(name: str) -> int:
query = f"INSERT INTO project (name) VALUES ('{name}')"
database = Database()
return database.write(query=query)
def setup_method(self, test_method):
"""Create/Wipe a table called test_data where tests will be run"""
self.connection = Database(database=TEST_DATABASE)
self.connection.query('DROP TABLE IF EXISTS test_data')
self.connection.query('CREATE TABLE test_data (variable INTEGER)')
def get_experiment(experiment_id: int) -> dict:
query = f"SELECT * FROM experiment WHERE id = {experiment_id}"
database = Database()
return database.read(query=query)[0]
import os
from flask import jsonify
from api import app
from api.database import Database
db = Database()
config_name = os.getenv('APP_SETTINGS')
@app.errorhandler(405)
def url_not_found(error):
return jsonify({'message': 'Requested method not allowed'}), 405
@app.errorhandler(404)
def page_not_found(error):
return jsonify({'message': 'page not found, check the url'}), 404
@app.errorhandler(500)
def internal_error(error):
return "500 error"
if __name__ == '__main__':
db.create_tables()
app.run()
def get_experiments() -> dict:
query = "SELECT * FROM experiment"
database = Database()
return database.read(query=query)
