def get_experiment_target(experiment_id: int) -> pd.DataFrame:
database = Database()
query = f"SELECT project_id FROM experiment WHERE id = {experiment_id}"
project_id = database.read(query=query)[0]['project_id']
registry = Registry()
return registry.get_dataset(path=f"{project_id}-{experiment_id}",
key='target')
def get_experiment_model(experiment_id: int) -> object:
database = Database()
query = f"SELECT project_id FROM experiment WHERE id = {experiment_id}"
project_id = database.read(query=query)[0]['project_id']
registry = Registry()
return registry.get_model(path=f"{project_id}-{experiment_id}",
key='model')
def __init__(self, data):
self.link = Database("Intelligence")
self.search_type = list(data.keys())[0]
self.search_value = data[self.search_type]
self.result = {}
self.result["data"] = []
super().__init__()
def test_rollback(self):
"""Ensure that the database rollback method removes changes to the data
"""
# insert test data and rollback
self.connection.query('INSERT INTO test_data (variable) VALUES (1)')
self.connection.rollback()
self.connection.close()
# reopen connection and query to ensure data was removed
self.connection = Database(database=TEST_DATABASE)
result = self.connection.query('SELECT * FROM test_data')
self.assertEqual(result, [])
def test_commit(self):
"""Ensure that the database commit method retains
the data after reconnection.
"""
# insert test data and commit
self.connection.query('INSERT INTO test_data (variable) VALUES (1)')
self.connection.commit()
self.connection.close()
# reopen connection and query to ensure data was retained
self.connection = Database(database=TEST_DATABASE)
result = self.connection.query('SELECT * FROM test_data')
self.assertEqual(result, [(1, )])
def decorated(*args, **kwargs):
"""creates the decorator"""
token = None
if 'Authorization' in request.headers:
token = request.headers['Authorization']
if not token:
return jsonify({'message': 'Token is missing!'}), 401
try:
data = jwt.decode(token, 'mysecret')
database = Database()
query = database.get_by_argument('users', 'email', data['email'])
current_user = User(query[0], query[1], query[2], query[3])
except:
return jsonify({'message': 'Token is invalid!'}), 401
return f(current_user, *args, **kwargs)
def get_availability(self):
with Database() as db:
sql = 'SELECT "day", start, "type" FROM availability WHERE user_id = %s'
result = db.query(sql, self.id)
results = []
for r in result:
results.append([r[0], convert_time(r[1]), r[2]])
return results
def by_email(cls, email):
with Database() as db:
if db.exists("user", email=email):
sql = 'SELECT id FROM "user" WHERE email = %s'
id = db.query(sql, email)[0][0]
else:
raise KeyError('User {} not found'.format(email))
return User(id)
def create(cls, course, start, end, day, location):
with Database() as db:
id = uuid.uuid4().int
id = int(str(id)[:10])
sql = 'INSERT INTO "session" (id, course, start, "end", "day", location) VALUES (%s, %s, %s, %s, %s, %s)'
db.query(sql, id, course, start, end, day, location)
return Session(id, course)
def create(cls, name):
with Database() as db:
id = uuid.uuid4().int
id = int(str(id)[:10])
sql = 'INSERT INTO "course" (id, "name") VALUES (%s, %s)'
db.query(sql, id, name)
return Course(id)
def __init__(self, id):
self.id = id
self.name = "CSSE1001"
with Database() as db:
if db.exists("course", id=self.id):
sql = 'SELECT "name" FROM "course" WHERE id = %s'
self.name = db.query(sql, self.id, limit=1)[0][0]
else:
raise KeyError('Course {} not found'.format(self.id))
def login(cls, email, password):
with Database() as db:
password = hashlib.sha224(bytes(password, "utf-8")).hexdigest()
sql = 'SELECT "id", password FROM "user" WHERE email = %s'
user_id, db_password = db.query(sql, email)[0]
if password == db_password:
return User(user_id)
return None
def test_with_commit(self):
"""Ensure that an error free with statement commits when exiting
Performs an error free with block and ensures the data remains
"""
with Database(database=TEST_DATABASE) as db:
db.query('INSERT INTO test_data (variable) VALUES (1)')
result = self.connection.query('SELECT * FROM test_data')
self.assertEqual(result, [(1, )])
class Display_Intelligence():
def __init__(self):
self.link = Database("Intelligence")
super().__init__()
def display_ioc(self, type):
result = {}
result["data"] = []
select_result = self.link.select(type,
query={},
sort="disclosure_time")
if select_result:
for alert in select_result:
alert.pop("_id")
result["data"].append(alert)
self.link.close()
result["data"] = list_dict_duplicate_removal(result["data"])
return result
def ioc_count(self):
result = {}
count = 0
for ioc_type in ioc:
result[ioc_type] = self.link.count(ioc_type)
count += count + result[ioc_type]
result["count"] = count
self.link.close()
# result["data"] = list_dict_duplicate_removal(result["data"])
return result
def test_with_rollback(self):
"""Ensure that an error occurring in a database with context forces the database to rollback
Raises an error in the with context and ensures that modifications were removed
"""
try:
with Database(database=TEST_DATABASE) as db:
db.query('INSERT INTO test_data (variable) VALUES (1)')
raise Exception
except:
pass
result = self.connection.query('SELECT * FROM test_data')
self.assertEqual(result, [])
def __init__(self, id, course):
self.id = id
self.course = course
self.start = datetime.now()
self.end = datetime.now()
self.day = datetime.now()
self.location = "SAD"
with Database() as db:
if db.exists("session", id=self.id):
sql = 'SELECT start, "end", "day", location FROM "session" WHERE id = %s AND course_id = %s'
self.start, self.end, self.day, self.location = db.query(
sql, self.id, self.course, limit=1)[0]
else:
raise KeyError('Session Time {} not found'.format(self.id))
def create(cls, email, password, permissions):
with Database() as db:
if db.exists("user", email=email):
return None
password = hashlib.sha224(bytes(password, "utf-8")).hexdigest()
id = uuid.uuid4().int
id = int(str(id)[:10])
sql = 'INSERT INTO "user" (id, email, permission, password) VALUES (%s, %s, %s, %s)'
db.query(sql, id, email, permissions, password)
return User(id)
def __init__(self, id):
self.id = id
self.email = "*****@*****.**"
self.permissions = 0
with Database() as db:
if db.exists("user", id=self.id):
sql = 'SELECT "email", permission FROM "user" WHERE id = %s'
self.email, self.permissions = db.query(sql, self.id,
limit=1)[0]
else:
raise KeyError('User {} not found'.format(self.id))
self.is_authenticated = True
self.is_active = True
self.is_anonymous = False
def run_experiment(project_id: int, engine: str, model: object,
target_col: str, train_data: pd.DataFrame,
test_data: pd.DataFrame, metrics: dict) -> int:
_ENGINES = ['sklearn']
if engine not in _ENGINES:
raise Exception(
f'Engine not registered, must be one of the following: {",".join(_ENGINES)}'
)
database = Database()
query = f"INSERT INTO experiment (project_id, engine) VALUES ('{project_id}', '{engine}')"
experiment_id = database.write(query=query)
registry = Registry()
registry.put_model(path=f"{project_id}-{experiment_id}",
key='pre-model',
model=model)
registry.put_metrics(path=f"{project_id}-{experiment_id}",
key='metrics',
metrics=metrics)
registry.put_dataset(path=f"{project_id}-{experiment_id}",
key='test',
dataset=test_data)
registry.put_dataset(path=f"{project_id}-{experiment_id}",
key='train',
dataset=train_data)
pipeline = Pipeline()
conf = {
'experiment_id': experiment_id,
'project_id': project_id,
'target_col': target_col
}
triggered = pipeline.trigger_dag(dag_id=f'{engine}-pipeline',
data=dict(conf=conf))
if triggered:
query = f"UPDATE experiment SET status = 'submitted' WHERE id = {experiment_id}"
database.write(query=query)
else:
query = f"UPDATE experiment SET status = 'submission-failed' WHERE id = {experiment_id}"
database.write(query=query)
raise Exception('Pipeline failed to run experiment')
return experiment_id
import pickle
import time
UDP_IP = socket.gethostname() # returns the IP of this device
UDP_PORT = 5015 # port number
print(UDP_IP)
# initialize UDP socket
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 2)
# bind socket to address
sock.bind((socket.gethostname(), UDP_PORT))
print("waiting for incoming messages...")
print("press CTRL+C to exit")
db = Database()
con, cursor = db.connect()
START_TIME = time.time()
while True:
data, addr = sock.recvfrom(136) # receive data with certain buffer size
data = pickle.loads(data)
print(f"received following data: {data} from {addr}\n"
) # decode incoming message
print('Adding record to database...')
try:
cursor.execute(
"""INSERT INTO "sensor_data" (name, value, date) VALUES (%s,%s,%s)""",
(data["name"], data["value"], data["date"]))
from api.database import Database
from api.models import LoggedInResponse, ShortURL, SuccessfulShortURLResponse
from api.utils import authenticate_token, create_short_url, create_user
load_dotenv(".env")
API_BASE = os.environ.get("API_BASE_URL", "http://*****:*****@app.on_event("startup")
async def startup() -> None:
await db.connect()
await db.initialize_tables()
@app.on_event("shutdown")
async def shutdown() -> None:
await db.disconnect()
@app.get("/discord", response_model=LoggedInResponse)
import tornado.ioloop
import tornado.web
import json
import sys
import os
from api.database import Database
from api.service import Service, UnknownInstanceError
from api.import_data import import_local_data
from api.endpoint import Endpoint
if __name__ == "__main__":
try:
# initialize database schema (SQLAlchemy)
db = Database("hivery.db")
# pre-process raw data files and load into database
import_local_data(db, "data/companies.json", "data/people.json", "data/foods.json")
# pass database to service
service = Service(db)
# construct the API endpoint
endpoint = Endpoint(service)
# start listening on the API endpoint
endpoint.run(port_num=8888)
except ImportError as err:
print("data import failed because:: " + str(err))
from flask_restful import Resource, Api
from flask import Flask, session, request, url_for, render_template, redirect, abort, escape, flash
from flask import current_app as app
from werkzeug.security import generate_password_hash, check_password_hash
from functools import wraps
from datetime import datetime
from api.database import Database
database = Database()
users = database.get_users()
def login_required(f):
@wraps(f)
def wrap(*args, **kwargs):
print(request.data)
print(request.environ)
if session.get("authenticated"):
return f(*args, **kwargs)
else:
return {"error": "You need to authenticate to use this endpoint"}, 401
return wrap
class Login(Resource):
def __init__(self):
super().__init__()
def post(self):
data = request.json
def get(self): members = Database.getmembers() return members
def setup_method(self, test_method):
"""Create/Wipe a table called test_data where tests will be run"""
self.connection = Database(database=TEST_DATABASE)
self.connection.query('DROP TABLE IF EXISTS test_data')
self.connection.query('CREATE TABLE test_data (variable INTEGER)')
class TestDatabase(unittest.TestCase):
def setup_method(self, test_method):
"""Create/Wipe a table called test_data where tests will be run"""
self.connection = Database(database=TEST_DATABASE)
self.connection.query('DROP TABLE IF EXISTS test_data')
self.connection.query('CREATE TABLE test_data (variable INTEGER)')
def teardown_method(self, test_method):
"""Drop the test_data table used to perform tests"""
self.connection.query('DROP TABLE test_data')
self.connection.close()
def test_commit(self):
"""Ensure that the database commit method retains
the data after reconnection.
"""
# insert test data and commit
self.connection.query('INSERT INTO test_data (variable) VALUES (1)')
self.connection.commit()
self.connection.close()
# reopen connection and query to ensure data was retained
self.connection = Database(database=TEST_DATABASE)
result = self.connection.query('SELECT * FROM test_data')
self.assertEqual(result, [(1, )])
def test_rollback(self):
"""Ensure that the database rollback method removes changes to the data
"""
# insert test data and rollback
self.connection.query('INSERT INTO test_data (variable) VALUES (1)')
self.connection.rollback()
self.connection.close()
# reopen connection and query to ensure data was removed
self.connection = Database(database=TEST_DATABASE)
result = self.connection.query('SELECT * FROM test_data')
self.assertEqual(result, [])
def test_close(self):
"""Ensure that the close method closes the database connection"""
with self.assertRaises(InterfaceError):
self.connection.close()
self.connection.query('SELECT * FROM test_data')
def test_with_commit(self):
"""Ensure that an error free with statement commits when exiting
Performs an error free with block and ensures the data remains
"""
with Database(database=TEST_DATABASE) as db:
db.query('INSERT INTO test_data (variable) VALUES (1)')
result = self.connection.query('SELECT * FROM test_data')
self.assertEqual(result, [(1, )])
def test_with_rollback(self):
"""Ensure that an error occurring in a database with context forces the database to rollback
Raises an error in the with context and ensures that modifications were removed
"""
try:
with Database(database=TEST_DATABASE) as db:
db.query('INSERT INTO test_data (variable) VALUES (1)')
raise Exception
except:
pass
result = self.connection.query('SELECT * FROM test_data')
self.assertEqual(result, [])
def test_limit(self):
"""Ensure that the limit keyword of the query method works as expected
Performs several queries with varying limits and compares the results to the expected results
"""
self.connection.query(
'INSERT INTO test_data (variable) VALUES (1), (2), (3), (4), (5)')
result = self.connection.query('SELECT * FROM test_data', limit=1)
self.assertEqual(result, [(1, )])
result = self.connection.query('SELECT * FROM test_data', limit=3)
self.assertEqual(result, [(1, ), (2, ), (3, )])
result = self.connection.query('SELECT * FROM test_data')
self.assertEqual(result, [(1, ), (2, ), (3, ), (4, ), (5, )])
def test_exists(self):
"""Ensure that the exists method accurately reports the existence of rows in the database"""
self.connection.query(
'INSERT INTO test_data (variable) VALUES (1), (2), (3), (4), (5)')
self.assertTrue(self.connection.exists('test_data'))
self.assertTrue(self.connection.exists('test_data', variable=3))
self.assertFalse(self.connection.exists('test_data', variable=6))
def test_bool(self):
"""Ensure that the database returns the correct boolean value based on whether it is open or closed"""
self.assertTrue(self.connection)
self.connection.close()
self.assertFalse(self.connection)
def test_del(self):
"""Ensure that the __del__ magic method works in the same way the close works"""
with self.assertRaises(InterfaceError):
self.connection.__del__()
self.connection.query('SELECT * FROM test_data')
def test_database_init(app):
from api.database import Database
db = Database(app)
assert db.app == app
def create_project(name: str) -> int:
query = f"INSERT INTO project (name) VALUES ('{name}')"
database = Database()
return database.write(query=query)
from flask_restful import Resource, Api
from flask import Flask, session, request, url_for, render_template, redirect, abort, escape, flash
from flask import current_app as app
from datetime import datetime
from random import choice
import string
from api.database import Database
from api.account import login_required
database = Database()
posts = database.get_posts()
class Post(Resource):
def __init__(self):
super().__init__()
@login_required
def post(self):
data = request.json
if data.get("operation") == "add":
post_type = data.get("type")
post = {
"post_id": "".join([choice(string.ascii_letters + string.digits) for char in range(8)]),
"username": session["user"].get("username"),
"location": {
"type": "Point",
"coordinates": data.get("coordinates")
def get_experiment(experiment_id: int) -> dict:
query = f"SELECT * FROM experiment WHERE id = {experiment_id}"
database = Database()
return database.read(query=query)[0]
import os
from flask import jsonify
from api import app
from api.database import Database
db = Database()
config_name = os.getenv('APP_SETTINGS')
@app.errorhandler(405)
def url_not_found(error):
return jsonify({'message': 'Requested method not allowed'}), 405
@app.errorhandler(404)
def page_not_found(error):
return jsonify({'message': 'page not found, check the url'}), 404
@app.errorhandler(500)
def internal_error(error):
return "500 error"
if __name__ == '__main__':
db.create_tables()
app.run()
def get(self):
return {'status': 'OK' if Database.ping() else 'BROKEN'}