def wrapper(*args, **kwargs):
global query_counter
start_time = time.perf_counter()
with lock:
query_counter += 1
query_id = query_counter
Logger.info(
f'Query: query_id: <{query_id}> method: <{request.method}>; path=<{request.path}>'
)
return func(start_time=start_time, query_id=query_id, *args, **kwargs)
def change_file(fileid, start_time, query_id):
if not Files.query.filter_by(fileid=fileid).all():
code = 404
Logger.info(
f"Response: Query failed. query_id: <{query_id}>; err_code: <{code}>; "
f"time: <{calc_time(start_time)} ms>")
abort(code)
if 'file' not in request.files or not request.files['file'].filename:
code = 400
Logger.info(
f"Response: Query failed. query_id: <{query_id}>; err_code: <{code}>; "
f"time: <{calc_time(start_time)} ms>")
abort(code)
# Изменение в бд
with transaction():
try:
Data.query.filter_by(fileid=fileid).delete()
Files.query.filter_by(fileid=fileid).update(
{'filename': request.files['file'].filename})
handleFile(fileid, request.files['file'])
except HTTPException as ex:
Logger.info(
f"Response: Query failed. query_id: <{query_id}>; err_code: <{ex.code}>; "
f"time: <{calc_time(start_time)} ms>")
raise
db.session.commit()
Logger.info(f"Response: Query successed. query_id: <{query_id}>; "
f"time: <{calc_time(start_time)} ms>")
return '', 204
def create_app():
Logger.info('Creating app...')
app = Flask(__name__)
Logger.info('App created')
Logger.info('Configurating config...')
app.config.from_object(app_config.currentConfig)
Logger.info('Config configured')
import app.server.queries.move_handler as move_handler
app.register_blueprint(move_handler.move_handler)
Logger.info('Blueprint registered')
return app
def file_info(fileid, start_time, query_id):
if not Files.query.filter_by(fileid=fileid).all():
code = 404
Logger.info(
f"Response: Query failed. query_id: <{query_id}>; err_code: <{code}>; "
f"time: <{calc_time(start_time)} ms>")
abort(code)
try:
# Соединение таблиц и получение информации
fileinf = db.session.query(Files, func.count(Data.fileid).label('count_rows'))\
.join(Files.data)\
.group_by(Files.fileid)\
.having(Files.fileid == fileid)\
.first()
except HTTPException as ex:
Logger.info(
f"Response: Query failed. query_id: <{query_id}>; err_code: <{ex.code}>; "
f"time: <{calc_time(start_time)} ms>")
raise
Logger.info(f"Response: Query successed. query_id: <{query_id}>; "
f"time: <{calc_time(start_time)} ms>")
return jsonify(fileid=fileid,
filename=fileinf[0].filename,
first_download=fileinf[0].first_download,
last_download=fileinf[0].last_download,
data_count=fileinf.count_rows), 200
def upload_file(start_time, query_id):
if 'file' not in request.files or not request.files['file'].filename:
code = 400
Logger.info(
f"Response: Query failed. query_id: <{query_id}>; err_code: <{code}>; "
f"time: <{calc_time(start_time)} ms>")
abort(code)
# Добавление в бд
with transaction():
try:
file = Files(filename=request.files['file'].filename)
with transaction():
db.session.add(file)
handleFile(file.fileid, request.files['file'])
except HTTPException as ex:
Logger.info(
f"Response: Query failed. query_id: <{query_id}>; err_code: <{ex.code}>; "
f"time: <{calc_time(start_time)} ms>")
raise
db.session.commit()
Logger.info(f"Response: Query successed. query_id: <{query_id}>; "
f"time: <{calc_time(start_time)} ms>")
return jsonify(fileid=file.fileid), 200
def delete_file(fileid, start_time, query_id):
if not Files.query.filter_by(fileid=fileid).all():
code = 404
Logger.info(
f"Response: Query failed. query_id: <{query_id}>; err_code: <{code}>; "
f"time: <{calc_time(start_time)} ms>")
abort(code)
# Удаление из бд
with transaction():
try:
Data.query.filter_by(fileid=fileid).delete()
Files.query.filter_by(fileid=fileid).delete()
except HTTPException as ex:
Logger.info(
f"Response: Query failed. query_id: <{query_id}>; err_code: <{ex.code}>; "
f"time: <{calc_time(start_time)} ms>")
raise
db.session.commit()
Logger.info(f"Response: Query successed. query_id: <{query_id}>; "
f"time: <{calc_time(start_time)} ms>")
return '', 204
class OrdersWorker:
def __init__(self):
self.name = 'orders_worker'
self.extractor = Extractor(self.name)
self.transformer = Transformer(self.name)
self.loader = Loader(self.name)
self.logger = Logger(self.name)
def process(self):
self.logger.info('Started')
orders = self.extractor.extract_orders('orders')
self.logger.info(f'Extracted orders: {orders.count()}')
users = self.extractor.extract_users('users')
data = self.transformer.merge_users_and_orders(users, orders)
self.loader.load(data)
self.logger.info(f'Written records: {len(data)}')
self.logger.info('Finished')
#!/usr/bin/env python
# coding: utf-8
from celery import Celery
from appconfig import getConfig
from app.logger import Logger
Logger.info('Configuring celery...')
celery_api = Celery(broker=getConfig().CELERY_BROKER_URL)
celery_api.config_from_object(getConfig())
celery_api.autodiscover_tasks(['app.celery_tasks.run_ml_task'], force=True)
Logger.info('Celery configured')
def create_app():
Logger.info('Creating app...')
app = Flask(__name__)
Logger.info('App created')
Logger.info('Configurating config...')
app.config.from_object(appconfig.currentConfig)
Logger.info('Config configured')
Logger.info('Initialize database...')
db.init_app(app)
Logger.info('Database initialized')
import app.queries.data_handler as data_handler
app.register_blueprint(data_handler.data_handler)
return app
#!/usr/bin/env python
# coding: utf-8
from flask_script import Manager
from app.logger import Logger
from app.fl_app import application
manager = Manager(application)
if __name__ == "__main__":
Logger.info('Starting server')
manager.run()
#!/usr/bin/env python
# coding: utf-8
from flask_script import Manager
from app.logger import Logger
from app import create_app
from app.server.connection.robot_connection import connect_robot
from robot_config import robot_connect
app = create_app()
manager = Manager(app)
if __name__ == "__main__":
Logger.info('Setting connection with robot')
success = False
for i in range(robot_connect.get("attempts")):
Logger.info(f'Try to set connection, attempt: {i+1}')
success = connect_robot()
if success:
break
if not success:
Logger.info('Connection has not been setted. Terminate')
exit(0)
Logger.info('Connection setted')
Logger.info('Starting server')
manager.run()
def migrate(app):
engine = create_engine(getConfig().SQLALCHEMY_DATABASE_URI)
Logger.info('Check database existence...')
if not database_exists(engine.url):
Logger.info('Creating database...')
create_database(engine.url)
Logger.info('Database created')
else:
Logger.info('Database already exists. Not need to create')
assert database_exists(engine.url)
Logger.info('Database exists')
Logger.info('Creating tables...')
with app.test_request_context():
db.create_all()
Logger.info('Tables created')
def failLog():
Logger.info(f'{datetime.now().strftime("%Y-%m-%d %H:%M:%S")};Fail')
def successLog():
Logger.info(f'{datetime.now().strftime("%Y-%m-%d %H:%M:%S")};Success')
def celeryLogFailAndEmail(task_id, start_time, email, ex_name):
message = f"Your request was failed. To know why - you can make a status request with your task ID: {task_id}"
sendEmail(email, message)
Logger.info(f"Response: Celery task failed. task_id: <{task_id}>; exc_name: <{ex_name}>; "
f"time: <{calc_time(start_time)} ms>")
def celeryLogSuccessAndEmail(task_id, start_time, email, result):
message = f"Your request successed! The result is:\n" \
f"{result}"
sendEmail(email, message)
Logger.info(f"Response: Query successed. query_id: <{task_id}>; "
f"time: <{calc_time(start_time)} ms>")
def logFail(query_id, start_time, code):
Logger.info(
f"Response: Query failed. query_id: <{query_id}>; err_code: <{code}>; "
f"time: <{calc_time(start_time)} ms>")
def create_app():
Logger.info('Creating app...')
app = Flask(__name__)
Logger.info('App created')
Logger.info('Configure app...')
app.config.from_object(appconfig.getConfig())
Logger.info('App configured')
Logger.info('Initialize database...')
db.init_app(app)
Logger.info('Database initialized')
import app.queries.preprocessors_handler as preprocessors_handler
import app.queries.resources_handler as resources_handler
import app.queries.models_handler as models_handler
import app.queries.ml_handler as ml_handler
app.register_blueprint(preprocessors_handler.preprocessors_handler)
app.register_blueprint(resources_handler.resources_handler)
app.register_blueprint(models_handler.models_handler)
app.register_blueprint(ml_handler.ml_handler)
return app
def logSuccess(query_id, start_time):
Logger.info(f"Response: Query successed. query_id: <{query_id}>; "
f"time: <{calc_time(start_time)} ms>")
