import pymongo.errors
from aiohttp import web
from multidict import MultiDict
import cache_manager
import settings
import skd
from classes import Check, Task, Cache
from classes.ttl_dict import TTLDictNew
from utils import authorization as auth
from utils import aio, app_log, json_util
# from utils.db_client import get_db
# hello master
# hello rebase
LOG = app_log.get_logger()
MEM_LOG = app_log.get_logger('memory')
# db = get_db()
async def index(request):
return 'SKD rest api'
async def receive_task(request):
# return await skd.register_task(request['body'])
task = Task(request['body'])
await task.save()
query = {
async def decorated_func(check, task, cached_code, port):
# task = copy.deepcopy(_task)
# imp.reload(aio)
# await aio.lock.acquire()
# imp.reload(motor.motor_asyncio)
# db = db_client.get_db(loop)
# db = motor.motor_asyncio.AsyncIOMotorClient(settings.DATABASE, io_loop=loop).get_default_database()
# check.set_db(db)
# print(loop.call_soon_threadsafe(db.checks.find_one, {'system': 'NSS'}), '_id')
# LOG.info('hello env3')
# check.update(
# task_id=task['_id'],
# key=task['key'],
# started=datetime.datetime.now()
# )
# cached_code = check.pop('content')
# loop.call_soon_threadsafe(check.save)
# LOG.info('hello env4')
try:
result = await target(cached_code, check, task, port)
if isinstance(result, abc.Sequence) and not isinstance(result, str):
# deconstruct complex result
if result and isinstance(result[0], bool) or result[0] is None:
logical_result, result = result
elif result and isinstance(result[-1], bool) or result[-1] is None:
result, logical_result = result
else:
logical_result = None
# then save the f**k out of it!
if 'output_file_descriptor' in check:
raise Exception('cannot save file twice!')
# if len(result) > 10000:
# result = [('Слишком много записей, вывожу 10000 строк',)] + result[:10000]
if True or len(result) <= 100001:
# await check.generate_filename('xlsx')
wb = xlsxwriter.Workbook(check.filename, {'default_date_format': 'dd-mm-yyyy'})
sh = wb.add_worksheet(check['name'][:31])
for i, row in enumerate(result):
# for j, el in enumerate(row):
sh.write_row(i, 0, row)
wb.close()
# await aio.proc_run(write_xlsx, check.filename, check['name'], result)
# await check.calc_crc32()
else:
await check.generate_filename('csv')
adapter = Adapter()
writer = csv.writer(adapter, delimiter=';', lineterminator='\n',
quoting=csv.QUOTE_MINIMAL)
writer.writerows(result)
async with aiofiles.open(check.filename, 'w') as fd:
await fd.write(adapter.lines)
else:
logical_result = result
except Exception as exc:
app_log.get_logger(f'env_{port}').error('check: {}.{} failed with error: {}', check['name'], check['extension'], traceback.format_exc())
logical_result = 'runtime error: {}'.format(exc)
# update на время, когда выполнилась проверка
# await check.finish(result=logical_result)
# aio.lock.release()
return logical_result
async def index(request):
return web.Response(text=request.app['port'])
async def entry(request):
data = json_util.to_object_id(await request.json())
_check, task, cached_code = data['check'], data['task'], data[
'cached_code']
check = Check.restore(_check)
await check.put(running=True)
aio.aio.ensure_future(
run_check(check, task, cached_code, request.app['port']))
return web.Response(text="ok")
def start(port_str):
app = web.Application()
app.router.add_get('/', index)
app.router.add_post('/entry/', entry)
app['port'] = port_str
web.run_app(app, port=int(port_str))
if __name__ == '__main__':
port_str = sys.argv[1]
LOG = app_log.get_logger(f'worker_{port_str}')
start(port_str)
import collections
import hashlib
import weakref
from datetime import datetime, timedelta
from utils import app_log, json_util
from utils.aio import aio
# from utils.db_client import get_db
from . import Cache, Task, Check
LOG = app_log.get_logger()
# db = get_db()
# TEN_SECONDS = timedelta(seconds=10)
PROLONG_RUNNER_SECONDS = 10
def hash_obj(obj):
dummy = json_util.dumps(obj).encode('utf-8')
return hashlib.sha1(dummy).hexdigest()
class SingleQueryRunner(object):
sort_by = (('name', 1), ('extension', 1))
project = {'content': 0}
def __init__(self, task):
self.checks_tmpls_query = {'system': task['system']}
if 'operation' in task:
self.checks_tmpls_query.update(operation=task['operation'])
import time
import json
import numpy
import base64
import cv2
from flask import Flask, Response, request
from view.black_box import BlackBox
from utils import constants as C
from utils.app_log import get_logger
application = Flask(__name__)
logger = get_logger("application")
blackbox = BlackBox()
def dump_json(passed_json):
return json.dumps(passed_json, indent=4, sort_keys=True, default=str)
def return_failed_response(ex):
return Response(dump_json({"error": ex}),
status=C.STATUS_ERROR,
mimetype='application/json')
@application.route("/", methods=["GET", "POST"])
def check_everthing_is_ok():
return Response(dump_json(