import pymongo.errors from aiohttp import web from multidict import MultiDict import cache_manager import settings import skd from classes import Check, Task, Cache from classes.ttl_dict import TTLDictNew from utils import authorization as auth from utils import aio, app_log, json_util # from utils.db_client import get_db # hello master # hello rebase LOG = app_log.get_logger() MEM_LOG = app_log.get_logger('memory') # db = get_db() async def index(request): return 'SKD rest api' async def receive_task(request): # return await skd.register_task(request['body']) task = Task(request['body']) await task.save() query = {
async def decorated_func(check, task, cached_code, port): # task = copy.deepcopy(_task) # imp.reload(aio) # await aio.lock.acquire() # imp.reload(motor.motor_asyncio) # db = db_client.get_db(loop) # db = motor.motor_asyncio.AsyncIOMotorClient(settings.DATABASE, io_loop=loop).get_default_database() # check.set_db(db) # print(loop.call_soon_threadsafe(db.checks.find_one, {'system': 'NSS'}), '_id') # LOG.info('hello env3') # check.update( # task_id=task['_id'], # key=task['key'], # started=datetime.datetime.now() # ) # cached_code = check.pop('content') # loop.call_soon_threadsafe(check.save) # LOG.info('hello env4') try: result = await target(cached_code, check, task, port) if isinstance(result, abc.Sequence) and not isinstance(result, str): # deconstruct complex result if result and isinstance(result[0], bool) or result[0] is None: logical_result, result = result elif result and isinstance(result[-1], bool) or result[-1] is None: result, logical_result = result else: logical_result = None # then save the f**k out of it! if 'output_file_descriptor' in check: raise Exception('cannot save file twice!') # if len(result) > 10000: # result = [('Слишком много записей, вывожу 10000 строк',)] + result[:10000] if True or len(result) <= 100001: # await check.generate_filename('xlsx') wb = xlsxwriter.Workbook(check.filename, {'default_date_format': 'dd-mm-yyyy'}) sh = wb.add_worksheet(check['name'][:31]) for i, row in enumerate(result): # for j, el in enumerate(row): sh.write_row(i, 0, row) wb.close() # await aio.proc_run(write_xlsx, check.filename, check['name'], result) # await check.calc_crc32() else: await check.generate_filename('csv') adapter = Adapter() writer = csv.writer(adapter, delimiter=';', lineterminator='\n', quoting=csv.QUOTE_MINIMAL) writer.writerows(result) async with aiofiles.open(check.filename, 'w') as fd: await fd.write(adapter.lines) else: logical_result = result except Exception as exc: app_log.get_logger(f'env_{port}').error('check: {}.{} failed with error: {}', check['name'], check['extension'], traceback.format_exc()) logical_result = 'runtime error: {}'.format(exc) # update на время, когда выполнилась проверка # await check.finish(result=logical_result) # aio.lock.release() return logical_result
async def index(request): return web.Response(text=request.app['port']) async def entry(request): data = json_util.to_object_id(await request.json()) _check, task, cached_code = data['check'], data['task'], data[ 'cached_code'] check = Check.restore(_check) await check.put(running=True) aio.aio.ensure_future( run_check(check, task, cached_code, request.app['port'])) return web.Response(text="ok") def start(port_str): app = web.Application() app.router.add_get('/', index) app.router.add_post('/entry/', entry) app['port'] = port_str web.run_app(app, port=int(port_str)) if __name__ == '__main__': port_str = sys.argv[1] LOG = app_log.get_logger(f'worker_{port_str}') start(port_str)
import collections import hashlib import weakref from datetime import datetime, timedelta from utils import app_log, json_util from utils.aio import aio # from utils.db_client import get_db from . import Cache, Task, Check LOG = app_log.get_logger() # db = get_db() # TEN_SECONDS = timedelta(seconds=10) PROLONG_RUNNER_SECONDS = 10 def hash_obj(obj): dummy = json_util.dumps(obj).encode('utf-8') return hashlib.sha1(dummy).hexdigest() class SingleQueryRunner(object): sort_by = (('name', 1), ('extension', 1)) project = {'content': 0} def __init__(self, task): self.checks_tmpls_query = {'system': task['system']} if 'operation' in task: self.checks_tmpls_query.update(operation=task['operation'])
import time import json import numpy import base64 import cv2 from flask import Flask, Response, request from view.black_box import BlackBox from utils import constants as C from utils.app_log import get_logger application = Flask(__name__) logger = get_logger("application") blackbox = BlackBox() def dump_json(passed_json): return json.dumps(passed_json, indent=4, sort_keys=True, default=str) def return_failed_response(ex): return Response(dump_json({"error": ex}), status=C.STATUS_ERROR, mimetype='application/json') @application.route("/", methods=["GET", "POST"]) def check_everthing_is_ok(): return Response(dump_json(