def serve( host, port, app, request_handler, error_handler, before_start=None, after_start=None, before_stop=None, after_stop=None, debug=False, request_timeout=60, response_timeout=60, keep_alive_timeout=5, ssl=None, sock=None, request_max_size=None, request_buffer_queue_size=100, reuse_port=False, loop=None, protocol=HttpProtocol, backlog=100, register_sys_signals=True, run_multiple=False, run_async=False, connections=None, signal=Signal(), request_class=None, access_log=True, keep_alive=True, is_request_stream=False, router=None, websocket_max_size=None, websocket_max_queue=None, websocket_read_limit=2 ** 16, websocket_write_limit=2 ** 16, state=None, graceful_shutdown_timeout=15.0, asyncio_server_kwargs=None, ): """Start asynchronous HTTP Server on an individual process. :param host: Address to host on :param port: Port to host on :param request_handler: Sanic request handler with middleware :param error_handler: Sanic error handler with middleware :param before_start: function to be executed before the server starts listening. Takes arguments `app` instance and `loop` :param after_start: function to be executed after the server starts listening. Takes arguments `app` instance and `loop` :param before_stop: function to be executed when a stop signal is received before it is respected. Takes arguments `app` instance and `loop` :param after_stop: function to be executed when a stop signal is received after it is respected. Takes arguments `app` instance and `loop` :param debug: enables debug output (slows server) :param request_timeout: time in seconds :param response_timeout: time in seconds :param keep_alive_timeout: time in seconds :param ssl: SSLContext :param sock: Socket for the server to accept connections from :param request_max_size: size in bytes, `None` for no limit :param reuse_port: `True` for multiple workers :param loop: asyncio compatible event loop :param protocol: subclass of asyncio protocol class :param request_class: Request class to use :param access_log: disable/enable access log :param websocket_max_size: enforces the maximum size for incoming messages in bytes. :param websocket_max_queue: sets the maximum length of the queue that holds incoming messages. :param websocket_read_limit: sets the high-water limit of the buffer for incoming bytes, the low-water limit is half the high-water limit. :param websocket_write_limit: sets the high-water limit of the buffer for outgoing bytes, the low-water limit is a quarter of the high-water limit. :param is_request_stream: disable/enable Request.stream :param request_buffer_queue_size: streaming request buffer queue size :param router: Router object :param graceful_shutdown_timeout: How long take to Force close non-idle connection :param asyncio_server_kwargs: key-value args for asyncio/uvloop create_server method :return: Nothing """ if not run_async: # create new event_loop after fork loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) if debug: loop.set_debug(debug) connections = connections if connections is not None else set() server = partial( protocol, loop=loop, connections=connections, signal=signal, app=app, request_handler=request_handler, error_handler=error_handler, request_timeout=request_timeout, response_timeout=response_timeout, keep_alive_timeout=keep_alive_timeout, request_max_size=request_max_size, request_class=request_class, access_log=access_log, keep_alive=keep_alive, is_request_stream=is_request_stream, router=router, websocket_max_size=websocket_max_size, websocket_max_queue=websocket_max_queue, websocket_read_limit=websocket_read_limit, websocket_write_limit=websocket_write_limit, state=state, debug=debug, ) asyncio_server_kwargs = ( asyncio_server_kwargs if asyncio_server_kwargs else {} ) server_coroutine = loop.create_server( server, host, port, ssl=ssl, reuse_port=reuse_port, sock=sock, backlog=backlog, **asyncio_server_kwargs ) if run_async: return server_coroutine trigger_events(before_start, loop) try: http_server = loop.run_until_complete(server_coroutine) except BaseException: logger.exception("Unable to start server") return trigger_events(after_start, loop) # Ignore SIGINT when run_multiple if run_multiple: signal_func(SIGINT, SIG_IGN) # Register signals for graceful termination if register_sys_signals: _singals = (SIGTERM,) if run_multiple else (SIGINT, SIGTERM) for _signal in _singals: try: loop.add_signal_handler(_signal, loop.stop) except NotImplementedError: logger.warning( "Sanic tried to use loop.add_signal_handler " "but it is not implemented on this platform." ) pid = os.getpid() try: logger.info("Starting worker [%s]", pid) loop.run_forever() finally: logger.info("Stopping worker [%s]", pid) # Run the on_stop function if provided trigger_events(before_stop, loop) # Wait for event loop to finish and all connections to drain http_server.close() loop.run_until_complete(http_server.wait_closed()) # Complete all tasks on the loop signal.stopped = True for connection in connections: connection.close_if_idle() # Gracefully shutdown timeout. # We should provide graceful_shutdown_timeout, # instead of letting connection hangs forever. # Let's roughly calcucate time. start_shutdown = 0 while connections and (start_shutdown < graceful_shutdown_timeout): loop.run_until_complete(asyncio.sleep(0.1)) start_shutdown = start_shutdown + 0.1 # Force close non-idle connection after waiting for # graceful_shutdown_timeout coros = [] for conn in connections: if hasattr(conn, "websocket") and conn.websocket: coros.append(conn.websocket.close_connection()) else: conn.close() _shutdown = asyncio.gather(*coros, loop=loop) loop.run_until_complete(_shutdown) trigger_events(after_stop, loop) loop.close()
async def robot_add_friend_callback(data): """ 1. 获取机器人最后一次操作记录 0. 拉群 1. 入群 2. 根据最后一次操作类型做相应处理 参数示例: { "robot_id": "20181111222222", "user_id": "20181111222222", "nickname": "用户", "avatar": "http://xxx/xxx.png", "add_time": "1970-01-01T00:00:00" } """ async def bind_user_code(user, code): async with db.conn.acquire() as con: update_st = await con.prepare('update "user" set code = $1 where id = $2 ') await update_st.fetchrow(code, user) async def save_robot_add_friend(robot_id, user_id): async with db.conn.acquire() as con: insert_robot_friend_stmt = await con.prepare( 'insert into "robot_add_friend" (id, robot_id, user_id, status) values (uuid_generate_v4(), $1, $2, 1)') await insert_robot_friend_stmt.fetch(robot_id, user_id) robot_code = data.get('robot_id') user_code = data.get('user_id') nickname = data.get('nickname') nickname_md5 = data_md5(nickname) robot_info = await get_robot_by_code(robot_code) if robot_info is None: return robot_id = str(robot_info['id']) last_operate_key = CACHE_ROBOT_LASTED_OPERATE.format(robot_code=robot_code, nickname_md5=nickname_md5) user_operation_record = await redis.conn.get(last_operate_key) if user_operation_record is None: logger.warning(f'robot: {robot_code} add friend but user: {nickname}, nickname_md5: {nickname_md5} is not found') return user_operation_record = ujson.loads(user_operation_record) user_id = user_operation_record['user_id'] operation_type = int(user_operation_record['type']) await bind_user_code(user_id, user_code) await save_robot_add_friend(robot_id, user_id) logger.info(f'user: {user_code}, robot: {robot_code} last operation is :{operation_type}') if operation_type == TYPE.ROBOT_ENTER_GROUP: await send_text_msg(robot_code, user_code, PULL_GROUP_TEXT) guide_picture = settings['EXTERNAL_HOST'] + '/static/image/guide.png' await send_image_msg(robot_code, user_code, guide_picture) await robot_distribute_and_update_count(robot_id, user_id, user_code) async with await redis.conn.pipeline() as pipe: await pipe.decr(CACHE_TEMP_ROBOT_AMOUNT_USED.format(robot_id=robot_id), 1) await pipe.expire(CACHE_TEMP_ROBOT_AMOUNT_USED.format(robot_id=robot_id), until_tomorrow_expire_time()) await pipe.delete(CACHE_TEMP_USER_ROBOT_DISTRIBUTE.format(user_id=user_id)) await pipe.execute() elif operation_type == TYPE.PULL_USER_GROUP: redis_key = CACHE_BIND_USER_ROBOT_GROUP_MAP.format(user_id=user_id, robot_code=robot_code) group_code = await redis.conn.get(redis_key) if group_code is not None: invite_data = {"group_id": group_code, "robot_id": robot_code, "user_id": user_code} resp = await robot_invite_user(invite_data) logger.info(f'robot invite user join group data:{invite_data} and response:{resp}') await redis.conn.delete(last_operate_key)
def tus_file_upload(self, request: request): response = Response.HTTPResponse(status=200) logger.info( f"Metadata: {request.headers.get('Upload-Metadata', 'None')}") if request.method == 'GET': metadata = {} if "Upload-Metadata" not in request.headers: logger.error("Upload-Metadata header is mandatory") for kv in request.headers.get("Upload-Metadata").split(","): if len(kv.split(" ")) > 1: key, value = kv.split(" ") metadata[key] = base64.b64decode(value).decode("utf-8") if metadata.get("filename", None) is None: return Response.text("metadata filename is not set", 404) (filename_name, _) = os.path.splitext(metadata.get("filename")) if filename_name.upper() in [ os.path.splitext(f)[0].upper() for f in os.listdir( os.path.dirname(self.tus_upload_folder)) ]: response.headers['Tus-File-Name'] = metadata.get("filename") response.headers['Tus-File-Exists'] = True else: response.headers['Tus-File-Exists'] = False return response elif request.method == 'OPTIONS' and 'Access-Control-Request-Method' in request.headers: # CORS option request, return 200 return response if request.headers.get("Tus-Resumable") is not None: response.headers['Tus-Resumable'] = self.tus_api_version response.headers['Tus-Version'] = self.tus_api_version_supported if request.method == 'OPTIONS': response.headers['Tus-Extension'] = ",".join( self.tus_api_extensions) response.headers['Tus-Max-Size'] = self.tus_max_file_size response.status = 204 return response # process upload metadata metadata = {} if "Upload-Metadata" not in request.headers: logger.error("Upload-Metadata header is mandatory") for kv in request.headers.get("Upload-Metadata").split(","): if len(kv.split(" ")) > 1: key, value = kv.split(" ") metadata[key] = base64.b64decode(value).decode("utf-8") if metadata.get("filename") and os.path.lexists( os.path.join( self.tus_upload_folder, metadata.get( "filename"))) and self.tus_file_overwrite is False: response.status = 409 return response file_size = int(request.headers.get("Upload-Length", "0")) with db_session: File = TusFile(filename=metadata.get("filename", " "), file_size=file_size, metadata=metadata) try: f = open(os.path.join(self.tus_upload_folder, str(File.fid)), "w") f.seek(file_size - 1) f.write("\0") f.close() except IOError as e: logger.error("Unable to create file: {}".format(e)) response.status = 500 return response response.status = 201 response.headers['Location'] = '{}/{}'.format( request.url, str(File.fid)) response.headers['Tus-Temp-Filename'] = str(File.fid) else: logger.warning( "Received File upload for unsupported file transfer protocol") response.body = b"Received File upload for unsupported file transfer protocol" response.status = 500 return response
def serve( host, port, app, request_handler, error_handler, before_start=None, after_start=None, before_stop=None, after_stop=None, debug=False, request_timeout=60, response_timeout=60, keep_alive_timeout=5, ssl=None, sock=None, request_max_size=None, request_buffer_queue_size=100, reuse_port=False, loop=None, protocol=HttpProtocol, backlog=100, register_sys_signals=True, run_multiple=False, run_async=False, connections=None, signal=Signal(), request_class=None, access_log=True, keep_alive=True, is_request_stream=False, router=None, websocket_max_size=None, websocket_max_queue=None, websocket_read_limit=2 ** 16, websocket_write_limit=2 ** 16, state=None, graceful_shutdown_timeout=15.0, asyncio_server_kwargs=None, ): """Start asynchronous HTTP Server on an individual process. :param host: Address to host on :param port: Port to host on :param request_handler: Sanic request handler with middleware :param error_handler: Sanic error handler with middleware :param before_start: function to be executed before the server starts listening. Takes arguments `app` instance and `loop` :param after_start: function to be executed after the server starts listening. Takes arguments `app` instance and `loop` :param before_stop: function to be executed when a stop signal is received before it is respected. Takes arguments `app` instance and `loop` :param after_stop: function to be executed when a stop signal is received after it is respected. Takes arguments `app` instance and `loop` :param debug: enables debug output (slows server) :param request_timeout: time in seconds :param response_timeout: time in seconds :param keep_alive_timeout: time in seconds :param ssl: SSLContext :param sock: Socket for the server to accept connections from :param request_max_size: size in bytes, `None` for no limit :param reuse_port: `True` for multiple workers :param loop: asyncio compatible event loop :param protocol: subclass of asyncio protocol class :param run_async: bool: Do not create a new event loop for the server, and return an AsyncServer object rather than running it :param request_class: Request class to use :param access_log: disable/enable access log :param websocket_max_size: enforces the maximum size for incoming messages in bytes. :param websocket_max_queue: sets the maximum length of the queue that holds incoming messages. :param websocket_read_limit: sets the high-water limit of the buffer for incoming bytes, the low-water limit is half the high-water limit. :param websocket_write_limit: sets the high-water limit of the buffer for outgoing bytes, the low-water limit is a quarter of the high-water limit. :param is_request_stream: disable/enable Request.stream :param request_buffer_queue_size: streaming request buffer queue size :param router: Router object :param graceful_shutdown_timeout: How long take to Force close non-idle connection :param asyncio_server_kwargs: key-value args for asyncio/uvloop create_server method :return: Nothing """ if not run_async: # create new event_loop after fork loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) if debug: loop.set_debug(debug) app.asgi = False connections = connections if connections is not None else set() server = partial( protocol, loop=loop, connections=connections, signal=signal, app=app, request_handler=request_handler, error_handler=error_handler, request_timeout=request_timeout, response_timeout=response_timeout, keep_alive_timeout=keep_alive_timeout, request_max_size=request_max_size, request_buffer_queue_size=request_buffer_queue_size, request_class=request_class, access_log=access_log, keep_alive=keep_alive, is_request_stream=is_request_stream, router=router, websocket_max_size=websocket_max_size, websocket_max_queue=websocket_max_queue, websocket_read_limit=websocket_read_limit, websocket_write_limit=websocket_write_limit, state=state, debug=debug, ) asyncio_server_kwargs = ( asyncio_server_kwargs if asyncio_server_kwargs else {} ) server_coroutine = loop.create_server( server, host, port, ssl=ssl, reuse_port=reuse_port, sock=sock, backlog=backlog, **asyncio_server_kwargs ) if run_async: return AsyncioServer( loop, server_coroutine, connections, after_start, before_stop, after_stop, ) trigger_events(before_start, loop) try: http_server = loop.run_until_complete(server_coroutine) except BaseException: logger.exception("Unable to start server") return trigger_events(after_start, loop) # Ignore SIGINT when run_multiple if run_multiple: signal_func(SIGINT, SIG_IGN) # Register signals for graceful termination if register_sys_signals: _singals = (SIGTERM,) if run_multiple else (SIGINT, SIGTERM) for _signal in _singals: try: loop.add_signal_handler(_signal, loop.stop) except NotImplementedError: logger.warning( "Sanic tried to use loop.add_signal_handler " "but it is not implemented on this platform." ) pid = os.getpid() try: logger.info("Starting worker [%s]", pid) loop.run_forever() finally: logger.info("Stopping worker [%s]", pid) # Run the on_stop function if provided trigger_events(before_stop, loop) # Wait for event loop to finish and all connections to drain http_server.close() loop.run_until_complete(http_server.wait_closed()) # Complete all tasks on the loop signal.stopped = True for connection in connections: connection.close_if_idle() # Gracefully shutdown timeout. # We should provide graceful_shutdown_timeout, # instead of letting connection hangs forever. # Let's roughly calcucate time. start_shutdown = 0 while connections and (start_shutdown < graceful_shutdown_timeout): loop.run_until_complete(asyncio.sleep(0.1)) start_shutdown = start_shutdown + 0.1 # Force close non-idle connection after waiting for # graceful_shutdown_timeout coros = [] for conn in connections: if hasattr(conn, "websocket") and conn.websocket: coros.append(conn.websocket.close_connection()) else: conn.close() _shutdown = asyncio.gather(*coros, loop=loop) loop.run_until_complete(_shutdown) trigger_events(after_stop, loop) loop.close()
async def asr_view(request, user_cache): # 解包相关信息,供之后逻辑调用` token, pivot = user_cache["token"], user_cache["pivot"] processed_id = user_cache["processed"] audio_bytes = request.files.get("file") if not audio_bytes: _Logger.debug("empty audio bytes") return response.json( { "status": False, "msg": "音频为空", "ret": Ret.EMPTY_ASR_AUDIO } ) res = await aip_asr(audio_bytes) # 请求失败 if not res or res.get("err_no") == 2000: _Logger.warning("err happened when processing baidu asr") return response.json( { "status": False, "msg": "请求百度 asr 失败", "ret": Ret.BAIDU_ASR_ERR } ) # 请求成功 asr_res = res.get("result") or [""] sentence = asr_res[0] _Logger.debug(f"successfully processed baidu asr res = {res}") if not sentence: return response.json( { "status": False, "msg": "语音内容为空", "ret": Ret.EMPTY_ASR_AUDIO } ) sentence = split_sentence(sentence, pivot) _Logger.debug(f"get baidu asr sentence = {sentence}") if not sentence: resp = {"msg":f"需要说含有{pivot}的诗句哦", "ret": Ret.FAIL_NOT_VALID_SENTENCE, "status": False} return response.json(resp) result = await get_sentence_by_text(sentence) _Logger.debug(f"mysql search result = {result}") if not result: resp = {"msg":"没听过这句诗哦", "ret": Ret.FAIL_NOT_VALID_SENTENCE, "status": False} return response.json(resp) sent_id, poetry_id, _ = result sent_id = int(sent_id) if sent_id in processed_id: resp = { "msg": "这句已经说过了", "ret": Ret.FAIL_REPEATED, "status": False } return response.json(resp) user_cache["processed"].append(sent_id) user_cache["count"] += 1 await set_user_cache(token, user_cache) author, title, poetry_text = await get_poetry_by_id(poetry_id) _Logger.debug(f"get poetry from sql text = {poetry_text}, author = {author}, {title}") return response.json( { "status": True, "ret": Ret.SUCCESS_CODE, "data": { "author": author, "title": title, "text": poetry_text } } )
def register_plugin(self, plugin, *args, name=None, skip_reg=False, **kwargs): assert not self._running, "Cannot add, remove, or change plugins " \ "after the App has started serving." assert plugin, "Plugin must be a valid type! Do not pass in `None` " \ "or `False`" if isinstance(plugin, type): # We got passed in a Class. That's ok, we can handle this! module_name = getattr(plugin, '__module__') class_name = getattr(plugin, '__name__') lower_class = to_snake_case(class_name) try: mod = importlib.import_module(module_name) try: plugin = getattr(mod, lower_class) except AttributeError: plugin = mod # try the module-based resolution next except ImportError: raise if ismodule(plugin): # We got passed in a module. That's ok, we can handle this! try: # look for '.instance' on the module plugin = getattr(plugin, 'instance') assert plugin is not None except (AttributeError, AssertionError): # now look for the same name, # like my_module.my_module on the module. try: plugin_module_name = getattr(plugin, '__name__') assert plugin_module_name and len(plugin_module_name) > 0 plugin_module_name = plugin_module_name.split('.')[-1] plugin = getattr(plugin, plugin_module_name) assert plugin is not None except (AttributeError, AssertionError): raise RuntimeError( "Cannot import this module as a Sanic Plugin.") assert isinstance(plugin, SanicPlugin),\ "Plugin must be derived from SanicPlugin" if name is None: try: name = str(plugin.__class__.__name__) assert name is not None except (AttributeError, AssertionError, ValueError, KeyError): logger.warning( "Cannot determine a name for {}, using UUID.".format( repr(plugin))) name = str(uuid1(None, None)) assert isinstance(name, str), \ "Plugin name must be a python unicode string!" associated_tuple = plugin.AssociatedTuple if name in self._plugin_names: # we're already registered on this SPF reg = plugin.find_plugin_registration(self) assoc = associated_tuple(plugin, reg) raise ValueError("Plugin {:s} is already registered!".format(name), assoc) if plugin.is_registered_on_framework(self): raise RuntimeError("Plugin already shows it is registered to this " "spf, maybe under a different name?") self._plugin_names.add(name) shared_context = self.shared_context self._contexts[name] = context = SanicContext( self, shared_context, {'shared': shared_context}) _p_context = self._plugins_context _plugin_reg = _p_context.get(name, None) if _plugin_reg is None: _p_context[name] = _plugin_reg = _p_context.create_child_context() _plugin_reg['name'] = name _plugin_reg['context'] = context if skip_reg: dummy_reg = PluginRegistration(spf=self, plugin_name=name, url_prefix=None) context['log'] = partial(self.log, reg=dummy_reg) context['url_for'] = partial(self.url_for, reg=dummy_reg) plugin.registrations.add(dummy_reg) # This indicates the plugin is not registered on the app _plugin_reg['instance'] = None _plugin_reg['reg'] = None return associated_tuple(plugin, dummy_reg) if _plugin_reg.get('instance', False): raise RuntimeError("The plugin we are trying to register already " "has a known instance!") reg = self._register_helper(plugin, context, *args, _spf=self, _plugin_name=name, **kwargs) _plugin_reg['instance'] = plugin _plugin_reg['reg'] = reg return associated_tuple(plugin, reg)
async def rabbit_pt_callback(message: aio_pika.IncomingMessage): with message.process(): pieces = message.routing_key.split(".") #print(" [x] %r:%r" % ( # message.routing_key, # message.body.decode('utf-8') #)) if pieces[1] == "status": try: if pieces[3] == "create_payload_with_code": # this means we should be getting back the finished payload or an error query = await db_model.payload_query() payload = await db_objects.get(query, uuid=pieces[4]) agent_message = json.loads(message.body.decode()) if agent_message['status'] == 'success': file = open(payload.file_id.path, 'wb') file.write(base64.b64decode(agent_message['payload'])) file.close() code = base64.b64decode(agent_message['payload']) md5 = await hash_MD5(code) sha1 = await hash_SHA1(code) payload.file_id.md5 = md5 payload.file_id.sha1 = sha1 await db_objects.update(payload.file_id) query = await db_model.buildparameterinstance_query() current_instances = await db_objects.execute(query.where(db_model.BuildParameterInstance.payload == payload)) for ci in current_instances: if ci.build_parameter.name in agent_message['build_parameter_instances']: ci.parameter = agent_message['build_parameter_instances'][ci.build_parameter.name] await db_objects.update(ci) del agent_message['build_parameter_instances'][ci.build_parameter.name] query = await db_model.buildparameter_query() for k, v in agent_message['build_parameter_instances'].items(): # now create entries that were set to default in the build script that weren't supplied by the user try: bp = await db_objects.get(query, name=k, payload_type=payload.payload_type) await db_objects.create(db_model.BuildParameterInstance, parameter=v, payload=payload, build_parameter=bp) except Exception as e: agent_message['message'] += "Failed to find build parameter for name {}".format(k) payload.build_phase = agent_message['status'] payload.build_message = agent_message['message'] await db_objects.update(payload) elif pieces[3] == "command_transform": query = await db_model.task_query() task = await db_objects.get(query, id=pieces[5]) if pieces[4] == "error": # create a response that there was an error and set task to processed task.status = "error" task.completed = True task.timestamp = datetime.datetime.utcnow() task.status_timestamp_processed = task.timestamp await db_objects.update(task) await db_objects.create(db_model.Response, task=task, response=message.body.decode('utf-8')) else: task.params = message.body task.timestamp = datetime.datetime.utcnow() if pieces[4] == "success": task.status = "submitted" elif pieces[4] == "completed": task.status = "processed" task.completed = True else: task.status = pieces[4] task.status_timestamp_submitted = task.timestamp await db_objects.update(task) await add_command_attack_to_task(task, task.command) elif pieces[3] == "sync_classes": operation_query = await db_model.operation_query() operations = await db_objects.execute(operation_query.where(db_model.Operation.complete == False)) if pieces[5] == "" or pieces[5] is None: # this was an auto sync from starting a container operator = None else: operator_query = await db_model.operator_query() operator = await db_objects.get(operator_query, username=base64.b64decode(pieces[5]).decode()) if pieces[4] == "success": from app.api.payloadtype_api import import_payload_type_func try: status = await import_payload_type_func(json.loads(message.body.decode()), operator) if status['status'] == "success": if pieces[2] != status['ptype']: logger.warning( "Imported ptype {} from container {}".format(status['ptype'], pieces[2])) logger.info("Successfully updated database from docker code for {}".format(pieces[2])) for o in operations: await db_objects.create(db_model.OperationEventLog, operator=operator, level="info", operation=o, message="Successfully Sync-ed database with {} payload files".format(pieces[2])) # for a successful checkin, we need to find all wrapper payload types and get them to re-check in if status['wrapper'] is False: query = await db_model.payloadtype_query() pts = await db_objects.execute(query.where(db_model.PayloadType.wrapper == True)) sync_operator = "" if operator is None else operator.username for pt in pts: await send_pt_rabbitmq_message(pt.ptype, "sync_classes", "", sync_operator) else: logger.error("Failed to update database from docker code for {}: {}".format(pieces[2], status['error'])) for o in operations: await db_objects.create(db_model.OperationEventLog, operator=operator, level="info", operation=o, message="Failed Sync-ed database with {} payload files: {}".format( pieces[2], status['error'])) except Exception as i: logger.error("Failed to update database from syncing classes: {}".format(str(i))) for o in operations: await db_objects.create(db_model.OperationEventLog, operator=operator, level="error", operation=o, message="Failed Sync-ed database with {} payload files: {}".format( pieces[2], status['error'])) else: logger.error("Failed to get new payload and command info from container for {}: {}".format(pieces[0], message.body.decode())) for o in operations: await db_objects.create(db_model.OperationEventLog, operator=operator, level="error", operation=o, message="Failed getting information for payload {} with error: {}".format( pieces[2], message.body.decode())) except Exception as e: logger.exception("Exception in rabbit_pt_callback: " + str(e)) print(str(sys.exc_info()[-1].tb_lineno) + " " + str(e))
async def _train_async_internal( file_importer: TrainingDataImporter, train_path: Text, output_path: Text, force_training: bool, fixed_model_name: Optional[Text], persist_nlu_training_data: bool, additional_arguments: Optional[Dict], ) -> Optional[Text]: """Trains a Rasa model (Core and NLU). Use only from `train_async`. Args: file_importer: `TrainingDataImporter` which supplies the training data. train_path: Directory in which to train the model. output_path: Output path. force_training: If `True` retrain model even if data has not changed. persist_nlu_training_data: `True` if the NLU training data should be persisted with the model. fixed_model_name: Name of model to be stored. additional_arguments: Additional training parameters. Returns: Path of the trained model archive. """ stories, nlu_data = await asyncio.gather(file_importer.get_stories(), file_importer.get_nlu_data()) if stories.is_empty() and nlu_data.is_empty(): _logger.error( "No training data given. Please provide stories and NLU data in " "order to train a Rasa model using the '--data' argument.") return if stories.is_empty(): _logger.warning( "No stories present. Just a Rasa NLU model will be trained.") return await _train_nlu_with_validated_data( file_importer, output=output_path, fixed_model_name=fixed_model_name, persist_nlu_training_data=persist_nlu_training_data, ) if nlu_data.is_empty(): _logger.warning( "No NLU data present. Just a Rasa Core model will be trained.") return await _train_core_with_validated_data( file_importer, output=output_path, fixed_model_name=fixed_model_name, additional_arguments=additional_arguments, ) new_fingerprint = await model.model_fingerprint(file_importer) old_model = model.get_latest_model(output_path) fingerprint_comparison = FingerprintComparisonResult( force_training=force_training) if not force_training: fingerprint_comparison = model.should_retrain(new_fingerprint, old_model, train_path) if fingerprint_comparison.is_training_required(): await _do_training( file_importer, output_path=output_path, train_path=train_path, fingerprint_comparison_result=fingerprint_comparison, fixed_model_name=fixed_model_name, persist_nlu_training_data=persist_nlu_training_data, additional_arguments=additional_arguments, ) return model.package_model( fingerprint=new_fingerprint, output_directory=output_path, train_path=train_path, fixed_model_name=fixed_model_name, ) print_success("Nothing changed. You can use the old model stored at '{}'." "".format(os.path.abspath(old_model))) _logger.info("Nothing changed. You can use the old model stored at '{}'." "".format(os.path.abspath(old_model))) return old_model
def run( self, host: Optional[str] = None, port: Optional[int] = None, *, debug: bool = False, auto_reload: Optional[bool] = None, ssl: Union[Dict[str, str], SSLContext, None] = None, sock: Optional[socket] = None, workers: int = 1, protocol: Optional[Type[Protocol]] = None, backlog: int = 100, register_sys_signals: bool = True, access_log: Optional[bool] = None, unix: Optional[str] = None, loop: None = None, reload_dir: Optional[Union[List[str], str]] = None, ) -> None: """ Run the HTTP Server and listen until keyboard interrupt or term signal. On termination, drain connections before closing. :param host: Address to host on :type host: str :param port: Port to host on :type port: int :param debug: Enables debug output (slows server) :type debug: bool :param auto_reload: Reload app whenever its source code is changed. Enabled by default in debug mode. :type auto_relaod: bool :param ssl: SSLContext, or location of certificate and key for SSL encryption of worker(s) :type ssl: SSLContext or dict :param sock: Socket for the server to accept connections from :type sock: socket :param workers: Number of processes received before it is respected :type workers: int :param protocol: Subclass of asyncio Protocol class :type protocol: type[Protocol] :param backlog: a number of unaccepted connections that the system will allow before refusing new connections :type backlog: int :param register_sys_signals: Register SIG* events :type register_sys_signals: bool :param access_log: Enables writing access logs (slows server) :type access_log: bool :param unix: Unix socket to listen on instead of TCP port :type unix: str :return: Nothing """ if reload_dir: if isinstance(reload_dir, str): reload_dir = [reload_dir] for directory in reload_dir: direc = Path(directory) if not direc.is_dir(): logger.warning( f"Directory {directory} could not be located") self.reload_dirs.add(Path(directory)) if loop is not None: raise TypeError( "loop is not a valid argument. To use an existing loop, " "change to create_server().\nSee more: " "https://sanic.readthedocs.io/en/latest/sanic/deploying.html" "#asynchronous-support") if auto_reload or auto_reload is None and debug: self.auto_reload = True if os.environ.get("SANIC_SERVER_RUNNING") != "true": return reloader_helpers.watchdog(1.0, self) if sock is None: host, port = host or "127.0.0.1", port or 8000 if protocol is None: protocol = (WebSocketProtocol if self.websocket_enabled else HttpProtocol) # if access_log is passed explicitly change config.ACCESS_LOG if access_log is not None: self.config.ACCESS_LOG = access_log server_settings = self._helper( host=host, port=port, debug=debug, ssl=ssl, sock=sock, unix=unix, workers=workers, protocol=protocol, backlog=backlog, register_sys_signals=register_sys_signals, auto_reload=auto_reload, ) try: self.is_running = True self.is_stopping = False if workers > 1 and os.name != "posix": logger.warn( f"Multiprocessing is currently not supported on {os.name}," " using workers=1 instead") workers = 1 if workers == 1: serve_single(server_settings) else: serve_multiple(server_settings, workers) except BaseException: error_logger.exception( "Experienced exception while trying to serve") raise finally: self.is_running = False logger.info("Server Stopped")
def sign(self, envvars: dict, workdir: str) -> bool: env = copy.deepcopy(envvars) curpath = os.getenv('PATH') env['PATH'] = self.toolspath if curpath: env['PATH'] += ':' + curpath env['MACHINE'] = self.machine bupgen = 'BUPGEN' in env self._symlink_scripts(workdir) # We want to return the minimal set of artifacts possible, so # make sure we remove: # - the files that were sent over # - files generated that are not needed: # - tegraflash will sign and encrypt the 'kernel', but cboot does # not boot the encrypted+signed copy, just the signed one # - the subdirectories generated by the signing program # - other housekeeping files to_remove = os.listdir(workdir) + [ 'signed', 'encrypted_signed', 'flash.xml.tmp' ] pkc = self.keys.get('rsa_priv.pem') if self.soctype == 'tegra210': sbk = None else: to_remove.append('flash.xml') kernelname, kernelext = os.path.splitext(env['LNXFILE']) to_remove.append(kernelname + '_sigheader' + kernelext + '.encrypt.signed') try: sbk = self.keys.get('sbk.txt') except FileNotFoundError: sbk = None cmd = [ "{}-flash-helper".format(self.soctype), '--bup' if bupgen else '--no-flash', '-u', pkc ] if sbk: cmd += ['-v', sbk] cmd += [ 'flash.xml.in', env['DTBFILE'], '{}.cfg'.format(self.machine), env['ODMDATA'] ] if self.soctype == 'tegra210': cmd.append(env['boardcfg']) cmd.append(env['LNXFILE']) try: logger.info("Running: {}".format(cmd)) proc = subprocess.run(cmd, stdin=subprocess.DEVNULL, cwd=workdir, env=env, check=True, capture_output=True, encoding='utf-8') self.keys.cleanup() logger.debug("stdout: {}".format(proc.stdout)) logger.debug("stderr: {}".format(proc.stderr)) # For BUP generation, just return the payloads. # For flashing, we return all the signed/encrypted files if bupgen: utils.remove_files([ os.path.join(workdir, fname) for fname in os.listdir(workdir) if not fname.startswith('payloads') ]) else: utils.remove_files( [os.path.join(workdir, fname) for fname in to_remove]) return True except subprocess.CalledProcessError as e: self.keys.cleanup() logger.warning("signing error, stdout: {}\nstderr: {}".format( e.stdout, e.stderr)) return False
async def handle_404(request, exception): logger.warning(f"URL {request.path} not found, sending current UTC Time...") current_time = int(round(datetime.now(tz=pytz.timezone("UTC")).timestamp())) return json({"time": current_time}, status=404)
async def group_activated_callback(data): """ 1. 判断是否群宠用户和是否封号 * 不是群宠用户 或被封号 注销由创群 * 是群宠用户并且没有被封号 执行步骤2 2. 该群是否被激活(status!= 3) * 激活,激活群失败,发私聊消息 * 没有被激活,执行步骤3 3. 该群是否以前被激活, * 获取结算天数 执行步骤4 4. 插入group记录,map记录 更新robot表机器人激活额度(事务) 5. redis记录机器人当日激活群数,并判断该机器人的额度是否到达当日最大的额度 * 是 删除机器人快照缓存 6. 判断该用户是否是首次导群 * 是 修改user 表是否导群记录 7. 给用户发送群激活成功的私聊消息 8. 调用分配机器人接口,传入激活群机器人,以及要分配的额度,获取一个机器人 * 返回机器人是当前机器人 不做操作 * 返回机器人不是当前机器人 给用户发送私聊消息,推送机器人二维码 :param : :return: """ group_code = data.get('group_id') robot_code = data.get('robot_id') mem_code = data.get('user_id') name = data.get('name', '未命名') logger.info( f'robot activated group callback, robot_code:{robot_code}, group_code:{group_code},mem_code:{mem_code}' ) # 1 user_info = await get_user_by_code(mem_code) if user_info is None or user_info['status'] == 2: await cancel_group(data) logger.debug('user is not exist') return user_id = str(user_info['id']) channel = user_info['channel'] robot_info = await get_robot_by_code(robot_code) robot_id = str(robot_info['id']) # 2 key = 'bind_group:' + group_code if not await acquire_lock_group(key, release_time=30): return if await check_group_activated(group_code, robot_code, mem_code, name): return group_id = await add_group_record(group_code, name, user_id, robot_id) logger.info(f'activate group [{group_code}] for user [{mem_code}]') redis_val = { "group_code": group_code, "user_id": user_id, "user_code": mem_code, "robot_id": robot_id, "robot_code": robot_code, "group_id": group_id } await redis.conn.hset(GROUP_USER_ROBOT_MAP_REDIS_KEY, group_code, ujson.dumps(redis_val)) # 发起同步群成员 await update_cache_after_bind_group(robot_id) await sync_group_members({"group_id": group_code}) # 判断用户是否是首次导群 await checkout_user_has_group(user_id) await send_text_msg(robot_code, mem_code, GROUP_WAIT_STAR_RATE.format(group_name=name)) result, distribute_robot_id = await check_robot_amount_distribute( user_id, mem_code, channel, robot_code) if distribute_robot_id is not None: if distribute_robot_id != robot_id: robot = await get_robot_by_id(distribute_robot_id) robot_name = robot.get('name') qr_code = robot.get('qr_code') send_msg = DISTRIBUTE_NEW_ROBOT.format(robot_name=robot_name) await send_text_msg(robot_code, mem_code, send_msg) await send_image_msg(robot_code, mem_code, qr_code) else: logger.warning( f'request [robot_distribution] to distribute robot result is None') await send_text_msg(robot_code, mem_code, NO_ROBOT_WORD) return
import grp import os import socket from sanic.log import logger from admin.views import app if __name__ == "__main__": if app.config["DEBUG"]: app.run(host="127.0.0.1", port=8000, debug=True) else: # Remove old socket (is any). try: os.unlink(app.config["SOCKET_FILE"]) except FileNotFoundError as e: logger.info(f"No old socket file found: {e}") # Create socket and run app. with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as sock: try: sock.bind(app.config["SOCKET_FILE"]) os.chmod(app.config["SOCKET_FILE"], 0o775) os.chown(app.config["SOCKET_FILE"], -1, grp.getgrnam("nginx").gr_gid) app.run(sock=sock, access_log=False) except OSError as e: logger.warning(e)
async def request(self, *args, **kwargs): request_hooks = kwargs.pop('request_hooks', self.request_hooks) response_hooks = kwargs.pop('response_hooks', self.response_hooks) for request_hook in request_hooks: getattr(self, request_hook)(kwargs) http_status = 599 http_client_name = type(self).__name__ try: async with aiohttp.ClientSession( connector=aiohttp.TCPConnector(verify_ssl=False), trace_configs=[self.trace_config]) as session: with async_timeout.timeout(self.timeout): async with session.request(*args, **kwargs) as response: http_status = response.status _response_text = await response.text() try: _response = await response.json() except ContentTypeError: _response = {} except JSONDecodeError: logger.error('request JSONDecodeError text %s', _response_text) _response = {} if http_status >= 400: message = ClientHttpError.MESSAGE.format( http_client_name, kwargs['url'], (args, kwargs), response.status, _response, _response_text) if http_status >= 500: logger.error( f'BaseClient 5xx {http_client_name}: status_code={response.status} ' 'url=%s request=%s response=%s response_raw=%', kwargs['url'], (args, kwargs), _response, _response_text) else: logger.warning(message) raise ClientHttpError(http_client=http_client_name, url=kwargs['url'], request=(args, kwargs), status_code=http_status, response=_response, response_text=_response_text) for response_hook in response_hooks: getattr(self, response_hook)(_response) return _response except Exception as ex: try: logger.exception( f"BaseClient.request HTTP CLIENT ERROR {http_client_name}: " f"status_code={http_status}; url=%s; args=%s; kwargs=%s", kwargs.get('url'), args, kwargs) raise ex except TimeoutError: logger.error( f'BaseClient Timeout {http_client_name}: status_code={http_status} ' 'url=%s request=%s ', kwargs['url'], (args, kwargs)) raise ClientHttpError(http_client=http_client_name, url=kwargs['url'], request=(args, kwargs), status_code=http_status, response=_response, response_text=_response_text)
def prepare( self, host: Optional[str] = None, port: Optional[int] = None, *, dev: bool = False, debug: bool = False, auto_reload: Optional[bool] = None, ssl: Union[None, SSLContext, dict, str, list, tuple] = None, sock: Optional[socket] = None, workers: int = 1, protocol: Optional[Type[Protocol]] = None, backlog: int = 100, register_sys_signals: bool = True, access_log: Optional[bool] = None, unix: Optional[str] = None, loop: AbstractEventLoop = None, reload_dir: Optional[Union[List[str], str]] = None, noisy_exceptions: Optional[bool] = None, motd: bool = True, fast: bool = False, verbosity: int = 0, motd_display: Optional[Dict[str, str]] = None, ) -> None: if dev: debug = True auto_reload = True self.state.verbosity = verbosity if not self.state.auto_reload: self.state.auto_reload = bool(auto_reload) if fast and workers != 1: raise RuntimeError("You cannot use both fast=True and workers=X") if motd_display: self.config.MOTD_DISPLAY.update(motd_display) if reload_dir: if isinstance(reload_dir, str): reload_dir = [reload_dir] for directory in reload_dir: direc = Path(directory) if not direc.is_dir(): logger.warning( f"Directory {directory} could not be located" ) self.state.reload_dirs.add(Path(directory)) if loop is not None: raise TypeError( "loop is not a valid argument. To use an existing loop, " "change to create_server().\nSee more: " "https://sanic.readthedocs.io/en/latest/sanic/deploying.html" "#asynchronous-support" ) if ( self.__class__.should_auto_reload() and os.environ.get("SANIC_SERVER_RUNNING") != "true" ): # no cov return if sock is None: host, port = host or "127.0.0.1", port or 8000 if protocol is None: protocol = ( WebSocketProtocol if self.websocket_enabled else HttpProtocol ) # Set explicitly passed configuration values for attribute, value in { "ACCESS_LOG": access_log, "AUTO_RELOAD": auto_reload, "MOTD": motd, "NOISY_EXCEPTIONS": noisy_exceptions, }.items(): if value is not None: setattr(self.config, attribute, value) if fast: self.state.fast = True try: workers = len(os.sched_getaffinity(0)) except AttributeError: # no cov workers = os.cpu_count() or 1 server_settings = self._helper( host=host, port=port, debug=debug, ssl=ssl, sock=sock, unix=unix, workers=workers, protocol=protocol, backlog=backlog, register_sys_signals=register_sys_signals, ) self.state.server_info.append( ApplicationServerInfo(settings=server_settings) ) if self.config.USE_UVLOOP is True or ( self.config.USE_UVLOOP is _default and not OS_IS_WINDOWS ): try_use_uvloop()
def run(self): is_debug = self.log.loglevel == logging.DEBUG protocol = ( self.websocket_protocol if self.app.callable.websocket_enabled else self.http_protocol ) self._server_settings = self.app.callable._helper( loop=self.loop, debug=is_debug, protocol=protocol, ssl=self.ssl_context, run_async=True, ) self._server_settings["signal"] = self.signal self._server_settings.pop("sock") self._await(self.app.callable._startup()) self._await( self.app.callable._server_event("init", "before", loop=self.loop) ) main_start = self._server_settings.pop("main_start", None) main_stop = self._server_settings.pop("main_stop", None) if main_start or main_stop: # noqa logger.warning( "Listener events for the main process are not available " "with GunicornWorker" ) try: self._await(self._run()) self.app.callable.is_running = True self._await( self.app.callable._server_event( "init", "after", loop=self.loop ) ) self.loop.run_until_complete(self._check_alive()) self._await( self.app.callable._server_event( "shutdown", "before", loop=self.loop ) ) self.loop.run_until_complete(self.close()) except BaseException: traceback.print_exc() finally: try: self._await( self.app.callable._server_event( "shutdown", "after", loop=self.loop ) ) except BaseException: traceback.print_exc() finally: self.loop.close() sys.exit(self.exit_code)
async def create_server( self, host: Optional[str] = None, port: Optional[int] = None, *, debug: bool = False, ssl: Union[None, SSLContext, dict, str, list, tuple] = None, sock: Optional[socket] = None, protocol: Type[Protocol] = None, backlog: int = 100, access_log: Optional[bool] = None, unix: Optional[str] = None, return_asyncio_server: bool = False, asyncio_server_kwargs: Dict[str, Any] = None, noisy_exceptions: Optional[bool] = None, ) -> Optional[AsyncioServer]: """ Asynchronous version of :func:`run`. This method will take care of the operations necessary to invoke the *before_start* events via :func:`trigger_events` method invocation before starting the *sanic* app in Async mode. .. note:: This does not support multiprocessing and is not the preferred way to run a :class:`Sanic` application. :param host: Address to host on :type host: str :param port: Port to host on :type port: int :param debug: Enables debug output (slows server) :type debug: bool :param ssl: SSLContext, or location of certificate and key for SSL encryption of worker(s) :type ssl: SSLContext or dict :param sock: Socket for the server to accept connections from :type sock: socket :param protocol: Subclass of asyncio Protocol class :type protocol: type[Protocol] :param backlog: a number of unaccepted connections that the system will allow before refusing new connections :type backlog: int :param access_log: Enables writing access logs (slows server) :type access_log: bool :param return_asyncio_server: flag that defines whether there's a need to return asyncio.Server or start it serving right away :type return_asyncio_server: bool :param asyncio_server_kwargs: key-value arguments for asyncio/uvloop create_server method :type asyncio_server_kwargs: dict :param noisy_exceptions: Log exceptions that are normally considered to be quiet/silent :type noisy_exceptions: bool :return: AsyncioServer if return_asyncio_server is true, else Nothing """ if sock is None: host, port = host or "127.0.0.1", port or 8000 if protocol is None: protocol = ( WebSocketProtocol if self.websocket_enabled else HttpProtocol ) # Set explicitly passed configuration values for attribute, value in { "ACCESS_LOG": access_log, "NOISY_EXCEPTIONS": noisy_exceptions, }.items(): if value is not None: setattr(self.config, attribute, value) server_settings = self._helper( host=host, port=port, debug=debug, ssl=ssl, sock=sock, unix=unix, loop=get_event_loop(), protocol=protocol, backlog=backlog, run_async=return_asyncio_server, ) if self.config.USE_UVLOOP is not _default: error_logger.warning( "You are trying to change the uvloop configuration, but " "this is only effective when using the run(...) method. " "When using the create_server(...) method Sanic will use " "the already existing loop." ) main_start = server_settings.pop("main_start", None) main_stop = server_settings.pop("main_stop", None) if main_start or main_stop: logger.warning( "Listener events for the main process are not available " "with create_server()" ) return await serve( asyncio_server_kwargs=asyncio_server_kwargs, **server_settings )
async def fetch_data(self, url, session): try: async with session.get(url) as response: data = await response.text() data = pjson.loads(data) cnfy_id = 'cnfy-{}'.format(str(uuid.uuid4())) for tx in data['transactions']: if tx['type'] in [4] and tx['feeAssetId'] == os.environ['ASSET_ID']: attachment_base58 = base58.b58decode(tx['attachment']).decode('utf-8') attachment = None try: attachment = requests.get('{0}:{1}/ipfs/{2}'.format(config['ipfs']['host'], config['ipfs']['port'], attachment_base58), timeout=2).text except Exception as error: logger.error('IPFS Error: {0}'.format(error)) if attachment == None: logger.warning('CONTINUE ON IPFS HASH {0}'.format(attachment_base58) ) continue attachment_hash = hashlib.sha256(attachment.encode('utf-8')).hexdigest() root = ET.fromstring(attachment) version = root.findall('version')[0].text if len(root.findall('version')) > 0 else None blockchain = root.findall('blockchain')[0].text if len(root.findall('blockchain')) > 0 else None network = root.findall('network')[0].text if len(root.findall('network')) > 0 else None operations = root.findall('operations')[0] if len(root.findall('operations')) > 0 else [] if str(version) != str(os.environ['CDM_VERSION']): continue operation_create = operations.findall('create')[0] if len(operations.findall('create')) > 0 else None operation_insert = operations.findall('insert')[0] if len(operations.findall('insert')) > 0 else None if (operation_create): table_ciphertext = None table_sha256hash = None table = operation_create.findall('table')[0] if len(operation_create.findall('table')) > 0 else None recipient_public_key = None recipient = operation_create.findall('recipient')[0] if len(operation_create.findall('recipient')) > 0 else None if recipient: recipient_public_key = recipient.findall('publickey')[0].text if len(recipient.findall('publickey')) > 0 else None if table: table_ciphertext = table.findall('ciphertext')[0].text if len(table.findall('ciphertext')) > 0 else None table_sha256hash = table.findall('sha256')[0].text if len(table.findall('sha256')) > 0 else None self.sql_data_tables.append(( table_sha256hash, tx['id'], table_ciphertext, recipient_public_key )) columns = operation_create.findall('columns')[0] if len(operation_create.findall('columns')) > 0 else None if columns: cols = columns.findall('column') if len(columns.findall('column')) > 0 else None for col in cols: col_ciphertext = None col_sha256hash = None if col: col_ciphertext = col.findall('ciphertext')[0].text if len(col.findall('ciphertext')) > 0 else None col_sha256hash = col.findall('sha256')[0].text if len(col.findall('sha256')) > 0 else None self.sql_data_columns.append(( col_sha256hash, table_sha256hash, col_ciphertext, recipient_public_key )) if (operation_insert): table_ciphertext = None table_sha256hash = None table = operation_insert.findall('table')[0] if len(operation_insert.findall('table')) > 0 else None recipient_public_key = None recipient = operation_insert.findall('recipient')[0] if len(operation_insert.findall('recipient')) > 0 else None if recipient: recipient_public_key = recipient.findall('publickey')[0].text if len(recipient.findall('publickey')) > 0 else None columns = operation_insert.findall('columns')[0] if len(operation_insert.findall('columns')) > 0 else None if columns: cols = columns.findall('column') if len(columns.findall('column')) > 0 else None for col in cols: col_ciphertext = None col_sha256hash = None val_ciphertext = None val_sha256hash = None if col: col_ciphertext = col.findall('ciphertext')[0].text if len(col.findall('ciphertext')) > 0 else None col_sha256hash = col.findall('sha256')[0].text if len(col.findall('sha256')) > 0 else None value = col.findall('value')[0] if len(col.findall('value')) > 0 else None if value: val_ciphertext = value.findall('ciphertext')[0].text if len(value.findall('ciphertext')) > 0 else None val_sha256hash = value.findall('sha256')[0].text if len(value.findall('sha256')) > 0 else None self.sql_data_values.append(( val_sha256hash, col_sha256hash, val_ciphertext, col_ciphertext, recipient_public_key )) print(self.sql_data_values) tx_data = ( tx['id'], data['height'], tx['type'], tx['sender'], tx['senderPublicKey'], tx['recipient'], tx['amount'], tx['assetId'], tx['feeAssetId'], tx['feeAsset'], tx['fee'], tx['attachment'], tx['version'], datetime.fromtimestamp(tx['timestamp'] / 1e3), cnfy_id, attachment_hash ) self.sql_data_transactions.append(tx_data) for proof in tx['proofs']: proof_id = 'proof-' + str(uuid.uuid4()) self.sql_data_proofs.append((tx['id'], proof, proof_id)) except asyncio.CancelledError: logger.info('Parser has been stopped') raise except Exception as error: logger.error('Fetching data error: {}'.format(error)) pass
}) if request.method == 'HEAD': return response.HTTPResponse(content_type=MIME_TS, status=200) log.info(f'[{request.ip}] Start: {u7d_msg}') with closing(await asyncio_dgram.bind((IPTV, client_port))) as stream: timedout = False respond = await request.respond(content_type=MIME_TS) try: await respond.send((await asyncio.wait_for(stream.recv(), 1))[0]) while True: await respond.send((await asyncio.wait_for(stream.recv(), 0.25))[0]) except Exception as ex: log.warning(f'[{request.ip}] {repr(ex)}') if isinstance(ex, TimeoutError): timedout = True finally: log.info(f'[{request.ip}] End: {u7d_msg}') try: if timedout: await respond.send(end_stream=True) else: u7d.send_signal(signal.SIGINT) except Exception as ex: log.warning(f'[{request.ip}] {repr(ex)}') @app.get('/favicon.ico') async def handle_notfound(request):
def serve(host, port, request_handler, error_handler, before_start=None, after_start=None, before_stop=None, after_stop=None, debug=False, request_timeout=60, response_timeout=60, keep_alive_timeout=60, ssl=None, sock=None, request_max_size=None, reuse_port=False, loop=None, protocol=HttpProtocol, backlog=100, register_sys_signals=True, run_async=False, connections=None, signal=Signal(), request_class=None, access_log=True, keep_alive=True, is_request_stream=False, router=None, websocket_max_size=None, websocket_max_queue=None, state=None, graceful_shutdown_timeout=15.0): """Start asynchronous HTTP Server on an individual process. :param host: Address to host on :param port: Port to host on :param request_handler: Sanic request handler with middleware :param error_handler: Sanic error handler with middleware :param before_start: function to be executed before the server starts listening. Takes arguments `app` instance and `loop` :param after_start: function to be executed after the server starts listening. Takes arguments `app` instance and `loop` :param before_stop: function to be executed when a stop signal is received before it is respected. Takes arguments `app` instance and `loop` :param after_stop: function to be executed when a stop signal is received after it is respected. Takes arguments `app` instance and `loop` :param debug: enables debug output (slows server) :param request_timeout: time in seconds :param ssl: SSLContext :param sock: Socket for the server to accept connections from :param request_max_size: size in bytes, `None` for no limit :param reuse_port: `True` for multiple workers :param loop: asyncio compatible event loop :param protocol: subclass of asyncio protocol class :param request_class: Request class to use :param access_log: disable/enable access log :param is_request_stream: disable/enable Request.stream :param router: Router object :return: Nothing """ if not run_async: loop = async_loop.new_event_loop() asyncio.set_event_loop(loop) if debug: loop.set_debug(debug) connections = connections if connections is not None else set() server = partial( protocol, loop=loop, connections=connections, signal=signal, request_handler=request_handler, error_handler=error_handler, request_timeout=request_timeout, response_timeout=response_timeout, keep_alive_timeout=keep_alive_timeout, request_max_size=request_max_size, request_class=request_class, access_log=access_log, keep_alive=keep_alive, is_request_stream=is_request_stream, router=router, websocket_max_size=websocket_max_size, websocket_max_queue=websocket_max_queue, state=state, debug=debug, ) server_coroutine = loop.create_server( server, host, port, ssl=ssl, reuse_port=reuse_port, sock=sock, backlog=backlog ) # Instead of pulling time at the end of every request, # pull it once per minute loop.call_soon(partial(update_current_time, loop)) if run_async: return server_coroutine trigger_events(before_start, loop) try: http_server = loop.run_until_complete(server_coroutine) except: logger.exception("Unable to start server") return trigger_events(after_start, loop) # Register signals for graceful termination if register_sys_signals: for _signal in (SIGINT, SIGTERM): try: loop.add_signal_handler(_signal, loop.stop) except NotImplementedError: logger.warning('Sanic tried to use loop.add_signal_handler ' 'but it is not implemented on this platform.') pid = os.getpid() try: logger.info('Starting worker [%s]', pid) loop.run_forever() finally: logger.info("Stopping worker [%s]", pid) # Run the on_stop function if provided trigger_events(before_stop, loop) # Wait for event loop to finish and all connections to drain http_server.close() loop.run_until_complete(http_server.wait_closed()) # Complete all tasks on the loop signal.stopped = True for connection in connections: connection.close_if_idle() # Gracefully shutdown timeout. # We should provide graceful_shutdown_timeout, # instead of letting connection hangs forever. # Let's roughly calcucate time. start_shutdown = 0 while connections and (start_shutdown < graceful_shutdown_timeout): loop.run_until_complete(asyncio.sleep(0.1)) start_shutdown = start_shutdown + 0.1 # Force close non-idle connection after waiting for # graceful_shutdown_timeout coros = [] for conn in connections: if hasattr(conn, "websocket") and conn.websocket: coros.append(conn.websocket.close_connection(force=True)) else: conn.close() _shutdown = asyncio.gather(*coros, loop=loop) loop.run_until_complete(_shutdown) trigger_events(after_stop, loop) loop.close()
async def robot_distribution_view(request): """ - 分配机器人 - 更新或绑定师徒关系 - 创建机器人用户展示表 """ async def exists_user_access(union_id): async with db.conn.acquire() as con: select_stmt = await con.prepare( '''select "robot_access".id from "robot_access" join "robot" on "robot".id="robot_access".robot_id where "robot_access".union_id=$1 and "robot".status<>3 order by "robot_access".create_date desc limit 1''') access_id = await select_stmt.fetchval(union_id) return access_id async def update_user_access(user_access_id, sharing_user_id=None, channel=None): if not sharing_user_id: is_sharing_user = False else: is_sharing_user = True async with db.conn.acquire() as con: update_stmt = await con.prepare( '''update "robot_access" set sharing_user_id=case when $1 then $2 else sharing_user_id end, channel=case when $3 then $4 else channel end, update_date=now() where id=$5 returning robot_id''') return await update_stmt.fetchval(is_sharing_user, sharing_user_id, is_sharing_user, channel, user_access_id) async def create_user_access(open_id, union_id, robot_id, sharing_user_id=None, channel=None): async with db.conn.acquire() as con: insert_stmt = await con.prepare('''insert into "robot_access" (id, open_id, union_id, robot_id, sharing_user_id, channel, status) values (uuid_generate_v4(), $1, $2, $3, $4, $5, $6)''') await insert_stmt.fetch(open_id, union_id, robot_id, sharing_user_id, channel, CREATE_STATUS) async def robot_info(robot_id): '''查询机器人信息''' async with db.conn.acquire() as con: select_stmt = await con.prepare( 'select head_url, qr_code, wechat_no, name from "robot" where id=$1' ) robot = await select_stmt.fetchrow(robot_id) return robot open_id = request.json.get('open_id') union_id = request.json.get('union_id') if not open_id or not union_id: return response_json(None, code=PARAMS_ERROR_CODE, msg='缺少必要参数open_id或union_id') sharing_user_id = request.json.get('sharing_user_id', None) channel = request.json.get('channel', None) access_id = await exists_user_access(union_id) if access_id: if not sharing_user_id or sharing_user_id == 'null': sharing_user_id = None if not channel: channel = None robot_id = await update_user_access(access_id, sharing_user_id, channel) else: robot_id = await robot_distribution(channel=channel) if not robot_id: logger.warning(f'not match distribute robot for user [{open_id}]') return response_json(None, code=ROBOT_NOT_ENOUGH_CODE, msg='没有可分配的机器人') await create_user_access(open_id, union_id, robot_id, sharing_user_id, channel) await robot_today_display_record(robot_id) logger.info(f'distribute robot [{robot_id}] for user [{open_id}]') robot = await robot_info(robot_id) return response_json(dict(robot))
def not_found(request, exception): logger.warning(f'Unhandled route {request.method} {request.path}') return json({'error': 'Unexpected route'}, status=404)
async def after_server_start(_: Sanic, __: AbstractEventLoop): logger.warning('Server successfully started!')
async def create_server( self, host: Optional[str] = None, port: Optional[int] = None, *, debug: bool = False, ssl: Union[dict, SSLContext, None] = None, sock: Optional[socket] = None, protocol: Type[Protocol] = None, backlog: int = 100, access_log: Optional[bool] = None, unix: Optional[str] = None, return_asyncio_server: bool = False, asyncio_server_kwargs: Dict[str, Any] = None, ) -> Optional[AsyncioServer]: """ Asynchronous version of :func:`run`. This method will take care of the operations necessary to invoke the *before_start* events via :func:`trigger_events` method invocation before starting the *sanic* app in Async mode. .. note:: This does not support multiprocessing and is not the preferred way to run a :class:`Sanic` application. :param host: Address to host on :type host: str :param port: Port to host on :type port: int :param debug: Enables debug output (slows server) :type debug: bool :param ssl: SSLContext, or location of certificate and key for SSL encryption of worker(s) :type ssl: SSLContext or dict :param sock: Socket for the server to accept connections from :type sock: socket :param protocol: Subclass of asyncio Protocol class :type protocol: type[Protocol] :param backlog: a number of unaccepted connections that the system will allow before refusing new connections :type backlog: int :param access_log: Enables writing access logs (slows server) :type access_log: bool :param return_asyncio_server: flag that defines whether there's a need to return asyncio.Server or start it serving right away :type return_asyncio_server: bool :param asyncio_server_kwargs: key-value arguments for asyncio/uvloop create_server method :type asyncio_server_kwargs: dict :return: AsyncioServer if return_asyncio_server is true, else Nothing """ if sock is None: host, port = host or "127.0.0.1", port or 8000 if protocol is None: protocol = (WebSocketProtocol if self.websocket_enabled else HttpProtocol) # if access_log is passed explicitly change config.ACCESS_LOG if access_log is not None: self.config.ACCESS_LOG = access_log server_settings = self._helper( host=host, port=port, debug=debug, ssl=ssl, sock=sock, unix=unix, loop=get_event_loop(), protocol=protocol, backlog=backlog, run_async=return_asyncio_server, ) # Trigger before_start events await self.trigger_events( server_settings.get("before_start", []), server_settings.get("loop"), ) main_start = server_settings.pop("main_start", None) main_stop = server_settings.pop("main_stop", None) if main_start or main_stop: logger.warning( "Listener events for the main process are not available " "with create_server()") return await serve(asyncio_server_kwargs=asyncio_server_kwargs, **server_settings)
async def after_server_stop(_: Sanic, __: AbstractEventLoop): from .core import mongo mongo.close() logger.warning('Stopped.')
async def http1_response_header(self, data: bytes, end_stream: bool) -> None: res = self.response # Compatibility with simple response body if not data and getattr(res, "body", None): data, end_stream = res.body, True # type: ignore size = len(data) headers = res.headers status = res.status if not isinstance(status, int) or status < 200: raise RuntimeError(f"Invalid response status {status!r}") if not has_message_body(status): # Header-only response status self.response_func = None if (data or not end_stream or "content-length" in headers or "transfer-encoding" in headers): data, size, end_stream = b"", 0, True headers.pop("content-length", None) headers.pop("transfer-encoding", None) logger.warning( f"Message body set in response on {self.request.path}. " f"A {status} response may only have headers, no body.") elif self.head_only and "content-length" in headers: self.response_func = None elif end_stream: # Non-streaming response (all in one block) headers["content-length"] = size self.response_func = None elif "content-length" in headers: # Streaming response with size known in advance self.response_bytes_left = int(headers["content-length"]) - size self.response_func = self.http1_response_normal else: # Length not known, use chunked encoding headers["transfer-encoding"] = "chunked" data = b"%x\r\n%b\r\n" % (size, data) if size else b"" self.response_func = self.http1_response_chunked if self.head_only: # Head request: don't send body data = b"" self.response_func = self.head_response_ignored headers["connection"] = "keep-alive" if self.keep_alive else "close" ret = format_http1_response(status, res.processed_headers) if data: ret += data # Send a 100-continue if expected and not Expectation Failed if self.expecting_continue: self.expecting_continue = False if status != 417: ret = HTTP_CONTINUE + ret # Send response if self.protocol.access_log: self.log_response() await self._send(ret) self.stage = Stage.IDLE if end_stream else Stage.RESPONSE
async def fetch_data(self, url, session): try: async with session.get(url) as response: data = await response.text() data = pjson.loads(data) cnfy_id = 'cnfy-{}'.format(str(uuid.uuid4())) for tx in data['transactions']: if tx['type'] in [ 4 ] and tx['feeAssetId'] == os.environ['ASSET_ID']: attachment_base58 = base58.b58decode( tx['attachment']).decode('utf-8') attachment = None try: attachment = requests.get( '{0}:{1}/ipfs/{2}'.format( config['ipfs']['host'], config['ipfs']['port'], attachment_base58), timeout=2).text except Exception as error: logger.error('IPFS Error: {0}'.format(error)) if attachment == None: logger.warning('CONTINUE ON IPFS HASH {0}'.format( attachment_base58)) continue else: self.transactions_in_batch += 1 logger.info( 'TX detected. Height: {0}, IPFS Hash: {1}'. format(self.height, attachment_base58)) root = ET.fromstring(attachment) version = root.findall('version')[0].text if len( root.findall('version')) > 0 else None blockchain = root.findall('blockchain')[0].text if len( root.findall('blockchain')) > 0 else None network = root.findall('network')[0].text if len( root.findall('network')) > 0 else None messages = root.findall('messages')[0] if len( root.findall('messages')) > 0 else [] # if str(version) != str(os.environ['CDM_VERSION']): # continue for message in messages: to_public_key = None cc_public_key = None to_public_key_ciphertext = None to_public_key_sha256hash = None cc_public_key_ciphertext = None cc_public_key_sha256hash = None to = message.findall('to')[0] if len( message.findall('to')) > 0 else None cc = message.findall('cc')[0] if len( message.findall('cc')) > 0 else None if to: to_public_key = to.findall( 'publickey')[0].text if len( to.findall('publickey')) > 0 else None to_public_key_ciphertext = to.findall( 'ciphertext')[0].text if len( to.findall('ciphertext')) > 0 else None to_public_key_sha256hash = to.findall( 'sha256')[0].text if len( to.findall('sha256')) > 0 else None if cc: cc_public_key = cc.findall( 'publickey')[0].text if len( cc.findall('publickey')) > 0 else None cc_public_key_ciphertext = cc.findall( 'ciphertext')[0].text if len( cc.findall('ciphertext')) > 0 else None cc_public_key_sha256hash = cc.findall( 'sha256')[0].text if len( cc.findall('sha256')) > 0 else None subject_ciphertext = None subject_sha256hash = None subject = message.findall('subject')[0] if len( message.findall('subject')) > 0 else None if subject: subject_ciphertext = subject.findall( 'ciphertext')[0].text if len( subject.findall( 'ciphertext')) > 0 else None subject_sha256hash = subject.findall( 'sha256' )[0].text if len( subject.findall('sha256')) > 0 else None body_ciphertext = None body_sha256hash = None body = message.findall('body')[0] if len( message.findall('body')) > 0 else None if body: body_ciphertext = body.findall( 'ciphertext' )[0].text if len( body.findall('ciphertext')) > 0 else None body_sha256hash = body.findall( 'sha256')[0].text if len( body.findall('sha256')) > 0 else None if version == '0.7': recipient_public_key = to_public_key if to else cc_public_key recipient_public_key_ciphertext = recipient_public_key recipient_public_key_sha256hash = hashlib.sha256( recipient_public_key.encode( 'utf-8')).hexdigest() else: recipient_public_key_ciphertext = to_public_key_ciphertext if to else cc_public_key_ciphertext recipient_public_key_sha256hash = to_public_key_sha256hash if to else cc_public_key_sha256hash recipient_type = 'to' if to else 'cc' thread_hash = hashlib.sha256(''.join([ subject_sha256hash or '', body_sha256hash or '' ]).encode('utf-8')).hexdigest() re_subject_hash = None re_message_hash = None regarding = message.findall('regarding')[0] if len( message.findall('regarding')) > 0 else None if regarding: re_subject_hash = regarding.findall( 'subjecthash')[0].text if len( regarding.findall( 'subjecthash')) > 0 else None re_message_hash = regarding.findall( 'messagehash')[0].text if len( regarding.findall( 'messagehash')) > 0 else None thread_hash = hashlib.sha256(''.join([ re_subject_hash or '', re_message_hash or '' ]).encode('utf-8')).hexdigest() fwd_subject_hash = None fwd_message_hash = None forwarded = message.findall('forwarded')[0] if len( message.findall('forwarded')) > 0 else None if forwarded: fwd_subject_hash = forwarded.findall( 'subjecthash')[0].text if len( forwarded.findall( 'subjecthash')) > 0 else None fwd_message_hash = forwarded.findall( 'messagehash')[0].text if len( forwarded.findall( 'messagehash')) > 0 else None cdm_id = 'cdm-' + str(uuid.uuid4()) self.sql_data_cdms.append( (cdm_id, tx['id'], recipient_public_key_ciphertext, recipient_public_key_sha256hash, subject_ciphertext, subject_sha256hash, body_ciphertext, body_sha256hash, thread_hash, blockchain, network, recipient_type, re_subject_hash, re_message_hash, fwd_subject_hash, fwd_message_hash, datetime.fromtimestamp(tx['timestamp'] / 1e3), version)) from_block = message.findall('from')[0] if len( message.findall('from')) > 0 else None if from_block: senders = from_block.findall('sender') if len( from_block.findall('sender')) > 0 else None for sender in senders: sender_public_key = sender.findall( 'publickey')[0].text if len( sender.findall( 'publickey')) > 0 else None sender_public_key_ciphertext = sender.findall( 'ciphertext')[0].text if len( sender.findall( 'ciphertext')) > 0 else None sender_public_key_sha256hash = sender.findall( 'sha256' )[0].text if len( sender.findall('sha256')) > 0 else None sender_signature = sender.findall( 'signature')[0].text if len( sender.findall( 'signature')) > 0 else None if version == '0.7': sender_public_key_ciphertext = sender_public_key sender_public_key_sha256hash = hashlib.sha256( sender_public_key.encode( 'utf-8')).hexdigest() sender_id = str(uuid.uuid4()) self.sql_data_senders.append( (sender_id, cdm_id, sender_public_key_ciphertext, sender_public_key_sha256hash, sender_signature, True)) tx_data = (tx['id'], data['height'], tx['type'], tx['sender'], tx['senderPublicKey'], hashlib.sha256(tx['senderPublicKey'].encode( 'utf-8')).hexdigest(), tx['recipient'], tx['amount'], tx['assetId'], tx['feeAssetId'], tx['feeAsset'], tx['fee'], tx['attachment'], tx['version'], datetime.fromtimestamp(tx['timestamp'] / 1e3), cnfy_id, hashlib.sha256( attachment.encode('utf-8')).hexdigest()) self.sql_data_transactions.append(tx_data) for proof in tx['proofs']: proof_id = 'proof-' + str(uuid.uuid4()) self.sql_data_proofs.append( (tx['id'], proof, proof_id)) except asyncio.CancelledError: logger.info('Parser has been stopped') raise except Exception as error: logger.error('Fetching data error: {}'.format(error)) pass
def serve(host, port, request_handler, error_handler, before_start=None, after_start=None, before_stop=None, after_stop=None, debug=False, request_timeout=60, response_timeout=60, keep_alive_timeout=5, ssl=None, sock=None, request_max_size=None, reuse_port=False, loop=None, protocol=HttpProtocol, backlog=100, register_sys_signals=True, run_async=False, connections=None, signal=Signal(), request_class=None, access_log=True, keep_alive=True, is_request_stream=False, router=None, websocket_max_size=None, websocket_max_queue=None, state=None, graceful_shutdown_timeout=15.0): """Start asynchronous HTTP Server on an individual process. :param host: Address to host on :param port: Port to host on :param request_handler: Sanic request handler with middleware :param error_handler: Sanic error handler with middleware :param before_start: function to be executed before the server starts listening. Takes arguments `app` instance and `loop` :param after_start: function to be executed after the server starts listening. Takes arguments `app` instance and `loop` :param before_stop: function to be executed when a stop signal is received before it is respected. Takes arguments `app` instance and `loop` :param after_stop: function to be executed when a stop signal is received after it is respected. Takes arguments `app` instance and `loop` :param debug: enables debug output (slows server) :param request_timeout: time in seconds :param response_timeout: time in seconds :param keep_alive_timeout: time in seconds :param ssl: SSLContext :param sock: Socket for the server to accept connections from :param request_max_size: size in bytes, `None` for no limit :param reuse_port: `True` for multiple workers :param loop: asyncio compatible event loop :param protocol: subclass of asyncio protocol class :param request_class: Request class to use :param access_log: disable/enable access log :param is_request_stream: disable/enable Request.stream :param router: Router object :return: Nothing """ if not run_async: loop = async_loop.new_event_loop() asyncio.set_event_loop(loop) if debug: loop.set_debug(debug) connections = connections if connections is not None else set() server = partial( protocol, loop=loop, connections=connections, signal=signal, request_handler=request_handler, error_handler=error_handler, request_timeout=request_timeout, response_timeout=response_timeout, keep_alive_timeout=keep_alive_timeout, request_max_size=request_max_size, request_class=request_class, access_log=access_log, keep_alive=keep_alive, is_request_stream=is_request_stream, router=router, websocket_max_size=websocket_max_size, websocket_max_queue=websocket_max_queue, state=state, debug=debug, ) server_coroutine = loop.create_server( server, host, port, ssl=ssl, reuse_port=reuse_port, sock=sock, backlog=backlog ) # Instead of pulling time at the end of every request, # pull it once per minute loop.call_soon(partial(update_current_time, loop)) if run_async: return server_coroutine trigger_events(before_start, loop) try: http_server = loop.run_until_complete(server_coroutine) except BaseException: logger.exception("Unable to start server") return trigger_events(after_start, loop) # Register signals for graceful termination if register_sys_signals: for _signal in (SIGINT, SIGTERM): try: loop.add_signal_handler(_signal, loop.stop) except NotImplementedError: logger.warning('Sanic tried to use loop.add_signal_handler ' 'but it is not implemented on this platform.') pid = os.getpid() try: logger.info('Starting worker [%s]', pid) loop.run_forever() finally: logger.info("Stopping worker [%s]", pid) # Run the on_stop function if provided trigger_events(before_stop, loop) # Wait for event loop to finish and all connections to drain http_server.close() loop.run_until_complete(http_server.wait_closed()) # Complete all tasks on the loop signal.stopped = True for connection in connections: connection.close_if_idle() # Gracefully shutdown timeout. # We should provide graceful_shutdown_timeout, # instead of letting connection hangs forever. # Let's roughly calcucate time. start_shutdown = 0 while connections and (start_shutdown < graceful_shutdown_timeout): loop.run_until_complete(asyncio.sleep(0.1)) start_shutdown = start_shutdown + 0.1 # Force close non-idle connection after waiting for # graceful_shutdown_timeout coros = [] for conn in connections: if hasattr(conn, "websocket") and conn.websocket: coros.append( conn.websocket.close_connection(after_handshake=True) ) else: conn.close() _shutdown = asyncio.gather(*coros, loop=loop) loop.run_until_complete(_shutdown) trigger_events(after_stop, loop) loop.close()
async def fetch_data(self, url, session): try: async with session.get(url) as response: data = await response.text() data = pjson.loads(data) cnfy_id = 'cnfy-{}'.format(str(uuid.uuid4())) for tx in data['transactions']: if tx['type'] in [ 4 ] and tx['feeAssetId'] == os.environ['ASSET_ID']: attachment_base58 = base58.b58decode( tx['attachment']).decode('utf-8') attachment = None try: attachment = requests.get( '{0}:{1}/ipfs/{2}'.format( config['ipfs']['host'], config['ipfs']['port'], attachment_base58), timeout=2).text except Exception as error: logger.error('IPFS Error: {0}'.format(error)) if attachment == None: logger.warning('CONTINUE ON IPFS HASH {0}'.format( attachment_base58)) continue attachment_hash = hashlib.sha256( attachment.encode('utf-8')).hexdigest() root = ET.fromstring(attachment) version = root.findall('version')[0].text if len( root.findall('version')) > 0 else None blockchain = root.findall('blockchain')[0].text if len( root.findall('blockchain')) > 0 else None network = root.findall('network')[0].text if len( root.findall('network')) > 0 else None opcodes = root.findall('opcodes')[0] if len( root.findall('opcodes')) > 0 else [] if str(version) != str(os.environ['CDM_VERSION']): continue opcode = opcodes.findall('opcode')[0] if len( opcodes.findall('opcode')) > 0 else None if opcode: traffic_id = opcode.findall( 'trafficlight' )[0].text if len( opcode.findall('trafficlight')) > 0 else None print('traffic_id', traffic_id) tr_data = (traffic_id, 'GODMODE') self.sql_data_traffic.append(tr_data) tx_data = (tx['id'], data['height'], tx['type'], tx['sender'], tx['senderPublicKey'], tx['recipient'], tx['amount'], tx['assetId'], tx['feeAssetId'], tx['feeAsset'], tx['fee'], tx['attachment'], tx['version'], datetime.fromtimestamp(tx['timestamp'] / 1e3), cnfy_id, attachment_hash) self.sql_data_transactions.append(tx_data) for proof in tx['proofs']: proof_id = 'proof-' + str(uuid.uuid4()) self.sql_data_proofs.append( (tx['id'], proof, proof_id)) except asyncio.CancelledError: logger.info('Parser has been stopped') raise except Exception as error: logger.error('Fetching data error: {}'.format(error)) pass