def _upsert(self, doc, namespace, timestamp=util.utc_now(), *, doc_id=None, is_update=False): """ index or update document :param doc: native object :param namespace: :param timestamp: :param is_update: :return: """ # if param doc_id gaven, use it as doc id if doc_id: doc['_id'] = doc_id # Get operate type, 'index' and 'update' was supported in this function _op_type = ElasticOperate.update if is_update else ElasticOperate.index # format doc doc = self._formatter.format_document(doc) # Generate the action and action log by doc info action, action_log = self._gen_action(_op_type, namespace, timestamp, doc) # Get actions for the doc in bulk buffer pre_action = self.bulk_buffer.get_action(doc_id) # Merge the pre_action in bulk buffer and current action action = self._action_merge(pre_action, action) # Push new action to bulk buffer self._push_to_buffer(action, action_log)
async def create_notification(self, request): request: CreateNotificationRequest = convert_request( CreateNotificationRequest, await request.json(), ) try: conditions = dataclasses.asdict( deserialize.deserialize(ConditionClause, request.conditions)) except deserialize.exceptions.DeserializeException as error: return json_response(reason=f'wrong condition clause {error}', status=400) current_datetime = utc_now() if request.scheduled_at is None: scheduled_at = current_datetime else: scheduled_at = request.scheduled_at if current_datetime > scheduled_at: return json_response( reason=f'scheduled_at should later than current time', status=400) notification = await create_notification( title=request.title, body=request.body, scheduled_at=scheduled_at, deep_link=request.deep_link, image_url=request.image_url, icon_url=request.icon_url, conditions=conditions, ) return json_response(result=notification_model_to_dict(notification))
async def change_notification_status( target_notification: Notification, status: NotificationStatus) -> Notification: target_notification.status = status target_notification.modified_at = utc_now() await target_notification.save() return target_notification
def __init__(self, prev_block_hash, actions, create_time=util.utc_now(), status=ActionLogBlockStatus.processing, serializer=JSONSerializer()): super().__init__(prev_block_hash=prev_block_hash, create_time=create_time, status=status, actions=actions, serializer=serializer) self.first_action = { 'ns': self.first_action.get('ns'), 'ts': self.first_action.get('ts') } self.last_action = { 'ns': self.last_action.get('ns'), 'ts': self.last_action.get('ts') } # Simple verify block do NOT has merkle_tree and actions self.merkle_tree = None self.actions = [] self.status = status
def index(self, doc, namespace, timestamp=util.utc_now()): """ Index document :param doc: :param namespace: :param timestamp: :return: """ self._upsert(doc, namespace, timestamp) logger.debug('[Index document] ns: %s' % namespace)
def delete(self, doc, namespace, timestamp=util.utc_now()): """ Delete document by doc_id :param doc: :param namespace: :param timestamp: :return: """ action, action_log = self._gen_action(ElasticOperate.delete, namespace, timestamp, doc) self._push_to_buffer(action, action_log) logger.debug('[Delete document] ns: %s' % namespace)
def update(self, doc_id, doc, namespace, timestamp=util.utc_now()): """ Update document :param doc_id: :param doc: :param namespace: :param timestamp: :return: """ self._upsert(doc, namespace, timestamp, doc_id=doc_id, is_update=True) logger.debug('[Update document] ns: %s' % namespace)
async def update_notification( target_notification: Notification, **kwargs ) -> Notification: available_fields = ['title', 'body', 'icon_url', 'image_url', 'deep_link', 'scheduled_at'] for k, v in kwargs.items(): if k in available_fields and v is not None: setattr(target_notification, k, v) target_notification.modified_at = utc_now() await target_notification.save() return target_notification
async def update_device( target_device: Device, push_token: str = None, send_platform: SendPlatform = SendPlatform.UNKNOWN, device_platform: DevicePlatform = DevicePlatform.UNKNOWN, ) -> Device: target_device.send_platform = send_platform target_device.push_token = push_token target_device.device_platform = device_platform target_device.modified_at = utc_now() await target_device.save() return target_device
def _get_last_block(self): block = None if self.docman.es_sync.indices.exists_type(index=self.docman.log_index, doc_type=self.docman.log_type): first_non_valid_block_query_body = { "size": 1, "query": { "term": { "status.keyword": { "value": ActionLogBlockStatus.processing } } }, "sort": [ { "create_time": { "order": "asc" } } ] } rn = self.docman.es_sync.search(index=self.docman.log_index, doc_type=self.docman.log_type, body=first_non_valid_block_query_body) t = util.utc_now() if rn.get('hits').get('total') > 0: t = rn.get('hits').get('hits')[0].get('_source').get('create_time') or t self._clean_invalid_block(t) last_valid_block_query_body = { "size": 1, "sort": [ { "create_time": { "order": "desc" } } ] } rl = self.docman.es_sync.search(index=self.docman.log_index, doc_type=self.docman.log_type, body=last_valid_block_query_body) if rl.get('hits').get('total') > 0: block = self._gen_sv_block_from_dict(rl.get('hits').get('hits')[0].get('_source')) if not block: logger.info('Last block does NOT exist') else: logger.info('Get last block success') return block
async def job(self): # real working job mysql_config = config.notification_worker.mysql await init_db( host=mysql_config.host, port=mysql_config.port, user=mysql_config.user, password=mysql_config.password, db=mysql_config.database, ) self.redis_pool = await aioredis.create_pool( f'redis://{self.redis_host}:{self.redis_port}', password=self.redis_password, db=int( config.notification_worker.redis.notification_queue.database), minsize=5, maxsize=10, ) while True: with await self.redis_pool as redis_conn: job_json = await blocking_get_notification_job( redis_conn=redis_conn, timeout=self.REDIS_TIMEOUT) logger.debug(multiprocessing.current_process()) if not job_json: continue logger.debug(job_json) job: NotificationJob = deserialize.deserialize( NotificationJob, json.loads(job_json)) scheduled_at = string_to_utc_datetime(job.scheduled_at) current_datetime = utc_now() if scheduled_at > current_datetime: await publish_notification_job( redis_conn=redis_conn, job=object_to_dict(job), priority=NotificationPriority.SCHEDULED, ) logger.debug('scheduled notification (passed)') await asyncio.sleep(1) continue logger.info('new task') asyncio.create_task(self.process_job(job=job))
def __init__(self, prev_block_hash, actions, create_time=util.utc_now(), status=ActionLogBlockStatus.processing, serializer=JSONSerializer()): self._serializer = serializer self.prev_block_hash = prev_block_hash # unix timestamp, 13 bytes self.create_time = create_time self.status = status self.actions = actions self.actions_count = len(self.actions) self.first_action = actions[0] if self.actions_count > 0 else {} self.last_action = actions[-1] if self.actions_count > 0 else {} # Merkle tree for actions self.merkle_tree = self._make_merkle_tree() self.merkle_root_hash = self._get_merkle_root( ) # depend on self.actions self.id = self.__hash__( ) # depend self.on prev_block_hash, self.create_time and self.merkle_root_hash
async def _process_bulk_chunk(client: AsyncElasticsearch, bulk_actions, bulk_data, max_retries, initial_backoff, max_backoff, **kwargs): """ Send a bulk request to elasticsearch and process the output, it will retry when exception raised. """ attempted = 0 succeed, failed = [], [] while attempted <= max_retries: # send the actual request future = client.bulk('\n'.join(bulk_actions) + '\n', **kwargs) attempted += 1 # if raise on error is set, we need to collect errors per chunk before raising them try: result = await future except TransportError as e: logger.warning('[Elasticsearch] %r', e) if type(e) in NO_RETRY_EXCEPTIONS or attempted > max_retries: # if we are not propagating, mark all actions in current chunk as failed err_message = str(e) for data in bulk_data: # collect all the information about failed actions op_type, action = data[0].copy().popitem() info = {"error": err_message, "status": e.status_code, "create_time": util.utc_now()} if op_type != 'delete': info['data'] = data[1] info['action'] = action failed.append(info) except Exception as e: logger.warning('[AsyncHelper] %r', e) if attempted > max_retries: # if we are not propagating, mark all actions in current chunk as failed err_message = str(e) for data in bulk_data: # collect all the information about failed actions op_type, action = data[0].copy().popitem() info = {"error": err_message, "status": 500, "create_time": util.utc_now(), "action": action} # if op_type != 'delete': # info['data'] = data[1] failed.append(info) else: to_retry, to_retry_data = [], [] # go through request-response pairs and detect failures for (action, data), (ok, info) in zip(bulk_data, _chunk_result(bulk_data, result)): op, info = info.popitem() if not ok and info.get('status') != 404: if attempted < max_retries: to_retry.append(client.transport.serializer.dumps(action)) if data: to_retry.append(client.transport.serializer.dumps(data)) to_retry_data.append((action, data)) else: info = { 'error': str(info.get('error')), 'status': info.get('status'), 'action': action, # 'data': data, 'create_time': util.utc_now() } failed.append(info) else: # succeed or max retry succeed.append(info) # retry only subset of documents that didn't succeed if attempted < max_retries: bulk_actions, bulk_data = to_retry, to_retry_data if not to_retry: # all success, no need to retry break finally: delay = min(max_backoff, initial_backoff * 2 ** (attempted - 1)) await asyncio.sleep(delay) if attempted <= max_retries: logger.debug('Elasticsearch bulk request retry') return succeed, failed
def _log_block_commit(self, block): logger.debug('Log block commit') action, _ = self._gen_action(ElasticOperate.index, '.'.join([self.log_index, self.log_type]), util.utc_now(), block.to_dict(), gen_log=False) return async_helpers.bulk(client=self.es, actions=[action], max_retries=3, initial_backoff=0.1, max_backoff=1)
async def bulk_index(self, docs, namespace, params=None, chunk_size=None, doc_process=None, ): """ Insert multiple documents into Elasticsearch directly. :return: """ if not docs: return None if doc_process: docs = stream.map(docs, doc_process) docs = stream.map(docs, self._formatter.format_document) async def bulk(docs): succeed_total, failed_total = 0, 0 async for (succeed, failed) in self._chunk(actions=docs, chunk_size=self.chunk_size, params=params): succeed_total += len(succeed) failed_total += len(failed) self.monitor.increase_succeed(len(succeed)) self.monitor.increase_failed(len(failed)) logger.info('[Direct bulk] ns:%s succeed:%d' % (namespace, len(succeed))) if failed: logger.warning('[Direct bulk] ns:%s failed:%d' % (namespace, len(failed))) _, failed = await asyncio.ensure_future(self._failed_actions_commit(failed)) if not failed: logger.debug('Failed actions commit success') else: logger.warning('Failed actions commit failed') return succeed_total, failed_total return await bulk(stream.map(docs, lambda doc: self._gen_action(ElasticOperate.index, namespace, util.utc_now(), doc, False)[0]))
async def launch_notification(self, request): notification_uuid = request.match_info['notification_uuid'] notification = await find_notification_by_id(uuid=notification_uuid) if notification is None: return json_response( reason=f'notification not found {notification_uuid}', status=404) notification = await change_notification_status( target_notification=notification, status=NotificationStatus.LAUNCHED, ) conditions = deserialize.deserialize(ConditionClause, notification.conditions) device_total = await get_device_total_by_conditions( conditions=conditions) notification_job_capacity = math.ceil(device_total / self.NOTIFICATION_WORKER_COUNT) try: tasks = [] with await self.redis_pool as redis_conn: current_datetime = utc_now() scheduled_at_utc = datetime_to_utc_datetime( notification.scheduled_at) priority = NotificationPriority.IMMEDIATE if scheduled_at_utc > current_datetime: priority = NotificationPriority.SCHEDULED for job_index in range(self.NOTIFICATION_WORKER_COUNT): job: NotificationJob = deserialize.deserialize( NotificationJob, { 'notification': { 'id': notification.id, 'uuid': str(notification.uuid), 'title': notification.title, 'body': notification.body, 'image_url': notification.image_url, 'icon_url': notification.icon_url, 'deep_link': notification.deep_link, 'conditions': object_to_dict(conditions), 'devices': { 'start': job_index * notification_job_capacity, 'size': notification_job_capacity, } }, 'scheduled_at': scheduled_at_utc.isoformat() }) tasks.append( publish_notification_job( redis_conn=redis_conn, job=object_to_dict(job), priority=priority, )) await asyncio.gather(*tasks) except Exception as e: # rollback if queue pushing failed logger.warning(f'rollback because of queue pushing failed {e}') notification = await change_notification_status( target_notification=notification, status=NotificationStatus.ERROR, ) return json_response(result=notification_model_to_dict(notification))
async def change_notification_status(uuid: str, status: NotificationStatus) -> int: return await Notification.filter(uuid=uuid).update( status=status, modified_at=utc_now(), )