def _upsert(self, doc, namespace, timestamp=util.utc_now(), *, doc_id=None, is_update=False):
"""
index or update document
:param doc: native object
:param namespace:
:param timestamp:
:param is_update:
:return:
"""
# if param doc_id gaven, use it as doc id
if doc_id:
doc['_id'] = doc_id
# Get operate type, 'index' and 'update' was supported in this function
_op_type = ElasticOperate.update if is_update else ElasticOperate.index
# format doc
doc = self._formatter.format_document(doc)
# Generate the action and action log by doc info
action, action_log = self._gen_action(_op_type, namespace, timestamp, doc)
# Get actions for the doc in bulk buffer
pre_action = self.bulk_buffer.get_action(doc_id)
# Merge the pre_action in bulk buffer and current action
action = self._action_merge(pre_action, action)
# Push new action to bulk buffer
self._push_to_buffer(action, action_log)
async def create_notification(self, request):
request: CreateNotificationRequest = convert_request(
CreateNotificationRequest,
await request.json(),
)
try:
conditions = dataclasses.asdict(
deserialize.deserialize(ConditionClause, request.conditions))
except deserialize.exceptions.DeserializeException as error:
return json_response(reason=f'wrong condition clause {error}',
status=400)
current_datetime = utc_now()
if request.scheduled_at is None:
scheduled_at = current_datetime
else:
scheduled_at = request.scheduled_at
if current_datetime > scheduled_at:
return json_response(
reason=f'scheduled_at should later than current time',
status=400)
notification = await create_notification(
title=request.title,
body=request.body,
scheduled_at=scheduled_at,
deep_link=request.deep_link,
image_url=request.image_url,
icon_url=request.icon_url,
conditions=conditions,
)
return json_response(result=notification_model_to_dict(notification))
async def change_notification_status(
target_notification: Notification,
status: NotificationStatus) -> Notification:
target_notification.status = status
target_notification.modified_at = utc_now()
await target_notification.save()
return target_notification
def __init__(self,
prev_block_hash,
actions,
create_time=util.utc_now(),
status=ActionLogBlockStatus.processing,
serializer=JSONSerializer()):
super().__init__(prev_block_hash=prev_block_hash,
create_time=create_time,
status=status,
actions=actions,
serializer=serializer)
self.first_action = {
'ns': self.first_action.get('ns'),
'ts': self.first_action.get('ts')
}
self.last_action = {
'ns': self.last_action.get('ns'),
'ts': self.last_action.get('ts')
}
# Simple verify block do NOT has merkle_tree and actions
self.merkle_tree = None
self.actions = []
self.status = status
def index(self, doc, namespace, timestamp=util.utc_now()):
"""
Index document
:param doc:
:param namespace:
:param timestamp:
:return:
"""
self._upsert(doc, namespace, timestamp)
logger.debug('[Index document] ns: %s' % namespace)
def delete(self, doc, namespace, timestamp=util.utc_now()):
"""
Delete document by doc_id
:param doc:
:param namespace:
:param timestamp:
:return:
"""
action, action_log = self._gen_action(ElasticOperate.delete, namespace, timestamp, doc)
self._push_to_buffer(action, action_log)
logger.debug('[Delete document] ns: %s' % namespace)
def update(self, doc_id, doc, namespace, timestamp=util.utc_now()):
"""
Update document
:param doc_id:
:param doc:
:param namespace:
:param timestamp:
:return:
"""
self._upsert(doc, namespace, timestamp, doc_id=doc_id, is_update=True)
logger.debug('[Update document] ns: %s' % namespace)
async def update_notification(
target_notification: Notification,
**kwargs
) -> Notification:
available_fields = ['title', 'body', 'icon_url', 'image_url', 'deep_link', 'scheduled_at']
for k, v in kwargs.items():
if k in available_fields and v is not None:
setattr(target_notification, k, v)
target_notification.modified_at = utc_now()
await target_notification.save()
return target_notification
async def update_device(
target_device: Device,
push_token: str = None,
send_platform: SendPlatform = SendPlatform.UNKNOWN,
device_platform: DevicePlatform = DevicePlatform.UNKNOWN,
) -> Device:
target_device.send_platform = send_platform
target_device.push_token = push_token
target_device.device_platform = device_platform
target_device.modified_at = utc_now()
await target_device.save()
return target_device
def _get_last_block(self):
block = None
if self.docman.es_sync.indices.exists_type(index=self.docman.log_index,
doc_type=self.docman.log_type):
first_non_valid_block_query_body = {
"size": 1,
"query": {
"term": {
"status.keyword": {
"value": ActionLogBlockStatus.processing
}
}
},
"sort": [
{
"create_time": {
"order": "asc"
}
}
]
}
rn = self.docman.es_sync.search(index=self.docman.log_index, doc_type=self.docman.log_type,
body=first_non_valid_block_query_body)
t = util.utc_now()
if rn.get('hits').get('total') > 0:
t = rn.get('hits').get('hits')[0].get('_source').get('create_time') or t
self._clean_invalid_block(t)
last_valid_block_query_body = {
"size": 1,
"sort": [
{
"create_time": {
"order": "desc"
}
}
]
}
rl = self.docman.es_sync.search(index=self.docman.log_index, doc_type=self.docman.log_type,
body=last_valid_block_query_body)
if rl.get('hits').get('total') > 0:
block = self._gen_sv_block_from_dict(rl.get('hits').get('hits')[0].get('_source'))
if not block:
logger.info('Last block does NOT exist')
else:
logger.info('Get last block success')
return block
async def job(self): # real working job
mysql_config = config.notification_worker.mysql
await init_db(
host=mysql_config.host,
port=mysql_config.port,
user=mysql_config.user,
password=mysql_config.password,
db=mysql_config.database,
)
self.redis_pool = await aioredis.create_pool(
f'redis://{self.redis_host}:{self.redis_port}',
password=self.redis_password,
db=int(
config.notification_worker.redis.notification_queue.database),
minsize=5,
maxsize=10,
)
while True:
with await self.redis_pool as redis_conn:
job_json = await blocking_get_notification_job(
redis_conn=redis_conn, timeout=self.REDIS_TIMEOUT)
logger.debug(multiprocessing.current_process())
if not job_json:
continue
logger.debug(job_json)
job: NotificationJob = deserialize.deserialize(
NotificationJob, json.loads(job_json))
scheduled_at = string_to_utc_datetime(job.scheduled_at)
current_datetime = utc_now()
if scheduled_at > current_datetime:
await publish_notification_job(
redis_conn=redis_conn,
job=object_to_dict(job),
priority=NotificationPriority.SCHEDULED,
)
logger.debug('scheduled notification (passed)')
await asyncio.sleep(1)
continue
logger.info('new task')
asyncio.create_task(self.process_job(job=job))
def __init__(self,
prev_block_hash,
actions,
create_time=util.utc_now(),
status=ActionLogBlockStatus.processing,
serializer=JSONSerializer()):
self._serializer = serializer
self.prev_block_hash = prev_block_hash
# unix timestamp, 13 bytes
self.create_time = create_time
self.status = status
self.actions = actions
self.actions_count = len(self.actions)
self.first_action = actions[0] if self.actions_count > 0 else {}
self.last_action = actions[-1] if self.actions_count > 0 else {}
# Merkle tree for actions
self.merkle_tree = self._make_merkle_tree()
self.merkle_root_hash = self._get_merkle_root(
) # depend on self.actions
self.id = self.__hash__(
) # depend self.on prev_block_hash, self.create_time and self.merkle_root_hash
async def _process_bulk_chunk(client: AsyncElasticsearch,
bulk_actions,
bulk_data,
max_retries,
initial_backoff,
max_backoff,
**kwargs):
"""
Send a bulk request to elasticsearch and process the output, it will retry when exception raised.
"""
attempted = 0
succeed, failed = [], []
while attempted <= max_retries:
# send the actual request
future = client.bulk('\n'.join(bulk_actions) + '\n', **kwargs)
attempted += 1
# if raise on error is set, we need to collect errors per chunk before raising them
try:
result = await future
except TransportError as e:
logger.warning('[Elasticsearch] %r', e)
if type(e) in NO_RETRY_EXCEPTIONS or attempted > max_retries:
# if we are not propagating, mark all actions in current chunk as failed
err_message = str(e)
for data in bulk_data:
# collect all the information about failed actions
op_type, action = data[0].copy().popitem()
info = {"error": err_message, "status": e.status_code, "create_time": util.utc_now()}
if op_type != 'delete':
info['data'] = data[1]
info['action'] = action
failed.append(info)
except Exception as e:
logger.warning('[AsyncHelper] %r', e)
if attempted > max_retries:
# if we are not propagating, mark all actions in current chunk as failed
err_message = str(e)
for data in bulk_data:
# collect all the information about failed actions
op_type, action = data[0].copy().popitem()
info = {"error": err_message, "status": 500, "create_time": util.utc_now(), "action": action}
# if op_type != 'delete':
# info['data'] = data[1]
failed.append(info)
else:
to_retry, to_retry_data = [], []
# go through request-response pairs and detect failures
for (action, data), (ok, info) in zip(bulk_data, _chunk_result(bulk_data, result)):
op, info = info.popitem()
if not ok and info.get('status') != 404:
if attempted < max_retries:
to_retry.append(client.transport.serializer.dumps(action))
if data:
to_retry.append(client.transport.serializer.dumps(data))
to_retry_data.append((action, data))
else:
info = {
'error': str(info.get('error')),
'status': info.get('status'),
'action': action,
# 'data': data,
'create_time': util.utc_now()
}
failed.append(info)
else:
# succeed or max retry
succeed.append(info)
# retry only subset of documents that didn't succeed
if attempted < max_retries:
bulk_actions, bulk_data = to_retry, to_retry_data
if not to_retry:
# all success, no need to retry
break
finally:
delay = min(max_backoff, initial_backoff * 2 ** (attempted - 1))
await asyncio.sleep(delay)
if attempted <= max_retries:
logger.debug('Elasticsearch bulk request retry')
return succeed, failed
def _log_block_commit(self, block):
logger.debug('Log block commit')
action, _ = self._gen_action(ElasticOperate.index, '.'.join([self.log_index, self.log_type]), util.utc_now(),
block.to_dict(), gen_log=False)
return async_helpers.bulk(client=self.es, actions=[action], max_retries=3,
initial_backoff=0.1, max_backoff=1)
async def bulk_index(self, docs, namespace, params=None, chunk_size=None, doc_process=None, ):
"""
Insert multiple documents into Elasticsearch directly.
:return:
"""
if not docs:
return None
if doc_process:
docs = stream.map(docs, doc_process)
docs = stream.map(docs, self._formatter.format_document)
async def bulk(docs):
succeed_total, failed_total = 0, 0
async for (succeed, failed) in self._chunk(actions=docs, chunk_size=self.chunk_size, params=params):
succeed_total += len(succeed)
failed_total += len(failed)
self.monitor.increase_succeed(len(succeed))
self.monitor.increase_failed(len(failed))
logger.info('[Direct bulk] ns:%s succeed:%d' % (namespace, len(succeed)))
if failed:
logger.warning('[Direct bulk] ns:%s failed:%d' % (namespace, len(failed)))
_, failed = await asyncio.ensure_future(self._failed_actions_commit(failed))
if not failed:
logger.debug('Failed actions commit success')
else:
logger.warning('Failed actions commit failed')
return succeed_total, failed_total
return await bulk(stream.map(docs,
lambda doc:
self._gen_action(ElasticOperate.index, namespace, util.utc_now(), doc, False)[0]))
async def launch_notification(self, request):
notification_uuid = request.match_info['notification_uuid']
notification = await find_notification_by_id(uuid=notification_uuid)
if notification is None:
return json_response(
reason=f'notification not found {notification_uuid}',
status=404)
notification = await change_notification_status(
target_notification=notification,
status=NotificationStatus.LAUNCHED,
)
conditions = deserialize.deserialize(ConditionClause,
notification.conditions)
device_total = await get_device_total_by_conditions(
conditions=conditions)
notification_job_capacity = math.ceil(device_total /
self.NOTIFICATION_WORKER_COUNT)
try:
tasks = []
with await self.redis_pool as redis_conn:
current_datetime = utc_now()
scheduled_at_utc = datetime_to_utc_datetime(
notification.scheduled_at)
priority = NotificationPriority.IMMEDIATE
if scheduled_at_utc > current_datetime:
priority = NotificationPriority.SCHEDULED
for job_index in range(self.NOTIFICATION_WORKER_COUNT):
job: NotificationJob = deserialize.deserialize(
NotificationJob, {
'notification': {
'id': notification.id,
'uuid': str(notification.uuid),
'title': notification.title,
'body': notification.body,
'image_url': notification.image_url,
'icon_url': notification.icon_url,
'deep_link': notification.deep_link,
'conditions': object_to_dict(conditions),
'devices': {
'start':
job_index * notification_job_capacity,
'size': notification_job_capacity,
}
},
'scheduled_at': scheduled_at_utc.isoformat()
})
tasks.append(
publish_notification_job(
redis_conn=redis_conn,
job=object_to_dict(job),
priority=priority,
))
await asyncio.gather(*tasks)
except Exception as e: # rollback if queue pushing failed
logger.warning(f'rollback because of queue pushing failed {e}')
notification = await change_notification_status(
target_notification=notification,
status=NotificationStatus.ERROR,
)
return json_response(result=notification_model_to_dict(notification))
async def change_notification_status(uuid: str,
status: NotificationStatus) -> int:
return await Notification.filter(uuid=uuid).update(
status=status,
modified_at=utc_now(),
)
