def task_process_bulk(self, r):
index = _.get(r, 'index')
doc_type = _.get(r, 'doc_type')
body = _.get(r, 'body')
params = _.get(r, 'params') or {}
result = es_client.bulk(body=body,
index=index,
doc_type=doc_type,
params=params)
if result['errors']:
raise elasticsearch_util.bulk_error_2_elasticsearch_exception(
result['items'])
return result
def index():
body = []
index = 0
course = wx_order.populates(filter=wx_order_cursor.filter,
pop_fields=wx_order_cursor.pop_fields,
field_value_filter=lambda v: str(v))
wx_order_cursor.total = len(course)
for item in course:
order = item.pop('order')
body.append({
'index': {
'_id': str(item.pop('_id')),
'_parent': _.get(order, '_id')
}
})
body.append(item)
index += 1
if (index %
wx_order_cursor.limit) == 0 or index >= wx_order_cursor.total:
res = es_client.bulk(index=opt['index'],
doc_type=opt['type'],
params=opt['params'],
body=body)
if res['errors']:
raise elasticsearch_util.bulk_error_2_elasticsearch_exception(
res['items'])
else:
wx_order_cursor.count += len(res['items'])
logger.info('carChangePlan order wxorder indexed:{0}'.format(
wx_order_cursor.count))
body = []
def index():
course = order.find(filter=order_cursor.filter, projection=order_cursor.projection,
batch_size=order_cursor.limit)
def gen_body(_course):
result = None
while result is None or len(result) == order_cursor.limit:
result = [
[{'index': {'_id': str(v.pop('_id'))}}, v]
for v in itertools.islice(_course, order_cursor.limit)
]
if len(result) > 0:
yield _.flatten(result)
for body in gen_body(course):
res = es_client.bulk(index=opt['index'], doc_type=opt['type'], params=opt['params'], body=body)
if res['errors']:
raise elasticsearch_util.bulk_error_2_elasticsearch_exception(res['items'])
else:
order_cursor.count += len(res['items'])
logger.info('activity order indexed:{0}'.format(order_cursor.count))
def index():
body = []
index = 0
course = used_car_offer.populates(
filter=used_car_offer_cursor.filter,
projection=used_car_offer_cursor.projection,
pop_fields=used_car_offer_cursor.pop_fields)
used_car_offer_cursor.total = len(course)
for item in course:
body.append({
'index': {
'_id': str(item.pop('_id')),
'_parent': str(item.get('used_car_id'))
}
})
body.append(item)
index += 1
if (index % used_car_offer_cursor.limit
) == 0 or index >= used_car_offer_cursor.total:
res = es_client.bulk(index=opt['index'],
doc_type=opt['type'],
params=opt['params'],
body=body)
if res['errors']:
raise elasticsearch_util.bulk_error_2_elasticsearch_exception(
res['items'])
else:
used_car_offer_cursor.count += len(res['items'])
logger.info('carChangePlan used_car_offer indexed:{0}'.format(
used_car_offer_cursor.count))
body = []
def index():
es_sync_util.index_exists_create(index=opt['index'],
mappings=opt['mappings'],
settings=opt['settings'])
body = []
index = 0
course = impression_track.find(
filter=impression_track_cursor.filter,
projection=impression_track_cursor.projection,
batch_size=impression_track_cursor.limit)
impression_track_cursor.total = course.count()
for item in course:
body.append({'index': {'_id': str(item.pop('_id'))}})
body.append(_.assign(item, track_util.url_split(_.get(item, 'uri'))))
index += 1
if (index % impression_track_cursor.limit
) == 0 or index >= impression_track_cursor.total:
res = es_client.bulk(index=opt['index'],
doc_type=opt['type'],
params=opt['params'],
body=body)
if res['errors']:
raise elasticsearch_util.bulk_error_2_elasticsearch_exception(
res['items'])
else:
impression_track_cursor.count += len(res['items'])
logger.info('tracks impression_track indexed:{0}'.format(
impression_track_cursor.count))
body = []
def index():
body = []
index = 0
course = order.find(filter=order_cursor.filter,
projection=order_cursor.projection,
batch_size=order_cursor.limit)
order_cursor.total = course.count()
for item in course:
body.append({
'index': {
'_id': str(item.pop('_id')),
'_parent': str(item.get('car_change_plan_id'))
}
})
body.append(item)
index += 1
if (index % order_cursor.limit) == 0 or index >= order_cursor.total:
res = es_client.bulk(index=opt['index'],
doc_type=opt['type'],
params=opt['params'],
body=body)
if res['errors']:
raise elasticsearch_util.bulk_error_2_elasticsearch_exception(
res['items'])
else:
order_cursor.count += len(res['items'])
logger.info('carChangePlan order indexed:{0}'.format(
order_cursor.count))
body = []
