def task_process_bulk(self, r): index = _.get(r, 'index') doc_type = _.get(r, 'doc_type') body = _.get(r, 'body') params = _.get(r, 'params') or {} result = es_client.bulk(body=body, index=index, doc_type=doc_type, params=params) if result['errors']: raise elasticsearch_util.bulk_error_2_elasticsearch_exception( result['items']) return result
def index(): body = [] index = 0 course = wx_order.populates(filter=wx_order_cursor.filter, pop_fields=wx_order_cursor.pop_fields, field_value_filter=lambda v: str(v)) wx_order_cursor.total = len(course) for item in course: order = item.pop('order') body.append({ 'index': { '_id': str(item.pop('_id')), '_parent': _.get(order, '_id') } }) body.append(item) index += 1 if (index % wx_order_cursor.limit) == 0 or index >= wx_order_cursor.total: res = es_client.bulk(index=opt['index'], doc_type=opt['type'], params=opt['params'], body=body) if res['errors']: raise elasticsearch_util.bulk_error_2_elasticsearch_exception( res['items']) else: wx_order_cursor.count += len(res['items']) logger.info('carChangePlan order wxorder indexed:{0}'.format( wx_order_cursor.count)) body = []
def index(): course = order.find(filter=order_cursor.filter, projection=order_cursor.projection, batch_size=order_cursor.limit) def gen_body(_course): result = None while result is None or len(result) == order_cursor.limit: result = [ [{'index': {'_id': str(v.pop('_id'))}}, v] for v in itertools.islice(_course, order_cursor.limit) ] if len(result) > 0: yield _.flatten(result) for body in gen_body(course): res = es_client.bulk(index=opt['index'], doc_type=opt['type'], params=opt['params'], body=body) if res['errors']: raise elasticsearch_util.bulk_error_2_elasticsearch_exception(res['items']) else: order_cursor.count += len(res['items']) logger.info('activity order indexed:{0}'.format(order_cursor.count))
def index(): body = [] index = 0 course = used_car_offer.populates( filter=used_car_offer_cursor.filter, projection=used_car_offer_cursor.projection, pop_fields=used_car_offer_cursor.pop_fields) used_car_offer_cursor.total = len(course) for item in course: body.append({ 'index': { '_id': str(item.pop('_id')), '_parent': str(item.get('used_car_id')) } }) body.append(item) index += 1 if (index % used_car_offer_cursor.limit ) == 0 or index >= used_car_offer_cursor.total: res = es_client.bulk(index=opt['index'], doc_type=opt['type'], params=opt['params'], body=body) if res['errors']: raise elasticsearch_util.bulk_error_2_elasticsearch_exception( res['items']) else: used_car_offer_cursor.count += len(res['items']) logger.info('carChangePlan used_car_offer indexed:{0}'.format( used_car_offer_cursor.count)) body = []
def index(): es_sync_util.index_exists_create(index=opt['index'], mappings=opt['mappings'], settings=opt['settings']) body = [] index = 0 course = impression_track.find( filter=impression_track_cursor.filter, projection=impression_track_cursor.projection, batch_size=impression_track_cursor.limit) impression_track_cursor.total = course.count() for item in course: body.append({'index': {'_id': str(item.pop('_id'))}}) body.append(_.assign(item, track_util.url_split(_.get(item, 'uri')))) index += 1 if (index % impression_track_cursor.limit ) == 0 or index >= impression_track_cursor.total: res = es_client.bulk(index=opt['index'], doc_type=opt['type'], params=opt['params'], body=body) if res['errors']: raise elasticsearch_util.bulk_error_2_elasticsearch_exception( res['items']) else: impression_track_cursor.count += len(res['items']) logger.info('tracks impression_track indexed:{0}'.format( impression_track_cursor.count)) body = []
def index(): body = [] index = 0 course = order.find(filter=order_cursor.filter, projection=order_cursor.projection, batch_size=order_cursor.limit) order_cursor.total = course.count() for item in course: body.append({ 'index': { '_id': str(item.pop('_id')), '_parent': str(item.get('car_change_plan_id')) } }) body.append(item) index += 1 if (index % order_cursor.limit) == 0 or index >= order_cursor.total: res = es_client.bulk(index=opt['index'], doc_type=opt['type'], params=opt['params'], body=body) if res['errors']: raise elasticsearch_util.bulk_error_2_elasticsearch_exception( res['items']) else: order_cursor.count += len(res['items']) logger.info('carChangePlan order indexed:{0}'.format( order_cursor.count)) body = []