def test_bulk_write(self): self.db.test.collection.bulk_write([ DeleteOne({'noCollation': 42}), DeleteMany({'noCollation': 42}), DeleteOne({'foo': 42}, collation=self.collation), DeleteMany({'foo': 42}, collation=self.collation), ReplaceOne({'noCollation': 24}, {'bar': 42}), UpdateOne({'noCollation': 84}, {'$set': {'bar': 10}}, upsert=True), UpdateMany({'noCollation': 45}, {'$set': {'bar': 42}}), ReplaceOne({'foo': 24}, {'foo': 42}, collation=self.collation), UpdateOne({'foo': 84}, {'$set': {'foo': 10}}, upsert=True, collation=self.collation), UpdateMany({'foo': 45}, {'$set': {'foo': 42}}, collation=self.collation) ]) delete_cmd = self.listener.results['started'][0].command update_cmd = self.listener.results['started'][1].command def check_ops(ops): for op in ops: if 'noCollation' in op['q']: self.assertNotIn('collation', op) else: self.assertEqual(self.collation.document, op['collation']) check_ops(delete_cmd['deletes']) check_ops(update_cmd['updates'])
def non_retryable_single_statement_ops(coll): return [ (coll.bulk_write, [[UpdateOne({}, {'$set': {'a': 1}}), UpdateMany({}, {'$set': {'a': 1}})]], {}), (coll.bulk_write, [[DeleteOne({}), DeleteMany({})]], {}), (coll.update_many, [{}, {'$set': {'a': 1}}], {}), (coll.delete_many, [{}], {}), # Deprecated methods. # Multi remove. (coll.remove, [{}], {}), # Multi update. (coll.update, [{}, {'$set': {'a': 1}}], {'multi': True}), # Unacknowledged deprecated methods. (coll.insert, [{}], {'w': 0}), # Unacknowledged Non-multi update. (coll.update, [{}, {'$set': {'a': 1}}], {'w': 0}), # Unacknowledged Non-multi remove. (coll.remove, [{}], {'multi': False, 'w': 0}), # Unacknowledged Replace. (coll.find_and_modify, [{}, {'a': 3}], {'writeConcern': {'w': 0}}), # Unacknowledged Update. (coll.find_and_modify, [{}, {'$set': {'a': 1}}], {'writeConcern': {'w': 0}}), # Unacknowledged Delete. (coll.find_and_modify, [{}, {}], {'remove': True, 'writeConcern': {'w': 0}}), ]
def denormalize_pipeline_on_all_runs(pipeline: dict) -> UpdateMany: """ Finds all pipeline runs whose pipeline reference matches this pipeline, and replaces the reference with a copy of the actual pipeline document. """ return UpdateMany( filter={ "$and": [ { "pipeline.id": pipeline["id"] }, { "pipeline.digest": pipeline["digest"] }, # We check for existence of this field to identify pipeline # runs whose pipeline hasn't been copied over yet. { "pipeline.schema": { "$exists": False } }, ] }, update={"$set": { "pipeline": pipeline }}, )
def denormalize_problem_on_all_runs(problem: dict) -> UpdateMany: """ Finds all pipeline runs whose problem reference matches this problem, and replaces the reference with a copy of the actual problem document. """ return UpdateMany( filter={ "$and": [ { "problem.id": problem["id"] }, { "problem.digest": problem["digest"] }, # We check for existence of this field to identify pipeline # runs whose problem hasn't been copied over yet. { "problem.name": { "$exists": False } }, ] }, update={"$set": { "problem": problem }}, )
def denormalize_dataset_on_all_runs(dataset: dict) -> UpdateMany: """ Finds all pipeline runs having a dataset reference matches this dataset, and replaces the reference with a copy of the actual dataset document. """ return UpdateMany( filter={ "$and": [ { "datasets.id": dataset["id"] }, { "datasets.digest": dataset["digest"] }, # We check for existence of this field to identify pipeline # runs whose datasets haven't been copied over yet. { "datasets.name": { "$exists": False } }, ] }, update={"$set": { "datasets.$[matcher]": dataset }}, array_filters=[{ "matcher.id": dataset["id"], "matcher.digest": dataset["digest"] }], )
def non_retryable_single_statement_ops(coll): return [ (coll.bulk_write, [[UpdateOne({}, {'$set': {'a': 1}}), UpdateMany({}, {'$set': {'a': 1}})]], {}), (coll.bulk_write, [[DeleteOne({}), DeleteMany({})]], {}), (coll.update_many, [{}, {'$set': {'a': 1}}], {}), (coll.delete_many, [{}], {}), ]
def write_batch(term_dict): ops = [ UpdateMany({"term": t}, {"$set": { 'doc_count': term_dict[t] }}) for t in term_dict.keys() ] bulk_write_result = index.bulk_write(ops) print("{} updated.".format(bulk_write_result.matched_count)) return bulk_write_result.matched_count
def test_UpdateOneAndMany(self): result = yield self.coll.bulk_write([ UpdateOne({'x': {"$exists": True}}, {"$inc": {'x': 1}}), # simple UpdateOne({'y': 123}, {"$inc": {'y': -1}, "$set": {'a': "hi"}}), # set UpdateOne({'z': 322}, {"$inc": {'z': 1}}), # missed UpdateOne({'w': 5}, {"$set": {"u": 7}}, upsert=True), # upsert UpdateMany({}, {"$set": {"m": 1}}), # many ]) docs = yield self.coll.find(fields={"_id": 0}) self.assertEqual(len(docs), 4) self.assertIn({'x': 43, 'm': 1}, docs) self.assertIn({'y': 122, 'a': "hi", 'm': 1}, docs) self.assertIn({'z': 321, 'm': 1}, docs) self.assertIn({'w': 5, 'u': 7, 'm': 1}, docs) self.assertIsInstance(result, BulkWriteResult) self.assertEqual(result.inserted_count, 0) self.assertEqual(result.matched_count, 6) self.assertEqual(result.modified_count, 6) self.assertEqual(result.upserted_count, 1) self.assertEqual(set(result.upserted_ids), {3})
def test_UpdateOneNotEqualsUpdateMany(self): self.assertNotEqual(UpdateOne({'foo': 42}, {'$set': { 'bar': 42 }}), UpdateMany({'foo': 42}, {'$set': { 'bar': 42 }}))
def test_UpdateManyEquals(self): self.assertEqual(UpdateMany({'foo': 42}, {'$set': { 'bar': 42 }}), UpdateMany({'foo': 42}, {'$set': { 'bar': 42 }}))
def process_skus(self, skus, show_product_id, from_site): """统一下架""" crawl_bulk_operations = [] shop_bulk_operations = [] crawl_bulk_operations.append(UpdateMany({'from_site': from_site, 'show_product_id': show_product_id}, {'$set': {'is_outof_stock': True}})) shop_bulk_operations.append(UpdateMany({'from_site': from_site, 'show_product_id': show_product_id}, {'$set': {'is_outof_stock': True}})) min_list_price = None min_current_price = None for item in skus: sku_detail = { 'scan_time': int(time.time()), 'scan_date': time.strftime('%Y-%m-%d',time.localtime(time.time())), 'show_product_id': str(item.get('show_product_id')), 'from_site': item.get('from_site'), 'id': item.get('id'), 'list_price': handle_price(str(item.get('list_price'))), 'current_price': handle_price(str(item.get('current_price'))), 'color': item.get('color'), 'size_num': item.get('size_num'), 'groupbuy_price': item.get('groupbuy_price'), 's_t': item.get('s_t') } s_color = sku_detail['color'].lower().strip() for c in s_color: s_color = re.sub(" ", " ", s_color) sku_detail['s_color'] = s_color for key in sku_detail.keys(): if not sku_detail[key]: del sku_detail[key] else: if type(sku_detail[key]) == str: sku_detail[key] = sku_detail[key].strip() if 'is_outof_stock' in item.keys(): sku_detail['is_outof_stock'] = item.get('is_outof_stock') else: sku_detail['is_outof_stock'] = False if sku_detail['is_outof_stock'] == False: float_list_price = float(sku_detail['list_price']) if not min_list_price or min_list_price > float_list_price: min_list_price = float_list_price float_current_price = float(sku_detail['current_price']) if not min_current_price or min_current_price > float_current_price: min_current_price = float_current_price sku_size = item.get('size') if type(sku_size) == dict: for key in sku_size: sku_detail[key] = sku_size[key] else: sku_detail['size'] = sku_size '''单个爬虫更新单个商品(测试使用)''' if self.settings['SIZE_TRANSFORM_TEST'] == 3: Singletransform().singletransform(from_site, sku_detail, self.settings['SIZE_TRANSFORM_LIST']) if 'quantity' in item.keys(): sku_detail['quantity'] = item.get('quantity') sku_flag = str(sku_detail['id']) sku_flag = "".join(sku_flag.split()) sku_flag = sku_flag.lower() spaces = ['\n', '\t', '\r'] for s in spaces: if s in sku_flag: sku_flag = re.sub(s, '', sku_flag) sku_detail['sku_flag'] = sku_flag sku_mongo = self.shop_db['skus'].find_one({'from_site': sku_detail['from_site'], 'show_product_id':sku_detail['show_product_id'], 'sku_flag': sku_detail['sku_flag']}) if sku_detail['from_site'] in self.settings['SIZE_TRANSFORM_LIST']: if sku_mongo and 'size' in sku_mongo.keys(): if sku_detail['size'] != sku_mongo['size']: sku_detail['handle_size'] = '1' else: sku_detail['handle_size'] = '1' if sku_detail['from_site'] == 'ralphlauren': if not sku_mongo: crawl_bulk_operations.append(UpdateOne( {'from_site': sku_detail['from_site'], 'show_product_id':sku_detail['show_product_id'], 'color': sku_detail['color'], 'size': sku_detail['size']}, {'$set': sku_detail}, upsert=True )) shop_bulk_operations.append(UpdateOne( {'from_site': sku_detail['from_site'], 'show_product_id':sku_detail['show_product_id'], 'color': sku_detail['color'], 'size': sku_detail['size']}, {'$set': sku_detail}, upsert=True )) else: crawl_bulk_operations.append(UpdateOne( {'from_site': sku_detail['from_site'], 'show_product_id':sku_detail['show_product_id'], 'color': sku_detail['color'], 'size': sku_detail['size']}, {'$set': sku_detail} )) shop_bulk_operations.append(UpdateOne( {'from_site': sku_detail['from_site'], 'show_product_id':sku_detail['show_product_id'], 'color': sku_detail['color'], 'size': sku_detail['size']}, {'$set': sku_detail} )) else: if not sku_mongo: crawl_bulk_operations.append(UpdateOne( {'from_site': sku_detail['from_site'], 'show_product_id':sku_detail['show_product_id'], 'sku_flag': sku_detail['sku_flag']}, {'$set': sku_detail}, upsert=True )) shop_bulk_operations.append(UpdateOne( {'from_site': sku_detail['from_site'], 'show_product_id':sku_detail['show_product_id'], 'sku_flag': sku_detail['sku_flag']}, {'$set': sku_detail}, upsert=True )) else: crawl_bulk_operations.append(UpdateOne( {'from_site': sku_detail['from_site'], 'show_product_id':sku_detail['show_product_id'], 'sku_flag': sku_detail['sku_flag']}, {'$set': sku_detail} )) shop_bulk_operations.append(UpdateOne( {'from_site': sku_detail['from_site'], 'show_product_id':sku_detail['show_product_id'], 'sku_flag': sku_detail['sku_flag']}, {'$set': sku_detail} )) self.db['skus'].bulk_write(crawl_bulk_operations) self.shop_db['skus'].bulk_write(shop_bulk_operations) return min_list_price, min_current_price
def bulk_update(self, filter, document, upsert, multi): if multi: self._requests.append(UpdateMany(filter, document, upsert=upsert)) else: self._requests.append(UpdateOne(filter, document, upsert=upsert))