def test_bulk_write(self):
self.db.test.collection.bulk_write([
DeleteOne({'noCollation': 42}),
DeleteMany({'noCollation': 42}),
DeleteOne({'foo': 42}, collation=self.collation),
DeleteMany({'foo': 42}, collation=self.collation),
ReplaceOne({'noCollation': 24}, {'bar': 42}),
UpdateOne({'noCollation': 84}, {'$set': {'bar': 10}}, upsert=True),
UpdateMany({'noCollation': 45}, {'$set': {'bar': 42}}),
ReplaceOne({'foo': 24}, {'foo': 42}, collation=self.collation),
UpdateOne({'foo': 84}, {'$set': {'foo': 10}}, upsert=True,
collation=self.collation),
UpdateMany({'foo': 45}, {'$set': {'foo': 42}},
collation=self.collation)
])
delete_cmd = self.listener.results['started'][0].command
update_cmd = self.listener.results['started'][1].command
def check_ops(ops):
for op in ops:
if 'noCollation' in op['q']:
self.assertNotIn('collation', op)
else:
self.assertEqual(self.collation.document,
op['collation'])
check_ops(delete_cmd['deletes'])
check_ops(update_cmd['updates'])
def non_retryable_single_statement_ops(coll):
return [
(coll.bulk_write, [[UpdateOne({}, {'$set': {'a': 1}}),
UpdateMany({}, {'$set': {'a': 1}})]], {}),
(coll.bulk_write, [[DeleteOne({}), DeleteMany({})]], {}),
(coll.update_many, [{}, {'$set': {'a': 1}}], {}),
(coll.delete_many, [{}], {}),
# Deprecated methods.
# Multi remove.
(coll.remove, [{}], {}),
# Multi update.
(coll.update, [{}, {'$set': {'a': 1}}], {'multi': True}),
# Unacknowledged deprecated methods.
(coll.insert, [{}], {'w': 0}),
# Unacknowledged Non-multi update.
(coll.update, [{}, {'$set': {'a': 1}}], {'w': 0}),
# Unacknowledged Non-multi remove.
(coll.remove, [{}], {'multi': False, 'w': 0}),
# Unacknowledged Replace.
(coll.find_and_modify, [{}, {'a': 3}], {'writeConcern': {'w': 0}}),
# Unacknowledged Update.
(coll.find_and_modify, [{}, {'$set': {'a': 1}}],
{'writeConcern': {'w': 0}}),
# Unacknowledged Delete.
(coll.find_and_modify, [{}, {}],
{'remove': True, 'writeConcern': {'w': 0}}),
]
def denormalize_pipeline_on_all_runs(pipeline: dict) -> UpdateMany:
"""
Finds all pipeline runs whose pipeline reference matches this
pipeline, and replaces the reference with a copy of the actual
pipeline document.
"""
return UpdateMany(
filter={
"$and": [
{
"pipeline.id": pipeline["id"]
},
{
"pipeline.digest": pipeline["digest"]
},
# We check for existence of this field to identify pipeline
# runs whose pipeline hasn't been copied over yet.
{
"pipeline.schema": {
"$exists": False
}
},
]
},
update={"$set": {
"pipeline": pipeline
}},
)
def denormalize_problem_on_all_runs(problem: dict) -> UpdateMany:
"""
Finds all pipeline runs whose problem reference matches this
problem, and replaces the reference with a copy of the actual
problem document.
"""
return UpdateMany(
filter={
"$and": [
{
"problem.id": problem["id"]
},
{
"problem.digest": problem["digest"]
},
# We check for existence of this field to identify pipeline
# runs whose problem hasn't been copied over yet.
{
"problem.name": {
"$exists": False
}
},
]
},
update={"$set": {
"problem": problem
}},
)
def denormalize_dataset_on_all_runs(dataset: dict) -> UpdateMany:
"""
Finds all pipeline runs having a dataset reference matches this
dataset, and replaces the reference with a copy of the actual
dataset document.
"""
return UpdateMany(
filter={
"$and": [
{
"datasets.id": dataset["id"]
},
{
"datasets.digest": dataset["digest"]
},
# We check for existence of this field to identify pipeline
# runs whose datasets haven't been copied over yet.
{
"datasets.name": {
"$exists": False
}
},
]
},
update={"$set": {
"datasets.$[matcher]": dataset
}},
array_filters=[{
"matcher.id": dataset["id"],
"matcher.digest": dataset["digest"]
}],
)
def non_retryable_single_statement_ops(coll):
return [
(coll.bulk_write, [[UpdateOne({}, {'$set': {'a': 1}}),
UpdateMany({}, {'$set': {'a': 1}})]], {}),
(coll.bulk_write, [[DeleteOne({}), DeleteMany({})]], {}),
(coll.update_many, [{}, {'$set': {'a': 1}}], {}),
(coll.delete_many, [{}], {}),
]
def write_batch(term_dict):
ops = [
UpdateMany({"term": t}, {"$set": {
'doc_count': term_dict[t]
}}) for t in term_dict.keys()
]
bulk_write_result = index.bulk_write(ops)
print("{} updated.".format(bulk_write_result.matched_count))
return bulk_write_result.matched_count
def test_UpdateOneAndMany(self):
result = yield self.coll.bulk_write([
UpdateOne({'x': {"$exists": True}}, {"$inc": {'x': 1}}), # simple
UpdateOne({'y': 123}, {"$inc": {'y': -1}, "$set": {'a': "hi"}}), # set
UpdateOne({'z': 322}, {"$inc": {'z': 1}}), # missed
UpdateOne({'w': 5}, {"$set": {"u": 7}}, upsert=True), # upsert
UpdateMany({}, {"$set": {"m": 1}}), # many
])
docs = yield self.coll.find(fields={"_id": 0})
self.assertEqual(len(docs), 4)
self.assertIn({'x': 43, 'm': 1}, docs)
self.assertIn({'y': 122, 'a': "hi", 'm': 1}, docs)
self.assertIn({'z': 321, 'm': 1}, docs)
self.assertIn({'w': 5, 'u': 7, 'm': 1}, docs)
self.assertIsInstance(result, BulkWriteResult)
self.assertEqual(result.inserted_count, 0)
self.assertEqual(result.matched_count, 6)
self.assertEqual(result.modified_count, 6)
self.assertEqual(result.upserted_count, 1)
self.assertEqual(set(result.upserted_ids), {3})
def test_UpdateOneNotEqualsUpdateMany(self):
self.assertNotEqual(UpdateOne({'foo': 42}, {'$set': {
'bar': 42
}}), UpdateMany({'foo': 42}, {'$set': {
'bar': 42
}}))
def test_UpdateManyEquals(self):
self.assertEqual(UpdateMany({'foo': 42}, {'$set': {
'bar': 42
}}), UpdateMany({'foo': 42}, {'$set': {
'bar': 42
}}))
def process_skus(self, skus, show_product_id, from_site):
"""统一下架"""
crawl_bulk_operations = []
shop_bulk_operations = []
crawl_bulk_operations.append(UpdateMany({'from_site': from_site, 'show_product_id': show_product_id}, {'$set': {'is_outof_stock': True}}))
shop_bulk_operations.append(UpdateMany({'from_site': from_site, 'show_product_id': show_product_id}, {'$set': {'is_outof_stock': True}}))
min_list_price = None
min_current_price = None
for item in skus:
sku_detail = {
'scan_time': int(time.time()),
'scan_date': time.strftime('%Y-%m-%d',time.localtime(time.time())),
'show_product_id': str(item.get('show_product_id')),
'from_site': item.get('from_site'),
'id': item.get('id'),
'list_price': handle_price(str(item.get('list_price'))),
'current_price': handle_price(str(item.get('current_price'))),
'color': item.get('color'),
'size_num': item.get('size_num'),
'groupbuy_price': item.get('groupbuy_price'),
's_t': item.get('s_t')
}
s_color = sku_detail['color'].lower().strip()
for c in s_color:
s_color = re.sub(" ", " ", s_color)
sku_detail['s_color'] = s_color
for key in sku_detail.keys():
if not sku_detail[key]:
del sku_detail[key]
else:
if type(sku_detail[key]) == str:
sku_detail[key] = sku_detail[key].strip()
if 'is_outof_stock' in item.keys():
sku_detail['is_outof_stock'] = item.get('is_outof_stock')
else:
sku_detail['is_outof_stock'] = False
if sku_detail['is_outof_stock'] == False:
float_list_price = float(sku_detail['list_price'])
if not min_list_price or min_list_price > float_list_price:
min_list_price = float_list_price
float_current_price = float(sku_detail['current_price'])
if not min_current_price or min_current_price > float_current_price:
min_current_price = float_current_price
sku_size = item.get('size')
if type(sku_size) == dict:
for key in sku_size:
sku_detail[key] = sku_size[key]
else:
sku_detail['size'] = sku_size
'''单个爬虫更新单个商品（测试使用）'''
if self.settings['SIZE_TRANSFORM_TEST'] == 3:
Singletransform().singletransform(from_site, sku_detail, self.settings['SIZE_TRANSFORM_LIST'])
if 'quantity' in item.keys():
sku_detail['quantity'] = item.get('quantity')
sku_flag = str(sku_detail['id'])
sku_flag = "".join(sku_flag.split())
sku_flag = sku_flag.lower()
spaces = ['\n', '\t', '\r']
for s in spaces:
if s in sku_flag:
sku_flag = re.sub(s, '', sku_flag)
sku_detail['sku_flag'] = sku_flag
sku_mongo = self.shop_db['skus'].find_one({'from_site': sku_detail['from_site'], 'show_product_id':sku_detail['show_product_id'], 'sku_flag': sku_detail['sku_flag']})
if sku_detail['from_site'] in self.settings['SIZE_TRANSFORM_LIST']:
if sku_mongo and 'size' in sku_mongo.keys():
if sku_detail['size'] != sku_mongo['size']:
sku_detail['handle_size'] = '1'
else:
sku_detail['handle_size'] = '1'
if sku_detail['from_site'] == 'ralphlauren':
if not sku_mongo:
crawl_bulk_operations.append(UpdateOne(
{'from_site': sku_detail['from_site'], 'show_product_id':sku_detail['show_product_id'], 'color': sku_detail['color'], 'size': sku_detail['size']},
{'$set': sku_detail},
upsert=True
))
shop_bulk_operations.append(UpdateOne(
{'from_site': sku_detail['from_site'], 'show_product_id':sku_detail['show_product_id'], 'color': sku_detail['color'], 'size': sku_detail['size']},
{'$set': sku_detail},
upsert=True
))
else:
crawl_bulk_operations.append(UpdateOne(
{'from_site': sku_detail['from_site'], 'show_product_id':sku_detail['show_product_id'], 'color': sku_detail['color'], 'size': sku_detail['size']},
{'$set': sku_detail}
))
shop_bulk_operations.append(UpdateOne(
{'from_site': sku_detail['from_site'], 'show_product_id':sku_detail['show_product_id'], 'color': sku_detail['color'], 'size': sku_detail['size']},
{'$set': sku_detail}
))
else:
if not sku_mongo:
crawl_bulk_operations.append(UpdateOne(
{'from_site': sku_detail['from_site'], 'show_product_id':sku_detail['show_product_id'], 'sku_flag': sku_detail['sku_flag']},
{'$set': sku_detail},
upsert=True
))
shop_bulk_operations.append(UpdateOne(
{'from_site': sku_detail['from_site'], 'show_product_id':sku_detail['show_product_id'], 'sku_flag': sku_detail['sku_flag']},
{'$set': sku_detail},
upsert=True
))
else:
crawl_bulk_operations.append(UpdateOne(
{'from_site': sku_detail['from_site'], 'show_product_id':sku_detail['show_product_id'], 'sku_flag': sku_detail['sku_flag']},
{'$set': sku_detail}
))
shop_bulk_operations.append(UpdateOne(
{'from_site': sku_detail['from_site'], 'show_product_id':sku_detail['show_product_id'], 'sku_flag': sku_detail['sku_flag']},
{'$set': sku_detail}
))
self.db['skus'].bulk_write(crawl_bulk_operations)
self.shop_db['skus'].bulk_write(shop_bulk_operations)
return min_list_price, min_current_price
def bulk_update(self, filter, document, upsert, multi):
if multi:
self._requests.append(UpdateMany(filter, document, upsert=upsert))
else:
self._requests.append(UpdateOne(filter, document, upsert=upsert))
