def post(self):
datas = self.request.get("data")
secret = self.request.get("secret")
datas = datas.split(",")
userdata = UserData.all().filter("secret = ", secret).get()
if userdata:
email = userdata.email.lower()
else:
return
# merge
imagedatas = {}
for data in datas:
hash, tags = data.split(":")
if hash in imagedatas:
imagedatas[hash] += ";" + tags
else:
imagedatas[hash] = tags
hkeys = imagedatas.keys()
keys = [db.Key.from_path("Image", "hash:" + hash) for hash in hkeys]
userkeys = [db.Key.from_path("UserImage", "email:" + email + "#hash:" + hash) for hash in hkeys]
images = Image.get(keys)
userimages = UserImage.get(userkeys)
updated = []
for hash, image, userimage in zip(hkeys, images, userimages):
tags = imagedatas[hash]
tags = [kk for kk in [k.strip() for k in tags.split(";")] if kk]
if not image:
image = Image(key_name="hash:" + hash, hash=hash)
image.tags = tags
updated.append(image)
elif not set(image.tags) == set(tags):
image.tags = list(set(image.tags).union(tags))
updated.append(image)
if not userimage:
userimage = UserImage(key_name="email:" + email + "#hash:" + hash, email=email, hash=hash)
userimage.tags = tags
updated.append(userimage)
elif not set(userimage.tags) == set(tags):
userimage.tags = list(set(image.tags).union(tags))
updated.append(userimage)
db.put(updated)
return
def _insert_collection_data_to_db(self,
collection_data: tuple,
collection_num=''):
""" 添加合集的数据"""
# 添加合集的标签
name, tag_names, total_num, img_first_url, width, height = collection_data
tags = []
for tag_name in tag_names:
t = self.session.query(Tag).filter_by(tag_name=tag_name)
if t.count() == 0: # tag不存在
self.session.add(Tag(tag_name=tag_name))
self.session.commit()
tags.append(
self.session.query(Tag).filter_by(
tag_name=tag_name).first())
else: # tag 存在
tags.append(t.first())
collection = Collection(
collection_num=collection_num,
name=name,
total_num=total_num,
tags=tags,
)
self.session.add(collection)
self.session.commit()
# 添加该合集的图片信息
images = []
for count in range(1, int(total_num) + 1):
if 1 <= count <= 9:
index = '0' + str(count)
else:
index = str(count)
info = img_first_url.split("/")
year = info[-3]
month = info[-2]
day = info[-1][:2]
img_url = f"https://i.meizitu.net/{year}/{month}/{day}{info[-1][2]}{index}.{info[-1].split('.')[-1]}"
image = Image(year=year,
month=month,
day=day,
width=width,
height=height,
meizitu_url=img_url,
collection_num=collection_num)
images.append(image)
self.session.add_all(images)
# self.session.commit()
download_record = self.session.query(DownloadRecord).filter_by(
collection_num=collection_num)
download_record.update({'status': 1})
self.session.commit()
def process_item(self, item, spider):
'''
:param item: item是从spiders通过yield发射过来的对象
:param spider: spider是指的不同爬虫 （spider.name）
:return:
'''
try:
item_name = item.get_name()
if item_name == "OrangemallCategory":
cate = Category(cate_id=item['cate_id'],
parent_id=item['parent_id'],
level=item['level'],
name=item['name'],
create_time=item['create_time'],
is_delete=item['is_delete'])
minst.add_records(session, cate)
elif item_name == "OrangemallShop":
shop = Shop(shop_id=item['shop_id'],
name=item['name'],
original_price=item['original_price'],
promote_price=item['promote_price'],
stock=item['stock'],
cate_id=item['cate_id'],
create_date=item['create_date'],
sale=item['sale'],
sort=item['sort'],
is_hot=item['is_hot'],
is_delete=item['is_delete'])
minst.add_records(session, shop)
elif item_name == "OrangeMallProperty":
property = Property(property_id=item['property_id'],
name=item['name'],
shop_id=item['shop_id'],
is_delete=item['is_delete'])
minst.add_records(session, property)
elif item_name == "OrangeMallImage":
image = Image(img_id=item['img_id'],
shop_id=item['shop_id'],
type=item['type'],
img_url=item['img_url'],
is_delete=item['is_delete'])
minst.add_records(session, image)
# return item
except Exception as e:
print(f"MySQLBookPipeLine:process_item has error: {e}")
# return item
finally:
return item
def post(self):
hashs = self.request.get("hash")
hashs = hashs.split(",")
keys = [db.Key.from_path("Image", "hash:" + hash) for hash in hashs]
images = Image.get(keys)
returned = []
for image in images:
if image:
returned.append(";".join(image.tags))
else:
returned.append("")
self.response.out.write(",".join(returned))
