def weibo_to_full_item(item: WeiboItem):
return FullItem(
service=ServiceType.Weibo,
item_id=item.weibo_id,
content=item.content,
image_urls=item.images,
source_id=str(item.user_id),
url=f'https://weibo.com/{item.user_id}/{item.weibo_id}')
def to_full_item(self):
return FullItem(item_id=self.item_id,
service=self.service,
content=self.content,
source_id=self.source_id,
url=self.url,
image_urls=self.image_urls,
tags=self.tags,
attachment_urls=self.attachment_urls)
def item2fullitem(t: TwitterItem) -> FullItem:
name, uid = t.author
url = f"https://twitter.com/{uid}/status/{t.id}"
return FullItem(content=t.text,
item_id=str(t.id),
service=ServiceType.Twitter,
source_id=uid,
image_urls=t.images,
url=url)
def test_jsonify(self):
print(
json.dumps(
FullItem(service=ServiceType.Twitter,
item_id='233',
source_id='666',
content='content',
image_urls=['image.png'],
url='url').to_dict()))
def push_item(self, item: FullItem, images: Iterable[IO], channel: str, converted_username: str):
d = self.root / item.service.value / item.source_id
self.ensure_dir(d)
json_buffer = BytesIO(json.dumps(item.to_dict(), ensure_ascii=False).encode('utf-8'))
json_fp = d / f"{item.item_id}_info.json"
self.write_file(json_fp, json_buffer)
PostRecord.put_record(item.service, item.item_id, ServiceType.Mega, str(json_fp), channel)
for idx, img in enumerate(images):
fp = d / f"{item.item_id}_{idx:03d}.png"
self.write_file(fp, img)
PostRecord.put_record(item.service, item.item_id, ServiceType.Mega, str(fp), channel)
def pull_item(self, index_item: IndexItem) -> Optional[FullItem]:
data = self.api.get_illustrate(int(index_item.item_id))
if data is None:
return None
content = f"{' '.join(f'#{t}' for t in data.tags)}\n\n{data.description}"
self.get_nickname(data.author_id)
return FullItem(
service=ServiceType.Pixiv,
item_id=str(data.id),
source_id=str(data.author_id),
content=content,
image_urls=data.images,
url=f"https://www.pixiv.net/artworks/{index_item.item_id}",
tags=data.tags)
def push_item(self, item: FullItem, images: Iterable[IO], channel: str,
converted_username: str):
parent = self.root / item.service.value.replace(
'/', '_') / converted_username.replace('/', '_')
parent.mkdir(parents=True, exist_ok=True)
meta_file = parent / f"{item.item_id}_info.json"
with meta_file.open('w') as f:
json.dump(item.to_dict(), f, ensure_ascii=False)
if len(item.content) > 100:
content_file = parent / f"{item.item_id}_content.txt"
with content_file.open('w') as f:
f.write(item.content)
for idx, buf in enumerate(images):
fp = parent / f"{item.item_id}_{idx:03d}.png"
with fp.open('wb') as f:
f.write(buf.read())
def pull_item(self, index_item: IndexItem) -> Optional[FullItem]:
artist, article_id = self.split_item_id(index_item.item_id)
data = self.api.get_page(artist, article_id)
if data is None:
return None
content = f"{' '.join(f'#{t}' for t in data.tags)}\n\n{data.content}"
self.get_nickname(artist)
return FullItem(
service=ServiceType.Fanbox,
item_id=index_item.item_id,
source_id=artist,
content=content,
image_urls=data.images,
url=f"https://www.fanbox.cc/@{artist}/posts/{article_id}",
tags=data.tags,
attachment_urls=data.files
)
def push_item(self, item: FullItem, images: Iterable[IO], channel: str,
converted_username: str):
json_buffer = BytesIO(
json.dumps(item.to_dict(), ensure_ascii=False).encode('utf-8'))
# nickname = UserInfo.get_nickname(item.service, item.source_id)
# if nickname is None:
# dir_name = f"({item.source_id})"
# else:
# dir_name = f"{nickname}({item.source_id})"
dir_name = item.source_id
fp = self.write_file(channel, item.service, dir_name,
f"{item.item_id}_info.json", json_buffer)
PostRecord.put_record(item.service, item.item_id, ServiceType.WebDav,
fp, channel)
for idx, img in enumerate(images):
fp = self.write_file(channel, item.service, dir_name,
f"{item.item_id}_{idx:03d}.png", img)
PostRecord.put_record(item.service, item.item_id,
ServiceType.WebDav, fp, channel)