def apiDownload(api: pixivpy3.ByPassSniApi, item: dict) -> None:
# item = {"illust": illust, "title": title,
# "count": count, "tags": tags, "url": url, "suffix": suffix}
logger.info("尝试下载 " + str(item['illust']) + ':' + item['title'])
# 检测PATH是否存在，不存在则创建
# 当日图片根文件夹
basePath = os.path.join(define.STATICPATH, config.IMGPATH)
# 存放原图的文件夹，按时间分类好
localtime = time.localtime()
origPath = os.path.join(basePath, "orig",
str(localtime.tm_year).zfill(4),
str(localtime.tm_mon).zfill(2),
str(localtime.tm_mday).zfill(2))
# 存放缩略图的文件夹
thumPath = os.path.join(basePath, "thum")
if not os.path.exists(origPath):
logger.debug("创建路径 " + origPath)
os.makedirs(origPath)
if not os.path.exists(thumPath):
logger.debug("创建路径 " + thumPath)
os.makedirs(thumPath)
# 遍历下载
for i in range(item['count']):
if config.USECAT: # 判断是否使用代理镜像站下载
url = item['url'][i].replace("i.pximg.net", define.PIXIVCAT)
else:
url = item['url'][i]
name = str(item['illust']) + "-" + str(i) + "." + item['suffix']
logger.debug("下载orig " + name)
origName = os.path.join(origPath, name)
if not (os.path.exists(origName)
and isValidImage(origName)): # 若图片不完整则下载
api.download(url, path=origPath, name=name, replace=True) # 下载器
# 封面创建缩略图
thumName = os.path.join(thumPath, str(item['illust']) + '.png')
if i == 0 and not (os.path.exists(thumName)
and isValidImage(thumName)):
logger.debug("创建thum " + str(item['illust']))
img = Image.open(os.path.join(origPath, name))
width = img.size[0] # 获取宽度
height = img.size[1] # 获取高度
if 1.3 * width < height:
x0 = int(0)
y0 = int((height - 1.3 * width) / 2)
x1 = int(width)
y1 = int(y0 + 1.3 * width)
elif 1.3 * width > height:
x0 = (width - height / 1.3) / 2
y0 = int(0)
x1 = int(x0 + height / 1.3)
y1 = int(height)
img = img.crop((x0, y0, x1, y1))
img = img.resize((192, 250), Image.ANTIALIAS) # 按比例缩放，高度固定为250px
img.save(thumName)
time.sleep(3) # 延时3s，防止下载过快卡死
logger.info("下载完成 " + str(item['illust']) + ':' + item['title'])
def login_pixiv(self):
self.api = ByPassSniApi() # Same as AppPixivAPI, but bypass the GFW
self.api.require_appapi_hosts()
account_data = None
with open("account.json", "r") as f:
account_data = json.loads(f.read())
self.api.login(account_data["username"], account_data["password"])
self.has_logged_in = True
def apiRanking(api: pixivpy3.ByPassSniApi, t: str) -> list:
logger.info("尝试获取 " + t + " 排行榜信息")
json_result = api.illust_ranking(mode=t)
rt = []
for json_item in json_result.illusts:
if json_item.type != 'illust':
continue
illust = json_item.id
title = json_item.title
count = json_item.page_count
if count == 1:
url = [json_item.meta_single_page.original_image_url]
else:
url = [i.image_urls.original for i in json_item.meta_pages]
tags = [i.name for i in json_item.tags]
temp = {
"illust": illust,
"title": title,
"count": count,
"tags": tags,
"url": url,
"suffix": url[0].split('.')[-1]
}
logger.debug(temp)
rt.append(temp)
logger.info(t + " 排行榜信息获取完成")
return rt
def main():
api = ByPassSniApi() # Same as AppPixivAPI, but bypass the GFW
api.require_appapi_hosts()
# api.set_additional_headers({'Accept-Language':'en-US'})
api.set_accept_language("en-us")
# api.login(_USERNAME, _PASSWORD)
print(api.auth(refresh_token=_REFRESH_TOKEN))
json_result = api.illust_ranking(
"day", date=(datetime.now() - timedelta(days=5)).strftime("%Y-%m-%d"))
print(
"Printing image titles and tags with English tag translations present when available"
)
for illust in json_result.illusts[:3]:
print('Illustration: "' + str(illust.title) + '"\nTags: ' +
str(illust.tags) + "\n")
def main():
sni = False
if not sni:
api = AppPixivAPI()
else:
api = ByPassSniApi() # Same as AppPixivAPI, but bypass the GFW
api.require_appapi_hosts()
api.auth(refresh_token=_REFRESH_TOKEN)
# get rankings
json_result = api.illust_ranking("day", date="2019-01-01")
directory = "illusts"
if not os.path.exists(directory):
os.makedirs(directory)
# download top3 day rankings to 'illusts' dir
for idx, illust in enumerate(json_result.illusts[:4]):
image_url = illust.meta_single_page.get("original_image_url",
illust.image_urls.large)
print("%s: %s" % (illust.title, image_url))
# try four args in MR#102
if idx == 0:
api.download(image_url, path=directory, name=None)
elif idx == 1:
url_basename = os.path.basename(image_url)
extension = os.path.splitext(url_basename)[1]
name = "illust_id_%d_%s%s" % (illust.id, illust.title, extension)
api.download(image_url, path=directory, name=name)
elif idx == 2:
api.download(image_url,
path=directory,
fname="illust_%s.jpg" % (illust.id))
else:
# path will not work due to fname is a handler
api.download(
image_url,
path="/foo/bar",
fname=open("%s/illust_%s.jpg" % (directory, illust.id), "wb"),
)
import nonebot
from nonebot import MatcherGroup
from nonebot.log import logger
from nonebot.permission import MESSAGE
from nonebot.adapters.cqhttp import Bot, Event, MessageSegment
from nonebot_plugin_rauthman import isInService
from urllib.parse import quote
# from pixivpy3 import AppPixivAPI
from pixivpy3 import ByPassSniApi
from .config import *
import random
import datetime
sv = MatcherGroup(type='message', rule=isInService('pixiv', 1))
# aapi = AppPixivAPI()
aapi = ByPassSniApi()
aapi.require_appapi_hosts(hostname="public-api.secure.pixiv.net")
aapi.set_accept_language('jp')
sv_search = sv.on_startswith(msg='/pixiv', permission=MESSAGE, block=True)
@sv_search.handle()
async def pixiv_handle(bot: Bot, event: Event, state: dict):
msg = str(event.message).strip()
msg = msg.replace('/pixiv', '')
if not msg:
logger.info('pixiv_handle | no keyword')
await sv_search.finish()
state['keyword'] = msg
import asyncio
from pixivpy3 import AppPixivAPI, ByPassSniApi
from utils import settings, log, launch
if settings["pixiv"]["proxy"] is not None:
proxies = {
'http': settings["pixiv"]["proxy"],
'https': settings["pixiv"]["proxy"]
}
else:
proxies = None
if settings["pixiv"]["bypass"]:
papi = ByPassSniApi(proxies=proxies)
papi.require_appapi_hosts()
else:
papi = AppPixivAPI(proxies=proxies)
papi.set_accept_language('zh-cn')
def auth():
papi.login(settings["pixiv"]["username"], settings["pixiv"]["password"])
log.info("pixiv login succeeded")
def start_auto_auth():
async def watchman():
while True:
import config
import requests
from pixivpy3 import ByPassSniApi
RECOMMENDED = 0
KONACHAN = 1
YANDERE = 2
DANBOORU = 3
PIXIV = 4
pixivApi = ByPassSniApi()
pixivApi.require_appapi_hosts()
def login():
if config.pixiv_login_mode == 0:
pixivApi.auth(refresh_token=config.pixiv_refresh_token)
else:
pixivApi.login(username=config.pixiv_username,
password=config.pixiv_password)
if config.pixiv_print_refresh_token:
print("Your pixiv account refresh_token is '{}'.".format(
pixivApi.refresh_token))
class PixivDownloader(object):
def __init__(self):
self.dir = os.path.dirname(os.path.realpath(__file__))
self.has_logged_in = False
self.lock = threading.Lock()
self.image_dir = 'download'
# 创建下载目录
full_image_dir = os.path.join(self.dir, self.image_dir)
if not os.path.exists(full_image_dir):
os.mkdir(full_image_dir)
def login_pixiv(self):
self.api = ByPassSniApi() # Same as AppPixivAPI, but bypass the GFW
self.api.require_appapi_hosts()
account_data = None
with open("account.json", "r") as f:
account_data = json.loads(f.read())
self.api.login(account_data["username"], account_data["password"])
self.has_logged_in = True
def download_illust(self, illust):
info = IllustInfo(
url= {
'squareMedium': illust.image_urls.get('square_medium'),
'medium': illust.image_urls.get('medium'),
'large': illust.image_urls.get('large'),
'original' : illust.meta_single_page.get('original_image_url'),
},
id=illust.id
)
self.download_image(info)
def download_image(self, info: IllustInfo) -> DownloadResult:
# 按以下优先级对URL进行下载。如果下载出来是404则尝试下载下一个URL
urls = [info.url['large'], info.url['original'], info.url['medium'], info.url['squareMedium']]
for url in urls:
result = self.download_image_using_url(url=url, id=info.id)
if result in (DownloadResult.SUCCESS, DownloadResult.SKIPPED):
return result
elif result == DownloadResult.ERROR_404:
print('[WARNING] 404: %s' % url)
continue
elif result == DownloadResult.ERROR_UNKNOWN:
break
# 下载失败
print('[ERROR] Download failed!!! image id=%s' % url)
return DownloadResult.ERROR_UNKNOWN
def download_image_using_url(self, url: str, id: int) -> DownloadResult:
'''
从给定的URL下载图片
返回下载结果
'''
retry_times = TOTAL_RETRY_TIMES
while retry_times > 0:
try:
ext = os.path.splitext(url)[1]
img_name = 'pixiv_%d%s' % (id, ext)
full_image_name = os.path.join(self.dir, self.image_dir, img_name)
if os.path.exists(full_image_name):
if self.is_valid_image(full_image_name):
print('skip %s' % url)
return DownloadResult.SKIPPED
# 是损坏的图片文件
os.remove(full_image_name)
# 因为下载过程可能是多线程并发执行的，而登录只需要全局进行一次，所以用锁保护起来
# 判断两次是典型的单例模式的写法
if self.has_logged_in is False:
self.lock.acquire()
if self.has_logged_in is False:
self.login_pixiv()
self.lock.release()
if retry_times == TOTAL_RETRY_TIMES:
# 首次尝试下载
print('Download', url)
else:
print('Retry download', url)
self.api.download(url, path=self.image_dir, name=img_name, replace=False)
time.sleep(random.random() * 10.0 + 10.0) # 防止下载过于频繁被封
if self.is_404(full_image_name):
os.remove(full_image_name)
return DownloadResult.ERROR_404
if self.is_valid_image(full_image_name):
return DownloadResult.SUCCESS
print('[WARNING]Invalid image: %s' % url)
except Exception as e:
print("[WARNING]download %s failed: %s" % (url, str(e)))
retry_times -= 1
# 几次反复尝试后全部失败
if os.path.exists(full_image_name):
os.remove(full_image_name)
return DownloadResult.ERROR_UNKNOWN
def is_404(self, filename: str):
try:
with open(filename, 'r') as f:
text = f.read()
return '404 Not Found' in text
except:
return False
def is_valid_image(self, filename: str):
try:
im = Image.open(filename)
im.save(filename)
return True
except IOError:
# filename not an image file
return False
def download_tag_most_popular(self, tag: str, number: int):
'''
下载某个标签的收藏最多的xx张图片。
由于pixiv按热度排序需要超级会员，此接口只有在bot拥有超级会员时有效
'''
if self.has_logged_in is False:
self.login_pixiv()
json_result = self.api.search_illust(tag, search_target='partial_match_for_tags', sort='popular_desc')
if hasattr(json_result, 'error'):
raise PixivDownloadError(json_result.error)
illusts = json_result.illusts
for illust in illusts[:number]:
self.download_illust(illust)
def download_tag_most_popular_using_pixivic(self, tag: str, number: int, offset=0):
'''
下载某个标签的收藏最多的xx张图片。
使用http://www.pixivic.com 提供的API
'''
end = offset + number
start_page = offset // ONE_PAGE + 1
end_page = (end-1) // ONE_PAGE + 1
# 获取插画的信息。分页来处理
illust_infos = []
for i in range(start_page, end_page + 1):
# 构建报文
params = {
'page': i,
'keyword': tag
}
headers = {
'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36',
'origin': 'https://pixivic.com',
'referer': 'https://pixivic.com/popSearch'
}
url = 'https://api.pixivic.com/illustrations'
# 获取数据
response = requests.get(url, params=params, headers=headers)
result = response.json()
# 根据offset和end，把需要的插画数据添加到illust_infos数组中
offset_in_this_page = offset - ONE_PAGE * (i - 1)
offset_in_this_page = max([offset_in_this_page, 0])
end_in_this_page = end - ONE_PAGE * (i - 1)
end_in_this_page = min(end_in_this_page, ONE_PAGE)
illust_infos += result['data']['illustrations'][offset_in_this_page:end_in_this_page]
infos = [IllustInfo(
id=info['id'],
url={
'squareMedium': info['imageUrls'][0]['squareMedium'],
'medium': info['imageUrls'][0]['medium'],
'large': info['imageUrls'][0]['large'],
'original': info['imageUrls'][0]['original'],
}
) for info in illust_infos]
loop = asyncio.get_event_loop()
loop.run_until_complete(self.download_illusts_async(infos))
async def download_illusts_async(self, illust_infos: list):
# 启用线程池来异步下载
loop = asyncio.get_event_loop()
results = []
with futures.ThreadPoolExecutor(8) as executor:
tasks = [loop.run_in_executor(executor, self.download_image, info) for info in illust_infos]
results = await asyncio.gather(*tasks)
# 统计结果
failed_images = []
succeeded_count = 0
failed_count = 0
skipped_count = 0
for i, result in enumerate(results):
info = illust_infos[i]
if result == DownloadResult.SUCCESS:
succeeded_count += 1
elif result == DownloadResult.SKIPPED:
skipped_count += 1
else:
failed_images.append(info.id)
failed_count += 1
# 打印结果
print('=======Download result==========')
print('Total succeeded files: %d' % succeeded_count)
print('Total skipped files: %d' % skipped_count)
print('Total failed files: %d' % failed_count)
if failed_count > 0:
print('Failed File are:')
for id in failed_images:
print('id=%d' % id)
print('=======Download result end======')
def download_recommended(self, number: int):
# get recommended
if self.has_logged_in is False:
self.login_pixiv()
json_result = self.api.illust_recommended()
if hasattr(json_result, 'error'):
raise PixivDownloadError(json_result.error)
illusts = json_result.illusts
for illust in illusts[:number]:
self.download_illust(illust)
from typing import Any
from pixivpy3 import ByPassSniApi
from setu_viewer.config import Config
api = ByPassSniApi()
# api.require_appapi_hosts(hostname="public-api.secure.pixiv.net")
api.hosts = Config.PIXIV_HOST # TODO: CF DOH is broken, manual set hosts
api.auth(refresh_token=Config.PIXIV_REFRESH_KEY)
def popular_preview(word: str) -> Any:
"""P 站热度搜索预览"""
params = {
"filter": "for_android",
"include_translated_tag_results": "true",
"merge_plain_keyword_results": "true",
"word": word,
"search_target": "exact_match_for_tags"
}
raw_result = api.no_auth_requests_call(
'GET', f'{api.hosts}/v1/search/popular-preview/illust', params=params)
return raw_result.json()
from pixivpy3 import ByPassSniApi
import nonebot
api = ByPassSniApi()
def pixiv_login():
global api
api.require_appapi_hosts(hostname="public-api.secure.pixiv.net")
# api.set_additional_headers({'Accept-Language':'en-US'})
api.set_accept_language("en-us")
config = nonebot.get_driver().config
api.login(config.pixivusername, config.pixivpassword)
def pixiv_api(api_action, *args, **kwargs):
action = getattr(api, api_action, None)
if action:
res = action(*args, **kwargs)
return res