def __init__(self, download_dir, token): self.__download_dir = download_dir self.__token = token self.__client = Client() self.__client.authenticate(self.__token) pathlib.Path(self.__download_dir).mkdir(parents=True, exist_ok=True) self.__download_dir = pathlib.Path(self.__download_dir)
def __init__(self, name):
Thread.__init__(self, daemon=True)
self.name = name
self.client = Client(client_id=g.PixivClientID,
client_secret=g.PixivClientSecret)
self.allranking = []
self.artpath = Path('flask/images/pixiv/')
self.start()
class PixApi:
# 初始化
def __init__(self, items, index):
# Multi-Threading
print("Page:%d" % index)
print(len(items))
self.items = items
self.threadID = index
self.threadLocal = threading.local()
self.client = Client()
self.illuData = None
setattr(self.threadLocal, 'client', self.client)
setattr(self.threadLocal, 'items', self.items)
self.client.login(pixiv_id, password)
time.sleep(5)
thread = threading.Thread(target=self.processItem, name=("Page%d" % index))
threads.append(thread)
# 取得每一頁最高收藏數的插圖
def processItem(self):
index = 0
errorCount = 0
item = None
getVal = False
while index < len(self.items):
try:
timer = 0
while timer < 3 and index < len(self.items):
item = self.items[index]
try:
self.illuData = self.client.fetch_illustration(int(item))
except:
pass
if self.illuData != None:
getVal = True
break
time.sleep(1)
timer = timer + 1
except Exception as E:
index = index - 1
print(E)
errorCount = errorCount + 1
if getVal:
value = int(self.illuData.total_bookmarks)
bestKeepList.append((int(item), value))
getVal = False
if errorCount >= 3:
break
index = index + 1
print("Thread%d OK!------------------------------------------" % self.threadID)
def login(client: Client, username, password):
try:
token = open('.token', 'r', encoding='utf-8').read()
client.authenticate(token)
except Exception:
logging.info(
"try token login failed, use username and password instead...")
client.login(username, password)
# save token
with open('.token', 'w', encoding='utf-8') as f:
f.write(cast(str, client.refresh_token))
logging.info("new token saved to ./.token")
class pixivComm:
token = ""
pixivClient = None
def __init__(self, pixivCreds):
self.pixivClient = Client()
self.pixivClient.login(pixivCreds["user"], pixivCreds["pw"])
self.token = self.pixivClient.refresh_token
def searchImage(self, tag):
self.pixivClient.authenticate(self.token)
searchStr = tag.replace('_', ' ')
print(searchStr, tag)
results = self.pixivClient.search_illustrations(searchStr)
illustrations = results["illustrations"]
randIllust = illustrations[randint(0,len(illustrations) - 1)]
print(dir(randIllust))
while randIllust.type != enums.ContentType.ILLUSTRATION:
randIllust = illustrations[randint(0,len(illustrations) - 1)]
self.getImage(randIllust)
imageURL = randIllust.image_urls[enums.Size.ORIGINAL]
imageURL = glob.glob("./pixiv/" + str(randIllust.id) + "*")[0]
return imageURL
def getImage(self, illustration):
illustration.download(pathlib.Path("./pixiv", size=enums.Size.ORIGINAL), filename = illustration.id)
def testFetch(self):
self.pixivClient.authenticate(self.token)
illus = self.pixivClient.fetch_illustration(82417326)
self.getImage(illus)
def __init__(self, items, index):
# Multi-Threading
print("Page:%d" % index)
print(len(items))
self.items = items
self.threadID = index
self.threadLocal = threading.local()
self.client = Client()
self.illuData = None
setattr(self.threadLocal, 'client', self.client)
setattr(self.threadLocal, 'items', self.items)
self.client.login(pixiv_id, password)
time.sleep(5)
thread = threading.Thread(target=self.processItem, name=("Page%d" % index))
threads.append(thread)
from pixivapi import Client, Size
import random
from os import path
import pathlib
from pixivconfig import *
from config import temp_path
client = Client()
def __init__():
client.login(username, password)
def getUrl(origin='https://i.pximg.net/'):
return origin.replace('pximg.net','pixiv.cat')
def getRecommended():
illus = client.fetch_illustrations_recommended()
illu = illus['illustrations'][random.randint(
0, len(illus['illustrations'])-1)]
p = path.join(temp_path, 'pixiv')
p = pathlib.Path(p)
# illu.download(p)
# ext = path.splitext(illu.image_urls[Size.ORIGINAL])[1]
# print(illu.image_urls)
return illu, getUrl(illu.image_urls[Size.ORIGINAL])
if __name__ == '__main__':
__init__()
def __init__(self, pixivCreds):
self.pixivClient = Client()
self.pixivClient.login(pixivCreds["user"], pixivCreds["pw"])
self.token = self.pixivClient.refresh_token
def start(self, tag, page):
global bestKeepList
bestKeepList.clear()
# 開始時間
print("start time: " + str(datetime.datetime.now()))
items = [] # 儲存這一頁的所有插圖Id
pageNum = 1 # 從第一頁開始
toNext = True
path = "html/body/div/div/div/div/div/div/section/div/ul" # xPath in html to get illustration id
errorCount = 0 # 紀錄數據取得失敗的次數，達到三次將退出
while (pageNum < (page + 1)):
if toNext:
if self.safeMode:
url = "https://www.pixiv.net/tags/%s/illustrations?p=%d&mode=safe" % (tag, pageNum)
else:
url = "https://www.pixiv.net/tags/%s/illustrations?p=%d" % (tag, pageNum)
print(url)
mainDriver.get(url)
toNext = False
print('正在處理第%d頁' % pageNum)
try:
timer = 0
while True:
timer = timer + 1
time.sleep(0.5)
inner_html = mainDriver.find_element_by_xpath(path).get_attribute("innerHTML")
pattern = re.compile('href="/artworks/(.*?)"', re.S)
self.items = list(set(re.findall(pattern, inner_html)))
if len(self.items) > 0:
break
elif timer >= 10:
raise Exception
except:
pageNum = pageNum - 1
print("超時，將會再嘗試%d次" % (2 - errorCount))
errorCount = errorCount + 1
else:
if len(self.items) > 0:
print("ID獲取完成!")
#MyThread(self.items, pageNum) # 建立新的Thread
PixApi(self.items, pageNum)
toNext = True
errorCount = 0
# 每五頁執行一次
if (pageNum % 5) == 0 or pageNum == page:
print("開始收集每張圖的收藏數!請稍等!")
self.runThread()
elif errorCount >= 3:
self.runThread()
break
pageNum = pageNum + 1
print('-----------------------------------------------------')
print("finish time: " + str(datetime.datetime.now()))
# 關閉Driver
for driver in drivers:
driver.close()
bestKeepList.sort(key=self.takeSecond, reverse=True)
#print(len(bestKeepList))
bestKeepList = bestKeepList[:100] # 保留前一百個
#print(bestKeepList)
self.resultClient = Client()
self.resultClient.login(pixiv_id, password)
class Pixiv:
safeMode = None
# 物件初始化
def __init__(self):
mainDriver.get(loginUrl)
# 利用Web driver模擬登入Pixiv
def loginWithSelenium(self, id, pw):
global pixiv_id
global password
pixiv_id = id
password = pw
fieldGroup = mainDriver.find_element_by_xpath("//div[@class='input-field-group']")
# 獲取User ID輸入欄
userNameField = fieldGroup.find_element_by_xpath("//div[@class='input-field']/input[@type='text'][@autocomplete='username']")
userNameField.clear()
userNameField.send_keys(pixiv_id)
# 獲取密碼輸入欄
passwordField = fieldGroup.find_element_by_xpath("//div[@class='input-field']/input[@type='password'][@autocomplete='current-password']")
passwordField.clear()
passwordField.send_keys(password)
# 獲取提交按鈕
submitButton = mainDriver.find_element_by_xpath("//div[@id='LoginComponent']/form/button[@type='submit'][@class='signup-form__submit']")
submitButton.click()
def ifLoginSuccess(self):
time.sleep(3)
return mainDriver.current_url == 'https://www.pixiv.net/'
# 用於取得一個頁面內所有圖片ID，目前一頁最多60個ID
def start(self, tag, page):
global bestKeepList
bestKeepList.clear()
# 開始時間
print("start time: " + str(datetime.datetime.now()))
items = [] # 儲存這一頁的所有插圖Id
pageNum = 1 # 從第一頁開始
toNext = True
path = "html/body/div/div/div/div/div/div/section/div/ul" # xPath in html to get illustration id
errorCount = 0 # 紀錄數據取得失敗的次數，達到三次將退出
while (pageNum < (page + 1)):
if toNext:
if self.safeMode:
url = "https://www.pixiv.net/tags/%s/illustrations?p=%d&mode=safe" % (tag, pageNum)
else:
url = "https://www.pixiv.net/tags/%s/illustrations?p=%d" % (tag, pageNum)
print(url)
mainDriver.get(url)
toNext = False
print('正在處理第%d頁' % pageNum)
try:
timer = 0
while True:
timer = timer + 1
time.sleep(0.5)
inner_html = mainDriver.find_element_by_xpath(path).get_attribute("innerHTML")
pattern = re.compile('href="/artworks/(.*?)"', re.S)
self.items = list(set(re.findall(pattern, inner_html)))
if len(self.items) > 0:
break
elif timer >= 10:
raise Exception
except:
pageNum = pageNum - 1
print("超時，將會再嘗試%d次" % (2 - errorCount))
errorCount = errorCount + 1
else:
if len(self.items) > 0:
print("ID獲取完成!")
#MyThread(self.items, pageNum) # 建立新的Thread
PixApi(self.items, pageNum)
toNext = True
errorCount = 0
# 每五頁執行一次
if (pageNum % 5) == 0 or pageNum == page:
print("開始收集每張圖的收藏數!請稍等!")
self.runThread()
elif errorCount >= 3:
self.runThread()
break
pageNum = pageNum + 1
print('-----------------------------------------------------')
print("finish time: " + str(datetime.datetime.now()))
# 關閉Driver
for driver in drivers:
driver.close()
bestKeepList.sort(key=self.takeSecond, reverse=True)
#print(len(bestKeepList))
bestKeepList = bestKeepList[:100] # 保留前一百個
#print(bestKeepList)
self.resultClient = Client()
self.resultClient.login(pixiv_id, password)
# 執行儲存於threads中的子執行序，並於完成後清除
def runThread(self):
# t.join(): 等待所有子執行序執行結束才會繼續跑主執行序
for t in threads:
t.start()
for t in threads:
t.join()
print("清空Thread!")
threads.clear()
# 用於排序所使用的compare函式
def takeSecond(self, element):
return element[1]
# 利用Pixiv api取得指定ID的圖片資訊
def getImage(self, index):
id = bestKeepList[index][0]
errorCount = 0
while errorCount < 3:
try:
illuData = self.resultClient.fetch_illustration(id)
return illuData
except:
print("fetch err")
errorCount = errorCount + 1
time.sleep(0.5)
return None
#illuData.download(directory=Path('D:/123'), size=Size.ORIGINAL)
# 取得儲存前100名的List之大小
def getListSize(self):
return bestKeepList.__len__()
class DowPixiv():
__page_data = ""
__files = []
def __init__(self, download_dir, token):
self.__download_dir = download_dir
self.__token = token
self.__client = Client()
self.__client.authenticate(self.__token)
pathlib.Path(self.__download_dir).mkdir(parents=True, exist_ok=True)
self.__download_dir = pathlib.Path(self.__download_dir)
def __load_page(self):
if self.__page_data == "":
self.__page_data = self.__client.fetch_user_bookmarks(self.__client.account.id)
else:
self.__page_data = self.__client.fetch_user_bookmarks(self.__client.account.id, max_bookmark_id=int (self.__page_data['next']))
self.__files = []
for ill in self.__page_data['illustrations']:
tags = ill.user.name
tags += " " + ill.user.account
for tag in ill.tags:
if tag['translated_name'] is not None:
tags += " " + str (tag['translated_name']).replace(' ', '_').replace("'", "''")
file_name = []
if ill.page_count == 1:
ec = str (ill.image_urls[Size.ORIGINAL]).split('/')[-1].split(".")[-1]
file_name.append(str (ill.image_urls[Size.ORIGINAL]).split('/')[-1].split("_")[0] + "." + ec)
for p in ill.meta_pages:
name = str (p[Size.ORIGINAL]).split('/')[-1]
file_name.append(name)
self.__files.append([ill, file_name, tags])
def __download_request(self, ill):
ill.download(directory=self.__download_dir, size=Size.ORIGINAL)
def GetFilesLen(self):
return len(self.__files)
def GetNextPage(self):
self.__load_page()
return self.__page_data['next'] is not None
def DownloadFile(self, file):
ill = file[0]
print("Download File: %d" % ill.id)
self.__download_request(ill)
result = True
for s in file[1]:
dow_dir = pathlib.Path(self.__download_dir)
if len (file[1]) > 1:
dow_dir = dow_dir.joinpath(str (ill.id))
if not dow_dir.joinpath(s).exists():
result = False
break
return result
def GetFile(self, index): # url, list of names, tags
if index > len(self.__files):
return None
else:
return self.__files[index]
def GetShortFileName(self, file):
return str (file[0].id)
class ThreadPixiv(Thread):
def __init__(self, name):
Thread.__init__(self, daemon=True)
self.name = name
self.client = Client(client_id=g.PixivClientID,
client_secret=g.PixivClientSecret)
self.allranking = []
self.artpath = Path('flask/images/pixiv/')
self.start()
def run(self):
self.pixiv_init()
def download_art(self, obj, size, filename):
obj.download(directory=self.artpath, size=size, filename=filename)
def random_pixiv_art(self): # download and set random pixiv art
try:
ranking = random.choice(self.allranking)
fetchmode = random.random() # ranked or ranked related art 20/80
if fetchmode > 0.2:
related_offset = 0
allrelated = []
for _ in range(4):
related = self.client.fetch_illustration_related(
ranking.id, offset=related_offset).get('illustrations')
allrelated = u.sort_pixiv_arts(related, allrelated)
related_offset += 30
illustration = random.choice(list(allrelated))
else:
illustration = ranking
print(f'art id: {illustration.id}')
artid = illustration.id
g.last_link = f'https://www.pixiv.net/en/artworks/{artid}'
g.last_rand_img = f'{artid}.png'
art = Path(f'flask/images/pixiv/{artid}.png')
if not art.is_file():
self.download_art(illustration, g.pixiv_size, artid)
if not art.is_file():
os.rename(f'flask/images/pixiv/{artid}.jpg',
f'flask/images/pixiv/{artid}.png')
set_image('pixiv/', f'{artid}.png')
except BadApiResponse as pixiv_exception: # reconnect
if 'Status code: 400' in str(pixiv_exception):
self.pixiv_init()
self.random_pixiv_art()
except Exception as e:
if 'RemoteDisconnected' in str(e):
self.random_pixiv_art()
def save_pixiv_art(self,
namesave,
owner,
artid,
folder='user/',
setpic=False,
save=False,
save_msg=False):
"""
save pixiv art by art id
:param save_msg: whether send <image saved> message
:param save: whether save image
:param setpic: whether set image
:param namesave: filename
:param owner: twitch username
:param artid: pixiv art id
:param folder: image save folder inside flask app static folder
"""
try:
print(f'art id: {artid}')
namesave = u.while_is_file(folder, namesave, '.png')
namesave = u.while_is_file(folder, namesave, '_p0.png')
savedart = self.client.fetch_illustration(int(artid))
self.download_art(savedart, g.pixiv_size, namesave)
if os.path.isdir('flask/images/pixiv/' + namesave):
mypath2 = 'flask/images/pixiv/' + namesave
onlyfiles = [
f for f in listdir(mypath2) if isfile(join(mypath2, f))
]
for i in onlyfiles:
os.rename(f'flask/images/pixiv/{namesave}/{i}',
f'flask/images/{folder}{namesave}{i[8:-4]}.png')
if save:
db.add_link(
f'https://www.pixiv.net/en/artworks/{artid}',
f'{namesave}{i[8:-4]}.png')
db.add_owner(f'{namesave}{i[8:-4]}.png', owner)
if setpic:
set_image(folder, f'{namesave}{i[8:-4]}.png')
time.sleep(1.5)
os.rmdir(f'flask/images/pixiv/{namesave}')
if save_msg:
u.send_message(f'{owner}, {namesave}.png saved')
return
art = Path(f'flask/images/pixiv/{namesave}.png')
filepath = f'flask/images/pixiv/{namesave}.png'
if not art.is_file():
filepath = f'flask/images/pixiv/{namesave}.jpg'
os.rename(filepath, f'flask/images/{folder}{namesave}.png')
if save:
db.add_link(f'https://www.pixiv.net/en/artworks/{artid}',
f'{namesave}.png')
db.add_owner(f'{namesave}.png', owner)
if setpic:
set_image(folder, f'{namesave}.png')
if save_msg:
u.send_message(f'{owner}, {namesave}.png saved')
except BadApiResponse as pixiv_exception: # reconnect
print(f'badapiresponse - {pixiv_exception}')
if 'Status code: 404' in str(pixiv_exception):
u.send_message(f'{owner}, {artid} not found')
return
if 'Status code: 400' in str(pixiv_exception):
self.pixiv_init()
self.save_pixiv_art(namesave, owner, artid, folder, setpic, save,
save_msg)
except Exception as e:
if 'RemoteDisconnected' in str(e):
self.save_pixiv_art(namesave, owner, artid, folder, setpic,
save, save_msg)
def pixiv_init(self):
try:
self.allranking *= 0
self.client.authenticate(g.PixivToken)
print('pixiv auth √')
rank_offset = 30
ranking1 = self.client.fetch_illustrations_ranking(
mode=RankingMode.DAY
).get('illustrations') # check 500 arts, filter by tags and ratio
self.allranking = u.sort_pixiv_arts(ranking1, self.allranking)
for i in range(16):
print(f'\rpixiv load={int(i / 16 * 100) + 7}%', end='')
ranking = self.client.fetch_illustrations_ranking(
mode=RankingMode.DAY,
offset=rank_offset).get('illustrations')
self.allranking = u.sort_pixiv_arts(ranking, self.allranking)
rank_offset += 30
print()
except BadApiResponse:
time.sleep(30)
self.run()
def client():
"""For use in post-auth content-related tests."""
c = Client()
c.access_token = "xxx"
return c
def unauthed_client():
"""For use in authentication-related tests."""
return Client()
import pixivapi
from pixivapi import ContentType
from pixivapi import Client
from datetime import datetime
from pathlib import Path
from pixivapi import Size
from pixivapi import Sort
from pixivapi import SearchTarget
from pixivapi import Visibility
#login section.
client = Client()
userid = input("Enter your userid.\n")
password = input("Enter your password.\n")
client.login(userid, password)
print(f"Logged in as {userid}")
needs = input("What would you like to do today?\n")
#fetch an illustration found, using the id provided.
if needs == "illustration download":
illustration_id = input("Enter the illustration id.\n")
illustration = client.fetch_illustration(illustration_id)
illustration.download(directory=Path.home() / "PixivImages",
size=Size.ORIGINAL)
print(f"{illustration_id} was downloaded.")
else: