class pixivComm:
token = ""
pixivClient = None
def __init__(self, pixivCreds):
self.pixivClient = Client()
self.pixivClient.login(pixivCreds["user"], pixivCreds["pw"])
self.token = self.pixivClient.refresh_token
def searchImage(self, tag):
self.pixivClient.authenticate(self.token)
searchStr = tag.replace('_', ' ')
print(searchStr, tag)
results = self.pixivClient.search_illustrations(searchStr)
illustrations = results["illustrations"]
randIllust = illustrations[randint(0,len(illustrations) - 1)]
print(dir(randIllust))
while randIllust.type != enums.ContentType.ILLUSTRATION:
randIllust = illustrations[randint(0,len(illustrations) - 1)]
self.getImage(randIllust)
imageURL = randIllust.image_urls[enums.Size.ORIGINAL]
imageURL = glob.glob("./pixiv/" + str(randIllust.id) + "*")[0]
return imageURL
def getImage(self, illustration):
illustration.download(pathlib.Path("./pixiv", size=enums.Size.ORIGINAL), filename = illustration.id)
def testFetch(self):
self.pixivClient.authenticate(self.token)
illus = self.pixivClient.fetch_illustration(82417326)
self.getImage(illus)
class PixApi:
# 初始化
def __init__(self, items, index):
# Multi-Threading
print("Page:%d" % index)
print(len(items))
self.items = items
self.threadID = index
self.threadLocal = threading.local()
self.client = Client()
self.illuData = None
setattr(self.threadLocal, 'client', self.client)
setattr(self.threadLocal, 'items', self.items)
self.client.login(pixiv_id, password)
time.sleep(5)
thread = threading.Thread(target=self.processItem, name=("Page%d" % index))
threads.append(thread)
# 取得每一頁最高收藏數的插圖
def processItem(self):
index = 0
errorCount = 0
item = None
getVal = False
while index < len(self.items):
try:
timer = 0
while timer < 3 and index < len(self.items):
item = self.items[index]
try:
self.illuData = self.client.fetch_illustration(int(item))
except:
pass
if self.illuData != None:
getVal = True
break
time.sleep(1)
timer = timer + 1
except Exception as E:
index = index - 1
print(E)
errorCount = errorCount + 1
if getVal:
value = int(self.illuData.total_bookmarks)
bestKeepList.append((int(item), value))
getVal = False
if errorCount >= 3:
break
index = index + 1
print("Thread%d OK!------------------------------------------" % self.threadID)
class Pixiv:
safeMode = None
# 物件初始化
def __init__(self):
mainDriver.get(loginUrl)
# 利用Web driver模擬登入Pixiv
def loginWithSelenium(self, id, pw):
global pixiv_id
global password
pixiv_id = id
password = pw
fieldGroup = mainDriver.find_element_by_xpath("//div[@class='input-field-group']")
# 獲取User ID輸入欄
userNameField = fieldGroup.find_element_by_xpath("//div[@class='input-field']/input[@type='text'][@autocomplete='username']")
userNameField.clear()
userNameField.send_keys(pixiv_id)
# 獲取密碼輸入欄
passwordField = fieldGroup.find_element_by_xpath("//div[@class='input-field']/input[@type='password'][@autocomplete='current-password']")
passwordField.clear()
passwordField.send_keys(password)
# 獲取提交按鈕
submitButton = mainDriver.find_element_by_xpath("//div[@id='LoginComponent']/form/button[@type='submit'][@class='signup-form__submit']")
submitButton.click()
def ifLoginSuccess(self):
time.sleep(3)
return mainDriver.current_url == 'https://www.pixiv.net/'
# 用於取得一個頁面內所有圖片ID，目前一頁最多60個ID
def start(self, tag, page):
global bestKeepList
bestKeepList.clear()
# 開始時間
print("start time: " + str(datetime.datetime.now()))
items = [] # 儲存這一頁的所有插圖Id
pageNum = 1 # 從第一頁開始
toNext = True
path = "html/body/div/div/div/div/div/div/section/div/ul" # xPath in html to get illustration id
errorCount = 0 # 紀錄數據取得失敗的次數，達到三次將退出
while (pageNum < (page + 1)):
if toNext:
if self.safeMode:
url = "https://www.pixiv.net/tags/%s/illustrations?p=%d&mode=safe" % (tag, pageNum)
else:
url = "https://www.pixiv.net/tags/%s/illustrations?p=%d" % (tag, pageNum)
print(url)
mainDriver.get(url)
toNext = False
print('正在處理第%d頁' % pageNum)
try:
timer = 0
while True:
timer = timer + 1
time.sleep(0.5)
inner_html = mainDriver.find_element_by_xpath(path).get_attribute("innerHTML")
pattern = re.compile('href="/artworks/(.*?)"', re.S)
self.items = list(set(re.findall(pattern, inner_html)))
if len(self.items) > 0:
break
elif timer >= 10:
raise Exception
except:
pageNum = pageNum - 1
print("超時，將會再嘗試%d次" % (2 - errorCount))
errorCount = errorCount + 1
else:
if len(self.items) > 0:
print("ID獲取完成!")
#MyThread(self.items, pageNum) # 建立新的Thread
PixApi(self.items, pageNum)
toNext = True
errorCount = 0
# 每五頁執行一次
if (pageNum % 5) == 0 or pageNum == page:
print("開始收集每張圖的收藏數!請稍等!")
self.runThread()
elif errorCount >= 3:
self.runThread()
break
pageNum = pageNum + 1
print('-----------------------------------------------------')
print("finish time: " + str(datetime.datetime.now()))
# 關閉Driver
for driver in drivers:
driver.close()
bestKeepList.sort(key=self.takeSecond, reverse=True)
#print(len(bestKeepList))
bestKeepList = bestKeepList[:100] # 保留前一百個
#print(bestKeepList)
self.resultClient = Client()
self.resultClient.login(pixiv_id, password)
# 執行儲存於threads中的子執行序，並於完成後清除
def runThread(self):
# t.join(): 等待所有子執行序執行結束才會繼續跑主執行序
for t in threads:
t.start()
for t in threads:
t.join()
print("清空Thread!")
threads.clear()
# 用於排序所使用的compare函式
def takeSecond(self, element):
return element[1]
# 利用Pixiv api取得指定ID的圖片資訊
def getImage(self, index):
id = bestKeepList[index][0]
errorCount = 0
while errorCount < 3:
try:
illuData = self.resultClient.fetch_illustration(id)
return illuData
except:
print("fetch err")
errorCount = errorCount + 1
time.sleep(0.5)
return None
#illuData.download(directory=Path('D:/123'), size=Size.ORIGINAL)
# 取得儲存前100名的List之大小
def getListSize(self):
return bestKeepList.__len__()
class ThreadPixiv(Thread):
def __init__(self, name):
Thread.__init__(self, daemon=True)
self.name = name
self.client = Client(client_id=g.PixivClientID,
client_secret=g.PixivClientSecret)
self.allranking = []
self.artpath = Path('flask/images/pixiv/')
self.start()
def run(self):
self.pixiv_init()
def download_art(self, obj, size, filename):
obj.download(directory=self.artpath, size=size, filename=filename)
def random_pixiv_art(self): # download and set random pixiv art
try:
ranking = random.choice(self.allranking)
fetchmode = random.random() # ranked or ranked related art 20/80
if fetchmode > 0.2:
related_offset = 0
allrelated = []
for _ in range(4):
related = self.client.fetch_illustration_related(
ranking.id, offset=related_offset).get('illustrations')
allrelated = u.sort_pixiv_arts(related, allrelated)
related_offset += 30
illustration = random.choice(list(allrelated))
else:
illustration = ranking
print(f'art id: {illustration.id}')
artid = illustration.id
g.last_link = f'https://www.pixiv.net/en/artworks/{artid}'
g.last_rand_img = f'{artid}.png'
art = Path(f'flask/images/pixiv/{artid}.png')
if not art.is_file():
self.download_art(illustration, g.pixiv_size, artid)
if not art.is_file():
os.rename(f'flask/images/pixiv/{artid}.jpg',
f'flask/images/pixiv/{artid}.png')
set_image('pixiv/', f'{artid}.png')
except BadApiResponse as pixiv_exception: # reconnect
if 'Status code: 400' in str(pixiv_exception):
self.pixiv_init()
self.random_pixiv_art()
except Exception as e:
if 'RemoteDisconnected' in str(e):
self.random_pixiv_art()
def save_pixiv_art(self,
namesave,
owner,
artid,
folder='user/',
setpic=False,
save=False,
save_msg=False):
"""
save pixiv art by art id
:param save_msg: whether send <image saved> message
:param save: whether save image
:param setpic: whether set image
:param namesave: filename
:param owner: twitch username
:param artid: pixiv art id
:param folder: image save folder inside flask app static folder
"""
try:
print(f'art id: {artid}')
namesave = u.while_is_file(folder, namesave, '.png')
namesave = u.while_is_file(folder, namesave, '_p0.png')
savedart = self.client.fetch_illustration(int(artid))
self.download_art(savedart, g.pixiv_size, namesave)
if os.path.isdir('flask/images/pixiv/' + namesave):
mypath2 = 'flask/images/pixiv/' + namesave
onlyfiles = [
f for f in listdir(mypath2) if isfile(join(mypath2, f))
]
for i in onlyfiles:
os.rename(f'flask/images/pixiv/{namesave}/{i}',
f'flask/images/{folder}{namesave}{i[8:-4]}.png')
if save:
db.add_link(
f'https://www.pixiv.net/en/artworks/{artid}',
f'{namesave}{i[8:-4]}.png')
db.add_owner(f'{namesave}{i[8:-4]}.png', owner)
if setpic:
set_image(folder, f'{namesave}{i[8:-4]}.png')
time.sleep(1.5)
os.rmdir(f'flask/images/pixiv/{namesave}')
if save_msg:
u.send_message(f'{owner}, {namesave}.png saved')
return
art = Path(f'flask/images/pixiv/{namesave}.png')
filepath = f'flask/images/pixiv/{namesave}.png'
if not art.is_file():
filepath = f'flask/images/pixiv/{namesave}.jpg'
os.rename(filepath, f'flask/images/{folder}{namesave}.png')
if save:
db.add_link(f'https://www.pixiv.net/en/artworks/{artid}',
f'{namesave}.png')
db.add_owner(f'{namesave}.png', owner)
if setpic:
set_image(folder, f'{namesave}.png')
if save_msg:
u.send_message(f'{owner}, {namesave}.png saved')
except BadApiResponse as pixiv_exception: # reconnect
print(f'badapiresponse - {pixiv_exception}')
if 'Status code: 404' in str(pixiv_exception):
u.send_message(f'{owner}, {artid} not found')
return
if 'Status code: 400' in str(pixiv_exception):
self.pixiv_init()
self.save_pixiv_art(namesave, owner, artid, folder, setpic, save,
save_msg)
except Exception as e:
if 'RemoteDisconnected' in str(e):
self.save_pixiv_art(namesave, owner, artid, folder, setpic,
save, save_msg)
def pixiv_init(self):
try:
self.allranking *= 0
self.client.authenticate(g.PixivToken)
print('pixiv auth √')
rank_offset = 30
ranking1 = self.client.fetch_illustrations_ranking(
mode=RankingMode.DAY
).get('illustrations') # check 500 arts, filter by tags and ratio
self.allranking = u.sort_pixiv_arts(ranking1, self.allranking)
for i in range(16):
print(f'\rpixiv load={int(i / 16 * 100) + 7}%', end='')
ranking = self.client.fetch_illustrations_ranking(
mode=RankingMode.DAY,
offset=rank_offset).get('illustrations')
self.allranking = u.sort_pixiv_arts(ranking, self.allranking)
rank_offset += 30
print()
except BadApiResponse:
time.sleep(30)
self.run()
client = Client()
userid = input("Enter your userid.\n")
password = input("Enter your password.\n")
client.login(userid, password)
print(f"Logged in as {userid}")
needs = input("What would you like to do today?\n")
#fetch an illustration found, using the id provided.
if needs == "illustration download":
illustration_id = input("Enter the illustration id.\n")
illustration = client.fetch_illustration(illustration_id)
illustration.download(directory=Path.home() / "PixivImages",
size=Size.ORIGINAL)
print(f"{illustration_id} was downloaded.")
else:
pass
#downloads the artist id's art.
if needs == "artist download":
downloaded = 0
artist_id = input("Enter artist id to download their art:\n")
num = input("How much art do you want to download?\n")
print(f"Looking for {num} images.")
directory = Path.home() / 'PixivImages'