class SetuService():
def __init__(self, username, password):
self._username = username
self._password = password
_fglfile = open('src/main/resources/fgl.txt', 'r')
self._fgl = []
for line in _fglfile.readlines():
self._fgl.append(line.strip())
_fglfile.close()
self._api = PixivAPI(**_REQUESTS_KWARGS)
self._api.login(self._username, self._password)
def search(self, keyword, mode):
_res = self._api.search_works(keyword,
types=["illustration"],
per_page=500,
mode=mode,
sort="popular")
_illusts = _res.response
#print(_res)
for i in _illusts:
if str(i.id) not in self._fgl:
f = open('src/main/resources/fgl.txt', 'a')
self._fgl.append(str(i.id))
f.write(str(i.id) + '\n')
f.close()
return i.image_urls.large.replace(
"i.pximg.net", "i.pixiv.cat") + " " + str(i.id)
class Pixiv(Thread):
search_on = 0.0 #最後查詢時間
client = None
def __init__(self):
Thread.__init__(self)
def __connect(self):
if self.client is None:
try:
self.client = PixivAPI()
self.client.login(cfg['pixiv']['帳號'], cfg['pixiv']['密碼'])
except Exception as e:
raise e
return False
return True
def run(self):
pass
def search(self, key, number=30):
if not self.__connect():
return 'Pixiv模組發生錯誤 暫時不能使用'
if number > 1000:
number = 1000
if key[0] == '@':
result = self.client.users_works(int(key[1:]))
else:
result = self.client.search_works(
key,
page=1,
per_page=number,
mode='tag', # text標題 tag標籤 exact_tag精準標籤 caption描述
period='all', # all所有 day一天內 week一週內 month一月內
order='desc', # desc新順序 asc舊順序
sort='date',
)
if result.status == 'failure':
return '找不到 <%s>' % (key)
result_rank = []
for i in result.response:
for i2 in result_rank:
if i.stats.views_count > i2.stats.views_count:
result_rank.insert(result_rank.index(i2), i)
break
else:
result_rank.append(i)
reply = []
for i in result_rank:
self.client.download(i.image_urls.px_480mw,
path=cfg['temp_dir'],
name=str(i.id)) #px_128x128 px_480mw
print('%s\\%s' % (cfg['temp_dir'], i.id))
url = imgur.upload('%s\\%s' % (cfg['temp_dir'], i.id))
#url = 'http://temp.maou.pw/%s' % (i.id)
reply.append(url)
if len(reply) >= 4:
break
url = 'https://www.pixiv.net/search.php?word=123&s_mode=s_tag_full'
reply = reply[:4]
reply.append(url)
return reply
def rss(self):
if not self.__connect():
return 'Pixiv模組錯誤'
class PixivCrawler:
KKRTAG = ['弦巻こころ']
def __init__(
self,
auth,
work_path=os.path.abspath('../pixiv/'),
):
self._api = PixivAPI()
self._api.login(*auth)
self._wd = work_path
def fetch_work(self, work_id, tag):
got = False
ri = self._api.works(work_id)
try:
r = ri.response[0]
except:
r = None
if not r:
return got
url_list = []
if r.metadata:
for p in r.metadata.pages:
url_list.append(p.image_urls.large)
else:
url_list.append(r.image_urls.large)
created_time = r.created_time[:10].replace('-', '')
wd = os.path.join(self._wd, created_time)
if not os.path.isdir(wd):
os.mkdir(wd)
fns = []
for url in url_list:
fn = os.path.basename(url)
final_fn = os.path.join(created_time, fn)
_logger.info('getting %s to %s', url, wd)
try:
if self._api.download(url, fname=fn, path=wd):
got = True
shutil.move(os.path.join(wd, fn),
os.path.join(wd, fn + '.download'))
fns.append(final_fn)
except:
import sys
sys.excepthook(*sys.exc_info())
if fns:
meta = json.dumps(r)
dmeta = {
'work_id': work_id,
'mode': tag,
'user': r.user.id,
'fn': fns,
'meta': meta,
}
PixivCursor.insert_update_one(dmeta)
return got
def get_by_tag(self, search_tag='', filter_tag=[], num=30, save_tag=''):
if not search_tag and not filter_tag:
return None
if filter_tag:
filter_tag = [x.strip().lower() for x in filter_tag]
if not search_tag:
search_tag = filter_tag[0]
filter_tag = filter_tag[1:]
if not save_tag:
save_tag = search_tag
filter_tag = set(filter_tag)
_logger.info('search: %s filter: %s', search_tag, filter_tag)
ret = 0
page = 1
while ret < num:
r = self._api.search_works(search_tag,
mode='tag',
page=page,
per_page=30)
try:
l = r.response
except:
l = None
if not l:
break
_logger.info('get %d illusts', len(l))
for i in l:
if i.type != 'illustration':
continue
tt = set([x.strip().lower() for x in i.tags])
if len(tt & filter_tag) != len(filter_tag):
continue
if self.fetch_work(i.id, save_tag):
ret += 1
if ret > num:
break
page += 1
return ret
def get_rank(self, mode='daily', num=30):
ret = 0
page = 1
while ret < num:
r = self._api.ranking_all(mode=mode, page=page, per_page=30)
try:
l = r.response[0].works
except:
l = None
if not l:
break
_logger.info('get %d ranking illust', len(l))
for i in l:
if i.work.type != 'illustration':
continue
if self.fetch_work(i.work.id, mode):
ret += 1
if ret >= num:
break
page += 1
return ret
class CustomPixivPy:
"""
A wrapper around PixivAPI and AppPixivAPI to facilitate automatic re-authentication
(for required methods) and custom result format
"""
TOKEN_LIFESPAN = datetime.timedelta(seconds=3600)
MAX_PIXIV_RESULTS = 3000
RESULTS_PER_QUERY = 50
MAX_RETRIES = 5
def __init__(self, **kwargs):
super().__init__(**kwargs)
# forces reauth() to trigger if any method is called:
self.last_auth = datetime.datetime.fromtimestamp(0)
self.refresh_token = ""
self.aapi = AppPixivAPI(**kwargs)
self.papi = PixivAPI(**kwargs)
def login(self, refresh_token):
self.refresh_token = refresh_token
self.aapi.auth(refresh_token=refresh_token)
self.papi.auth(refresh_token=refresh_token)
self.last_auth = datetime.datetime.now()
logger.debug('Pyxiv login done')
return self # allows chaining
@retry
def illust_ranking(self, mode='daily', offset=None):
self.reauth()
offset = (offset or 0) // self.RESULTS_PER_QUERY + 1
return self.papi.ranking('illust',
mode,
offset,
include_stats=False,
image_sizes=['medium', 'large'])
@retry
def search_illust(self,
word,
search_target='text',
sort='date',
offset=None):
self.reauth()
offset = (offset or 0) // self.RESULTS_PER_QUERY + 1
return self.papi.search_works(word,
offset,
mode=search_target,
types=['illustration'],
sort=sort,
include_stats=False,
image_sizes=['medium', 'large'])
@retry
def illust_detail(self, illust_id, req_auth=True):
self.reauth()
return self.aapi.illust_detail(illust_id, req_auth)
def reauth(self):
"""Re-authenticates with pixiv if the last login was more than TOKEN_LIFESPAN ago"""
if datetime.datetime.now() - self.last_auth > self.TOKEN_LIFESPAN:
self.login(self.refresh_token)
self.papi.auth(refresh_token=self.refresh_token)
logger.debug("Reauth successful")
self.last_auth = datetime.datetime.now()
def get_pixiv_results(self, offset=None, *, query="", nsfw=False):
"""
Get results from Pixiv as a dict
If no parameters are given, SFW daily ranking is returned
:param offset: Optional. page offset
:param query: Optional. Specify a search query
:param nsfw: Whether to allow NSFW illustrations, false by default
:return: list of dicts containing illustration information
"""
json_result, last_error = None, None
for attempt in range(1, self.MAX_RETRIES + 1):
try:
json_result = self.search_illust(query, offset=offset, sort='popular') \
if query else self.illust_ranking('daily_r18' if nsfw else 'daily', offset=offset)
except PixivError as e:
if attempt == self.MAX_RETRIES:
logger.warning("Failed fetching Pixiv data: %s", e)
raise e from None
else:
break
results = []
if json_result.get('has_error'):
return results
it = json_result.response if query else (
x['work'] for x in json_result.response[0]['works'])
for img in it:
if not nsfw and img['sanity_level'] == 'black':
continue # white = SFW, semi_black = questionable, black = NSFW
results.append({
'url':
img['image_urls']['large'],
'thumb_url':
img['image_urls']['medium'],
'title':
img['title'],
'user_name':
img['user']['name'],
'user_link':
f"https://www.pixiv.net/en/users/{img['user']['id']}"
})
logger.debug(results[-1])
return results
