def _create_and_start_downloader(self,
request_package_list: RequestPackageList,
sync: bool) -> None:
self._downloader = DownloadThread(
self._parse_request_result,
self._when_completing_a_round_of_requests,
sync=sync)
self._downloader.start(request_package_list)
class Task(BaseTask):
"""
The default implement of default downloader.
"""
def __init__(self, storage: StorageUnit):
super().__init__(storage)
self._increment = True
self._timeout_increment = True
self.set_result_convert_rules(self.rules_of_convert_params)
def _create_and_start_downloader(self,
request_package_list: RequestPackageList,
sync: bool) -> None:
self._downloader = DownloadThread(
self._parse_request_result,
self._when_completing_a_round_of_requests,
sync=sync)
self._downloader.start(request_package_list)
def _convert_request_result(
self,
result: Future,
round_end: bool = False) -> Union[ResultPackage, int]:
if round_end is False:
return result.result()
else:
return result.result().result
def _restart(self) -> None:
global CLIENT_SESSION_PARAMS
if self._timeout_increment:
now_timeout = CLIENT_SESSION_PARAMS.total_timeout
CLIENT_SESSION_PARAMS.set_total_timeout(now_timeout + 30)
return super()._restart()
def rules_of_convert_params(self, result: Any) -> None:
if type(result) is list:
self._data.extend(result)
else:
self._data.append(result)
# This function was not supposed to be here,
# but to avoid adding too many intermediate layers,
# it's better to put it here.
# If an intermediate layer is used,
# the hierarchy should look like this:
# class Task:
# ...
# class PixivTask(Task):
# def picture_parser(...):
# ...
# class TokenTask(PixivTask):
# ...
def picture_parser(self, result_package: ResultPackage) -> ParseResult:
with open(path.join(self._abs_save_path, result_package.msg[0]),
'wb') as file_:
file_.write(result_package.result)
return ParseResult(result_package.msg[1], result_package.msg[0])
class TokenTask(Task):
"""
Subclasses that inherit from the
Task class in order to implement the token function
"""
def __init__(self, storage: StorageUnit):
super().__init__(storage)
self._wait_token_fn = None
self._wait_token_fn_args = ()
self._wait_token_fn_kwargs = {}
self._sync = False
self._stopped = False
self._lock = Lock()
def get_token_callback(self, token: str) -> None:
self._lock.acquire()
if token == "Internet connection exception":
self._stopped = True
self._state = TaskState.Stopped
if self._stopped:
self._lock.release()
return
request_package_list = self._wait_token_fn(
self, *(token, *self._wait_token_fn_args),
**self._wait_token_fn_kwargs)
self._downloader = DownloadThread(
self._parse_request_result,
self._when_completing_a_round_of_requests,
sync=self._sync)
self._downloader.start(request_package_list)
self._state = TaskState.Downloading
self._lock.release()
# When you request pixiv api interface,
# you must be use sync downloader.
# Use async downloader will raise ConnectionResetError exception.
# I don't know how to fixed it.
def _start(self) -> None:
self._lock.acquire()
self._stopped = False
self._lock.release()
super()._start()
def _stop(self) -> None:
self._lock.acquire()
self._stopped = True
super()._stop()
self._lock.release()
def _safe_stop(self) -> None:
self._lock.acquire()
self._stopped = True
super()._safe_stop()
self._lock.release()
def download(photos, destfolder="photos", numthreads=4):
queue = Queue()
for photo in photos:
queue.put(photo)
for i in range(numthreads):
t = DownloadThread(queue, destfolder)
t.start()
queue.join()
def get_token_callback(self, token: str) -> None:
self._lock.acquire()
if token == "Internet connection exception":
self._stopped = True
self._state = TaskState.Stopped
if self._stopped:
self._lock.release()
return
request_package_list = self._wait_token_fn(
self, *(token, *self._wait_token_fn_args),
**self._wait_token_fn_kwargs)
self._downloader = DownloadThread(
self._parse_request_result,
self._when_completing_a_round_of_requests,
sync=self._sync)
self._downloader.start(request_package_list)
self._state = TaskState.Downloading
self._lock.release()
def refresh_auth_token(self,
username: str = None,
password: str = None,
refresh_token: str = None) -> bool:
url = 'https://oauth.secure.pixiv.net/auth/token'
now = datetime.now()
x_client_time = now.strftime('%Y-%m-%dT%H:%M:%S+08:00')
x_client_hash = md5(
(x_client_time + self._LoginHash).encode()).hexdigest()
headers = {
'User-Agent': 'PixivAndroidApp/5.0.64 (Android 6.0)',
'x-client-time': x_client_time,
'x-client-hash': x_client_hash
}
data = {
'get_secure_url': 1,
'client_id': self._ClientID,
'client_secret': self._ClientSecret,
}
if username is not None and password is not None:
data['grant_type'] = 'password'
data['username'] = username
data['password'] = password
elif refresh_token is not None or self._RefreshToken is not None:
data['grant_type'] = 'refresh_token'
data['refresh_token'] = refresh_token or self._RefreshToken
else:
raise Exception(
'Refresh token, but not contain username and password or contain refresh token.'
)
request_package = RequestPackage(url, 'POST', headers, data=data)
self._DownloadThread = DownloadThread(self._child_fetch_callback,
lambda future: print(end=''),
sync=True)
self._DownloadThread.start([request_package])
return True
class TokenManager:
instance = None
init = True
def __new__(cls, *args, **kwargs):
if cls.instance is None:
cls.instance = super().__new__(cls, *args, **kwargs)
return cls.instance
def __init__(self, refresh_token: str = None):
if not self.init:
return
self.init = False
# build params
self._ClientID = 'MOBrBDS8blbauoSck0ZfDbtuzpyT'
self._ClientSecret = 'lsACyCD94FhDUtGTXi3QzcFE2uU1hqtDaKeqrdwj'
self._LoginHash = '28c1fdd170a5204386cb1313c7077b34f83e4aaf4aa829ce78c231e05b0bae2c'
# token
self._AccessToken = None
self._RefreshToken = refresh_token
self._AccessTokenUpdateTime = 0
# network
self._DownloadThread = None
# operating mechanism
self._Refreshing = False
self._SendTokenList = []
self._load_setting()
core_logger.info('Create TokenManager instance.')
def _load_setting(self):
# Pay attention to the real search path
# Search path is running file path, not module path.
if 'Token.json' in listdir('.'):
with open('Token.json') as file:
data = load(file)
self._AccessToken = data['AccessToken']
self._RefreshToken = data['RefreshToken']
self._AccessTokenUpdateTime = data['AccessTokenUpdateTime']
core_logger.info('Find token file, loading.')
now_timestamp = int(datetime.now().timestamp())
if now_timestamp - self._AccessTokenUpdateTime >= AccessTokenUpdateExpiration:
self._AccessToken = None
self._RefreshToken = None
self._AccessTokenUpdateTime = 0
core_logger.info('According to the last update date '
'more than 30 days, the data is cleared')
else:
core_logger.info('Not find token file.')
def set_refresh_token(self, refresh_token):
self._RefreshToken = refresh_token
core_logger.debug('Call function set_refresh_token.')
@property
def msg(self) -> str:
if self._RefreshToken is None:
return "You haven't logged in yet."
update_time = datetime.fromtimestamp(
self._AccessTokenUpdateTime).strftime('%Y-%m-%d %H:%M:%S')
return f"AccessToken: {self._AccessToken}\n" \
f"RefreshToken: {self._RefreshToken}\n" \
f"AccessTokenUpdateTime: {update_time}"
@property
def has_refresh_token(self):
if self._RefreshToken is None:
return False
else:
return True
def get_token(self, instance_callback: Callable[[str], None]) -> str:
if int(datetime.now().timestamp()
) - self._AccessTokenUpdateTime < 45 * 60:
return self._AccessToken
else:
if not self._Refreshing:
self.refresh_auth_token()
self._Refreshing = True
self._SendTokenList.append(instance_callback)
return ''
def login(self, username: str, password: str) -> bool:
core_logger.info('Login account.')
return self.refresh_auth_token(username=username, password=password)
def refresh_auth_token(self,
username: str = None,
password: str = None,
refresh_token: str = None) -> bool:
url = 'https://oauth.secure.pixiv.net/auth/token'
now = datetime.now()
x_client_time = now.strftime('%Y-%m-%dT%H:%M:%S+08:00')
x_client_hash = md5(
(x_client_time + self._LoginHash).encode()).hexdigest()
headers = {
'User-Agent': 'PixivAndroidApp/5.0.64 (Android 6.0)',
'x-client-time': x_client_time,
'x-client-hash': x_client_hash
}
data = {
'get_secure_url': 1,
'client_id': self._ClientID,
'client_secret': self._ClientSecret,
}
if username is not None and password is not None:
data['grant_type'] = 'password'
data['username'] = username
data['password'] = password
elif refresh_token is not None or self._RefreshToken is not None:
data['grant_type'] = 'refresh_token'
data['refresh_token'] = refresh_token or self._RefreshToken
else:
raise Exception(
'Refresh token, but not contain username and password or contain refresh token.'
)
request_package = RequestPackage(url, 'POST', headers, data=data)
self._DownloadThread = DownloadThread(self._child_fetch_callback,
lambda future: print(end=''),
sync=True)
self._DownloadThread.start([request_package])
return True
# must be sync downloader
# use async download will raise ConnectionResetError exception.
# i don't know hwo to fixed it.
def _child_fetch_callback(self, future: Future):
result = future.result() # type: ResultPackage
if result.exception is None:
return self._extract_token(result.result.decode())
else:
core_logger.warning(
'Request refresh token failed, please check internet connection.'
)
self._Refreshing = False
while self._SendTokenList:
callback = self._SendTokenList.pop()
callback("Internet connection exception")
# FIXME: 未捕捉 son.decoder.JSONDecodeError 异常
# 该异常出现于使用代理登陆或者刷新 token 时被 cloudflare 拦截
def _extract_token(self, resp: str):
try:
resp = loads(resp) # type: dict
if resp.get('response', None) is not None:
self._AccessToken = resp['response']['access_token']
self._RefreshToken = resp['response']['refresh_token']
self._AccessTokenUpdateTime = int(datetime.now().timestamp())
# self._UserID = resp['user']['id'] # maybe will use it when multiple tokens
core_logger.debug('Refresh token.')
else:
raise OperationFailedException(
'Operation failed, '
'please check username and password and refresh token.')
except OperationFailedException:
core_logger.warning(
'Operation failed, please check username and password and refresh token.'
)
except KeyError:
# 该异常未被捕捉，异常级别应为 CRITICAL，应组织程序有序的退出
core_logger.critical('Extract access_token error!', exc_info=True)
raise KeyError
return self._write_token()
def _write_token(self):
with open('Token.json', 'w', encoding='utf-8') as file:
dump(
{
'AccessToken': self._AccessToken,
'RefreshToken': self._RefreshToken,
'AccessTokenUpdateTime': self._AccessTokenUpdateTime
}, file)
self._Refreshing = False
self._send_token()
def _send_token(self):
"""
This function will definitely end
because the callback functions are all functions that
create a download thread and start it
:return:
"""
while self._SendTokenList:
callback = self._SendTokenList.pop()
callback(self._AccessToken)