def api_request(url, data=None, method="get", json=True,
session=None, headers=headers):
"""
request and try
:param url:
:param data:
:param method:
:param json:
:param session:
:type session: requests.Session
:param headers:
:return:
"""
url = base_url + url
if data and method == 'get':
method = 'post'
request_obj = session or requests
request_method = getattr(request_obj, method, None) or request_obj.get
try:
req = request_method(url, data=data, headers=headers, timeout=10)
req.encoding = "UTF-8"
res = req.json() if json else req.text
# if session:
# session.cookies.save()
return res
except requests.exceptions.RequestException as e:
log.print_warn("request error: %s" % url)
return None
def config_spider(self):
config = Config()
config_dict = config.config # type: dict
is_config = False
try:
if self.app.pargs.path_download is not None:
paths = self.app.pargs.path_download.split(',') # type: list
if default_path_key in paths:
index = paths.index(default_path_key)
paths.remove(default_path_key)
paths.insert(index, default_download_dir)
final_paths = []
for p in paths:
try:
# some error need pass
if os.path.isdir(p) is False:
os.mkdir(p)
final_paths.append(p)
except:
log.print_warn("path may be wrong and be deleted: {}".format(p))
pass
if not final_paths:
final_paths.append(default_download_dir)
log.print_info('path will be set as: ' + ','.join(final_paths))
config_dict['download_path'] = final_paths
is_config = True
if self.app.pargs.mv_resolution is not None:
r = int(self.app.pargs.mv_resolution)
if r not in mv_resolutions:
log.print_warn("-mvr resolution config skip, value must be 240,480,720,1080")
config_dict['mv_def_resolution'] = r
is_config = True
if self.app.pargs.media_tag is not None:
config_dict['media_tag'] = True if self.app.pargs.media_tag.lower() == 'true'\
or self.app.pargs.media_tag == '1' else False
is_config = True
if self.app.pargs.media_tag_163 is not None:
config_dict['media_tag_163'] = True if self.app.pargs.media_tag_163.lower() == 'true' \
or self.app.pargs.media_tag_163 == '1' else False
is_config = True
except:
log.print_err("input error, pls check")
raise
if is_config:
config.save_config_file()
log.print_info("config success")
self.config_show()
def api_request(url,
data=None,
method="get",
json=True,
session=None,
headers=headers,
encrypt=True,
https=False):
"""
request and try
:param https:
:param encrypt:
:param url:
:param data:
:param method:
:param json:
:param session:
:type session: requests.Session
:param headers:
:return:
"""
url = base_https_url + url if https else base_url + url
request_obj = session or requests
# update cookies
if isinstance(request_obj, requests.Session):
for cookie in request_obj.cookies:
if cookie.name == '__csrf':
data['csrf_token'] = cookie.value
break
# encrypt
if encrypt:
data = encrypted_request(data)
method = 'get' if not data and method == 'get' else 'post'
request_method = getattr(request_obj, method, None) or request_obj.get
try:
req = request_method(url, data=data, headers=headers, timeout=10)
req.encoding = "UTF-8"
res = req.json() if json else req.text
# if session:
# session.cookies.save()
return res
except ValueError as e:
log.print_err("api do not return a valuable json")
return {}
except requests.exceptions.RequestException as e:
log.print_warn("request error: %s" % url)
return {}
def config_test(self):
result = True
try:
# check mongodb config
if self.config['no_mongo'] is False:
log.print_info('check mongodb config')
mongo = self.config['mongo']
for k in ['name', 'host', 'port']:
if k not in mongo:
log.print_err(
"mongo config error, key mongo.{} is not set yet".
format(k))
result = False
# try import model, which will connect to server and exit if server config wrong
import NXSpider.model.mongo_model
for k in [
'download_path', 'mv_def_resolution', 'media_tag',
'media_tag_163'
]:
if k not in self.config:
log.print_err(
"config error, key {} is not set yet".format(k))
result = False
# check type
type_check = {
'download_path': list,
'mv_def_resolution': int,
'media_tag': bool,
'media_tag_163': bool,
'download_file_check': bool,
'no_mongo': bool,
}
need_save = False
for k, v in type_check.items():
if not isinstance(self.config[k], v):
log.print_err("config error, {} is not a require type, "
"and is reset to default value: {}".format(
k, self.default_config[k]))
self.config[k] = self.default_config[k]
need_save = True
result = False
# download path check
final_paths = []
for p in self.config['download_path']:
try:
# some error need pass
if os.path.isdir(p) is False:
os.mkdir(p)
final_paths.append(p)
except:
log.print_warn(
"download path may be wrong and be deleted: {}".format(
p))
need_save = True
result = False
pass
# mv resolution check
if self.config['mv_def_resolution'] not in mv_resolutions:
log.print_warn(
"mv_def_resolution will be reset to default: {}".format(
self.default_config['mv_def_resolution']))
self.config['mv_def_resolution'] = self.default_config[
'mv_def_resolution']
need_save = True
result = False
if need_save:
self.config['download_path'] = final_paths
self.save_config_file()
return result
except Exception as e:
log.print_err(e)
return False