def api_request(url, data=None, method="get", json=True, session=None, headers=headers): """ request and try :param url: :param data: :param method: :param json: :param session: :type session: requests.Session :param headers: :return: """ url = base_url + url if data and method == 'get': method = 'post' request_obj = session or requests request_method = getattr(request_obj, method, None) or request_obj.get try: req = request_method(url, data=data, headers=headers, timeout=10) req.encoding = "UTF-8" res = req.json() if json else req.text # if session: # session.cookies.save() return res except requests.exceptions.RequestException as e: log.print_warn("request error: %s" % url) return None
def config_spider(self): config = Config() config_dict = config.config # type: dict is_config = False try: if self.app.pargs.path_download is not None: paths = self.app.pargs.path_download.split(',') # type: list if default_path_key in paths: index = paths.index(default_path_key) paths.remove(default_path_key) paths.insert(index, default_download_dir) final_paths = [] for p in paths: try: # some error need pass if os.path.isdir(p) is False: os.mkdir(p) final_paths.append(p) except: log.print_warn("path may be wrong and be deleted: {}".format(p)) pass if not final_paths: final_paths.append(default_download_dir) log.print_info('path will be set as: ' + ','.join(final_paths)) config_dict['download_path'] = final_paths is_config = True if self.app.pargs.mv_resolution is not None: r = int(self.app.pargs.mv_resolution) if r not in mv_resolutions: log.print_warn("-mvr resolution config skip, value must be 240,480,720,1080") config_dict['mv_def_resolution'] = r is_config = True if self.app.pargs.media_tag is not None: config_dict['media_tag'] = True if self.app.pargs.media_tag.lower() == 'true'\ or self.app.pargs.media_tag == '1' else False is_config = True if self.app.pargs.media_tag_163 is not None: config_dict['media_tag_163'] = True if self.app.pargs.media_tag_163.lower() == 'true' \ or self.app.pargs.media_tag_163 == '1' else False is_config = True except: log.print_err("input error, pls check") raise if is_config: config.save_config_file() log.print_info("config success") self.config_show()
def api_request(url, data=None, method="get", json=True, session=None, headers=headers, encrypt=True, https=False): """ request and try :param https: :param encrypt: :param url: :param data: :param method: :param json: :param session: :type session: requests.Session :param headers: :return: """ url = base_https_url + url if https else base_url + url request_obj = session or requests # update cookies if isinstance(request_obj, requests.Session): for cookie in request_obj.cookies: if cookie.name == '__csrf': data['csrf_token'] = cookie.value break # encrypt if encrypt: data = encrypted_request(data) method = 'get' if not data and method == 'get' else 'post' request_method = getattr(request_obj, method, None) or request_obj.get try: req = request_method(url, data=data, headers=headers, timeout=10) req.encoding = "UTF-8" res = req.json() if json else req.text # if session: # session.cookies.save() return res except ValueError as e: log.print_err("api do not return a valuable json") return {} except requests.exceptions.RequestException as e: log.print_warn("request error: %s" % url) return {}
def config_test(self): result = True try: # check mongodb config if self.config['no_mongo'] is False: log.print_info('check mongodb config') mongo = self.config['mongo'] for k in ['name', 'host', 'port']: if k not in mongo: log.print_err( "mongo config error, key mongo.{} is not set yet". format(k)) result = False # try import model, which will connect to server and exit if server config wrong import NXSpider.model.mongo_model for k in [ 'download_path', 'mv_def_resolution', 'media_tag', 'media_tag_163' ]: if k not in self.config: log.print_err( "config error, key {} is not set yet".format(k)) result = False # check type type_check = { 'download_path': list, 'mv_def_resolution': int, 'media_tag': bool, 'media_tag_163': bool, 'download_file_check': bool, 'no_mongo': bool, } need_save = False for k, v in type_check.items(): if not isinstance(self.config[k], v): log.print_err("config error, {} is not a require type, " "and is reset to default value: {}".format( k, self.default_config[k])) self.config[k] = self.default_config[k] need_save = True result = False # download path check final_paths = [] for p in self.config['download_path']: try: # some error need pass if os.path.isdir(p) is False: os.mkdir(p) final_paths.append(p) except: log.print_warn( "download path may be wrong and be deleted: {}".format( p)) need_save = True result = False pass # mv resolution check if self.config['mv_def_resolution'] not in mv_resolutions: log.print_warn( "mv_def_resolution will be reset to default: {}".format( self.default_config['mv_def_resolution'])) self.config['mv_def_resolution'] = self.default_config[ 'mv_def_resolution'] need_save = True result = False if need_save: self.config['download_path'] = final_paths self.save_config_file() return result except Exception as e: log.print_err(e) return False