def _init_session(session, retry_count=3): if session is None: session = requests.Session() session.mount('file://', FileAdapter()) session.mount('ftp://', FTPAdapter()) # do not set requests max_retries here to support arbitrary pause return session
def __init__( self, url, max_age=0, id_attr="id", path=None, str_template=None, include_attrs=None, exclude_attrs=None, no_verify=False, ): path = [] if path is None else path include_attrs = [] if include_attrs is None else include_attrs exclude_attrs = [] if exclude_attrs is None else exclude_attrs session = requests.Session() session.mount("file://", FileAdapter()) def load_cache(): r = session.get(url, verify=not no_verify) r.raise_for_status() return yaml.safe_load(r.text) cache_file = Path(tempfile.gettempdir(), "awsrun.dat") accts = PersistentExpiringValue(load_cache, cache_file, max_age=max_age) super().__init__( accts.value(), id_attr=id_attr, path=path, str_template=str_template, include_attrs=include_attrs, exclude_attrs=exclude_attrs, )
def perform_download_packages_from_repository(dispatcher, intent): """ See :class:`DownloadPackagesFromRepository`. """ rpm_version = make_rpm_version(intent.flocker_version) package_type = intent.distribution.package_type() s = requests.Session() # Tests use a local package repository s.mount('file://', FileAdapter()) downloaded_packages = set() for package in intent.packages: package_name = package_filename( package_type=package_type, package=package, architecture=PACKAGE_ARCHITECTURE[package], rpm_version=rpm_version, ) url = intent.source_repo + '/' + package_name local_path = intent.target_path.child(package_name).path download = s.get(url) download.raise_for_status() content = download.content with open(local_path, "wb") as local_file: local_file.write(content) downloaded_packages.add(package_name) return downloaded_packages
def write_pdf(self, outfile='ws.pdf'): doc = self._write_html() directory = Path(os.getcwd()) # delete=False is to prevent errno 10054 on Windows, # see https://stackoverflow.com/questions/23212435 and # https://docs.python.org/3.9/library/tempfile.html#tempfile.NamedTemporaryFile with tempfile.NamedTemporaryFile(mode="w+", prefix='tmp_', suffix='.html', dir=directory, delete=False) as tmp: tmp.write(f"{doc}") with HTMLSession() as session: session.mount("file://", FileAdapter()) site = session.get(f"file:///{Path(tmp.name).as_posix()}") site.html.render(timeout=15, keep_page=True) final_html = site.html.html # Release the file handle manually # This is to prevent errno 32 file in use site.raw.release_conn() # Delete the temporary file manually # Comment this line if you wish to not delete the HTML # for debugging purposes. Path(tmp.name).unlink() weasyprint.HTML(string=final_html).write_pdf(outfile)
def get_requests(cls): if cls._requests is None: cls._requests = Session() cls._requests.mount('file://', FileAdapter()) cls._requests.mount('ftp://', FTPAdapter()) #cls._requests.mount('ftps://', FTPAdapter(tls=True)) return cls._requests
def __init__(self, verbose=False): # Mount file:// to allow local path requests self.session = requests.Session() self.session.mount('file://', FileAdapter()) self.files = [] self._file_mapping = {} # Used to keep track of files and their respective metadata self.verbose = verbose
def __init__( self, url, max_age=0, delimiter=",", id_attr="id", str_template=None, include_attrs=None, exclude_attrs=None, no_verify=False, ): include_attrs = [] if include_attrs is None else include_attrs exclude_attrs = [] if exclude_attrs is None else exclude_attrs session = requests.Session() session.mount("file://", FileAdapter()) def load_cache(): r = session.get(url, verify=not no_verify) r.raise_for_status() buf = io.StringIO(r.text.strip()) return list(csv.DictReader(buf, delimiter=delimiter, skipinitialspace=True)) cache_file = Path(tempfile.gettempdir(), "awsrun.dat") accts = PersistentExpiringValue(load_cache, cache_file, max_age=max_age) super().__init__( accts.value(), id_attr=id_attr, str_template=str_template, include_attrs=include_attrs, exclude_attrs=exclude_attrs, )
def _fetch(self, url): s = requests.Session() s.mount("file:", FileAdapter()) # make sure file url is absolute u = urlparse(url) if u.scheme == "file": u = u._replace(path=str(Path(u.path).absolute())) url = u.geturl() try: doc = s.get(url) except Exception as e: raise ValueError("unable to fetch '{}', due to '{}'".format( url, str(e))) src = doc.text try: return loads(src, origin=url) except: try: return loads(src, parser=yaml) except Exception as e: print(src) raise ValueError("unable to parse '{}', due to '{}'".format( url, str(e)))
def url_get(url): """ Download an URL using a cache and return the response object :param url: :return: """ s = None info = dict() log.debug("GET URL {!s}".format(url)) if 'file://' in url: s = requests.session() s.mount('file://', FileAdapter()) else: s = CachedSession(cache_name="pyff_cache", backend=config.request_cache_backend, expire_after=config.request_cache_time, old_data_on_error=True) headers = {'User-Agent': "pyFF/{}".format(__version__), 'Accept': '*/*'} r = s.get(url, headers=headers, verify=False, timeout=config.request_timeout) if config.request_override_encoding is not None: r.encoding = config.request_override_encoding return r
def process_swagger_url(self, url): parsed_url = urlparse.urlparse(url) if not parsed_url.scheme: # Assume file relative to documentation env = self.state.document.settings.env relfn, absfn = env.relfn2path(url) env.note_dependency(relfn) with open(absfn) as fd: content = fd.read() if url[-4:] in ['.yml', 'yaml']: if yaml is None: raise SphinxError( 'YAML support is optional. If you choose to use it ' 'please install this distribution with the `yaml` ' 'requirements extra ' '(i.e. `sphinx-swaggerdoc[yaml]`). ' 'Or manually install PyYAML.') return yaml.load(content) return json.loads(content) else: s = requests.Session() s.mount('file://', FileAdapter()) r = s.get(url) return r.json()
def __init__(self, argv, ip_addr, baseline_type="OS", base_dir="/opt/apache-tomcat-8.5.35"): self.parse_argv(argv) session = HTMLSession() session.mount('file://', FileAdapter()) # Windows系统路径目录分隔符为反斜杠,但get需要正斜杠所以先进行一下替换 pwd = os.getcwd().replace("\\", "/") # 测试发现使用相对路径读不到文件,需要使用绝对路径 baseline_type = baseline_type.lower() self.ip_addr = ip_addr self.baseline_type = baseline_type self.base_dir = base_dir # ip_reg = "(\d{1,3}\.{1}){3}\d{1,3}" # full_reg = f"{ip_reg}_{baseline_type}\.html" # pwd_file_list = os.listdir() # for file in pwd_file_list: # if re.search(full_reg,file): # ip_addr = re.search(ip_reg,file).group() self.html_obj = session.get( f'file:///{pwd}/../4_report/{ip_addr}_{baseline_type}_report.html') self.shell_script_obj = open( f"../6_fix/{ip_addr}_{baseline_type}_fix.sh", "w+", encoding='utf-8', newline='\n') self.fix_item_list = {}
def default_requests_session( session: Optional[requests.Session] = None) -> requests.Session: """Create a requests.Session with suitable default values. This function is intended for use by functions that can utilize provided session, but do not require it. Example:: def download(url: str, *, session: Optional[requests.Session] = None): # Use provided session if any, or create new session session = default_requests_session(session) Keyword arguments: session: If not None, the session is passed unchanged. If None, create new session. """ if session is not None: return session session = requests.Session() # Add local file adapter session.mount("file://", FileAdapter()) return session
def Playlistparser(self): try: s = requests.Session() s.mount('file://', FileAdapter()) with s.get(config.url, headers=self.headers, proxies=config.proxies, stream=False, timeout=30) as playlist: if playlist.status_code != 304: if playlist.encoding is None: playlist.encoding = 'utf-8' playlist = playlist.json() self.headers['If-Modified-Since'] = gevent.time.strftime( '%a, %d %b %Y %H:%M:%S %Z', gevent.time.gmtime(self.playlisttime)) self.playlist = PlaylistGenerator( m3uchanneltemplate=config.m3uchanneltemplate) self.picons = picons.logomap self.channels = {} m = requests.auth.hashlib.md5() logging.info('[%s]: playlist %s downloaded' % (self.__class__.__name__, config.url)) try: urlpattern = requests.utils.re.compile( r'^(acestream|infohash)://[0-9a-f]{40}$|^(http|https)://.*.(acelive|acestream|acemedia|torrent)$' ) for channel in playlist['channels']: name = channel['name'] url = 'infohash://{url}'.format(**channel) channel['group'] = channel.pop('cat') channel['logo'] = self.picons[name] = channel.get( 'logo', picons.logomap.get(name)) channel['tvgid'] = channel.pop('program') if requests.utils.re.search(urlpattern, url): self.channels[name] = url channel['url'] = quote(ensure_str(name), '') self.playlist.addItem(channel) m.update(ensure_binary(name)) except Exception as e: logging.error("[%s]: can't parse JSON! %s" % (self.__class__.__name__, repr(e))) return False self.etag = '"' + m.hexdigest() + '"' logging.debug('[%s]: plugin playlist generated' % self.__class__.__name__) self.playlisttime = gevent.time.time() except requests.exceptions.RequestException: logging.error("[%s]: can't download %s playlist!" % (self.__class__.__name__, config.url)) return False except: logging.error(traceback.format_exc()) return False
def Playlistparser(self): try: s = requests.Session() s.mount('file://', FileAdapter()) with s.get(config.url, headers=self.headers, proxies=config.proxies, stream=False, timeout=30) as r: if r.status_code != 304: if r.encoding is None: r.encoding = 'utf-8' self.headers['If-Modified-Since'] = gevent.time.strftime( '%a, %d %b %Y %H:%M:%S %Z', gevent.time.gmtime(self.playlisttime)) self.playlist = PlaylistGenerator( m3uchanneltemplate=config.m3uchanneltemplate) self.picons = picons.logomap self.channels = {} m = requests.auth.hashlib.md5() logging.info('[%s]: playlist %s downloaded' % (self.__class__.__name__, config.url)) pattern = requests.utils.re.compile( r',(?P<name>.+) \((?P<group>.+)\)[\r\n]+(?P<url>[^\r\n]+)?' ) urlpattern = requests.utils.re.compile( r'^(acestream|infohash)://[0-9a-f]{40}$|^(http|https)://.*.(acelive|acestream|acemedia|torrent)$' ) for match in pattern.finditer(r.text, requests.auth.re.MULTILINE): itemdict = match.groupdict() name = itemdict.get('name', '') itemdict['logo'] = self.picons[name] = itemdict.get( 'logo', picons.logomap.get(name)) url = itemdict['url'] if requests.utils.re.search(urlpattern, url): self.channels[name] = url itemdict['url'] = quote(ensure_str(name), '') self.playlist.addItem(itemdict) m.update(ensure_binary(name)) self.etag = '"' + m.hexdigest() + '"' logging.debug('[%s]: plugin playlist generated' % self.__class__.__name__) self.playlisttime = gevent.time.time() except requests.exceptions.RequestException: logging.error("[%s]: can't download %s playlist!" % (self.__class__.__name__, config.url)) return False except: logging.error(traceback.format_exc()) return False return True
def __init__(self, email, password): self.session = requests.Session( ) # Session for downloading content from urls self.session.mount('https://', requests.adapters.HTTPAdapter(max_retries=3)) self.session.mount('file://', FileAdapter()) self.driver = None self.email = email self.password = password
def async_get(event_loop): """ AsyncSession cannot be created global since it will create a different loop from pytest-asyncio. """ async_session = AsyncHTMLSession() async_session.mount('file://', FileAdapter()) path = os.path.sep.join((os.path.dirname(os.path.abspath(__file__)), 'python.html')) url = 'file://{}'.format(path) return partial(async_session.get, url)
def Playlistparser(self): try: s = requests.Session() s.mount('file://', FileAdapter()) with s.get(config.url, headers=self.headers, proxies=config.proxies, stream=False, timeout=30) as r: if r.status_code != 304: if r.encoding is None: r.encoding = 'utf-8' self.headers['If-Modified-Since'] = gevent.time.strftime( '%a, %d %b %Y %H:%M:%S %Z', gevent.time.gmtime(self.playlisttime)) self.playlist = PlaylistGenerator( m3uchanneltemplate=config.m3uchanneltemplate) self.picons = picons.logomap self.channels = {} m = requests.auth.hashlib.md5() self.logger.info('Playlist %s downloaded' % config.url) pattern = requests.auth.re.compile( r',(?P<name>.+) \((?P<group>.+)\)[\r\n]+(?P<url>[^\r\n]+)?' ) for match in pattern.finditer(r.text, requests.auth.re.MULTILINE): itemdict = match.groupdict() name = itemdict.get('name', '') if not 'logo' in itemdict: itemdict['logo'] = picons.logomap.get(name) self.picons[name] = itemdict['logo'] url = itemdict['url'] if url.startswith(('acestream://', 'infohash://')) \ or (url.startswith(('http://','https://')) and url.endswith(('.acelive', '.acestream', '.acemedia', '.torrent'))): self.channels[name] = url itemdict['url'] = quote(ensure_str('%s.ts' % name), '') self.playlist.addItem(itemdict) m.update(name.encode('utf-8')) self.etag = '"' + m.hexdigest() + '"' self.logger.debug('torrenttv.m3u playlist generated') self.playlisttime = gevent.time.time() except requests.exceptions.RequestException: self.logger.error("Can't download %s playlist!" % config.url) return False except: self.logger.error(traceback.format_exc()) return False return True
def request(method, url, **kwargs): """ This method is to override the requests.request library method so that it will accept local files. Please see the documentation/code for requests.request (in api.py) for more information. :param method: :param url: :param kwargs: :return: """ with requests.sessions.Session() as session: session.mount('file://', FileAdapter()) return session.request(method=method, url=url, **kwargs)
def download(url: str) -> requests.Response: """Download an URL or file using :py:mod:`requests`. :raises: requests.exceptions.HTTPError """ session = requests.Session() session.mount('file://', FileAdapter()) res = session.get(url) res.raise_for_status() return res
def __init__(self, sh, *args, **kwargs): """ Initializes the plugin @param url: URL of the json data to fetch @param cycle: the polling interval in seconds """ self.logger = logging.getLogger(__name__) self._url = self.get_parameter_value('url') self._cycle = self.get_parameter_value('cycle') self._session = requests.Session() self._session.mount('file://', FileAdapter()) self._items = {}
def send_request(url): headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36', 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8', 'Accept-Language': 'en-US,en;q=0.8', 'Accept-Encoding': 'gzip', } s = requests.Session() s.mount('file://', FileAdapter()) content = s.get(url, headers=headers, timeout=1, stream=True) return content.text if hasattr(content, "text") else content.content
def url_get(url: str) -> Response: """ Download an URL using a cache and return the response object :param url: :return: """ s: Union[Session, CachedSession] if 'file://' in url: s = requests.session() s.mount('file://', FileAdapter()) elif 'dir://' in url: s = requests.session() s.mount('dir://', DirAdapter()) else: retry = Retry(total=3, backoff_factor=0.5) adapter = HTTPAdapter(max_retries=retry) s = CachedSession( cache_name="pyff_cache", backend=config.request_cache_backend, expire_after=config.request_cache_time, old_data_on_error=True, ) s.mount('http://', adapter) s.mount('https://', adapter) headers = {'User-Agent': "pyFF/{}".format(__version__), 'Accept': '*/*'} _etag = None if _etag is not None: headers['If-None-Match'] = _etag try: r = s.get(url, headers=headers, verify=False, timeout=config.request_timeout) except IOError as ex: s = requests.Session() r = s.get(url, headers=headers, verify=False, timeout=config.request_timeout) if six.PY2: r.encoding = "utf-8" log.debug("url_get({}) returns {} chrs encoded as {}".format( url, len(r.content), r.encoding)) if config.request_override_encoding is not None: r.encoding = config.request_override_encoding return r
def processSwaggerURL(self, path): try: # Adds support for relative path imports for swagger json objects r = open(path, 'r').read() return json.loads(r) except ValueError as e: # No valid JSON object decoded. print e except Exception: s = requests.Session() s.mount('file://', FileAdapter()) r = s.get(path) return r.json()
def download(url): """Uses requests to download an URL, maybe from a file""" session = requests.Session() session.mount('file://', FileAdapter()) try: res = session.get(url) except requests.exceptions.ConnectionError as e: raise e res.raise_for_status() return res
def fetch(self, url, use_proxy=False, allow_ftp=False, allow_file=False, output_file=None, **request_kwargs): """Fetch the URL using a custom HTTP handler supporting timeout. :param url: The URL to fetch. :param use_proxy: If True, use Launchpad's configured proxy. :param allow_ftp: If True, allow ftp:// URLs. :param allow_file: If True, allow file:// URLs. (Be careful to only pass this if the URL is trusted.) :param output_file: If not None, download the response content to this file object or path. :param request_kwargs: Additional keyword arguments passed on to `Session.request`. """ self.session = Session() # Always ignore proxy/authentication settings in the environment; we # configure that sort of thing explicitly. self.session.trust_env = False # Mount our custom adapters. self.session.mount("https://", CleanableHTTPAdapter()) self.session.mount("http://", CleanableHTTPAdapter()) # We can do FTP, but currently only via an HTTP proxy. if allow_ftp and use_proxy: self.session.mount("ftp://", CleanableHTTPAdapter()) if allow_file: self.session.mount("file://", FileAdapter()) request_kwargs.setdefault("method", "GET") if use_proxy and config.launchpad.http_proxy: request_kwargs.setdefault("proxies", {}) request_kwargs["proxies"]["http"] = config.launchpad.http_proxy request_kwargs["proxies"]["https"] = config.launchpad.http_proxy if allow_ftp: request_kwargs["proxies"]["ftp"] = config.launchpad.http_proxy if output_file is not None: request_kwargs["stream"] = True response = self.session.request(url=url, **request_kwargs) response.raise_for_status() if output_file is None: # Make sure the content has been consumed before returning. response.content else: # Download the content to the given file. stream.stream_response_to_file(response, path=output_file) # The responses library doesn't persist cookies in the session # (https://github.com/getsentry/responses/issues/80). Work around # this. session_cookies = request_kwargs.get("cookies") if session_cookies is not None and response.cookies: session_cookies.update(response.cookies) return response
def __init__(self, cache=None, timeout=300, operation_timeout=None, session=None): self.cache = cache self.load_timeout = timeout self.operation_timeout = operation_timeout self.logger = logging.getLogger(__name__) self.session = session or requests.Session() self.session.mount("file://", FileAdapter()) self.session.headers[ "User-Agent"] = "Zeep/%s (www.python-zeep.org)" % (get_version())
def processSwaggerURL(self, url): parsed_url = urlparse.urlparse(url) if not parsed_url.scheme: env = self.state.document.settings.env relfn, absfn = env.relfn2path(url) env.note_dependency(relfn) with open(absfn) as fd: content = fd.read() return json.loads(content) else: s = requests.Session() s.mount('file://', FileAdapter()) r = s.get(url) return r.json()
def _download(url, save_to: Union[str, os.PathLike, BinaryIO]) -> None: with requests.Session() as s: if FileAdapter: s.mount('file://', FileAdapter()) r = s.get(url) if not r: raise ValueError(r.content.decode('utf8')) if isinstance(save_to, str) or isinstance(save_to, PathLike): with open(save_to, 'wb') as fp: fp.write(r.content) else: save_to.write(r.content)
def download(url): """Uses requests to download an URL, maybe from a file :param str url: The URL to download :rtype: requests.Response :raises: requests.exceptions.HTTPError """ session = requests.Session() session.mount('file://', FileAdapter()) res = session.get(url) res.raise_for_status() return res
def _soup(url): """ Simple wrapper to BeautifulSoup object from url :param url: :return: BeautifulSoup """ s = requests.Session() s.mount('file://', FileAdapter()) r = s.get(url) r.raise_for_status() # force encoding into utf-8 as sometimes airbase comes back in 'ISO-8859-1' r.encoding = 'utf-8' return bs4.BeautifulSoup(r.text, 'html.parser')