def _init_session(session, retry_count=3):
if session is None:
session = requests.Session()
session.mount('file://', FileAdapter())
session.mount('ftp://', FTPAdapter())
# do not set requests max_retries here to support arbitrary pause
return session
def __init__(
self,
url,
max_age=0,
id_attr="id",
path=None,
str_template=None,
include_attrs=None,
exclude_attrs=None,
no_verify=False,
):
path = [] if path is None else path
include_attrs = [] if include_attrs is None else include_attrs
exclude_attrs = [] if exclude_attrs is None else exclude_attrs
session = requests.Session()
session.mount("file://", FileAdapter())
def load_cache():
r = session.get(url, verify=not no_verify)
r.raise_for_status()
return yaml.safe_load(r.text)
cache_file = Path(tempfile.gettempdir(), "awsrun.dat")
accts = PersistentExpiringValue(load_cache, cache_file, max_age=max_age)
super().__init__(
accts.value(),
id_attr=id_attr,
path=path,
str_template=str_template,
include_attrs=include_attrs,
exclude_attrs=exclude_attrs,
)
def perform_download_packages_from_repository(dispatcher, intent):
"""
See :class:`DownloadPackagesFromRepository`.
"""
rpm_version = make_rpm_version(intent.flocker_version)
package_type = intent.distribution.package_type()
s = requests.Session()
# Tests use a local package repository
s.mount('file://', FileAdapter())
downloaded_packages = set()
for package in intent.packages:
package_name = package_filename(
package_type=package_type,
package=package,
architecture=PACKAGE_ARCHITECTURE[package],
rpm_version=rpm_version,
)
url = intent.source_repo + '/' + package_name
local_path = intent.target_path.child(package_name).path
download = s.get(url)
download.raise_for_status()
content = download.content
with open(local_path, "wb") as local_file:
local_file.write(content)
downloaded_packages.add(package_name)
return downloaded_packages
def write_pdf(self, outfile='ws.pdf'):
doc = self._write_html()
directory = Path(os.getcwd())
# delete=False is to prevent errno 10054 on Windows,
# see https://stackoverflow.com/questions/23212435 and
# https://docs.python.org/3.9/library/tempfile.html#tempfile.NamedTemporaryFile
with tempfile.NamedTemporaryFile(mode="w+",
prefix='tmp_',
suffix='.html',
dir=directory,
delete=False) as tmp:
tmp.write(f"{doc}")
with HTMLSession() as session:
session.mount("file://", FileAdapter())
site = session.get(f"file:///{Path(tmp.name).as_posix()}")
site.html.render(timeout=15, keep_page=True)
final_html = site.html.html
# Release the file handle manually
# This is to prevent errno 32 file in use
site.raw.release_conn()
# Delete the temporary file manually
# Comment this line if you wish to not delete the HTML
# for debugging purposes.
Path(tmp.name).unlink()
weasyprint.HTML(string=final_html).write_pdf(outfile)
def get_requests(cls):
if cls._requests is None:
cls._requests = Session()
cls._requests.mount('file://', FileAdapter())
cls._requests.mount('ftp://', FTPAdapter())
#cls._requests.mount('ftps://', FTPAdapter(tls=True))
return cls._requests
def __init__(self, verbose=False):
# Mount file:// to allow local path requests
self.session = requests.Session()
self.session.mount('file://', FileAdapter())
self.files = []
self._file_mapping = {} # Used to keep track of files and their respective metadata
self.verbose = verbose
def __init__(
self,
url,
max_age=0,
delimiter=",",
id_attr="id",
str_template=None,
include_attrs=None,
exclude_attrs=None,
no_verify=False,
):
include_attrs = [] if include_attrs is None else include_attrs
exclude_attrs = [] if exclude_attrs is None else exclude_attrs
session = requests.Session()
session.mount("file://", FileAdapter())
def load_cache():
r = session.get(url, verify=not no_verify)
r.raise_for_status()
buf = io.StringIO(r.text.strip())
return list(csv.DictReader(buf, delimiter=delimiter, skipinitialspace=True))
cache_file = Path(tempfile.gettempdir(), "awsrun.dat")
accts = PersistentExpiringValue(load_cache, cache_file, max_age=max_age)
super().__init__(
accts.value(),
id_attr=id_attr,
str_template=str_template,
include_attrs=include_attrs,
exclude_attrs=exclude_attrs,
)
def _fetch(self, url):
s = requests.Session()
s.mount("file:", FileAdapter())
# make sure file url is absolute
u = urlparse(url)
if u.scheme == "file":
u = u._replace(path=str(Path(u.path).absolute()))
url = u.geturl()
try:
doc = s.get(url)
except Exception as e:
raise ValueError("unable to fetch '{}', due to '{}'".format(
url, str(e)))
src = doc.text
try:
return loads(src, origin=url)
except:
try:
return loads(src, parser=yaml)
except Exception as e:
print(src)
raise ValueError("unable to parse '{}', due to '{}'".format(
url, str(e)))
def url_get(url):
"""
Download an URL using a cache and return the response object
:param url:
:return:
"""
s = None
info = dict()
log.debug("GET URL {!s}".format(url))
if 'file://' in url:
s = requests.session()
s.mount('file://', FileAdapter())
else:
s = CachedSession(cache_name="pyff_cache",
backend=config.request_cache_backend,
expire_after=config.request_cache_time,
old_data_on_error=True)
headers = {'User-Agent': "pyFF/{}".format(__version__), 'Accept': '*/*'}
r = s.get(url,
headers=headers,
verify=False,
timeout=config.request_timeout)
if config.request_override_encoding is not None:
r.encoding = config.request_override_encoding
return r
def process_swagger_url(self, url):
parsed_url = urlparse.urlparse(url)
if not parsed_url.scheme: # Assume file relative to documentation
env = self.state.document.settings.env
relfn, absfn = env.relfn2path(url)
env.note_dependency(relfn)
with open(absfn) as fd:
content = fd.read()
if url[-4:] in ['.yml', 'yaml']:
if yaml is None:
raise SphinxError(
'YAML support is optional. If you choose to use it '
'please install this distribution with the `yaml` '
'requirements extra '
'(i.e. `sphinx-swaggerdoc[yaml]`). '
'Or manually install PyYAML.')
return yaml.load(content)
return json.loads(content)
else:
s = requests.Session()
s.mount('file://', FileAdapter())
r = s.get(url)
return r.json()
def __init__(self,
argv,
ip_addr,
baseline_type="OS",
base_dir="/opt/apache-tomcat-8.5.35"):
self.parse_argv(argv)
session = HTMLSession()
session.mount('file://', FileAdapter())
# Windows系统路径目录分隔符为反斜杠，但get需要正斜杠所以先进行一下替换
pwd = os.getcwd().replace("\\", "/")
# 测试发现使用相对路径读不到文件，需要使用绝对路径
baseline_type = baseline_type.lower()
self.ip_addr = ip_addr
self.baseline_type = baseline_type
self.base_dir = base_dir
# ip_reg = "(\d{1,3}\.{1}){3}\d{1,3}"
# full_reg = f"{ip_reg}_{baseline_type}\.html"
# pwd_file_list = os.listdir()
# for file in pwd_file_list:
# if re.search(full_reg,file):
# ip_addr = re.search(ip_reg,file).group()
self.html_obj = session.get(
f'file:///{pwd}/../4_report/{ip_addr}_{baseline_type}_report.html')
self.shell_script_obj = open(
f"../6_fix/{ip_addr}_{baseline_type}_fix.sh",
"w+",
encoding='utf-8',
newline='\n')
self.fix_item_list = {}
def default_requests_session(
session: Optional[requests.Session] = None) -> requests.Session:
"""Create a requests.Session with suitable default values.
This function is intended for use by functions that can utilize
provided session, but do not require it.
Example::
def download(url: str, *, session: Optional[requests.Session] = None):
# Use provided session if any, or create new session
session = default_requests_session(session)
Keyword arguments:
session: If not None, the session is passed unchanged.
If None, create new session.
"""
if session is not None:
return session
session = requests.Session()
# Add local file adapter
session.mount("file://", FileAdapter())
return session
def Playlistparser(self):
try:
s = requests.Session()
s.mount('file://', FileAdapter())
with s.get(config.url,
headers=self.headers,
proxies=config.proxies,
stream=False,
timeout=30) as playlist:
if playlist.status_code != 304:
if playlist.encoding is None: playlist.encoding = 'utf-8'
playlist = playlist.json()
self.headers['If-Modified-Since'] = gevent.time.strftime(
'%a, %d %b %Y %H:%M:%S %Z',
gevent.time.gmtime(self.playlisttime))
self.playlist = PlaylistGenerator(
m3uchanneltemplate=config.m3uchanneltemplate)
self.picons = picons.logomap
self.channels = {}
m = requests.auth.hashlib.md5()
logging.info('[%s]: playlist %s downloaded' %
(self.__class__.__name__, config.url))
try:
urlpattern = requests.utils.re.compile(
r'^(acestream|infohash)://[0-9a-f]{40}$|^(http|https)://.*.(acelive|acestream|acemedia|torrent)$'
)
for channel in playlist['channels']:
name = channel['name']
url = 'infohash://{url}'.format(**channel)
channel['group'] = channel.pop('cat')
channel['logo'] = self.picons[name] = channel.get(
'logo', picons.logomap.get(name))
channel['tvgid'] = channel.pop('program')
if requests.utils.re.search(urlpattern, url):
self.channels[name] = url
channel['url'] = quote(ensure_str(name), '')
self.playlist.addItem(channel)
m.update(ensure_binary(name))
except Exception as e:
logging.error("[%s]: can't parse JSON! %s" %
(self.__class__.__name__, repr(e)))
return False
self.etag = '"' + m.hexdigest() + '"'
logging.debug('[%s]: plugin playlist generated' %
self.__class__.__name__)
self.playlisttime = gevent.time.time()
except requests.exceptions.RequestException:
logging.error("[%s]: can't download %s playlist!" %
(self.__class__.__name__, config.url))
return False
except:
logging.error(traceback.format_exc())
return False
def Playlistparser(self):
try:
s = requests.Session()
s.mount('file://', FileAdapter())
with s.get(config.url,
headers=self.headers,
proxies=config.proxies,
stream=False,
timeout=30) as r:
if r.status_code != 304:
if r.encoding is None: r.encoding = 'utf-8'
self.headers['If-Modified-Since'] = gevent.time.strftime(
'%a, %d %b %Y %H:%M:%S %Z',
gevent.time.gmtime(self.playlisttime))
self.playlist = PlaylistGenerator(
m3uchanneltemplate=config.m3uchanneltemplate)
self.picons = picons.logomap
self.channels = {}
m = requests.auth.hashlib.md5()
logging.info('[%s]: playlist %s downloaded' %
(self.__class__.__name__, config.url))
pattern = requests.utils.re.compile(
r',(?P<name>.+) \((?P<group>.+)\)[\r\n]+(?P<url>[^\r\n]+)?'
)
urlpattern = requests.utils.re.compile(
r'^(acestream|infohash)://[0-9a-f]{40}$|^(http|https)://.*.(acelive|acestream|acemedia|torrent)$'
)
for match in pattern.finditer(r.text,
requests.auth.re.MULTILINE):
itemdict = match.groupdict()
name = itemdict.get('name', '')
itemdict['logo'] = self.picons[name] = itemdict.get(
'logo', picons.logomap.get(name))
url = itemdict['url']
if requests.utils.re.search(urlpattern, url):
self.channels[name] = url
itemdict['url'] = quote(ensure_str(name), '')
self.playlist.addItem(itemdict)
m.update(ensure_binary(name))
self.etag = '"' + m.hexdigest() + '"'
logging.debug('[%s]: plugin playlist generated' %
self.__class__.__name__)
self.playlisttime = gevent.time.time()
except requests.exceptions.RequestException:
logging.error("[%s]: can't download %s playlist!" %
(self.__class__.__name__, config.url))
return False
except:
logging.error(traceback.format_exc())
return False
return True
def __init__(self, email, password):
self.session = requests.Session(
) # Session for downloading content from urls
self.session.mount('https://',
requests.adapters.HTTPAdapter(max_retries=3))
self.session.mount('file://', FileAdapter())
self.driver = None
self.email = email
self.password = password
def async_get(event_loop):
""" AsyncSession cannot be created global since it will create
a different loop from pytest-asyncio. """
async_session = AsyncHTMLSession()
async_session.mount('file://', FileAdapter())
path = os.path.sep.join((os.path.dirname(os.path.abspath(__file__)), 'python.html'))
url = 'file://{}'.format(path)
return partial(async_session.get, url)
def Playlistparser(self):
try:
s = requests.Session()
s.mount('file://', FileAdapter())
with s.get(config.url,
headers=self.headers,
proxies=config.proxies,
stream=False,
timeout=30) as r:
if r.status_code != 304:
if r.encoding is None: r.encoding = 'utf-8'
self.headers['If-Modified-Since'] = gevent.time.strftime(
'%a, %d %b %Y %H:%M:%S %Z',
gevent.time.gmtime(self.playlisttime))
self.playlist = PlaylistGenerator(
m3uchanneltemplate=config.m3uchanneltemplate)
self.picons = picons.logomap
self.channels = {}
m = requests.auth.hashlib.md5()
self.logger.info('Playlist %s downloaded' % config.url)
pattern = requests.auth.re.compile(
r',(?P<name>.+) \((?P<group>.+)\)[\r\n]+(?P<url>[^\r\n]+)?'
)
for match in pattern.finditer(r.text,
requests.auth.re.MULTILINE):
itemdict = match.groupdict()
name = itemdict.get('name', '')
if not 'logo' in itemdict:
itemdict['logo'] = picons.logomap.get(name)
self.picons[name] = itemdict['logo']
url = itemdict['url']
if url.startswith(('acestream://', 'infohash://')) \
or (url.startswith(('http://','https://')) and url.endswith(('.acelive', '.acestream', '.acemedia', '.torrent'))):
self.channels[name] = url
itemdict['url'] = quote(ensure_str('%s.ts' % name),
'')
self.playlist.addItem(itemdict)
m.update(name.encode('utf-8'))
self.etag = '"' + m.hexdigest() + '"'
self.logger.debug('torrenttv.m3u playlist generated')
self.playlisttime = gevent.time.time()
except requests.exceptions.RequestException:
self.logger.error("Can't download %s playlist!" % config.url)
return False
except:
self.logger.error(traceback.format_exc())
return False
return True
def request(method, url, **kwargs):
"""
This method is to override the requests.request library method so that it will accept local files. Please see the
documentation/code for requests.request (in api.py) for more information.
:param method:
:param url:
:param kwargs:
:return:
"""
with requests.sessions.Session() as session:
session.mount('file://', FileAdapter())
return session.request(method=method, url=url, **kwargs)
def download(url: str) -> requests.Response:
"""Download an URL or file using :py:mod:`requests`.
:raises: requests.exceptions.HTTPError
"""
session = requests.Session()
session.mount('file://', FileAdapter())
res = session.get(url)
res.raise_for_status()
return res
def __init__(self, sh, *args, **kwargs):
"""
Initializes the plugin
@param url: URL of the json data to fetch
@param cycle: the polling interval in seconds
"""
self.logger = logging.getLogger(__name__)
self._url = self.get_parameter_value('url')
self._cycle = self.get_parameter_value('cycle')
self._session = requests.Session()
self._session.mount('file://', FileAdapter())
self._items = {}
def send_request(url):
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
'Accept-Language': 'en-US,en;q=0.8',
'Accept-Encoding': 'gzip',
}
s = requests.Session()
s.mount('file://', FileAdapter())
content = s.get(url, headers=headers, timeout=1, stream=True)
return content.text if hasattr(content, "text") else content.content
def url_get(url: str) -> Response:
"""
Download an URL using a cache and return the response object
:param url:
:return:
"""
s: Union[Session, CachedSession]
if 'file://' in url:
s = requests.session()
s.mount('file://', FileAdapter())
elif 'dir://' in url:
s = requests.session()
s.mount('dir://', DirAdapter())
else:
retry = Retry(total=3, backoff_factor=0.5)
adapter = HTTPAdapter(max_retries=retry)
s = CachedSession(
cache_name="pyff_cache",
backend=config.request_cache_backend,
expire_after=config.request_cache_time,
old_data_on_error=True,
)
s.mount('http://', adapter)
s.mount('https://', adapter)
headers = {'User-Agent': "pyFF/{}".format(__version__), 'Accept': '*/*'}
_etag = None
if _etag is not None:
headers['If-None-Match'] = _etag
try:
r = s.get(url,
headers=headers,
verify=False,
timeout=config.request_timeout)
except IOError as ex:
s = requests.Session()
r = s.get(url,
headers=headers,
verify=False,
timeout=config.request_timeout)
if six.PY2:
r.encoding = "utf-8"
log.debug("url_get({}) returns {} chrs encoded as {}".format(
url, len(r.content), r.encoding))
if config.request_override_encoding is not None:
r.encoding = config.request_override_encoding
return r
def processSwaggerURL(self, path):
try:
# Adds support for relative path imports for swagger json objects
r = open(path, 'r').read()
return json.loads(r)
except ValueError as e:
# No valid JSON object decoded.
print e
except Exception:
s = requests.Session()
s.mount('file://', FileAdapter())
r = s.get(path)
return r.json()
def download(url):
"""Uses requests to download an URL, maybe from a file"""
session = requests.Session()
session.mount('file://', FileAdapter())
try:
res = session.get(url)
except requests.exceptions.ConnectionError as e:
raise e
res.raise_for_status()
return res
def fetch(self, url, use_proxy=False, allow_ftp=False, allow_file=False,
output_file=None, **request_kwargs):
"""Fetch the URL using a custom HTTP handler supporting timeout.
:param url: The URL to fetch.
:param use_proxy: If True, use Launchpad's configured proxy.
:param allow_ftp: If True, allow ftp:// URLs.
:param allow_file: If True, allow file:// URLs. (Be careful to only
pass this if the URL is trusted.)
:param output_file: If not None, download the response content to
this file object or path.
:param request_kwargs: Additional keyword arguments passed on to
`Session.request`.
"""
self.session = Session()
# Always ignore proxy/authentication settings in the environment; we
# configure that sort of thing explicitly.
self.session.trust_env = False
# Mount our custom adapters.
self.session.mount("https://", CleanableHTTPAdapter())
self.session.mount("http://", CleanableHTTPAdapter())
# We can do FTP, but currently only via an HTTP proxy.
if allow_ftp and use_proxy:
self.session.mount("ftp://", CleanableHTTPAdapter())
if allow_file:
self.session.mount("file://", FileAdapter())
request_kwargs.setdefault("method", "GET")
if use_proxy and config.launchpad.http_proxy:
request_kwargs.setdefault("proxies", {})
request_kwargs["proxies"]["http"] = config.launchpad.http_proxy
request_kwargs["proxies"]["https"] = config.launchpad.http_proxy
if allow_ftp:
request_kwargs["proxies"]["ftp"] = config.launchpad.http_proxy
if output_file is not None:
request_kwargs["stream"] = True
response = self.session.request(url=url, **request_kwargs)
response.raise_for_status()
if output_file is None:
# Make sure the content has been consumed before returning.
response.content
else:
# Download the content to the given file.
stream.stream_response_to_file(response, path=output_file)
# The responses library doesn't persist cookies in the session
# (https://github.com/getsentry/responses/issues/80). Work around
# this.
session_cookies = request_kwargs.get("cookies")
if session_cookies is not None and response.cookies:
session_cookies.update(response.cookies)
return response
def __init__(self,
cache=None,
timeout=300,
operation_timeout=None,
session=None):
self.cache = cache
self.load_timeout = timeout
self.operation_timeout = operation_timeout
self.logger = logging.getLogger(__name__)
self.session = session or requests.Session()
self.session.mount("file://", FileAdapter())
self.session.headers[
"User-Agent"] = "Zeep/%s (www.python-zeep.org)" % (get_version())
def processSwaggerURL(self, url):
parsed_url = urlparse.urlparse(url)
if not parsed_url.scheme:
env = self.state.document.settings.env
relfn, absfn = env.relfn2path(url)
env.note_dependency(relfn)
with open(absfn) as fd:
content = fd.read()
return json.loads(content)
else:
s = requests.Session()
s.mount('file://', FileAdapter())
r = s.get(url)
return r.json()
def _download(url, save_to: Union[str, os.PathLike, BinaryIO]) -> None:
with requests.Session() as s:
if FileAdapter:
s.mount('file://', FileAdapter())
r = s.get(url)
if not r:
raise ValueError(r.content.decode('utf8'))
if isinstance(save_to, str) or isinstance(save_to, PathLike):
with open(save_to, 'wb') as fp:
fp.write(r.content)
else:
save_to.write(r.content)
def download(url):
"""Uses requests to download an URL, maybe from a file
:param str url: The URL to download
:rtype: requests.Response
:raises: requests.exceptions.HTTPError
"""
session = requests.Session()
session.mount('file://', FileAdapter())
res = session.get(url)
res.raise_for_status()
return res
def _soup(url):
"""
Simple wrapper to BeautifulSoup object from url
:param url:
:return: BeautifulSoup
"""
s = requests.Session()
s.mount('file://', FileAdapter())
r = s.get(url)
r.raise_for_status()
# force encoding into utf-8 as sometimes airbase comes back in 'ISO-8859-1'
r.encoding = 'utf-8'
return bs4.BeautifulSoup(r.text, 'html.parser')
