def getCookieJar(COOKIEFILE):
cookieJar = None
if COOKIEFILE:
try:
complete_path = os.path.join(profile, COOKIEFILE)
cookieJar = http_cookiejar.LWPCookieJar()
cookieJar.load(complete_path, ignore_discard=True)
except:
cookieJar = None
if not cookieJar:
cookieJar = http_cookiejar.LWPCookieJar()
return cookieJar
def __init__(self,
user,
passwd,
sp="",
idp=None,
metadata_file=None,
xmlsec_binary=None,
verbose=0,
ca_certs="",
disable_ssl_certificate_validation=True,
key_file=None,
cert_file=None,
config=None):
"""
:param user: user name
:param passwd: user password
:param sp: The SP URL
:param idp: The IdP PAOS endpoint
:param metadata_file: Where the metadata file is if used
:param xmlsec_binary: Where the xmlsec1 binary can be found (*)
:param verbose: Chatty or not
:param ca_certs: is the path of a file containing root CA certificates
for SSL server certificate validation (*)
:param disable_ssl_certificate_validation: If
disable_ssl_certificate_validation is true, SSL cert validation
will not be performed (*)
:param key_file: Private key filename (*)
:param cert_file: Certificate filename (*)
:param config: Config() instance, overrides all the parameters marked
with an asterisk (*) above
"""
if not config:
config = Config()
config.disable_ssl_certificate_validation = \
disable_ssl_certificate_validation
config.key_file = key_file
config.cert_file = cert_file
config.ca_certs = ca_certs
config.xmlsec_binary = xmlsec_binary
Entity.__init__(self, "sp", config)
self._idp = idp
self._sp = sp
self.user = user
self.passwd = passwd
self._verbose = verbose
if metadata_file:
self._metadata = MetadataStore([saml, samlp], None, config)
self._metadata.load("local", metadata_file)
logger.debug("Loaded metadata from '%s'", metadata_file)
else:
self._metadata = None
self.metadata = self._metadata
self.cookie_handler = None
self.done_ecp = False
self.cookie_jar = cookielib.LWPCookieJar()
def get(url, cookiepath=None, cookie=None, user_agent=None, referer=None):
# use cookies if cookiepath is set and if the cookiepath exists.
if cookiepath is not None:
# check if user has supplied only a folder path, or a full path
if not os.path.isfile(cookiepath):
# if the user supplied only a folder path, append on to the end
# of the path a common filename.
cookiepath = os.path.join(cookiepath, 'cookies.lwp')
# check that the cookie exists
if not os.path.exists(cookiepath):
with open(cookiepath, 'w') as f:
f.write('#LWP-Cookies-2.0\n')
cj = http_cookiejar.LWPCookieJar()
cj.load(cookiepath)
req = urllib_request.Request(url)
if user_agent:
req.add_header('User-Agent', user_agent)
else:
req.add_header('User-Agent', USER_AGENT_STRING)
if referer:
req.add_header('Referer', referer)
if cookie:
req.add_header('Cookie', cookie)
opener = urllib_request.build_opener(urllib_request.HTTPCookieProcessor(cj))
try:
response = opener.open(req)
except urllib_error.URLError as e:
xbmc.log('%s Error opening %s' % (e, url))
sys.exit(1)
link = response.read()
response.close()
return link
else:
return _loadwithoutcookies(url, user_agent)
def __init__(self, base_url, user, passwd):
self._base_url = base_url
# build cookies so we keep track of being logged in
cj = http_cookiejar.LWPCookieJar()
opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(cj))
urllib.request.install_opener(opener)
login = dict(email=user, password=passwd, login_button='Login')
req = urllib.request.Request("%s/user/login" % self._base_url,
urllib.parse.urlencode(login))
response = urllib.request.urlopen(req)
def __init__(self): #, creds, cfile):
self.cookies = cjar.LWPCookieJar()
#//
self.setProxyRef(self.pxy_def)
# cant set auth because we dont know yet what auth type to use, doesnt make sense to set up on default
# self.setAuthentication(creds, self.ath[self.ath_def], self.ath_def)
self.scheme = self.sch_def
self.auth = None
self.head = {}
def getUrl(url, proxy={}, timeout=TIMEOUT, cookies=True):
global cs
cookie = []
if proxy:
urllib_request.install_opener(
urllib_request.build_opener(urllib_request.ProxyHandler(proxy)))
elif cookies:
cookie = http_cookiejar.LWPCookieJar()
opener = urllib_request.build_opener(
urllib_request.HTTPCookieProcessor(cookie))
urllib_request.install_opener(opener)
req = urllib_request.Request(url)
req.add_header(
'User-Agent',
'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.116 Safari/537.36'
)
try:
response = urllib_request.urlopen(req, timeout=timeout)
linkSRC = response.read()
response.close()
except:
linkSRC = ''
cs = ''.join(['%s=%s;' % (c.name, c.value) for c in cookie])
return linkSRC
class Net:
"""
This class wraps :mod:`urllib2` and provides an easy way to make http
requests while taking care of cookies, proxies, gzip compression and
character encoding.
Example::
from addon.common.net import Net
net = Net()
response = net.http_GET('http://xbmc.org')
print response.content
"""
_cj = http_cookiejar.LWPCookieJar()
_proxy = None
_user_agent = 'Mozilla/5.0 (Windows NT 6.3; rv:36.0) Gecko/20100101 Firefox/36.0'
_http_debug = False
def __init__(self, cookie_file='', proxy='', user_agent='', ssl_verify=True, http_debug=False):
"""
Kwargs:
cookie_file (str): Full path to a file to be used to load and save
cookies to.
proxy (str): Proxy setting (eg.
``'http://*****:*****@example.com:1234'``)
user_agent (str): String to use as the User Agent header. If not
supplied the class will use a default user agent (chrome)
http_debug (bool): Set ``True`` to have HTTP header info written to
the XBMC log for all requests.
"""
if cookie_file:
self.set_cookies(cookie_file)
if proxy:
self.set_proxy(proxy)
if user_agent:
self.set_user_agent(user_agent)
self._ssl_verify = ssl_verify
self._http_debug = http_debug
self._update_opener()
def set_cookies(self, cookie_file):
"""
Set the cookie file and try to load cookies from it if it exists.
Args:
cookie_file (str): Full path to a file to be used to load and save
cookies to.
"""
try:
self._cj.load(cookie_file, ignore_discard=True)
self._update_opener()
return True
except:
return False
def get_cookies(self, as_dict=False):
"""Returns A dictionary containing all cookie information by domain."""
if as_dict:
return dict((cookie.name, cookie.value) for cookie in self._cj)
else:
return self._cj._cookies
def save_cookies(self, cookie_file):
"""
Saves cookies to a file.
Args:
cookie_file (str): Full path to a file to save cookies to.
"""
self._cj.save(cookie_file, ignore_discard=True)
def set_proxy(self, proxy):
"""
Args:
proxy (str): Proxy setting (eg.
``'http://*****:*****@example.com:1234'``)
"""
self._proxy = proxy
self._update_opener()
def get_proxy(self):
"""Returns string containing proxy details."""
return self._proxy
def set_user_agent(self, user_agent):
"""
Args:
user_agent (str): String to use as the User Agent header.
"""
self._user_agent = user_agent
def get_user_agent(self):
"""Returns user agent string."""
return self._user_agent
def _update_opener(self):
"""
Builds and installs a new opener to be used by all future calls to
:func:`urllib2.urlopen`.
"""
handlers = [urllib_request.HTTPCookieProcessor(self._cj), urllib_request.HTTPBasicAuthHandler()]
if self._http_debug:
handlers += [urllib_request.HTTPHandler(debuglevel=1)]
else:
handlers += [urllib_request.HTTPHandler()]
if self._proxy:
handlers += [urllib_request.ProxyHandler({'http': self._proxy})]
try:
import platform
node = platform.node().lower()
except:
node = ''
if not self._ssl_verify or node == 'xboxone':
try:
import ssl
ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
if self._http_debug:
handlers += [urllib_request.HTTPSHandler(context=ctx, debuglevel=1)]
else:
handlers += [urllib_request.HTTPSHandler(context=ctx)]
except:
pass
opener = urllib_request.build_opener(*handlers)
urllib_request.install_opener(opener)
def http_GET(self, url, headers={}, compression=True):
"""
Perform an HTTP GET request.
Args:
url (str): The URL to GET.
Kwargs:
headers (dict): A dictionary describing any headers you would like
to add to the request. (eg. ``{'X-Test': 'testing'}``)
compression (bool): If ``True`` (default), try to use gzip
compression.
Returns:
An :class:`HttpResponse` object containing headers and other
meta-information about the page and the page content.
"""
return self._fetch(url, headers=headers, compression=compression)
def http_POST(self, url, form_data, headers={}, compression=True):
"""
Perform an HTTP POST request.
Args:
url (str): The URL to POST.
form_data (dict): A dictionary of form data to POST.
Kwargs:
headers (dict): A dictionary describing any headers you would like
to add to the request. (eg. ``{'X-Test': 'testing'}``)
compression (bool): If ``True`` (default), try to use gzip
compression.
Returns:
An :class:`HttpResponse` object containing headers and other
meta-information about the page and the page content.
"""
return self._fetch(url, form_data, headers=headers, compression=compression)
def http_HEAD(self, url, headers={}):
"""
Perform an HTTP HEAD request.
Args:
url (str): The URL to GET.
Kwargs:
headers (dict): A dictionary describing any headers you would like
to add to the request. (eg. ``{'X-Test': 'testing'}``)
Returns:
An :class:`HttpResponse` object containing headers and other
meta-information about the page.
"""
request = urllib_request.Request(url)
request.get_method = lambda: 'HEAD'
request.add_header('User-Agent', self._user_agent)
for key in headers:
request.add_header(key, headers[key])
response = urllib_request.urlopen(request)
return HttpResponse(response)
def http_DELETE(self, url, headers={}):
"""
Perform an HTTP DELETE request.
Args:
url (str): The URL to GET.
Kwargs:
headers (dict): A dictionary describing any headers you would like
to add to the request. (eg. ``{'X-Test': 'testing'}``)
Returns:
An :class:`HttpResponse` object containing headers and other
meta-information about the page.
"""
request = urllib_request.Request(url)
request.get_method = lambda: 'DELETE'
request.add_header('User-Agent', self._user_agent)
for key in headers:
request.add_header(key, headers[key])
response = urllib_request.urlopen(request)
return HttpResponse(response)
def _fetch(self, url, form_data={}, headers={}, compression=True):
"""
Perform an HTTP GET or POST request.
Args:
url (str): The URL to GET or POST.
form_data (dict): A dictionary of form data to POST. If empty, the
request will be a GET, if it contains form data it will be a POST.
Kwargs:
headers (dict): A dictionary describing any headers you would like
to add to the request. (eg. ``{'X-Test': 'testing'}``)
compression (bool): If ``True`` (default), try to use gzip
compression.
Returns:
An :class:`HttpResponse` object containing headers and other
meta-information about the page and the page content.
"""
req = urllib_request.Request(url)
if form_data:
if isinstance(form_data, six.string_types):
form_data = form_data
else:
form_data = urllib_parse.urlencode(form_data, True)
form_data = form_data.encode('utf-8') if six.PY3 else form_data
req = urllib_request.Request(url, form_data)
req.add_header('User-Agent', self._user_agent)
for key in headers:
req.add_header(key, headers[key])
if compression:
req.add_header('Accept-Encoding', 'gzip')
host = req.host if six.PY3 else req.get_host()
req.add_unredirected_header('Host', host)
response = urllib_request.urlopen(req, timeout=15)
return HttpResponse(response)
def doLogin(cookiepath, username, password):
#check if user has supplied only a folder path, or a full path
if not os.path.isfile(cookiepath):
# if the user supplied only a folder path, append on to the end of the
# path a filename.
cookiepath = os.path.join(cookiepath, 'cookies.lwp')
#delete any old version of the cookie file
try:
os.remove(cookiepath)
except:
pass
if username and password:
#the url you will request to.
login_url = 'https://fantasti.cc/signin.php'
#the header used to pretend you are a browser
header_string = 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-GB; rv:1.9.0.3) Gecko/2008092417 Firefox/3.0.3'
#build the form data necessary for the login
login_data = urllib_parse.urlencode({
'user': username,
'pass': password,
'memento': 1,
'x': 0,
'y': 0,
'do': 'login',
'SSO': ''
})
#build the request we will make
req = urllib_request.Request(login_url)
req.add_header('User-Agent', header_string)
#initiate the cookielib class
cj = http_cookiejar.LWPCookieJar()
# Setup no redirects
class NoRedirection(urllib_request.HTTPRedirectHandler):
def redirect_request(self, req, fp, code, msg, headers, newurl):
return None
#install cookielib into the url opener, so that cookies are handled
opener = urllib_request.build_opener(
urllib_request.HTTPCookieProcessor(cj), NoRedirection())
urllib_request.install_opener(opener)
#do the login and get the response
try:
source = urllib_request.urlopen(req, login_data.encode()).read()
except urllib_error.HTTPError as e:
source = e.read()
#check the received html for a string that will tell us if the user is
# logged in
#pass the username, which can be used to do this.
login, avatar = check_login(source, username)
#if login suceeded, save the cookiejar to disk
if login:
cj.save(cookiepath)
#return whether we are logged in or not
return (login, avatar)
else:
return (False, False)
TRANSLATEPATH = xbmcvfs.translatePath if PY3 else xbmc.translatePath
LOGINFO = xbmc.LOGINFO if PY3 else xbmc.LOGNOTICE
base_hdrs = {'User-Agent': USER_AGENT,
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
'Accept-Encoding': 'gzip',
'Accept-Language': 'en-US,en;q=0.8',
'Connection': 'keep-alive'}
openloadhdr = base_hdrs
progress = xbmcgui.DialogProgress()
dialog = xbmcgui.Dialog()
urlopen = urllib_request.urlopen
cj = http_cookiejar.LWPCookieJar(TRANSLATEPATH(cookiePath))
Request = urllib_request.Request
handlers = [urllib_request.HTTPBasicAuthHandler(), urllib_request.HTTPHandler(), urllib_request.HTTPSHandler()]
ssl_context = ssl.create_default_context()
ssl_context.check_hostname = False
ssl_context.verify_mode = ssl.CERT_NONE
handlers.append(urllib_request.HTTPSHandler(context=ssl_context))
def kodilog(logvar, level=LOGINFO):
xbmc.log("@@@@Cumination: " + str(logvar), level)
@url_dispatcher.register()
def clear_cache():
import sys
import re
from six.moves import urllib_request, urllib_parse, http_cookiejar
from kodi_six import xbmc, xbmcaddon, xbmcplugin, xbmcgui
xbmcaddon.Addon(id='plugin.video.empflix')
cookiejar = http_cookiejar.LWPCookieJar()
cookie_handler = urllib_request.HTTPCookieProcessor(cookiejar)
urllib_request.build_opener(cookie_handler)
def CATEGORIES():
link = openURL('https://www.empflix.com/categories.php')
match = re.compile(r'/([^/]+)/\?a=1&d=" title="([^"]+)"').findall(link)
addDir('All', 'https://www.empflix.com/', 1, '', 1)
for channame, name in match:
addDir(name, ('https://www.empflix.com/' + channame), 2, '', 1)
xbmcplugin.endOfDirectory(int(sys.argv[1]))
def SORTMETHOD(url):
if url == 'https://www.empflix.com/':
addDir('Featured', url + 'featured', 2, '', 1)
addDir('Most Recent', url + 'new', 2, '', 1)
addDir('Most Viewed', url + 'popular', 2, '', 1)
addDir('Top Rated', url + 'toprated', 2, '', 1)
else:
match = re.compile('(https://www.empflix.com/channels/)'
'(.*)').findall(url)
for start, end in match:
addDir('Being Watched', start + 'watched-' + end, 2, '', 1)
def _get_cookiejar(cookie_file):
return cookielib.LWPCookieJar(cookie_file)
def request(url, timeout=None, debug=False):
"""Request the given URL using LIGO.ORG SAML authentication.
This requires an active Kerberos ticket for the user, to get one:
>>> from ligo.org import kinit
>>> kinit('albert.einstein')
Then request as follows
>>> from ligo.org import request
>>> response = request(myurl)
>>> print(response.read())
Parameters
----------
url : `str`
URL path for request
timeout : `int`, optional, default: no timeout
number of seconds to wait for server response,
debug : `bool`, optional, default: `False`
Query in verbose debugging mode
Returns
-------
response : `file`-like
the raw response from the URL, probably XML/HTML or JSON
Examples
--------
>>> from ligo.org import request
>>> response = request('https://ldas-jobs.ligo.caltech.edu/')
>>> print(response.read())
"""
# set debug to 1 to see all HTTP(s) traffic
debug = int(debug)
# need an instance of HTTPS handler to do HTTPS
httpshandler = urllib2.HTTPSHandler(debuglevel=debug)
# use a cookie jar to store session cookies
jar = http_cookiejar.LWPCookieJar()
# if a cookie jar exists open it and read the cookies
# and make sure it has the right permissions
if os.path.exists(COOKIE_JAR):
os.chmod(COOKIE_JAR, stat.S_IRUSR | stat.S_IWUSR)
# set ignore_discard so that session cookies are preserved
try:
jar.load(COOKIE_JAR, ignore_discard=True)
except http_cookiejar.LoadError as e:
warnings.warn('http_cookiejar.LoadError caught: %s' % str(e))
# create a cookie handler from the cookie jar
cookiehandler = urllib2.HTTPCookieProcessor(jar)
# need a redirect handler to follow redirects
redirecthandler = urllib2.HTTPRedirectHandler()
# need an auth handler that can do negotiation.
# input parameter is the Kerberos service principal.
auth_handler = HTTPNegotiateAuthHandler(service_principal='HTTP@%s' %
LIGO_LOGIN_URL)
# create the opener.
opener = urllib2.build_opener(auth_handler, cookiehandler, httpshandler,
redirecthandler)
# prepare the request object
req = urllib2.Request(url)
# use the opener and the request object to make the request.
if timeout is None:
timeout = socket._GLOBAL_DEFAULT_TIMEOUT
response = opener.open(req, timeout=timeout)
# save the session cookies to a file so that they can
# be used again without having to authenticate
jar.save(COOKIE_JAR, ignore_discard=True)
return response
def __init__(self):
self.cookies = {}
self.server_url = 'https://www.titulky.com'
opener = build_opener(HTTPCookieProcessor(http_cookiejar.LWPCookieJar()))
opener.addheaders = [('User-agent', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.2.3) Gecko/20100401 Firefox/3.6.3 ( .NET CLR 3.5.30729)')]
install_opener(opener)
def fetch_url(module,
url,
data=None,
headers=None,
method=None,
use_proxy=True,
force=False,
last_mod_time=None,
timeout=10):
"""Sends a request via HTTP(S) or FTP (needs the module as parameter)
:arg module: The AnsibleModule (used to get username, password etc. (s.b.).
:arg url: The url to use.
:kwarg data: The data to be sent (in case of POST/PUT).
:kwarg headers: A dict with the request headers.
:kwarg method: "POST", "PUT", etc.
:kwarg boolean use_proxy: Default: True
:kwarg boolean force: If True: Do not get a cached copy (Default: False)
:kwarg last_mod_time: Default: None
:kwarg int timeout: Default: 10
:returns: A tuple of (**response**, **info**). Use ``response.read()`` to read the data.
The **info** contains the 'status' and other meta data. When a HttpError (status > 400)
occurred then ``info['body']`` contains the error response data::
Example::
data={...}
resp, info = fetch_url(module,
"http://example.com",
data=module.jsonify(data)
header={Content-type': 'application/json'},
method="POST")
status_code = info["status"]
body = resp.read()
if status_code >= 400 :
body = info['body']
"""
if not HAS_URLPARSE:
module.fail_json(msg='urlparse is not installed')
# ensure we use proper tempdir
old_tempdir = tempfile.tempdir
tempfile.tempdir = module.tmpdir
# Get validate_certs from the module params
validate_certs = module.params.get('validate_certs', True)
username = module.params.get('url_username', '')
password = module.params.get('url_password', '')
http_agent = module.params.get('http_agent', 'ansible-httpget')
force_basic_auth = module.params.get('force_basic_auth', '')
follow_redirects = module.params.get('follow_redirects', 'urllib2')
client_cert = module.params.get('client_cert')
client_key = module.params.get('client_key')
cookies = cookiejar.LWPCookieJar()
r = None
info = dict(url=url)
try:
r = open_url(url,
data=data,
headers=headers,
method=method,
use_proxy=use_proxy,
force=force,
last_mod_time=last_mod_time,
timeout=timeout,
validate_certs=validate_certs,
url_username=username,
url_password=password,
http_agent=http_agent,
force_basic_auth=force_basic_auth,
follow_redirects=follow_redirects,
client_cert=client_cert,
client_key=client_key,
cookies=cookies)
info.update(r.info())
# parse the cookies into a nice dictionary
cookie_dict = dict()
for cookie in cookies:
cookie_dict[cookie.name] = cookie.value
info['cookies'] = cookie_dict
# finally update the result with a message about the fetch
info.update(
dict(msg="OK (%s bytes)" %
r.headers.get('Content-Length', 'unknown'),
url=r.geturl(),
status=r.code))
except NoSSLError as e:
# FIXME: can probably use python platform.py for this
# distribution = get_distribution()
# if distribution is not None and distribution.lower() == 'redhat':
# module.fail_json(msg='%s. You can also install python-ssl from EPEL' % to_native(e))
# else:
module.fail_json(msg='%s' % to_native(e))
except (ConnectionError, ValueError) as e:
module.fail_json(msg=to_native(e))
except urllib_error.HTTPError as e:
try:
body = e.read()
except AttributeError:
body = ''
# Try to add exception info to the output but don't fail if we can't
try:
info.update(dict(**e.info()))
except:
pass
info.update({'msg': to_native(e), 'body': body, 'status': e.code})
except urllib_error.URLError as e:
code = int(getattr(e, 'code', -1))
info.update(dict(msg="Request failed: %s" % to_native(e), status=code))
except socket.error as e:
info.update(
dict(msg="Connection failure: %s" % to_native(e), status=-1))
except httplib.BadStatusLine as e:
info.update(
dict(
msg=
"Connection failure: connection was closed before a valid response was received: %s"
% to_native(e.line),
status=-1))
except Exception as e:
info.update(dict(msg="An unknown error occurred: %s" % to_native(e),
status=-1),
exception=traceback.format_exc())
finally:
tempfile.tempdir = old_tempdir
return r, info
def eiss_download_report(period,
overwrite=False,
timeout=TIMEOUT,
report="DINFL04"):
"""Download all XML files"""
country, season = utils.period_to_country_season(period)
action = "EISS {0} - {1}".format(period, report)
years = utils.season_to_years(season)
eiss_season = "{0}/{1:02d}".format(years[0], years[1] % 100)
if country not in config.CONFIG["country"]:
logger.error("No valid country: {0}".format(country))
return
eiss_country = utils.get_value(config.CONFIG["country"][country], "eiss",
season)
if not eiss_country:
logger.error("Not configured: {0}".format(action))
return
eiss_file = os.path.join(
config.LOCAL["dir"]["download"], "eiss", "{period}{extra}.xml".format(
period=period, extra="" if report == "DINFL04" else "_" + report))
if os.path.exists(eiss_file) and not overwrite:
logger.info("Already local: {0}".format(action))
return
logger.info("Downloading: {0}".format(action))
tools.create_dir(eiss_file)
urllib.request.install_opener(
urllib.request.build_opener(
urllib.request.HTTPCookieProcessor(http_cookiejar.LWPCookieJar())))
params = (("report", report), ("Country", eiss_country),
("Season40", eiss_season), ("ShowILI", "true"), ("ShowARI",
"false"))
url = "{base}?{params}".format(
base="http://ecdc.europa.eu/_layouts/tessypdf.aspx",
params=urllib.parse.urlencode(params))
logger.debug("1st eiss url: {0}".format(url))
req = urllib.request.Request(url)
req.add_header('User-agent', 'Mozilla/5.0 (Linux i686)')
try:
html = urllib.request.urlopen(req,
timeout=timeout).read().decode("utf8")
except (urllib.error.HTTPError, urllib.error.URLError, socket.error):
logger.error("Failed downloading: {0}".format(action))
return
# except socket.timeout:
# if timeout < MAX_TIMEOUT:
# logger.info("Retrying with {0} timeout".format(timeout * 2))
# eiss_download(period, overwrite, 2 * timeout)
# else:
# logger.error("Timeout downloading: {0}".format(action))
# return
match = re.search(r"\"ExportUrlBase\":\"(.*?)\"", html)
# with io.open("/tmp/eiss.html", "w") as fobj:
# fobj.write(html)
if not match:
logger.error(("Element ExportUrlBase not found for Tessy html" +
" output for: {action}").format(action=action))
return
url = "{base}{export}{fileformat}".format(base="http://ecdc.europa.eu",
export=match.group(1).replace(
"FL03", "FL04"),
fileformat="XML")
# params = {"OpType": "Export",
# "FileName": "test",
# "ContentDisposition": "OnlyHtmlInline",
# "Format": "XML"}
# for var in ["ReportSession", "ControlID", "UICulture", "Culture",
# "ReportStack", "StreamID"]:
# if re2.search(r"{0}=([^&\"]*)".format(var), html):
# params[var] = re2.result.groups()[0]
# else:
# logger.error(("Element {var} not found for Tessy html output" +
# " for: {action}").format(var=var, action=action))
# return
# url = "{base}?{params}".format(
# base="http://ecdc.europa.eu/Reserved.ReportViewerWebControl.axd",
# params=urllib.urlencode(params))
logger.debug("2nd eiss url: {0}".format(url))
req = urllib.request.Request(url)
try:
html = urllib.request.urlopen(req,
timeout=timeout).read().decode("utf8")
except (urllib.error.HTTPError, socket.error):
logger.error("Failed downloading: {0}".format(action))
return
with io.open(eiss_file, "w") as fobj:
fobj.write(html)
