def get_auth_token(host, clientid, clientsecret,
proxy_host=None, proxy_port=None):
queryurl = '/oauth/access_token'
if proxy_host and proxy_port:
connection = HTTPSConnection(proxy_host, proxy_port)
connection.set_tunnel(host, 443)
else:
connection = HTTPSConnection(host)
authtoken = base64.b64encode(six.b('{0}:{1}'.format(clientid, clientsecret)))
authstring = b"Basic %s" % (authtoken,)
header = {"Authorization": authstring}
params = urlencode({'grant_type': 'client_credentials'})
connection.request("POST", queryurl, params, header)
response = connection.getresponse()
jsondata = bytes(response.read()).decode('utf-8')
data = json.loads(str(jsondata))
try:
key = data['access_token']
except KeyError:
print("We're having trouble getting a session token. Please check your API key.")
print("Error output: ")
print(data)
sys.exit()
finally:
connection.close()
return key
def send_email(email, template, kwargs):
"""Send an email via the mailgun service."""
maildir = os.path.join(topdir, 'emails')
env = jinja2.Environment(loader=jinja2.FileSystemLoader(maildir))
template = env.get_template(template)
rendered = template.render(**kwargs)
headers, message = parse_email(rendered)
mailargs = {'to': email,
'from': app.config['MAIL_FROM'],
'bcc': app.config.get('MAIL_BCC'),
'text': message}
mailargs.update(headers)
conn = HTTPSConnection('api.mailgun.net', 443)
conn.connect()
auth = b64enc('api:{0[MAILGUN_KEY]}'.format(app.config))
headers = {'Authorization': 'Basic {0}'.format(auth),
'Accept': 'application/json',
'Content-type': 'application/x-www-form-urlencoded'}
url = '/v2/{0[MAILGUN_DOMAIN]}/messages'.format(app.config)
body = urlencode(mailargs)
conn.request('POST', url, body, headers)
resp = conn.getresponse()
if resp.status != 200:
raise RuntimeError('could not send email')
conn.close()
def getRemoteDBModifiedTS():
"""
Performs a HEAD request to get the Last-Modified date-time
of a database dump file and parses it into a UNIX timestamp.
"""
debug_msg = 'Unable to get timestamp of remote database dump - {0}'
logging.info("Getting timestamp of database dump at '%s'", HOST + PATH)
try:
# Removing the scheme from the URL
conn = HTTPSConnection(HOST[8:], timeout=TIMEOUT)
conn.request('HEAD', PATH)
except gaierror as e:
logging.debug(debug_msg.format("Cannot connect to '%s', error: %s"),
HOST + PATH, e)
exit(1)
rsp = conn.getresponse()
if rsp.status != 200:
logging.debug(debug_msg.format('Server responded with: %d %s'),
rsp.status, rsp.reason)
exit(1)
last_modified = rsp.getheader('last-modified', None)
if last_modified is None:
logging.debug(
debug_msg.format('Response doesnt include Last-Modified Header'))
exit(1)
last_m_dt = datetime.strptime(
last_modified.split(', ')[1], '%d %b %Y %H:%M:%S %Z')
return timegm(last_m_dt.timetuple())
def getRemoteDBModifiedTS():
"""
Performs a HEAD request to get the Last-Modified date-time
of a database dump file and parses it into a UNIX timestamp.
"""
debug_msg = 'Unable to get timestamp of remote database dump - {0}'
logging.info("Getting timestamp of database dump at '%s'", HOST + PATH)
try:
# Removing the scheme from the URL
conn = HTTPSConnection(HOST[8:], timeout=TIMEOUT)
conn.request('HEAD', PATH)
except gaierror as e:
logging.debug(
debug_msg.format(
"Cannot connect to '%s', error: %s"),
HOST + PATH, e)
exit(1)
rsp = conn.getresponse()
if rsp.status != 200:
logging.debug(
debug_msg.format('Server responded with: %d %s'), rsp.status,
rsp.reason)
exit(1)
last_modified = rsp.getheader('last-modified', None)
if last_modified is None:
logging.debug(debug_msg.format(
'Response doesnt include Last-Modified Header'))
exit(1)
last_m_dt = datetime.strptime(
last_modified.split(', ')[1], '%d %b %Y %H:%M:%S %Z')
return timegm(last_m_dt.timetuple())
def make_request(method, uri, headers=None, body=None):
"""
This function should make a request, connection reuse, etc. is left to the implementation
:param str|unicode method: HTTP Verb
:param str|unicode uri: Server/path to make the request to
:param dict headers: Headers to send
:param str|unicode body: Body to send
:return: Request result
:rtype: Tuple of return_code (int), headers (list of (k,v) tuples), body (str|bytes)
"""
parsed_url = urlparse(uri)
if parsed_url.scheme == 'https':
connector = HTTPSConnection(host=parsed_url.netloc)
elif parsed_url.scheme == 'http':
connector = HTTPConnection(host=parsed_url.netloc)
else:
raise ValueError('Schema is not HTTP nor HTTPS')
_, full_path = uri.split(parsed_url.netloc, 1)
connector.request(method=method, url=full_path, body=body, headers=headers or {})
response = connector.getresponse()
status_code = response.status
body = response.read()
headers = response.getheaders()
return status_code, headers, body
def _new_conn(self):
# Performs the NTLM handshake that secures the connection. The socket
# must be kept open while requests are performed.
self.num_connections += 1
log.debug('Starting NTLM HTTPS connection no. %d: https://%s%s',
self.num_connections, self.host, self.authurl)
headers = {'Connection': 'Keep-Alive'}
req_header = 'Authorization'
resp_header = 'www-authenticate'
conn = HTTPSConnection(host=self.host, port=self.port)
# Send negotiation message
headers[req_header] = (
'NTLM %s' % ntlm.create_NTLM_NEGOTIATE_MESSAGE(self.rawuser))
log.debug('Request headers: %s', headers)
conn.request('GET', self.authurl, None, headers)
res = conn.getresponse()
reshdr = dict(res.getheaders())
log.debug('Response status: %s %s', res.status, res.reason)
log.debug('Response headers: %s', reshdr)
log.debug('Response data: %s [...]', res.read(100))
# Remove the reference to the socket, so that it can not be closed by
# the response object (we want to keep the socket open)
res.fp = None
# Server should respond with a challenge message
auth_header_values = reshdr[resp_header].split(', ')
auth_header_value = None
for s in auth_header_values:
if s[:5] == 'NTLM ':
auth_header_value = s[5:]
if auth_header_value is None:
raise Exception('Unexpected %s response header: %s' %
(resp_header, reshdr[resp_header]))
# Send authentication message
ServerChallenge, NegotiateFlags = \
ntlm.parse_NTLM_CHALLENGE_MESSAGE(auth_header_value)
auth_msg = ntlm.create_NTLM_AUTHENTICATE_MESSAGE(
ServerChallenge, self.user, self.domain, self.pw, NegotiateFlags)
headers[req_header] = 'NTLM %s' % auth_msg
log.debug('Request headers: %s', headers)
conn.request('GET', self.authurl, None, headers)
res = conn.getresponse()
log.debug('Response status: %s %s', res.status, res.reason)
log.debug('Response headers: %s', dict(res.getheaders()))
log.debug('Response data: %s [...]', res.read()[:100])
if res.status != 200:
if res.status == 401:
raise Exception('Server rejected request: wrong '
'username or password')
raise Exception('Wrong server response: %s %s' %
(res.status, res.reason))
res.fp = None
log.debug('Connection established')
return conn
def _send_prowl(prowl_api=None,
prowl_priority=None,
event=None,
message=None,
force=False):
if not sickbeard.USE_PROWL and not force:
return False
if prowl_api is None:
prowl_api = sickbeard.PROWL_API
if len(prowl_api) == 0:
return False
if prowl_priority is None:
prowl_priority = sickbeard.PROWL_PRIORITY
title = sickbeard.PROWL_MESSAGE_TITLE
logger.log(
"PROWL: Sending notice with details: title=\"{0}\" event=\"{1}\", message=\"{2}\", priority={3}, api={4}"
.format(title, event, message, prowl_priority,
prowl_api), logger.DEBUG)
http_handler = HTTPSConnection("api.prowlapp.com")
data = {
'apikey': prowl_api,
'application': title,
'event': event,
'description': message.encode('utf-8'),
'priority': prowl_priority
}
try:
http_handler.request(
"POST",
"/publicapi/add",
headers={'Content-type': "application/x-www-form-urlencoded"},
body=urlencode(data))
except (SSLError, HTTPException, socket.error):
logger.log("Prowl notification failed.", logger.ERROR)
return False
response = http_handler.getresponse()
request_status = response.status
if request_status == 200:
logger.log("Prowl notifications sent.", logger.INFO)
return True
elif request_status == 401:
logger.log("Prowl auth failed: {0}".format(response.reason),
logger.ERROR)
return False
else:
logger.log("Prowl notification failed.", logger.ERROR)
return False
def _send_prowl(prowl_api=None,
prowl_priority=None,
event=None,
message=None,
force=False):
if not app.USE_PROWL and not force:
return False
if prowl_api is None:
prowl_api = ','.join(app.PROWL_API)
if not prowl_api:
return False
if prowl_priority is None:
prowl_priority = app.PROWL_PRIORITY
title = app.PROWL_MESSAGE_TITLE
log.debug(
u'PROWL: Sending notice with details: title="{0}" event="{1}", message="{2}", priority={3}, api={4}',
title, event, message, prowl_priority, prowl_api)
http_handler = HTTPSConnection('api.prowlapp.com')
data = {
'apikey': prowl_api,
'application': title,
'event': event,
'description': message.encode('utf-8'),
'priority': prowl_priority
}
try:
http_handler.request(
'POST',
'/publicapi/add',
headers={'Content-type': 'application/x-www-form-urlencoded'},
body=urlencode(data))
except (SSLError, HTTPException, socket.error):
log.error(u'Prowl notification failed.')
return False
response = http_handler.getresponse()
request_status = response.status
if request_status == 200:
log.info(u'Prowl notifications sent.')
return True
elif request_status == 401:
log.error(u'Prowl auth failed: {0}', response.reason)
return False
else:
log.error(u'Prowl notification failed.')
return False
def execute_request(self):
""""Execute the request, and get a response.
Returns:
an HttpResponse object from httplib
"""
if self.is_secure_connection:
conn = HTTPSConnection(self.get_hostname(), timeout=self.timeout)
else:
conn = HTTPConnection(self.get_hostname(), timeout=self.timeout)
conn.request(self.method, self.url, self.body, self.headers)
response = conn.getresponse()
return response
def _send_prowl(prowl_api=None, prowl_priority=None, event=None, message=None, force=False):
if not sickbeard.USE_PROWL and not force:
return False
if prowl_api is None:
prowl_api = sickbeard.PROWL_API
if not prowl_api:
return False
if prowl_priority is None:
prowl_priority = sickbeard.PROWL_PRIORITY
title = sickbeard.PROWL_MESSAGE_TITLE
logger.log(u"PROWL: Sending notice with details: title=\"%s\" event=\"%s\", message=\"%s\", priority=%s, api=%s"
% (title, event, message, prowl_priority, prowl_api), logger.DEBUG)
http_handler = HTTPSConnection("api.prowlapp.com")
data = {'apikey': prowl_api,
'application': title,
'event': event,
'description': message.encode('utf-8'),
'priority': prowl_priority}
try:
http_handler.request("POST",
"/publicapi/add",
headers={'Content-type': "application/x-www-form-urlencoded"},
body=urlencode(data))
except (SSLError, HTTPException, socket.error):
logger.log(u"Prowl notification failed.", logger.ERROR)
return False
response = http_handler.getresponse()
request_status = response.status
if request_status == 200:
logger.log(u"Prowl notifications sent.", logger.INFO)
return True
elif request_status == 401:
logger.log(u"Prowl auth failed: %s" % response.reason, logger.ERROR)
return False
else:
logger.log(u"Prowl notification failed.", logger.ERROR)
return False
def _send_prowl(prowl_api=None, prowl_priority=None, event=None, message=None, force=False):
if not app.USE_PROWL and not force:
return False
if prowl_api is None:
prowl_api = ','.join(app.PROWL_API)
if not prowl_api:
return False
if prowl_priority is None:
prowl_priority = app.PROWL_PRIORITY
title = app.PROWL_MESSAGE_TITLE
log.debug(u'PROWL: Sending notice with details: title="{0}" event="{1}", message="{2}", priority={3}, api={4}',
title, event, message, prowl_priority, prowl_api)
http_handler = HTTPSConnection('api.prowlapp.com')
data = {'apikey': prowl_api,
'application': title,
'event': event,
'description': message.encode('utf-8'),
'priority': prowl_priority}
try:
http_handler.request('POST',
'/publicapi/add',
headers={'Content-type': 'application/x-www-form-urlencoded'},
body=urlencode(data))
except (SSLError, HTTPException, socket.error):
log.error(u'Prowl notification failed.')
return False
response = http_handler.getresponse()
request_status = response.status
if request_status == 200:
log.info(u'Prowl notifications sent.')
return True
elif request_status == 401:
log.error(u'Prowl auth failed: {0}', response.reason)
return False
else:
log.error(u'Prowl notification failed.')
return False
def getBestServer(servers):
"""Perform a speedtest.net latency request to determine which
speedtest.net server has the lowest latency
"""
results = {}
for server in servers:
cum = []
url = '%s/latency.txt' % os.path.dirname(server['url'])
urlparts = urlparse(url)
for i in range(0, 3):
try:
if urlparts[0] == 'https':
h = HTTPSConnection(urlparts[1])
else:
h = HTTPConnection(urlparts[1])
headers = {'User-Agent': user_agent}
start = timeit.default_timer()
h.request("GET", urlparts[2], headers=headers)
r = h.getresponse()
total = (timeit.default_timer() - start)
except (HTTPError, URLError, socket.error):
cum.append(3600)
continue
text = r.read(9)
if int(r.status) == 200 and text == 'test=test'.encode():
cum.append(total)
else:
cum.append(3600)
h.close()
avg = round((sum(cum) / 6) * 1000, 3)
results[avg] = server
fastest = sorted(results.keys())[0]
best = results[fastest]
best['latency'] = fastest
return best
def apihit(host,conntype,authtoken,queryurl,reqbody,
proxy_host=None, proxy_port=None):
retdata = ''
if proxy_host and proxy_port:
connection = HTTPSConnection(proxy_host, proxy_port)
connection.set_tunnel(host, 443)
else:
connection = HTTPSConnection(host)
tokenheader = {"Authorization": 'Bearer ' + authtoken, "Content-type": "application/json", "Accept": "text/plain"}
if conntype == "GET":
connection.request(conntype, queryurl, '', tokenheader)
else:
connection.request(conntype, queryurl, json.dumps(reqbody), tokenheader)
response = connection.getresponse()
respbody = response.read().decode('ascii', 'ignore')
try:
jsondata = respbody.decode()
retdata = json.loads(jsondata)
except AttributeError:
retdata = json.loads(respbody)
except:
raise
connection.close()
return retdata
def handle_event(e, p, counter):
# print(e)
if counter[0] % 1000 == 0:
print('processed ' + str(counter[0]) + ' objects')
counter[0] += 1
if __name__ == '__main__':
url = '/prust/wikipedia-movie-data/master/movies.json'
endpoint = HTTPSConnection('raw.githubusercontent.com',
'443',
context=ssl._create_unverified_context())
try:
ts1 = datetime.datetime.now().timestamp()
print('start time=' + str(ts1))
endpoint.request('GET', url)
response = endpoint.getresponse()
tokenizer = JsonTokenizer(response, 'ISO-8859-1', 65536)
JsonDecoder()\
.tokenizer(tokenizer)\
.root_class_name('Data')\
.event_handler(lambda e, p: handle_event(e, p, counter))\
.predicate('genres') \
.with_snake_cased_props()\
.decode()
ts2 = datetime.datetime.now().timestamp()
print('end time=' + str(ts2))
print('delta=' + str(ts2 - ts1))
def __init__(self, uri, basepath=None):
self.basepath = basepath
self.mimetype = None
self.file = None
self.data = None
self.uri = None
self.local = None
self.tmp_file = None
uri = uri or str()
uri = uri.encode('utf-8')
log.debug("FileObject %r, Basepath: %r", uri, basepath)
# Data URI
if isinstance(uri, binary_type):
uri = uri.decode('utf-8')
if uri.startswith("data:"):
m = _rx_datauri.match(uri)
self.mimetype = m.group("mime")
self.data = base64.decodestring(m.group("data"))
else:
# Check if we have an external scheme
if basepath and not urlparse(uri).scheme:
urlParts = urlparse(basepath)
else:
urlParts = urlparse(uri)
log.debug("URLParts: %r", urlParts)
if urlParts.scheme == 'file':
if basepath and uri.startswith('/'):
uri = urljoin(basepath, uri[1:])
urlResponse = urlopen(uri)
self.mimetype = urlResponse.info().get(
"Content-Type", '').split(";")[0]
self.uri = urlResponse.geturl()
self.file = urlResponse
# Drive letters have len==1 but we are looking
# for things like http:
elif urlParts.scheme in ('http', 'https'):
# External data
if basepath:
uri = urljoin(basepath, uri)
#path = urlsplit(url)[2]
#mimetype = getMimeType(path)
# Using HTTPLIB
parts = urlsplit(uri[uri.find("//"):])
server, path = parts.netloc, parts.path
if uri.startswith("https://"):
conn = HTTPSConnection(server)
else:
conn = HTTPConnection(server)
conn.request("GET", path)
r1 = conn.getresponse()
# log.debug("HTTP %r %r %r %r", server, path, uri, r1)
if (r1.status, r1.reason) == (200, "OK"):
self.mimetype = r1.getheader(
"Content-Type", '').split(";")[0]
self.uri = uri
if r1.getheader("content-encoding") == "gzip":
import gzip
self.file = gzip.GzipFile(mode="rb", fileobj=BytesIO(r1.read()))
else:
self.file = r1
else:
try:
urlResponse = urlopen(uri)
except HTTPError:
return
self.mimetype = urlResponse.info().get(
"Content-Type", '').split(";")[0]
self.uri = urlResponse.geturl()
self.file = urlResponse
else:
# Local data
if basepath:
uri = os.path.normpath(os.path.join(basepath, uri))
if os.path.isfile(uri):
self.uri = uri
self.local = uri
self.setMimeTypeByName(uri)
self.file = open(uri, "rb")
def retry_using_http_NTLM_auth(self, req, auth_header_field, realm, headers):
user, pw = self.passwd.find_user_password(realm, req.get_full_url())
if pw is not None:
user_parts = user.split('\\', 1)
if len(user_parts) == 1:
UserName = user_parts[0]
DomainName = ''
type1_flags = ntlm.NTLM_TYPE1_FLAGS & ~ntlm.NTLM_NegotiateOemDomainSupplied
else:
DomainName = user_parts[0].upper()
UserName = user_parts[1]
type1_flags = ntlm.NTLM_TYPE1_FLAGS
# ntlm secures a socket, so we must use the same socket for the complete handshake
headers = dict(req.headers)
headers.update(req.unredirected_hdrs)
auth = 'NTLM %s' % ntlm.create_NTLM_NEGOTIATE_MESSAGE(user, type1_flags)
if req.headers.get(self.auth_header, None) == auth:
return None
headers[self.auth_header] = auth
host = req.host
if not host:
raise urllib.request.URLError('no host given')
h = None
if req.get_full_url().startswith('https://'):
h = HTTPSConnection(host) # will parse host:port
else:
h = HTTPConnection(host) # will parse host:port
h.set_debuglevel(self._debuglevel)
# we must keep the connection because NTLM authenticates the connection, not single requests
headers["Connection"] = "Keep-Alive"
headers = dict((name.title(), val) for name, val in headers.items())
# For some reason, six doesn't do this translation correctly
# TODO rsanders low - find bug in six & fix it
try:
selector = req.selector
except AttributeError:
selector = req.get_selector()
h.request(req.get_method(), selector, req.data, headers)
r = h.getresponse()
r.begin()
r._safe_read(int(r.getheader('content-length')))
if r.getheader('set-cookie'):
# this is important for some web applications that store authentication-related info in cookies (it took a long time to figure out)
headers['Cookie'] = r.getheader('set-cookie')
r.fp = None # remove the reference to the socket, so that it can not be closed by the response object (we want to keep the socket open)
auth_header_value = r.getheader(auth_header_field, None)
# some Exchange servers send two WWW-Authenticate headers, one with the NTLM challenge
# and another with the 'Negotiate' keyword - make sure we operate on the right one
m = re.match('(NTLM [A-Za-z0-9+\-/=]+)', auth_header_value)
if m:
auth_header_value, = m.groups()
(ServerChallenge, NegotiateFlags) = ntlm.parse_NTLM_CHALLENGE_MESSAGE(auth_header_value[5:])
auth = 'NTLM %s' % ntlm.create_NTLM_AUTHENTICATE_MESSAGE(ServerChallenge, UserName, DomainName, pw,
NegotiateFlags)
headers[self.auth_header] = auth
headers["Connection"] = "Close"
headers = dict((name.title(), val) for name, val in headers.items())
try:
h.request(req.get_method(), selector, req.data, headers)
# none of the configured handlers are triggered, for example redirect-responses are not handled!
response = h.getresponse()
def notimplemented():
raise NotImplementedError
response.readline = notimplemented
infourl = urllib.response.addinfourl(response, response.msg, req.get_full_url())
infourl.code = response.status
infourl.msg = response.reason
return infourl
except socket.error as err:
raise urllib.error.URLError(err)
else:
return None
def _send_pushover(self,
msg,
title,
sound=None,
user_key=None,
api_key=None,
priority=None):
"""
Send a pushover notification to the address provided.
:param msg: The message to send (unicode)
:param title: The title of the message
:param sound: The notification sound to use
:param user_key: The pushover user id to send the message to (or to subscribe with)
:param api_key: The pushover api key to use
:param priority: The pushover priority to use
:return: True if the message succeeded, False otherwise
"""
if user_key is None:
user_key = app.PUSHOVER_USERKEY
if api_key is None:
api_key = app.PUSHOVER_APIKEY
if sound is None:
sound = app.PUSHOVER_SOUND
if priority is None:
priority = app.PUSHOVER_PRIORITY
# build up the URL and parameters
msg = msg.strip()
# default args
args = {
'token': api_key,
'user': user_key,
'title': title.encode('utf-8'),
'message': msg.encode('utf-8'),
'timestamp': int(time.time()),
'retry': 60,
'expire': 3600,
'priority': priority,
}
# If sound is not default, add it.
if sound != 'default':
args['sound'] = sound
if app.PUSHOVER_DEVICE:
args['device'] = ','.join(app.PUSHOVER_DEVICE)
log.debug(
'PUSHOVER: Sending notice with details: title="{0}" message="{1}", priority={2}, sound={3}',
args['title'], args['message'], priority, sound)
conn = HTTPSConnection('api.pushover.net:443')
conn.request('POST', '/1/messages.json', urlencode(args),
{'Content-type': 'application/x-www-form-urlencoded'})
conn_resp = conn.getresponse()
if conn_resp.status == 200:
log.info('Pushover notification successful.')
return True
# HTTP status 404 if the provided email address isn't a Pushover user.
elif conn_resp.status == 404:
log.warning(
'Username is wrong/not a pushover email. Pushover will send an email to it'
)
return False
# For HTTP status code 401's, it is because you are passing in either an invalid token, or the user has not added your service.
elif conn_resp.status == 401:
# HTTP status 401 if the user doesn't have the service added
subscribe_note = self._send_pushover(msg,
title,
sound=sound,
user_key=user_key,
api_key=api_key)
if subscribe_note:
log.debug('Subscription sent')
return True
else:
log.error('Subscription could not be sent')
# If you receive an HTTP status code of 400, it is because you failed to send the proper parameters
elif conn_resp.status == 400:
log.error('Wrong keys sent to pushover')
return False
# If you receive a HTTP status code of 429, it is because the message limit has been reached (free limit is 7,500)
elif conn_resp.status == 429:
log.error(
'Pushover API message limit reached - try a different API key')
return False
# Something else has gone wrong... who knows what's really happening
else:
log.error('Pushover notification failed. HTTP response code: {0}',
conn_resp.status)
return False
def retry_using_http_NTLM_auth(self, req, auth_header_field, realm, headers):
user, pw = self.passwd.find_user_password(realm, req.get_full_url())
if pw is not None:
user_parts = user.split('\\', 1)
if len(user_parts) == 1:
UserName = user_parts[0]
DomainName = ''
type1_flags = ntlm.NTLM_TYPE1_FLAGS & ~ntlm.NTLM_NegotiateOemDomainSupplied
else:
DomainName = user_parts[0].upper()
UserName = user_parts[1]
type1_flags = ntlm.NTLM_TYPE1_FLAGS
# ntlm secures a socket, so we must use the same socket for the complete handshake
headers = dict(req.headers)
headers.update(req.unredirected_hdrs)
auth = 'NTLM %s' % ntlm.create_NTLM_NEGOTIATE_MESSAGE(user, type1_flags).decode("utf-8")
if req.headers.get(self.auth_header, None) == auth:
return None
headers[self.auth_header] = auth
host = req.host
if not host:
raise urllib.request.URLError('no host given')
h = None
if req.get_full_url().startswith('https://'):
h = HTTPSConnection(host) # will parse host:port
else:
h = HTTPConnection(host) # will parse host:port
h.set_debuglevel(self._debuglevel)
# we must keep the connection because NTLM authenticates the connection, not single requests
headers["Connection"] = "Keep-Alive"
headers = dict((name.title(), val) for name, val in headers.items())
# For some reason, six doesn't do this translation correctly
# TODO rsanders low - find bug in six & fix it
try:
selector = req.selector
except AttributeError:
selector = req.get_selector()
h.request(req.get_method(), selector, req.data, headers)
r = h.getresponse()
r.begin()
r._safe_read(int(r.getheader('content-length')))
if r.getheader('set-cookie'):
# this is important for some web applications that store authentication-related info in cookies (it took a long time to figure out)
headers['Cookie'] = r.getheader('set-cookie')
r.fp = None # remove the reference to the socket, so that it can not be closed by the response object (we want to keep the socket open)
auth_header_value = r.getheader(auth_header_field, None)
# some Exchange servers send two WWW-Authenticate headers, one with the NTLM challenge
# and another with the 'Negotiate' keyword - make sure we operate on the right one
m = re.match('(NTLM [A-Za-z0-9+\-/=]+)', auth_header_value)
if m:
auth_header_value, = m.groups()
(ServerChallenge, NegotiateFlags) = ntlm.parse_NTLM_CHALLENGE_MESSAGE(auth_header_value[5:])
auth = 'NTLM %s' % ntlm.create_NTLM_AUTHENTICATE_MESSAGE(ServerChallenge, UserName, DomainName, pw,
NegotiateFlags).decode("utf-8")
headers[self.auth_header] = auth
headers["Connection"] = "Close"
headers = dict((name.title(), val) for name, val in headers.items())
try:
h.request(req.get_method(), selector, req.data, headers)
# none of the configured handlers are triggered, for example redirect-responses are not handled!
response = h.getresponse()
def notimplemented():
raise NotImplementedError
response.readline = notimplemented
infourl = urllib.response.addinfourl(response, response.msg, req.get_full_url())
infourl.code = response.status
infourl.msg = response.reason
return infourl
except socket.error as err:
raise urllib.error.URLError(err)
else:
return None
def _sendPushover(self, msg, title, sound=None, userKey=None, apiKey=None):
"""
Sends a pushover notification to the address provided
msg: The message to send (unicode)
title: The title of the message
sound: The notification sound to use
userKey: The pushover user id to send the message to (or to subscribe with)
apiKey: The pushover api key to use
returns: True if the message succeeded, False otherwise
"""
if userKey is None:
userKey = app.PUSHOVER_USERKEY
if apiKey is None:
apiKey = app.PUSHOVER_APIKEY
if sound is None:
sound = app.PUSHOVER_SOUND
logger.log(u"Pushover API KEY in use: " + apiKey, logger.DEBUG)
# build up the URL and parameters
msg = msg.strip()
# send the request to pushover
try:
if app.PUSHOVER_SOUND != "default":
args = {
"token": apiKey,
"user": userKey,
"title": title.encode('utf-8'),
"message": msg.encode('utf-8'),
"timestamp": int(time.time()),
"retry": 60,
"expire": 3600,
"sound": sound,
}
else:
# sound is default, so don't send it
args = {
"token": apiKey,
"user": userKey,
"title": title.encode('utf-8'),
"message": msg.encode('utf-8'),
"timestamp": int(time.time()),
"retry": 60,
"expire": 3600,
}
if app.PUSHOVER_DEVICE:
args["device"] = app.PUSHOVER_DEVICE
conn = HTTPSConnection("api.pushover.net:443")
conn.request("POST", "/1/messages.json",
urlencode(args), {"Content-type": "application/x-www-form-urlencoded"})
except HTTPError as e:
# if we get an error back that doesn't have an error code then who knows what's really happening
if not hasattr(e, 'code'):
logger.log(u"Pushover notification failed." + ex(e), logger.ERROR)
return False
else:
logger.log(u"Pushover notification failed. Error code: " + str(e.code), logger.ERROR)
# HTTP status 404 if the provided email address isn't a Pushover user.
if e.code == 404:
logger.log(u"Username is wrong/not a pushover email. Pushover will send an email to it", logger.WARNING)
return False
# For HTTP status code 401's, it is because you are passing in either an invalid token, or the user has not added your service.
elif e.code == 401:
# HTTP status 401 if the user doesn't have the service added
subscribeNote = self._sendPushover(msg, title, sound=sound, userKey=userKey, apiKey=apiKey)
if subscribeNote:
logger.log(u"Subscription sent", logger.DEBUG)
return True
else:
logger.log(u"Subscription could not be sent", logger.ERROR)
return False
# If you receive an HTTP status code of 400, it is because you failed to send the proper parameters
elif e.code == 400:
logger.log(u"Wrong data sent to pushover", logger.ERROR)
return False
# If you receive a HTTP status code of 429, it is because the message limit has been reached (free limit is 7,500)
elif e.code == 429:
logger.log(u"Pushover API message limit reached - try a different API key", logger.ERROR)
return False
logger.log(u"Pushover notification successful.", logger.INFO)
return True
def _update_index(self, url=None):
"""A helper function that ensures that self._index is
up-to-date. If the index is older than self.INDEX_TIMEOUT,
then download it again."""
# Check if the index is aleady up-to-date. If so, do nothing.
if not (self._index is None or url is not None or
time.time()-self._index_timestamp > self.INDEX_TIMEOUT):
return
# If a URL was specified, then update our URL.
self._url = url or self._url
source = self._source
assert source == 'google' or source == 'mirror'
# Download the index file.
if source == 'google':
host = "www.googleapis.com"
conn = HTTPSConnection(host)
conn.request("GET", "/storage/v1/b/{}/o".format(self._url))
r1 = conn.getresponse()
data = r1.read()
elif source == 'mirror':
index_url = path.join(self._url, 'index.json')
data = urlopen(index_url).read()
if six.PY3:
data = data.decode('utf-8')
data = loads(data)
objs = data["items"]
self._index_timestamp = time.time()
# Build a dictionary of packages.
packages = []
for p in objs:
P = Package.fromcsobj(p)
packages.append(P)
self._packages = dict((p.id, p) for p in packages)
# Build language collections.
langs = defaultdict(lambda: [])
for k in self._packages:
package = self._packages[k]
langs[package.language].append(package)
tasks = defaultdict(lambda: [])
for k in self._packages:
package = self._packages[k]
tasks[package.task].append(package)
collections = []
for lang in langs:
children = langs[lang]
name1 = Locale(lang).getDisplayLanguage()
try:
name2 = isoLangs[lang]['name']
except:
name2 = None
if name1 and name1 != lang:
name = name1
elif name2:
name = name2
else:
name = lang
id = "{}{}".format(Downloader.LANG_PREFIX, lang)
name = "{:<20} packages and models".format(name)
c = Collection(id=id, name=name, children=children)
collections.append(c)
for task in tasks:
children = tasks[task]
id = "{}{}".format(Downloader.TASK_PREFIX, task)
c = Collection(id=id, name=task, children=children)
collections.append(c)
self._collections = dict((c.id, c) for c in collections)
# Replace identifiers with actual children in collection.children.
for collection in self._collections.values():
for i, child_id in enumerate(collection.children):
if child_id in self._packages:
collection.children[i] = self._packages[child_id]
if child_id in self._collections:
collection.children[i] = self._collections[child_id]
# Fill in collection.packages for each collection.
for collection in self._collections.values():
packages = {}
queue = [collection]
for child in queue:
if isinstance(child, Collection):
queue.extend(child.children)
else:
packages[child.id] = child
collection.packages = packages.values()
# Flush the status cache
self._status_cache.clear()
# your CA certificate of the engine in the system, you need to pass it to HTTPSConnection.
context.load_verify_locations(cafile='ca.pem')
proxy_connection = HTTPSConnection(
proxy_url.hostname,
proxy_url.port,
context=context,
)
print("Downloading image...")
try:
# Send the request
proxy_connection.request(
'GET',
proxy_url.path,
headers=transfer_headers,
)
# Get response
r = proxy_connection.getresponse()
# Check the response status
if r.status not in (200, 204):
print("Error downloding (%s)" % (r.reason,))
try:
data = r.read(512)
except (EnvironmentError, HTTPException):
pass
else:
print("Response:")
print(data)
def _sendPushover(self, msg, title, sound=None, userKey=None, apiKey=None):
"""
Sends a pushover notification to the address provided
msg: The message to send (unicode)
title: The title of the message
sound: The notification sound to use
userKey: The pushover user id to send the message to (or to subscribe with)
apiKey: The pushover api key to use
returns: True if the message succeeded, False otherwise
"""
if userKey is None:
userKey = app.PUSHOVER_USERKEY
if apiKey is None:
apiKey = app.PUSHOVER_APIKEY
if sound is None:
sound = app.PUSHOVER_SOUND
log.debug(u'Pushover API KEY in use: {0}', apiKey)
# build up the URL and parameters
msg = msg.strip()
# send the request to pushover
try:
if app.PUSHOVER_SOUND != 'default':
args = {
'token': apiKey,
'user': userKey,
'title': title.encode('utf-8'),
'message': msg.encode('utf-8'),
'timestamp': int(time.time()),
'retry': 60,
'expire': 3600,
'sound': sound,
}
else:
# sound is default, so don't send it
args = {
'token': apiKey,
'user': userKey,
'title': title.encode('utf-8'),
'message': msg.encode('utf-8'),
'timestamp': int(time.time()),
'retry': 60,
'expire': 3600,
}
if app.PUSHOVER_DEVICE:
args['device'] = ','.join(app.PUSHOVER_DEVICE)
conn = HTTPSConnection('api.pushover.net:443')
conn.request('POST', '/1/messages.json', urlencode(args),
{'Content-type': 'application/x-www-form-urlencoded'})
except HTTPError as e:
# if we get an error back that doesn't have an error code then who knows what's really happening
if not hasattr(e, 'code'):
log.error(u'Pushover notification failed. {}', ex(e))
return False
else:
log.error(u'Pushover notification failed. Error code: {0}',
e.code)
# HTTP status 404 if the provided email address isn't a Pushover user.
if e.code == 404:
log.warning(
u'Username is wrong/not a pushover email. Pushover will send an email to it'
)
return False
# For HTTP status code 401's, it is because you are passing in either an invalid token, or the user has not added your service.
elif e.code == 401:
# HTTP status 401 if the user doesn't have the service added
subscribeNote = self._sendPushover(msg,
title,
sound=sound,
userKey=userKey,
apiKey=apiKey)
if subscribeNote:
log.debug(u'Subscription sent')
return True
else:
log.error(u'Subscription could not be sent')
return False
# If you receive an HTTP status code of 400, it is because you failed to send the proper parameters
elif e.code == 400:
log.error(u'Wrong data sent to pushover')
return False
# If you receive a HTTP status code of 429, it is because the message limit has been reached (free limit is 7,500)
elif e.code == 429:
log.error(
u'Pushover API message limit reached - try a different API key'
)
return False
log.info(u'Pushover notification successful.')
return True
def _send_pushover(self, msg, title, sound=None, user_key=None, api_key=None, priority=None):
"""
Send a pushover notification to the address provided.
:param msg: The message to send (unicode)
:param title: The title of the message
:param sound: The notification sound to use
:param user_key: The pushover user id to send the message to (or to subscribe with)
:param api_key: The pushover api key to use
:param priority: The pushover priority to use
:return: True if the message succeeded, False otherwise
"""
if user_key is None:
user_key = app.PUSHOVER_USERKEY
if api_key is None:
api_key = app.PUSHOVER_APIKEY
if sound is None:
sound = app.PUSHOVER_SOUND
if priority is None:
priority = app.PUSHOVER_PRIORITY
# build up the URL and parameters
msg = msg.strip()
# default args
args = {
'token': api_key,
'user': user_key,
'title': title.encode('utf-8'),
'message': msg.encode('utf-8'),
'timestamp': int(time.time()),
'retry': 60,
'expire': 3600,
'priority': priority,
}
# If sound is not default, add it.
if sound != 'default':
args['sound'] = sound
if app.PUSHOVER_DEVICE:
args['device'] = ','.join(app.PUSHOVER_DEVICE)
log.debug('PUSHOVER: Sending notice with details: title="{0}" message="{1}", priority={2}, sound={3}',
title, msg, priority, sound)
conn = HTTPSConnection('api.pushover.net:443')
conn.request('POST', '/1/messages.json',
urlencode(args), {'Content-type': 'application/x-www-form-urlencoded'})
conn_resp = conn.getresponse()
if conn_resp.status == 200:
log.info('Pushover notification successful.')
return True
# HTTP status 404 if the provided email address isn't a Pushover user.
elif conn_resp.status == 404:
log.warning('Username is wrong/not a pushover email. Pushover will send an email to it')
return False
# For HTTP status code 401's, it is because you are passing in either an invalid token, or the user has not added your service.
elif conn_resp.status == 401:
# HTTP status 401 if the user doesn't have the service added
subscribe_note = self._send_pushover(msg, title, sound=sound, user_key=user_key, api_key=api_key)
if subscribe_note:
log.debug('Subscription sent')
return True
else:
log.error('Subscription could not be sent')
# If you receive an HTTP status code of 400, it is because you failed to send the proper parameters
elif conn_resp.status == 400:
log.error('Wrong keys sent to pushover')
return False
# If you receive a HTTP status code of 429, it is because the message limit has been reached (free limit is 7,500)
elif conn_resp.status == 429:
log.error('Pushover API message limit reached - try a different API key')
return False
# Something else has gone wrong... who knows what's really happening
else:
log.error('Pushover notification failed. HTTP response code: {0}', conn_resp.status)
return False
def __init__(self, uri, basepath=None):
self.basepath = basepath
self.mimetype = None
self.file = None
self.data = None
self.uri = None
self.local = None
self.tmp_file = None
uri = uri or str()
uri = uri.encode('utf-8')
log.debug("FileObject %r, Basepath: %r", uri, basepath)
# Data URI
if isinstance(uri, binary_type):
uri = uri.decode('utf-8')
if uri.startswith("data:"):
m = _rx_datauri.match(uri)
self.mimetype = m.group("mime")
self.data = base64.decodestring(m.group("data"))
else:
# Check if we have an external scheme
if basepath and not urlparse(uri).scheme:
urlParts = urlparse(basepath)
else:
urlParts = urlparse(uri)
log.debug("URLParts: %r", urlParts)
if urlParts.scheme == 'file':
if basepath and uri.startswith('/'):
uri = urljoin(basepath, uri[1:])
urlResponse = urlopen(uri)
self.mimetype = urlResponse.info().get("Content-Type",
'').split(";")[0]
self.uri = urlResponse.geturl()
self.file = urlResponse
# Drive letters have len==1 but we are looking
# for things like http:
elif urlParts.scheme in ('http', 'https'):
# External data
if basepath:
uri = urljoin(basepath, uri)
#path = urlsplit(url)[2]
#mimetype = getMimeType(path)
# Using HTTPLIB
parts = urlsplit(uri[uri.find("//"):])
server, path = parts.netloc, parts.path
if uri.startswith("https://"):
conn = HTTPSConnection(server)
else:
conn = HTTPConnection(server)
conn.request("GET", path)
r1 = conn.getresponse()
# log.debug("HTTP %r %r %r %r", server, path, uri, r1)
if (r1.status, r1.reason) == (200, "OK"):
self.mimetype = r1.getheader("Content-Type",
'').split(";")[0]
self.uri = uri
if r1.getheader("content-encoding") == "gzip":
import gzip
self.file = gzip.GzipFile(mode="rb",
fileobj=BytesIO(r1.read()))
else:
self.file = r1
else:
try:
urlResponse = urlopen(uri)
except HTTPError:
return
self.mimetype = urlResponse.info().get("Content-Type",
'').split(";")[0]
self.uri = urlResponse.geturl()
self.file = urlResponse
else:
# Local data
if basepath:
uri = os.path.normpath(os.path.join(basepath, uri))
if os.path.isfile(uri):
self.uri = uri
self.local = uri
self.setMimeTypeByName(uri)
self.file = open(uri, "rb")
def _sendPushover(self, msg, title, sound=None, userKey=None, apiKey=None):
"""
Sends a pushover notification to the address provided
msg: The message to send (unicode)
title: The title of the message
sound: The notification sound to use
userKey: The pushover user id to send the message to (or to subscribe with)
apiKey: The pushover api key to use
returns: True if the message succeeded, False otherwise
"""
if userKey is None:
userKey = sickbeard.PUSHOVER_USERKEY
if apiKey is None:
apiKey = sickbeard.PUSHOVER_APIKEY
if sound is None:
sound = sickbeard.PUSHOVER_SOUND
logger.log(u"Pushover API KEY in use: " + apiKey, logger.DEBUG)
# build up the URL and parameters
msg = msg.strip()
# send the request to pushover
try:
if sickbeard.PUSHOVER_SOUND != "default":
args = {
"token": apiKey,
"user": userKey,
"title": title.encode('utf-8'),
"message": msg.encode('utf-8'),
"timestamp": int(time.time()),
"retry": 60,
"expire": 3600,
"sound": sound,
}
else:
# sound is default, so don't send it
args = {
"token": apiKey,
"user": userKey,
"title": title.encode('utf-8'),
"message": msg.encode('utf-8'),
"timestamp": int(time.time()),
"retry": 60,
"expire": 3600,
}
if sickbeard.PUSHOVER_DEVICE:
args["device"] = sickbeard.PUSHOVER_DEVICE
conn = HTTPSConnection("api.pushover.net:443")
conn.request("POST", "/1/messages.json",
urlencode(args), {"Content-type": "application/x-www-form-urlencoded"})
except HTTPError as e:
# if we get an error back that doesn't have an error code then who knows what's really happening
if not hasattr(e, 'code'):
logger.log(u"Pushover notification failed." + ex(e), logger.ERROR)
return False
else:
logger.log(u"Pushover notification failed. Error code: " + str(e.code), logger.ERROR)
# HTTP status 404 if the provided email address isn't a Pushover user.
if e.code == 404:
logger.log(u"Username is wrong/not a pushover email. Pushover will send an email to it", logger.WARNING)
return False
# For HTTP status code 401's, it is because you are passing in either an invalid token, or the user has not added your service.
elif e.code == 401:
# HTTP status 401 if the user doesn't have the service added
subscribeNote = self._sendPushover(msg, title, sound=sound, userKey=userKey, apiKey=apiKey)
if subscribeNote:
logger.log(u"Subscription sent", logger.DEBUG)
return True
else:
logger.log(u"Subscription could not be sent", logger.ERROR)
return False
# If you receive an HTTP status code of 400, it is because you failed to send the proper parameters
elif e.code == 400:
logger.log(u"Wrong data sent to pushover", logger.ERROR)
return False
# If you receive a HTTP status code of 429, it is because the message limit has been reached (free limit is 7,500)
elif e.code == 429:
logger.log(u"Pushover API message limit reached - try a different API key", logger.ERROR)
return False
logger.log(u"Pushover notification successful.", logger.INFO)
return True
def fetch_response(self, request):
"""Executes request and fetchs service response"""
connection = HTTPSConnection(self.SERVER_URL)
connection.request(request.method.upper(), request.to_url())
return connection.getresponse().read()
class TestLicenseClient(object):
def __init__(self, parser=None):
# type: (Optional[ArgumentParser]) -> None
"""
Class constructor for the test license client and HTMLParser
"""
self.log = logging.getLogger("License_Client")
self.setup_logging()
self.parser = parser
self.license_server_url = 'license.univention.de'
self.license_filename = 'ValidTest.license'
self.connection = None # type: Optional[HTTPSConnection]
self.server_username = '******'
self.server_password = ''
self.secret_file = '/etc/license.secret'
self.cookie = ''
self.license_shop = 'testing'
self.license_params = {
"kundeUnternehmen": "Univention",
"kundeEmail": "*****@*****.**",
"BaseDN": "",
"EndDate": "",
"Servers": 50,
"Support": 0,
"PremiumSupport": 0,
"Users": 50,
"ManagedClients": 50,
"CorporateClients": 50,
"VirtualDesktopUsers": 0,
"VirtualDesktopClients": 0,
"Type": "UCS",
} # type: Dict[str, Any]
def setup_logging(self):
# type: () -> None
"""
Creates and configures the logger with an INFO level
"""
self.log.setLevel(logging.INFO)
ch = logging.StreamHandler()
ch.setLevel(logging.DEBUG)
ch.setFormatter(
logging.Formatter(
"%(asctime)s - %(name)s - %(levelname)s - %(message)s"))
self.log.addHandler(ch)
def create_connection(self):
# type: () -> None
"""
Creates a HTTPS connection instance on a default port (443)
to the 'self.license_server_url'
"""
self.log.debug("In 'create_connection'")
self.connection = HTTPSConnection(self.license_server_url)
def close_connection(self):
# type: () -> None
"""
Closes the license server connection if the connection instance
was created
"""
self.log.debug("In 'close_connection'")
if self.connection:
try:
self.connection.close()
except HTTPException as exc:
self.log.exception(
"An HTTP Exception occurred while closing the connection: '%s'",
exc)
def get_server_password(self, secret_file='/etc/license.secret'):
# type: (str) -> None
"""
Opens and reads the 'secret_file'. Saves the result to a
'self.server_password'
"""
self.log.debug("In 'get_server_password': secret_file='%s'",
secret_file)
if not path.exists(secret_file):
self.log.critical(
"The '%s' secret file does not exist, cannot proceed without password",
secret_file)
raise CredentialsMissing("The '%s' secret file does not exist" %
secret_file)
try:
with open(secret_file, 'r') as password:
self.server_password = password.read()
except (IOError, ValueError) as exc:
self.log.exception(
"Failed to get the password from the '%s', an error occurred: %r",
secret_file, exc)
exit(1)
if not self.server_password:
self.log.critical(
"The password to access the license service cannot be empty")
exit(1)
def get_cookie(self):
# type: () -> None
"""
Makes a POST request with 'self.server_username' and
'self.server_password' into login forms and saves the
cookie from the response received.
"""
self.log.debug("In 'get_cookie' method")
body = {
"username": self.server_username,
"password": self.server_password,
}
headers = {
"Content-type": "application/x-www-form-urlencoded",
"Accept": "text/plain",
}
response = self.make_post_request('/shop/%s/' % (self.license_shop, ),
urlencode(body), headers)
self.log.debug(
"The response status is '%s', reason is '%s', headers are '%s'",
response.status, response.reason, response.getheaders())
self.cookie = response.getheader('set-cookie')
if not ('sessionid' in self.cookie):
self.log.critical(
"The 'sessionid' field was not found in the received cookie: '%s'",
self.cookie)
exit(1)
# extracting only the 'sessionid' part of the cookie:
self.cookie = self.cookie[:self.cookie.find(';')]
self.log.debug("The cookie is: '%s'", self.cookie)
# reading the response to avoid 'ResponseNotReady' exception later:
response.read()
def make_post_request(self, url, body, headers):
# type: (str, str, Dict[str, str]) -> HTTPResponse
"""
Makes a POST request with the given 'url', 'body', 'headers' and
returns the response
"""
self.log.debug(
"In 'make_post_request' method: url='%s', body='%s', headers='%s'",
url, body, headers)
assert self.connection
try:
self.connection.request("POST", url, body, headers)
return self.connection.getresponse()
except HTTPException as exc:
self.log.exception(
"An HTTP Exception occurred while making '%s' POST request: '%s'",
url, exc)
exit(1)
def make_get_request(self, url, headers):
# type: (str, Dict[str, str]) -> HTTPResponse
"""
Makes a GET request with the given 'url', 'headers' and
returns the response
"""
self.log.debug("In 'make_get_request' method: url='%s', headers='%s'",
url, headers)
assert self.connection
try:
self.connection.request("GET", url, headers=headers)
return self.connection.getresponse()
except HTTPException as exc:
self.log.exception(
"An HTTP Exception occurred while making '%s' GET request: '%s'",
url, exc)
exit(1)
def get_the_license(self, body):
# type: (str) -> None
"""
Processes the given 'body' with HTMLParser to find the link
to a created license file and downloads the license after.
"""
self.log.debug("In 'get_the_license' method: body='%s'", body)
parser = ShopParser(self.log)
parser.feed(body) # process the response 'body' for license link
if not parser.link_to_license:
self.log.critical(
"The link to the license file was not found in the body from server: '%s'",
body)
exit(1)
self.download_license_file(parser.link_to_license)
def order_a_license(self):
# type: () -> str
"""
Makes a POST request with encoded 'self.license_params' as a body to
order a new license. Returns the response body.
"""
self.log.debug("In 'order_a_license' method")
body = self.license_params
headers = {
"Cookie": self.cookie,
"Content-type": "application/x-www-form-urlencoded",
}
response = self.make_post_request(
'/shop/%s/order' % (self.license_shop, ), urlencode(body), headers)
assert response.status == 202
return self.get_body(response)
def get_body(self, response):
# type: (HTTPResponse) -> str
self.log.debug(
"The response status is '%s', reason is '%s', headers are '%s'",
response.status, response.reason, response.getheaders())
content_type = response.getheader('Content-Type')
mimetype, options = cgi.parse_header(content_type)
encoding = options.get("charset", "ascii")
return response.read().decode(encoding, "replace")
def download_license_file(self, link_to_license):
# type: (str) -> None
"""
Downloads the license located at `filename` and saves it
to the file with a 'self.license_filename'
"""
self.log.debug("In 'download_license_file' method")
headers = {
"Cookie": self.cookie,
"Accept": "text/plain",
}
response = self.make_get_request(
'/shop/%s/%s' % (self.license_shop, link_to_license), headers)
try:
body = self.get_body(response)
with open(self.license_filename, 'w') as license_file:
license_file.write(body)
self.log.info("The license was written to file '%s'",
self.license_filename)
except (IOError, ValueError) as exc:
self.log.exception(
"An error happened while writing the downloaded license to a file '%s': '%s'",
self.license_filename, exc)
exit(1)
def check_date_format(self):
# type: () -> None
"""
Checks if the 'EndDate' format is correct.
"""
try:
if self.license_params['EndDate'] != 'unlimited':
datetime.strptime(self.license_params['EndDate'], '%d.%m.%Y')
except ValueError as exc:
self.log.exception(
"The 'EndDate' for the license has a wrong format, supported format is 'dd.mm.yyyy': %r",
exc)
exit(1)
def update_with_parsed_args(self, args):
# type: (Dict[str, Any]) -> None
"""
Updates the loglevel and license filename settings if given
among the parsed arguments. Merges parsed data with default
license parameters.
"""
log_level = args.pop('LogLevel')
if log_level:
numeric_level = getattr(logging, log_level.upper(), None)
if isinstance(numeric_level, int):
self.log.setLevel(numeric_level)
else:
self.log.info(
"The LogLevel was not changed, unknown '%s' log level given",
log_level)
self.license_shop = args.pop('shop')
self.server_username = args.pop('username')
self.secret_file = args.pop('secret_file')
license_file = args.pop('FileName')
if license_file:
self.license_filename = license_file
self.log.debug("The filename for the license will be '%s'",
self.license_filename)
# merging parsed args with the default values:
self.license_params.update(
(key, val) for key, val in args.items() if val is not None)
self.log.info("Requested license parameters are: '%s'",
self.license_params)
def process_cmd_arguments(self):
# type: () -> None
"""
Populates self.parser class with positional and optional arguments and
processes the user input, checks the date format and than merges it
with the default values in the 'self.license_params' dictionary
"""
self.log.debug("In 'process_cmd_arguments' method")
assert self.parser
self.parser.add_argument("BaseDN", help="A base DN for the license")
self.parser.add_argument(
"EndDate",
help=
"The date till which the license will be valid (max 1 year from now)"
)
self.parser.add_argument(
"-f",
"--FileName",
help="The filename to be used for the issued license",
default="ValidTest.license")
self.parser.add_argument(
"-s",
"--Servers",
type=int,
help="Max amount of servers allowed with the license",
default=50)
self.parser.add_argument(
"-u",
"--Users",
type=int,
help="Max amount of users allowed with the license",
default=50)
self.parser.add_argument(
"-mc",
"--ManagedClients",
type=int,
help="Max amount of managed clients allowed with the license",
default=50)
self.parser.add_argument(
"-cc",
"--CorporateClients",
type=int,
help="Max amount of corporate clients allowed with the license",
default=50)
self.parser.add_argument("-ll",
"--LogLevel",
help="Logging level",
choices=("INFO", "DEBUG", "ERROR",
"CRITICAL"),
default="INFO")
self.parser.add_argument("--shop",
help="The shop",
default=self.license_shop)
self.parser.add_argument("--username",
help="username",
default=self.server_username)
self.parser.add_argument("--secret-file",
help="password file",
default=self.secret_file)
opts = self.parser.parse_args()
args = vars(opts) # converting Namespace to a dictionary
self.log.debug("Parsed arguments are: '%s'", args)
self.update_with_parsed_args(args)
def main(self, base_dn="", end_date="", server_url="", license_file=""):
# type: (str, str, str, str) -> None
"""
A method to order and download a test license from the license server.
'base_dn' and 'end_date' should be provided if argument parser is
not used.
'server_url' is an optional argument for the license shop server.
'license_file' is an optional argument for the license filename.
"""
self.log.debug(
"In 'main' method: server_url='%s', license_file='%s', base_dn='%s', end_date='%s'",
server_url, license_file, base_dn, end_date)
if self.parser:
self.process_cmd_arguments()
elif base_dn and end_date:
self.license_params['BaseDN'] = base_dn
self.license_params['EndDate'] = end_date
else:
self.log.error(
"The 'BaseDN' or/and 'EndDate' were not provided for the license to create"
)
exit(1)
self.check_date_format()
if server_url:
self.license_server_url = server_url
if license_file:
self.license_filename = license_file
self.get_server_password(self.secret_file)
try:
self.create_connection()
self.get_cookie()
self.get_the_license(self.order_a_license())
finally:
self.close_connection()
def _update_index(self, url=None):
"""A helper function that ensures that self._index is
up-to-date. If the index is older than self.INDEX_TIMEOUT,
then download it again."""
# Check if the index is aleady up-to-date. If so, do nothing.
if not (self._index is None or url is not None or
time.time()-self._index_timestamp > self.INDEX_TIMEOUT):
return
# If a URL was specified, then update our URL.
self._url = url or self._url
# Download the index file.
host = "www.googleapis.com"
conn = HTTPSConnection(host)
conn.request("GET", "/storage/v1/b/{}/o".format(self._url))
r1 = conn.getresponse()
data = r1.read()
if six.PY3:
data = data.decode('utf-8')
data = loads(data)
objs = data["items"]
self._index_timestamp = time.time()
# Build a dictionary of packages.
packages = []
for p in objs:
P = Package.fromcsobj(p)
packages.append(P)
self._packages = dict((p.id, p) for p in packages)
# Build language collections.
langs = defaultdict(lambda: [])
for k in self._packages:
package = self._packages[k]
langs[package.language].append(package)
tasks = defaultdict(lambda: [])
for k in self._packages:
package = self._packages[k]
tasks[package.task].append(package)
collections = []
for lang in langs:
children = langs[lang]
name1 = Locale(lang).getDisplayLanguage()
try:
name2 = isoLangs[lang]['name']
except:
name2 = None
if name1 and name1 != lang:
name = name1
elif name2:
name = name2
else:
name = lang
id = "{}{}".format(Downloader.LANG_PREFIX, lang)
name = "{:<20} packages and models".format(name)
c = Collection(id=id, name=name, children=children)
collections.append(c)
for task in tasks:
children = tasks[task]
id = "{}{}".format(Downloader.TASK_PREFIX, task)
c = Collection(id=id, name=task, children=children)
collections.append(c)
self._collections = dict((c.id, c) for c in collections)
# Replace identifiers with actual children in collection.children.
for collection in self._collections.values():
for i, child_id in enumerate(collection.children):
if child_id in self._packages:
collection.children[i] = self._packages[child_id]
if child_id in self._collections:
collection.children[i] = self._collections[child_id]
# Fill in collection.packages for each collection.
for collection in self._collections.values():
packages = {}
queue = [collection]
for child in queue:
if isinstance(child, Collection):
queue.extend(child.children)
else:
packages[child.id] = child
collection.packages = packages.values()
# Flush the status cache
self._status_cache.clear()
# your CA certificate of the engine in the system, you need to pass it to HTTPSConnection.
context.load_verify_locations(cafile='ca.pem')
proxy_connection = HTTPSConnection(
proxy_url.hostname,
proxy_url.port,
context=context,
)
print("Downloading image...")
try:
# Send the request
proxy_connection.request(
'GET',
proxy_url.path,
headers=transfer_headers,
)
# Get response
r = proxy_connection.getresponse()
# Check the response status
if r.status not in (200, 204):
print("Error downloding (%s)" % (r.reason, ))
try:
data = r.read(512)
except (EnvironmentError, HTTPException):
pass
else:
print("Response:")
print(data)