def get_auth_token(host, clientid, clientsecret,
proxy_host=None, proxy_port=None):
queryurl = '/oauth/access_token'
if proxy_host and proxy_port:
connection = HTTPSConnection(proxy_host, proxy_port)
connection.set_tunnel(host, 443)
else:
connection = HTTPSConnection(host)
authtoken = base64.b64encode(six.b('{0}:{1}'.format(clientid, clientsecret)))
authstring = b"Basic %s" % (authtoken,)
header = {"Authorization": authstring}
params = urlencode({'grant_type': 'client_credentials'})
connection.request("POST", queryurl, params, header)
response = connection.getresponse()
jsondata = bytes(response.read()).decode('utf-8')
data = json.loads(str(jsondata))
try:
key = data['access_token']
except KeyError:
print("We're having trouble getting a session token. Please check your API key.")
print("Error output: ")
print(data)
sys.exit()
finally:
connection.close()
return key
def send_email(email, template, kwargs):
"""Send an email via the mailgun service."""
maildir = os.path.join(topdir, 'emails')
env = jinja2.Environment(loader=jinja2.FileSystemLoader(maildir))
template = env.get_template(template)
rendered = template.render(**kwargs)
headers, message = parse_email(rendered)
mailargs = {'to': email,
'from': app.config['MAIL_FROM'],
'bcc': app.config.get('MAIL_BCC'),
'text': message}
mailargs.update(headers)
conn = HTTPSConnection('api.mailgun.net', 443)
conn.connect()
auth = b64enc('api:{0[MAILGUN_KEY]}'.format(app.config))
headers = {'Authorization': 'Basic {0}'.format(auth),
'Accept': 'application/json',
'Content-type': 'application/x-www-form-urlencoded'}
url = '/v2/{0[MAILGUN_DOMAIN]}/messages'.format(app.config)
body = urlencode(mailargs)
conn.request('POST', url, body, headers)
resp = conn.getresponse()
if resp.status != 200:
raise RuntimeError('could not send email')
conn.close()
def _new_conn(self):
# Performs the NTLM handshake that secures the connection. The socket
# must be kept open while requests are performed.
self.num_connections += 1
log.debug('Starting NTLM HTTPS connection no. %d: https://%s%s',
self.num_connections, self.host, self.authurl)
headers = {'Connection': 'Keep-Alive'}
req_header = 'Authorization'
resp_header = 'www-authenticate'
conn = HTTPSConnection(host=self.host, port=self.port)
# Send negotiation message
headers[req_header] = (
'NTLM %s' % ntlm.create_NTLM_NEGOTIATE_MESSAGE(self.rawuser))
log.debug('Request headers: %s', headers)
conn.request('GET', self.authurl, None, headers)
res = conn.getresponse()
reshdr = dict(res.getheaders())
log.debug('Response status: %s %s', res.status, res.reason)
log.debug('Response headers: %s', reshdr)
log.debug('Response data: %s [...]', res.read(100))
# Remove the reference to the socket, so that it can not be closed by
# the response object (we want to keep the socket open)
res.fp = None
# Server should respond with a challenge message
auth_header_values = reshdr[resp_header].split(', ')
auth_header_value = None
for s in auth_header_values:
if s[:5] == 'NTLM ':
auth_header_value = s[5:]
if auth_header_value is None:
raise Exception('Unexpected %s response header: %s' %
(resp_header, reshdr[resp_header]))
# Send authentication message
ServerChallenge, NegotiateFlags = \
ntlm.parse_NTLM_CHALLENGE_MESSAGE(auth_header_value)
auth_msg = ntlm.create_NTLM_AUTHENTICATE_MESSAGE(
ServerChallenge, self.user, self.domain, self.pw, NegotiateFlags)
headers[req_header] = 'NTLM %s' % auth_msg
log.debug('Request headers: %s', headers)
conn.request('GET', self.authurl, None, headers)
res = conn.getresponse()
log.debug('Response status: %s %s', res.status, res.reason)
log.debug('Response headers: %s', dict(res.getheaders()))
log.debug('Response data: %s [...]', res.read()[:100])
if res.status != 200:
if res.status == 401:
raise Exception('Server rejected request: wrong '
'username or password')
raise Exception('Wrong server response: %s %s' %
(res.status, res.reason))
res.fp = None
log.debug('Connection established')
return conn
def make_request(method, uri, headers=None, body=None):
"""
This function should make a request, connection reuse, etc. is left to the implementation
:param str|unicode method: HTTP Verb
:param str|unicode uri: Server/path to make the request to
:param dict headers: Headers to send
:param str|unicode body: Body to send
:return: Request result
:rtype: Tuple of return_code (int), headers (list of (k,v) tuples), body (str|bytes)
"""
parsed_url = urlparse(uri)
if parsed_url.scheme == 'https':
connector = HTTPSConnection(host=parsed_url.netloc)
elif parsed_url.scheme == 'http':
connector = HTTPConnection(host=parsed_url.netloc)
else:
raise ValueError('Schema is not HTTP nor HTTPS')
_, full_path = uri.split(parsed_url.netloc, 1)
connector.request(method=method, url=full_path, body=body, headers=headers or {})
response = connector.getresponse()
status_code = response.status
body = response.read()
headers = response.getheaders()
return status_code, headers, body
def retry_http_krb_sspi_auth(self, host, req):
url = req.full_url
scheme, _, host, path = url.split('/', 3)
h = HTTPConnection(host) if scheme == 'http:' else HTTPSConnection(
host)
headers = dict(req.unredirected_hdrs)
headers.update(
dict((k, v) for k, v in req.headers.items() if k not in headers))
try:
__, krb_context = kerberos.authGSSClientInit("HTTP@" + host)
kerberos.authGSSClientStep(krb_context, "")
negotiate_details = kerberos.authGSSClientResponse(krb_context)
headers["Connection"] = "Keep-Alive"
headers["Authorization"] = "Negotiate " + negotiate_details
h.request(req.get_method(), req.selector, req.data, headers)
response = h.getresponse()
return addinfourl(response, response.msg, req.get_full_url(),
response.status)
except:
# e = sys.exc_info()[0]
# _log.warning(str(e))
# _log.warning('Failed Kerberos authentication')
return None
def getRemoteDBModifiedTS():
"""
Performs a HEAD request to get the Last-Modified date-time
of a database dump file and parses it into a UNIX timestamp.
"""
debug_msg = 'Unable to get timestamp of remote database dump - {0}'
logging.info("Getting timestamp of database dump at '%s'", HOST + PATH)
try:
# Removing the scheme from the URL
conn = HTTPSConnection(HOST[8:], timeout=TIMEOUT)
conn.request('HEAD', PATH)
except gaierror as e:
logging.debug(debug_msg.format("Cannot connect to '%s', error: %s"),
HOST + PATH, e)
exit(1)
rsp = conn.getresponse()
if rsp.status != 200:
logging.debug(debug_msg.format('Server responded with: %d %s'),
rsp.status, rsp.reason)
exit(1)
last_modified = rsp.getheader('last-modified', None)
if last_modified is None:
logging.debug(
debug_msg.format('Response doesnt include Last-Modified Header'))
exit(1)
last_m_dt = datetime.strptime(
last_modified.split(', ')[1], '%d %b %Y %H:%M:%S %Z')
return timegm(last_m_dt.timetuple())
def __init__(self,
service_uri,
credentials=None,
key_file=None,
cert_file=None,
timeout=10,
buffer_size=65535,
error_factory=lambda x: x,
overload=False,
version=1):
self.version = version
self._headers = list()
self._headers.append((
'User-Agent',
'python-hessian/' + __version__,
))
self._headers.append((
'Content-Type',
'application/x-hessian',
))
if sys.version_info < (2, 6):
warn('HessianProxy timeout not enforceable before Python 2.6',
RuntimeWarning,
stacklevel=2)
kwargs = {}
else:
kwargs = {'timeout': timeout}
if six.PY2:
kwargs['strict'] = True
self._uri = urlparse(service_uri)
if self._uri.scheme == 'http':
self._client = HTTPConnection(self._uri.hostname, self._uri.port
or 80, **kwargs)
elif self._uri.scheme == 'https':
self._client = HTTPSConnection(self._uri.hostname,
self._uri.port or 443,
key_file=key_file,
cert_file=cert_file,
**kwargs)
else:
raise NotImplementedError(
"HessianProxy only supports http:// and https:// URIs")
# autofill credentials if they were passed via url instead of kwargs
if (self._uri.username and self._uri.password) and not credentials:
credentials = (self._uri.username, self._uri.password)
if credentials:
auth = 'Basic ' + base64.b64encode(':'.join(credentials))
self._headers.append(('Authorization', auth))
self._buffer_size = buffer_size
self._error_factory = error_factory
self._overload = overload
self._parser = Parser()
def create_connection(self):
# type: () -> None
"""
Creates a HTTPS connection instance on a default port (443)
to the 'self.license_server_url'
"""
self.log.debug("In 'create_connection'")
self.connection = HTTPSConnection(self.license_server_url)
def _send_prowl(prowl_api=None,
prowl_priority=None,
event=None,
message=None,
force=False):
if not sickbeard.USE_PROWL and not force:
return False
if prowl_api is None:
prowl_api = sickbeard.PROWL_API
if len(prowl_api) == 0:
return False
if prowl_priority is None:
prowl_priority = sickbeard.PROWL_PRIORITY
title = sickbeard.PROWL_MESSAGE_TITLE
logger.log(
"PROWL: Sending notice with details: title=\"{0}\" event=\"{1}\", message=\"{2}\", priority={3}, api={4}"
.format(title, event, message, prowl_priority,
prowl_api), logger.DEBUG)
http_handler = HTTPSConnection("api.prowlapp.com")
data = {
'apikey': prowl_api,
'application': title,
'event': event,
'description': message.encode('utf-8'),
'priority': prowl_priority
}
try:
http_handler.request(
"POST",
"/publicapi/add",
headers={'Content-type': "application/x-www-form-urlencoded"},
body=urlencode(data))
except (SSLError, HTTPException, socket.error):
logger.log("Prowl notification failed.", logger.ERROR)
return False
response = http_handler.getresponse()
request_status = response.status
if request_status == 200:
logger.log("Prowl notifications sent.", logger.INFO)
return True
elif request_status == 401:
logger.log("Prowl auth failed: {0}".format(response.reason),
logger.ERROR)
return False
else:
logger.log("Prowl notification failed.", logger.ERROR)
return False
def _send_prowl(prowl_api=None,
prowl_priority=None,
event=None,
message=None,
force=False):
if not app.USE_PROWL and not force:
return False
if prowl_api is None:
prowl_api = ','.join(app.PROWL_API)
if not prowl_api:
return False
if prowl_priority is None:
prowl_priority = app.PROWL_PRIORITY
title = app.PROWL_MESSAGE_TITLE
log.debug(
u'PROWL: Sending notice with details: title="{0}" event="{1}", message="{2}", priority={3}, api={4}',
title, event, message, prowl_priority, prowl_api)
http_handler = HTTPSConnection('api.prowlapp.com')
data = {
'apikey': prowl_api,
'application': title,
'event': event,
'description': message.encode('utf-8'),
'priority': prowl_priority
}
try:
http_handler.request(
'POST',
'/publicapi/add',
headers={'Content-type': 'application/x-www-form-urlencoded'},
body=urlencode(data))
except (SSLError, HTTPException, socket.error):
log.error(u'Prowl notification failed.')
return False
response = http_handler.getresponse()
request_status = response.status
if request_status == 200:
log.info(u'Prowl notifications sent.')
return True
elif request_status == 401:
log.error(u'Prowl auth failed: {0}', response.reason)
return False
else:
log.error(u'Prowl notification failed.')
return False
def execute_request(self):
""""Execute the request, and get a response.
Returns:
an HttpResponse object from httplib
"""
if self.is_secure_connection:
conn = HTTPSConnection(self.get_hostname(), timeout=self.timeout)
else:
conn = HTTPConnection(self.get_hostname(), timeout=self.timeout)
conn.request(self.method, self.url, self.body, self.headers)
response = conn.getresponse()
return response
def get_proxy_connection(proxy_url):
# At this stage, the SDK granted the permission to start transferring the disk, and the
# user should choose its preferred tool for doing it - regardless of the SDK.
# In this example, we will use Python's httplib.HTTPSConnection for transferring the data.
context = ssl.create_default_context()
# Note that ovirt-imageio-proxy by default checks the certificates, so if you don't have
# your CA certificate of the engine in the system, you need to pass it to HTTPSConnection.
context.load_verify_locations(cafile='ca.pem')
return HTTPSConnection(
proxy_url.hostname,
proxy_url.port,
context=context,
)
def apihit(host,conntype,authtoken,queryurl,reqbody,
proxy_host=None, proxy_port=None):
retdata = ''
if proxy_host and proxy_port:
connection = HTTPSConnection(proxy_host, proxy_port)
connection.set_tunnel(host, 443)
else:
connection = HTTPSConnection(host)
tokenheader = {"Authorization": 'Bearer ' + authtoken, "Content-type": "application/json", "Accept": "text/plain"}
if conntype == "GET":
connection.request(conntype, queryurl, '', tokenheader)
else:
connection.request(conntype, queryurl, json.dumps(reqbody), tokenheader)
response = connection.getresponse()
respbody = response.read().decode('ascii', 'ignore')
try:
jsondata = respbody.decode()
retdata = json.loads(jsondata)
except AttributeError:
retdata = json.loads(respbody)
except:
raise
connection.close()
return retdata
def retry_http_ntlm_sspi_auth(self, host, req, authenticatehdr,
auth_header):
url = req.full_url
scheme, _, host, path = url.split('/', 3)
h = HTTPConnection(host) if scheme == 'http:' else HTTPSConnection(
host)
headers = dict(req.unredirected_hdrs)
headers.update(
dict((k, v) for k, v in req.headers.items() if k not in headers))
headers["Connection"] = "Keep-Alive"
headers[auth_header] = "NTLM " + self.get_auth_req()
h.request(req.get_method(), req.selector, req.data, headers)
response = h.getresponse()
response.fp = None # keep-alive
ntlmauth = response.headers.get(authenticatehdr)
if ntlmauth is not None and ntlmauth.startswith('NTLM '):
challenge = ntlmauth[5:]
challenge_response = self.create_challenge_response(challenge)
if challenge_response is None:
_log.warning('Failed to authenticate using NTLM')
return None
headers["Connection"] = "Close"
headers[auth_header] = "NTLM " + challenge_response
h.request(req.get_method(), req.selector, req.data, headers)
response = h.getresponse()
return addinfourl(response, response.msg, req.get_full_url(),
response.status)
def indirect_caller():
conn = HTTPSConnection('example.com')
indirect_callee(conn)
def retry_using_http_NTLM_auth(self, req, auth_header_field, realm, headers):
user, pw = self.passwd.find_user_password(realm, req.get_full_url())
if pw is not None:
user_parts = user.split('\\', 1)
if len(user_parts) == 1:
UserName = user_parts[0]
DomainName = ''
type1_flags = ntlm.NTLM_TYPE1_FLAGS & ~ntlm.NTLM_NegotiateOemDomainSupplied
else:
DomainName = user_parts[0].upper()
UserName = user_parts[1]
type1_flags = ntlm.NTLM_TYPE1_FLAGS
# ntlm secures a socket, so we must use the same socket for the complete handshake
headers = dict(req.headers)
headers.update(req.unredirected_hdrs)
auth = 'NTLM %s' % ntlm.create_NTLM_NEGOTIATE_MESSAGE(user, type1_flags).decode("utf-8")
if req.headers.get(self.auth_header, None) == auth:
return None
headers[self.auth_header] = auth
host = req.host
if not host:
raise urllib.request.URLError('no host given')
h = None
if req.get_full_url().startswith('https://'):
h = HTTPSConnection(host) # will parse host:port
else:
h = HTTPConnection(host) # will parse host:port
h.set_debuglevel(self._debuglevel)
# we must keep the connection because NTLM authenticates the connection, not single requests
headers["Connection"] = "Keep-Alive"
headers = dict((name.title(), val) for name, val in headers.items())
# For some reason, six doesn't do this translation correctly
# TODO rsanders low - find bug in six & fix it
try:
selector = req.selector
except AttributeError:
selector = req.get_selector()
h.request(req.get_method(), selector, req.data, headers)
r = h.getresponse()
r.begin()
r._safe_read(int(r.getheader('content-length')))
if r.getheader('set-cookie'):
# this is important for some web applications that store authentication-related info in cookies (it took a long time to figure out)
headers['Cookie'] = r.getheader('set-cookie')
r.fp = None # remove the reference to the socket, so that it can not be closed by the response object (we want to keep the socket open)
auth_header_value = r.getheader(auth_header_field, None)
# some Exchange servers send two WWW-Authenticate headers, one with the NTLM challenge
# and another with the 'Negotiate' keyword - make sure we operate on the right one
m = re.match('(NTLM [A-Za-z0-9+\-/=]+)', auth_header_value)
if m:
auth_header_value, = m.groups()
(ServerChallenge, NegotiateFlags) = ntlm.parse_NTLM_CHALLENGE_MESSAGE(auth_header_value[5:])
auth = 'NTLM %s' % ntlm.create_NTLM_AUTHENTICATE_MESSAGE(ServerChallenge, UserName, DomainName, pw,
NegotiateFlags).decode("utf-8")
headers[self.auth_header] = auth
headers["Connection"] = "Close"
headers = dict((name.title(), val) for name, val in headers.items())
try:
h.request(req.get_method(), selector, req.data, headers)
# none of the configured handlers are triggered, for example redirect-responses are not handled!
response = h.getresponse()
def notimplemented():
raise NotImplementedError
response.readline = notimplemented
infourl = urllib.response.addinfourl(response, response.msg, req.get_full_url())
infourl.code = response.status
infourl.msg = response.reason
return infourl
except socket.error as err:
raise urllib.error.URLError(err)
else:
return None
def _sendPushover(self, msg, title, sound=None, userKey=None, apiKey=None):
"""
Sends a pushover notification to the address provided
msg: The message to send (unicode)
title: The title of the message
sound: The notification sound to use
userKey: The pushover user id to send the message to (or to subscribe with)
apiKey: The pushover api key to use
returns: True if the message succeeded, False otherwise
"""
if userKey is None:
userKey = app.PUSHOVER_USERKEY
if apiKey is None:
apiKey = app.PUSHOVER_APIKEY
if sound is None:
sound = app.PUSHOVER_SOUND
log.debug(u'Pushover API KEY in use: {0}', apiKey)
# build up the URL and parameters
msg = msg.strip()
# send the request to pushover
try:
if app.PUSHOVER_SOUND != 'default':
args = {
'token': apiKey,
'user': userKey,
'title': title.encode('utf-8'),
'message': msg.encode('utf-8'),
'timestamp': int(time.time()),
'retry': 60,
'expire': 3600,
'sound': sound,
}
else:
# sound is default, so don't send it
args = {
'token': apiKey,
'user': userKey,
'title': title.encode('utf-8'),
'message': msg.encode('utf-8'),
'timestamp': int(time.time()),
'retry': 60,
'expire': 3600,
}
if app.PUSHOVER_DEVICE:
args['device'] = ','.join(app.PUSHOVER_DEVICE)
conn = HTTPSConnection('api.pushover.net:443')
conn.request('POST', '/1/messages.json', urlencode(args),
{'Content-type': 'application/x-www-form-urlencoded'})
except HTTPError as e:
# if we get an error back that doesn't have an error code then who knows what's really happening
if not hasattr(e, 'code'):
log.error(u'Pushover notification failed. {}', ex(e))
return False
else:
log.error(u'Pushover notification failed. Error code: {0}',
e.code)
# HTTP status 404 if the provided email address isn't a Pushover user.
if e.code == 404:
log.warning(
u'Username is wrong/not a pushover email. Pushover will send an email to it'
)
return False
# For HTTP status code 401's, it is because you are passing in either an invalid token, or the user has not added your service.
elif e.code == 401:
# HTTP status 401 if the user doesn't have the service added
subscribeNote = self._sendPushover(msg,
title,
sound=sound,
userKey=userKey,
apiKey=apiKey)
if subscribeNote:
log.debug(u'Subscription sent')
return True
else:
log.error(u'Subscription could not be sent')
return False
# If you receive an HTTP status code of 400, it is because you failed to send the proper parameters
elif e.code == 400:
log.error(u'Wrong data sent to pushover')
return False
# If you receive a HTTP status code of 429, it is because the message limit has been reached (free limit is 7,500)
elif e.code == 429:
log.error(
u'Pushover API message limit reached - try a different API key'
)
return False
log.info(u'Pushover notification successful.')
return True
def fetch_response(self, request):
"""Executes request and fetchs service response"""
connection = HTTPSConnection(self.SERVER_URL)
connection.request(request.method.upper(), request.to_url())
return connection.getresponse().read()
def __init__(self, uri, basepath=None):
self.basepath = basepath
self.mimetype = None
self.file = None
self.data = None
self.uri = None
self.local = None
self.tmp_file = None
uri = uri or str()
uri = uri.encode('utf-8')
log.debug("FileObject %r, Basepath: %r", uri, basepath)
# Data URI
if isinstance(uri, binary_type):
uri = uri.decode('utf-8')
if uri.startswith("data:"):
m = _rx_datauri.match(uri)
self.mimetype = m.group("mime")
self.data = base64.decodestring(m.group("data"))
else:
# Check if we have an external scheme
if basepath and not urlparse(uri).scheme:
urlParts = urlparse(basepath)
else:
urlParts = urlparse(uri)
log.debug("URLParts: %r", urlParts)
if urlParts.scheme == 'file':
if basepath and uri.startswith('/'):
uri = urljoin(basepath, uri[1:])
urlResponse = urlopen(uri)
self.mimetype = urlResponse.info().get("Content-Type",
'').split(";")[0]
self.uri = urlResponse.geturl()
self.file = urlResponse
# Drive letters have len==1 but we are looking
# for things like http:
elif urlParts.scheme in ('http', 'https'):
# External data
if basepath:
uri = urljoin(basepath, uri)
#path = urlsplit(url)[2]
#mimetype = getMimeType(path)
# Using HTTPLIB
parts = urlsplit(uri[uri.find("//"):])
server, path = parts.netloc, parts.path
if uri.startswith("https://"):
conn = HTTPSConnection(server)
else:
conn = HTTPConnection(server)
conn.request("GET", path)
r1 = conn.getresponse()
# log.debug("HTTP %r %r %r %r", server, path, uri, r1)
if (r1.status, r1.reason) == (200, "OK"):
self.mimetype = r1.getheader("Content-Type",
'').split(";")[0]
self.uri = uri
if r1.getheader("content-encoding") == "gzip":
import gzip
self.file = gzip.GzipFile(mode="rb",
fileobj=BytesIO(r1.read()))
else:
self.file = r1
else:
try:
urlResponse = urlopen(uri)
except HTTPError:
return
self.mimetype = urlResponse.info().get("Content-Type",
'').split(";")[0]
self.uri = urlResponse.geturl()
self.file = urlResponse
else:
# Local data
if basepath:
uri = os.path.normpath(os.path.join(basepath, uri))
if os.path.isfile(uri):
self.uri = uri
self.local = uri
self.setMimeTypeByName(uri)
self.file = open(uri, "rb")
def _update_index(self, url=None):
"""A helper function that ensures that self._index is
up-to-date. If the index is older than self.INDEX_TIMEOUT,
then download it again."""
# Check if the index is aleady up-to-date. If so, do nothing.
if not (self._index is None or url is not None or
time.time()-self._index_timestamp > self.INDEX_TIMEOUT):
return
# If a URL was specified, then update our URL.
self._url = url or self._url
source = self._source
assert source == 'google' or source == 'mirror'
# Download the index file.
if source == 'google':
host = "www.googleapis.com"
conn = HTTPSConnection(host)
conn.request("GET", "/storage/v1/b/{}/o".format(self._url))
r1 = conn.getresponse()
data = r1.read()
elif source == 'mirror':
index_url = path.join(self._url, 'index.json')
data = urlopen(index_url).read()
if six.PY3:
data = data.decode('utf-8')
data = loads(data)
objs = data["items"]
self._index_timestamp = time.time()
# Build a dictionary of packages.
packages = []
for p in objs:
P = Package.fromcsobj(p)
packages.append(P)
self._packages = dict((p.id, p) for p in packages)
# Build language collections.
langs = defaultdict(lambda: [])
for k in self._packages:
package = self._packages[k]
langs[package.language].append(package)
tasks = defaultdict(lambda: [])
for k in self._packages:
package = self._packages[k]
tasks[package.task].append(package)
collections = []
for lang in langs:
children = langs[lang]
name1 = Locale(lang).getDisplayLanguage()
try:
name2 = isoLangs[lang]['name']
except:
name2 = None
if name1 and name1 != lang:
name = name1
elif name2:
name = name2
else:
name = lang
id = "{}{}".format(Downloader.LANG_PREFIX, lang)
name = "{:<20} packages and models".format(name)
c = Collection(id=id, name=name, children=children)
collections.append(c)
for task in tasks:
children = tasks[task]
id = "{}{}".format(Downloader.TASK_PREFIX, task)
c = Collection(id=id, name=task, children=children)
collections.append(c)
self._collections = dict((c.id, c) for c in collections)
# Replace identifiers with actual children in collection.children.
for collection in self._collections.values():
for i, child_id in enumerate(collection.children):
if child_id in self._packages:
collection.children[i] = self._packages[child_id]
if child_id in self._collections:
collection.children[i] = self._collections[child_id]
# Fill in collection.packages for each collection.
for collection in self._collections.values():
packages = {}
queue = [collection]
for child in queue:
if isinstance(child, Collection):
queue.extend(child.children)
else:
packages[child.id] = child
collection.packages = packages.values()
# Flush the status cache
self._status_cache.clear()
def _sendPushover(self, msg, title, sound=None, userKey=None, apiKey=None):
"""
Sends a pushover notification to the address provided
msg: The message to send (unicode)
title: The title of the message
sound: The notification sound to use
userKey: The pushover user id to send the message to (or to subscribe with)
apiKey: The pushover api key to use
returns: True if the message succeeded, False otherwise
"""
if userKey is None:
userKey = app.PUSHOVER_USERKEY
if apiKey is None:
apiKey = app.PUSHOVER_APIKEY
if sound is None:
sound = app.PUSHOVER_SOUND
logger.log(u"Pushover API KEY in use: " + apiKey, logger.DEBUG)
# build up the URL and parameters
msg = msg.strip()
# send the request to pushover
try:
if app.PUSHOVER_SOUND != "default":
args = {
"token": apiKey,
"user": userKey,
"title": title.encode('utf-8'),
"message": msg.encode('utf-8'),
"timestamp": int(time.time()),
"retry": 60,
"expire": 3600,
"sound": sound,
}
else:
# sound is default, so don't send it
args = {
"token": apiKey,
"user": userKey,
"title": title.encode('utf-8'),
"message": msg.encode('utf-8'),
"timestamp": int(time.time()),
"retry": 60,
"expire": 3600,
}
if app.PUSHOVER_DEVICE:
args["device"] = app.PUSHOVER_DEVICE
conn = HTTPSConnection("api.pushover.net:443")
conn.request("POST", "/1/messages.json",
urlencode(args), {"Content-type": "application/x-www-form-urlencoded"})
except HTTPError as e:
# if we get an error back that doesn't have an error code then who knows what's really happening
if not hasattr(e, 'code'):
logger.log(u"Pushover notification failed." + ex(e), logger.ERROR)
return False
else:
logger.log(u"Pushover notification failed. Error code: " + str(e.code), logger.ERROR)
# HTTP status 404 if the provided email address isn't a Pushover user.
if e.code == 404:
logger.log(u"Username is wrong/not a pushover email. Pushover will send an email to it", logger.WARNING)
return False
# For HTTP status code 401's, it is because you are passing in either an invalid token, or the user has not added your service.
elif e.code == 401:
# HTTP status 401 if the user doesn't have the service added
subscribeNote = self._sendPushover(msg, title, sound=sound, userKey=userKey, apiKey=apiKey)
if subscribeNote:
logger.log(u"Subscription sent", logger.DEBUG)
return True
else:
logger.log(u"Subscription could not be sent", logger.ERROR)
return False
# If you receive an HTTP status code of 400, it is because you failed to send the proper parameters
elif e.code == 400:
logger.log(u"Wrong data sent to pushover", logger.ERROR)
return False
# If you receive a HTTP status code of 429, it is because the message limit has been reached (free limit is 7,500)
elif e.code == 429:
logger.log(u"Pushover API message limit reached - try a different API key", logger.ERROR)
return False
logger.log(u"Pushover notification successful.", logger.INFO)
return True
# At this stage, the SDK granted the permission to start transferring the
# disk, and the user should choose its preferred tool for doing it -
# regardless of the SDK. In this example, we will use Python's
# httplib.HTTPSConnection for transferring the data.
proxy_url = urlparse(transfer.proxy_url)
context = ssl.create_default_context()
# Note that ovirt-imageio-proxy by default checks the certificates,
# so if you don't have your CA certificate of the engine in the system,
# you need to pass it to HTTPSConnection.
context.load_verify_locations(cafile='ca.pem')
proxy_connection = HTTPSConnection(
proxy_url.hostname,
proxy_url.port,
context=context,
)
# Find the disk entry among the OVA entries
for ova_entry in ova_entries[1:]:
if ova_entry.name == props['fileRef']:
disk_entry = ova_entry
break
print("Uploading disk %s..." % disk_entry.name)
disk_file = ova_file.extractfile(disk_entry)
image_size = disk_entry.size
# Send the request head. Note the following:
#
def _send_pushover(self,
msg,
title,
sound=None,
user_key=None,
api_key=None,
priority=None):
"""
Send a pushover notification to the address provided.
:param msg: The message to send (unicode)
:param title: The title of the message
:param sound: The notification sound to use
:param user_key: The pushover user id to send the message to (or to subscribe with)
:param api_key: The pushover api key to use
:param priority: The pushover priority to use
:return: True if the message succeeded, False otherwise
"""
if user_key is None:
user_key = app.PUSHOVER_USERKEY
if api_key is None:
api_key = app.PUSHOVER_APIKEY
if sound is None:
sound = app.PUSHOVER_SOUND
if priority is None:
priority = app.PUSHOVER_PRIORITY
# build up the URL and parameters
msg = msg.strip()
# default args
args = {
'token': api_key,
'user': user_key,
'title': title.encode('utf-8'),
'message': msg.encode('utf-8'),
'timestamp': int(time.time()),
'retry': 60,
'expire': 3600,
'priority': priority,
}
# If sound is not default, add it.
if sound != 'default':
args['sound'] = sound
if app.PUSHOVER_DEVICE:
args['device'] = ','.join(app.PUSHOVER_DEVICE)
log.debug(
'PUSHOVER: Sending notice with details: title="{0}" message="{1}", priority={2}, sound={3}',
args['title'], args['message'], priority, sound)
conn = HTTPSConnection('api.pushover.net:443')
conn.request('POST', '/1/messages.json', urlencode(args),
{'Content-type': 'application/x-www-form-urlencoded'})
conn_resp = conn.getresponse()
if conn_resp.status == 200:
log.info('Pushover notification successful.')
return True
# HTTP status 404 if the provided email address isn't a Pushover user.
elif conn_resp.status == 404:
log.warning(
'Username is wrong/not a pushover email. Pushover will send an email to it'
)
return False
# For HTTP status code 401's, it is because you are passing in either an invalid token, or the user has not added your service.
elif conn_resp.status == 401:
# HTTP status 401 if the user doesn't have the service added
subscribe_note = self._send_pushover(msg,
title,
sound=sound,
user_key=user_key,
api_key=api_key)
if subscribe_note:
log.debug('Subscription sent')
return True
else:
log.error('Subscription could not be sent')
# If you receive an HTTP status code of 400, it is because you failed to send the proper parameters
elif conn_resp.status == 400:
log.error('Wrong keys sent to pushover')
return False
# If you receive a HTTP status code of 429, it is because the message limit has been reached (free limit is 7,500)
elif conn_resp.status == 429:
log.error(
'Pushover API message limit reached - try a different API key')
return False
# Something else has gone wrong... who knows what's really happening
else:
log.error('Pushover notification failed. HTTP response code: {0}',
conn_resp.status)
return False
def __init__(self, url, key=None, secret=None, timeout=5, context=None):
"""
Create a new Crossbar.io push client.
The only mandatory argument is the Push service endpoint of the Crossbar.io
instance to push to.
For signed pushes, provide authentication key and secret. If those are not
given, unsigned pushes are performed.
:param url: URL of the HTTP bridge of Crossbar.io (e.g. http://example.com:8080/push).
:type url: str
:param key: Optional key to use for signing requests.
:type key: str
:param secret: When using signed request, the secret corresponding to key.
:type secret: str
:param timeout: Timeout for requests.
:type timeout: int
:param context: If the HTTP bridge is running on HTTPS (that is securely over TLS),
then the context provides the SSL settings the client should use (e.g. the
certificate chain against which to verify the server certificate). This parameter
is only available on Python 2.7.9+ and Python 3 (otherwise the parameter is silently
ignored!). See: https://docs.python.org/2/library/ssl.html#ssl.SSLContext
:type context: obj or None
"""
if six.PY2:
if type(url) == str:
url = six.u(url)
if type(key) == str:
key = six.u(key)
if type(secret) == str:
secret = six.u(secret)
assert (type(url) == six.text_type)
assert ((key and secret) or (not key and not secret))
assert (key is None or type(key) == six.text_type)
assert (secret is None or type(secret) == six.text_type)
assert (type(timeout) == int)
if _HAS_SSL and _HAS_SSL_CLIENT_CONTEXT:
assert (context is None or isinstance(context, ssl.SSLContext))
self._seq = 1
self._key = key
self._secret = secret
self._endpoint = _parse_url(url)
self._endpoint['headers'] = {
"Content-type": "application/json",
"User-agent": "crossbarconnect-python"
}
if self._endpoint['secure']:
if not _HAS_SSL:
raise Exception(
"Bridge URL is using HTTPS, but Python SSL module is missing"
)
if _HAS_SSL_CLIENT_CONTEXT:
self._connection = HTTPSConnection(self._endpoint['host'],
self._endpoint['port'],
timeout=timeout,
context=context)
else:
self._connection = HTTPSConnection(self._endpoint['host'],
self._endpoint['port'],
timeout=timeout)
else:
self._connection = HTTPConnection(self._endpoint['host'],
self._endpoint['port'],
timeout=timeout)
import datetime
counter = [0]
def handle_event(e, p, counter):
# print(e)
if counter[0] % 1000 == 0:
print('processed ' + str(counter[0]) + ' objects')
counter[0] += 1
if __name__ == '__main__':
url = '/prust/wikipedia-movie-data/master/movies.json'
endpoint = HTTPSConnection('raw.githubusercontent.com',
'443',
context=ssl._create_unverified_context())
try:
ts1 = datetime.datetime.now().timestamp()
print('start time=' + str(ts1))
endpoint.request('GET', url)
response = endpoint.getresponse()
tokenizer = JsonTokenizer(response, 'ISO-8859-1', 65536)
JsonDecoder()\
.tokenizer(tokenizer)\
.root_class_name('Data')\
.event_handler(lambda e, p: handle_event(e, p, counter))\
.predicate('genres') \
.with_snake_cased_props()\
