def getRemoteDBModifiedTS(): """ Performs a HEAD request to get the Last-Modified date-time of a database dump file and parses it into a UNIX timestamp. """ debug_msg = 'Unable to get timestamp of remote database dump - {0}' logging.info("Getting timestamp of database dump at '%s'", HOST + PATH) try: # Removing the scheme from the URL conn = HTTPSConnection(HOST[8:], timeout=TIMEOUT) conn.request('HEAD', PATH) except gaierror as e: logging.debug(debug_msg.format("Cannot connect to '%s', error: %s"), HOST + PATH, e) exit(1) rsp = conn.getresponse() if rsp.status != 200: logging.debug(debug_msg.format('Server responded with: %d %s'), rsp.status, rsp.reason) exit(1) last_modified = rsp.getheader('last-modified', None) if last_modified is None: logging.debug( debug_msg.format('Response doesnt include Last-Modified Header')) exit(1) last_m_dt = datetime.strptime( last_modified.split(', ')[1], '%d %b %Y %H:%M:%S %Z') return timegm(last_m_dt.timetuple())
def getRemoteDBModifiedTS(): """ Performs a HEAD request to get the Last-Modified date-time of a database dump file and parses it into a UNIX timestamp. """ debug_msg = 'Unable to get timestamp of remote database dump - {0}' logging.info("Getting timestamp of database dump at '%s'", HOST + PATH) try: # Removing the scheme from the URL conn = HTTPSConnection(HOST[8:], timeout=TIMEOUT) conn.request('HEAD', PATH) except gaierror as e: logging.debug( debug_msg.format( "Cannot connect to '%s', error: %s"), HOST + PATH, e) exit(1) rsp = conn.getresponse() if rsp.status != 200: logging.debug( debug_msg.format('Server responded with: %d %s'), rsp.status, rsp.reason) exit(1) last_modified = rsp.getheader('last-modified', None) if last_modified is None: logging.debug(debug_msg.format( 'Response doesnt include Last-Modified Header')) exit(1) last_m_dt = datetime.strptime( last_modified.split(', ')[1], '%d %b %Y %H:%M:%S %Z') return timegm(last_m_dt.timetuple())
def __init__(self, service_uri, credentials=None, key_file=None, cert_file=None, timeout=10, buffer_size=65535, error_factory=lambda x: x, overload=False, version=1): self.version = version self._headers = list() self._headers.append(( 'User-Agent', 'python-hessian/' + __version__, )) self._headers.append(( 'Content-Type', 'application/x-hessian', )) if sys.version_info < (2, 6): warn('HessianProxy timeout not enforceable before Python 2.6', RuntimeWarning, stacklevel=2) kwargs = {} else: kwargs = {'timeout': timeout} if six.PY2: kwargs['strict'] = True self._uri = urlparse(service_uri) if self._uri.scheme == 'http': self._client = HTTPConnection(self._uri.hostname, self._uri.port or 80, **kwargs) elif self._uri.scheme == 'https': self._client = HTTPSConnection(self._uri.hostname, self._uri.port or 443, key_file=key_file, cert_file=cert_file, **kwargs) else: raise NotImplementedError( "HessianProxy only supports http:// and https:// URIs") # autofill credentials if they were passed via url instead of kwargs if (self._uri.username and self._uri.password) and not credentials: credentials = (self._uri.username, self._uri.password) if credentials: auth = 'Basic ' + base64.b64encode(':'.join(credentials)) self._headers.append(('Authorization', auth)) self._buffer_size = buffer_size self._error_factory = error_factory self._overload = overload self._parser = Parser()
def create_connection(self): # type: () -> None """ Creates a HTTPS connection instance on a default port (443) to the 'self.license_server_url' """ self.log.debug("In 'create_connection'") self.connection = HTTPSConnection(self.license_server_url)
def _send_prowl(prowl_api=None, prowl_priority=None, event=None, message=None, force=False): if not sickbeard.USE_PROWL and not force: return False if prowl_api is None: prowl_api = sickbeard.PROWL_API if len(prowl_api) == 0: return False if prowl_priority is None: prowl_priority = sickbeard.PROWL_PRIORITY title = sickbeard.PROWL_MESSAGE_TITLE logger.log( "PROWL: Sending notice with details: title=\"{0}\" event=\"{1}\", message=\"{2}\", priority={3}, api={4}" .format(title, event, message, prowl_priority, prowl_api), logger.DEBUG) http_handler = HTTPSConnection("api.prowlapp.com") data = { 'apikey': prowl_api, 'application': title, 'event': event, 'description': message.encode('utf-8'), 'priority': prowl_priority } try: http_handler.request( "POST", "/publicapi/add", headers={'Content-type': "application/x-www-form-urlencoded"}, body=urlencode(data)) except (SSLError, HTTPException, socket.error): logger.log("Prowl notification failed.", logger.ERROR) return False response = http_handler.getresponse() request_status = response.status if request_status == 200: logger.log("Prowl notifications sent.", logger.INFO) return True elif request_status == 401: logger.log("Prowl auth failed: {0}".format(response.reason), logger.ERROR) return False else: logger.log("Prowl notification failed.", logger.ERROR) return False
def send_email(email, template, kwargs): """Send an email via the mailgun service.""" maildir = os.path.join(topdir, 'emails') env = jinja2.Environment(loader=jinja2.FileSystemLoader(maildir)) template = env.get_template(template) rendered = template.render(**kwargs) headers, message = parse_email(rendered) mailargs = {'to': email, 'from': app.config['MAIL_FROM'], 'bcc': app.config.get('MAIL_BCC'), 'text': message} mailargs.update(headers) conn = HTTPSConnection('api.mailgun.net', 443) conn.connect() auth = b64enc('api:{0[MAILGUN_KEY]}'.format(app.config)) headers = {'Authorization': 'Basic {0}'.format(auth), 'Accept': 'application/json', 'Content-type': 'application/x-www-form-urlencoded'} url = '/v2/{0[MAILGUN_DOMAIN]}/messages'.format(app.config) body = urlencode(mailargs) conn.request('POST', url, body, headers) resp = conn.getresponse() if resp.status != 200: raise RuntimeError('could not send email') conn.close()
def _send_prowl(prowl_api=None, prowl_priority=None, event=None, message=None, force=False): if not app.USE_PROWL and not force: return False if prowl_api is None: prowl_api = ','.join(app.PROWL_API) if not prowl_api: return False if prowl_priority is None: prowl_priority = app.PROWL_PRIORITY title = app.PROWL_MESSAGE_TITLE log.debug( u'PROWL: Sending notice with details: title="{0}" event="{1}", message="{2}", priority={3}, api={4}', title, event, message, prowl_priority, prowl_api) http_handler = HTTPSConnection('api.prowlapp.com') data = { 'apikey': prowl_api, 'application': title, 'event': event, 'description': message.encode('utf-8'), 'priority': prowl_priority } try: http_handler.request( 'POST', '/publicapi/add', headers={'Content-type': 'application/x-www-form-urlencoded'}, body=urlencode(data)) except (SSLError, HTTPException, socket.error): log.error(u'Prowl notification failed.') return False response = http_handler.getresponse() request_status = response.status if request_status == 200: log.info(u'Prowl notifications sent.') return True elif request_status == 401: log.error(u'Prowl auth failed: {0}', response.reason) return False else: log.error(u'Prowl notification failed.') return False
def connect(self): """ Override the connect() function to intercept calls to certain host/ports. If no app at host/port has been registered for interception then a normal HTTPSConnection is made. """ if debuglevel: sys.stderr.write('connect: %s, %s\n' % ( self.host, self.port, )) try: (app, script_name) = self.get_app(self.host, self.port) if app: if debuglevel: sys.stderr.write('INTERCEPTING call to %s:%s\n' % ( self.host, self.port, )) self.sock = wsgi_fake_socket(app, self.host, self.port, script_name, https=True) else: try: import ssl if not hasattr(self, 'key_file'): self.key_file = None if not hasattr(self, 'cert_file'): self.cert_file = None if not hasattr(self, '_context'): try: self._context = ssl.create_default_context() except AttributeError: self._context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) self._context.options |= ssl.OP_NO_SSLv2 if not hasattr(self, 'check_hostname'): self._check_hostname = ( self._context.verify_mode != ssl.CERT_NONE) else: self._check_hostname = self.check_hostname except (ImportError, AttributeError): pass HTTPSConnection.connect(self) except Exception: if debuglevel: # intercept & print out tracebacks traceback.print_exc() raise
def _new_conn(self): # Performs the NTLM handshake that secures the connection. The socket # must be kept open while requests are performed. self.num_connections += 1 log.debug('Starting NTLM HTTPS connection no. %d: https://%s%s', self.num_connections, self.host, self.authurl) headers = {'Connection': 'Keep-Alive'} req_header = 'Authorization' resp_header = 'www-authenticate' conn = HTTPSConnection(host=self.host, port=self.port) # Send negotiation message headers[req_header] = ( 'NTLM %s' % ntlm.create_NTLM_NEGOTIATE_MESSAGE(self.rawuser)) log.debug('Request headers: %s', headers) conn.request('GET', self.authurl, None, headers) res = conn.getresponse() reshdr = dict(res.getheaders()) log.debug('Response status: %s %s', res.status, res.reason) log.debug('Response headers: %s', reshdr) log.debug('Response data: %s [...]', res.read(100)) # Remove the reference to the socket, so that it can not be closed by # the response object (we want to keep the socket open) res.fp = None # Server should respond with a challenge message auth_header_values = reshdr[resp_header].split(', ') auth_header_value = None for s in auth_header_values: if s[:5] == 'NTLM ': auth_header_value = s[5:] if auth_header_value is None: raise Exception('Unexpected %s response header: %s' % (resp_header, reshdr[resp_header])) # Send authentication message ServerChallenge, NegotiateFlags = \ ntlm.parse_NTLM_CHALLENGE_MESSAGE(auth_header_value) auth_msg = ntlm.create_NTLM_AUTHENTICATE_MESSAGE( ServerChallenge, self.user, self.domain, self.pw, NegotiateFlags) headers[req_header] = 'NTLM %s' % auth_msg log.debug('Request headers: %s', headers) conn.request('GET', self.authurl, None, headers) res = conn.getresponse() log.debug('Response status: %s %s', res.status, res.reason) log.debug('Response headers: %s', dict(res.getheaders())) log.debug('Response data: %s [...]', res.read()[:100]) if res.status != 200: if res.status == 401: raise Exception('Server rejected request: wrong ' 'username or password') raise Exception('Wrong server response: %s %s' % (res.status, res.reason)) res.fp = None log.debug('Connection established') return conn
def retry_http_krb_sspi_auth(self, host, req): url = req.full_url scheme, _, host, path = url.split('/', 3) h = HTTPConnection(host) if scheme == 'http:' else HTTPSConnection( host) headers = dict(req.unredirected_hdrs) headers.update( dict((k, v) for k, v in req.headers.items() if k not in headers)) try: __, krb_context = kerberos.authGSSClientInit("HTTP@" + host) kerberos.authGSSClientStep(krb_context, "") negotiate_details = kerberos.authGSSClientResponse(krb_context) headers["Connection"] = "Keep-Alive" headers["Authorization"] = "Negotiate " + negotiate_details h.request(req.get_method(), req.selector, req.data, headers) response = h.getresponse() return addinfourl(response, response.msg, req.get_full_url(), response.status) except: # e = sys.exc_info()[0] # _log.warning(str(e)) # _log.warning('Failed Kerberos authentication') return None
def connect(self): """ Override the connect() function to intercept calls to certain host/ports. If no app at host/port has been registered for interception then a normal HTTPSConnection is made. """ if debuglevel: sys.stderr.write('connect: %s, %s\n' % (self.host, self.port,)) try: (app, script_name) = self.get_app(self.host, self.port) if app: if debuglevel: sys.stderr.write('INTERCEPTING call to %s:%s\n' % (self.host, self.port,)) self.sock = wsgi_fake_socket(app, self.host, self.port, script_name, https=True) else: try: import ssl if not hasattr(self, 'key_file'): self.key_file = None if not hasattr(self, 'cert_file'): self.cert_file = None if not hasattr(self, '_context'): try: self._context = ssl.create_default_context() except AttributeError: self._context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) self._context.options |= ssl.OP_NO_SSLv2 if not hasattr(self, 'check_hostname'): self._check_hostname = ( self._context.verify_mode != ssl.CERT_NONE ) else: self._check_hostname = self.check_hostname except (ImportError, AttributeError): pass HTTPSConnection.connect(self) except Exception: if debuglevel: # intercept & print out tracebacks traceback.print_exc() raise
def _send_prowl(prowl_api=None, prowl_priority=None, event=None, message=None, force=False): if not app.USE_PROWL and not force: return False if prowl_api is None: prowl_api = ','.join(app.PROWL_API) if not prowl_api: return False if prowl_priority is None: prowl_priority = app.PROWL_PRIORITY title = app.PROWL_MESSAGE_TITLE log.debug(u'PROWL: Sending notice with details: title="{0}" event="{1}", message="{2}", priority={3}, api={4}', title, event, message, prowl_priority, prowl_api) http_handler = HTTPSConnection('api.prowlapp.com') data = {'apikey': prowl_api, 'application': title, 'event': event, 'description': message.encode('utf-8'), 'priority': prowl_priority} try: http_handler.request('POST', '/publicapi/add', headers={'Content-type': 'application/x-www-form-urlencoded'}, body=urlencode(data)) except (SSLError, HTTPException, socket.error): log.error(u'Prowl notification failed.') return False response = http_handler.getresponse() request_status = response.status if request_status == 200: log.info(u'Prowl notifications sent.') return True elif request_status == 401: log.error(u'Prowl auth failed: {0}', response.reason) return False else: log.error(u'Prowl notification failed.') return False
def _send_prowl(prowl_api=None, prowl_priority=None, event=None, message=None, force=False): if not sickbeard.USE_PROWL and not force: return False if prowl_api is None: prowl_api = sickbeard.PROWL_API if not prowl_api: return False if prowl_priority is None: prowl_priority = sickbeard.PROWL_PRIORITY title = sickbeard.PROWL_MESSAGE_TITLE logger.log(u"PROWL: Sending notice with details: title=\"%s\" event=\"%s\", message=\"%s\", priority=%s, api=%s" % (title, event, message, prowl_priority, prowl_api), logger.DEBUG) http_handler = HTTPSConnection("api.prowlapp.com") data = {'apikey': prowl_api, 'application': title, 'event': event, 'description': message.encode('utf-8'), 'priority': prowl_priority} try: http_handler.request("POST", "/publicapi/add", headers={'Content-type': "application/x-www-form-urlencoded"}, body=urlencode(data)) except (SSLError, HTTPException, socket.error): logger.log(u"Prowl notification failed.", logger.ERROR) return False response = http_handler.getresponse() request_status = response.status if request_status == 200: logger.log(u"Prowl notifications sent.", logger.INFO) return True elif request_status == 401: logger.log(u"Prowl auth failed: %s" % response.reason, logger.ERROR) return False else: logger.log(u"Prowl notification failed.", logger.ERROR) return False
def __init__(self, service_uri, credentials=None, key_file=None, cert_file=None, timeout=10, buffer_size=65535, error_factory=lambda x: x, overload=False, version=1): self.version = version self._headers = list() self._headers.append(('User-Agent', 'python-hessian/' + __version__,)) self._headers.append(('Content-Type', 'application/x-hessian',)) if sys.version_info < (2, 6): warn('HessianProxy timeout not enforceable before Python 2.6', RuntimeWarning, stacklevel=2) kwargs = {} else: kwargs = {'timeout': timeout} if six.PY2: kwargs['strict'] = True self._uri = urlparse(service_uri) if self._uri.scheme == 'http': self._client = HTTPConnection(self._uri.hostname, self._uri.port or 80, **kwargs) elif self._uri.scheme == 'https': self._client = HTTPSConnection(self._uri.hostname, self._uri.port or 443, key_file=key_file, cert_file=cert_file, **kwargs) else: raise NotImplementedError("HessianProxy only supports http:// and https:// URIs") # autofill credentials if they were passed via url instead of kwargs if (self._uri.username and self._uri.password) and not credentials: credentials = (self._uri.username, self._uri.password) if credentials: auth = 'Basic ' + base64.b64encode(':'.join(credentials)) self._headers.append(('Authorization', auth)) self._buffer_size = buffer_size self._error_factory = error_factory self._overload = overload self._parser = Parser()
def make_request(method, uri, headers=None, body=None): """ This function should make a request, connection reuse, etc. is left to the implementation :param str|unicode method: HTTP Verb :param str|unicode uri: Server/path to make the request to :param dict headers: Headers to send :param str|unicode body: Body to send :return: Request result :rtype: Tuple of return_code (int), headers (list of (k,v) tuples), body (str|bytes) """ parsed_url = urlparse(uri) if parsed_url.scheme == 'https': connector = HTTPSConnection(host=parsed_url.netloc) elif parsed_url.scheme == 'http': connector = HTTPConnection(host=parsed_url.netloc) else: raise ValueError('Schema is not HTTP nor HTTPS') _, full_path = uri.split(parsed_url.netloc, 1) connector.request(method=method, url=full_path, body=body, headers=headers or {}) response = connector.getresponse() status_code = response.status body = response.read() headers = response.getheaders() return status_code, headers, body
def get_proxy_connection(proxy_url): # At this stage, the SDK granted the permission to start transferring the disk, and the # user should choose its preferred tool for doing it - regardless of the SDK. # In this example, we will use Python's httplib.HTTPSConnection for transferring the data. context = ssl.create_default_context() # Note that ovirt-imageio-proxy by default checks the certificates, so if you don't have # your CA certificate of the engine in the system, you need to pass it to HTTPSConnection. context.load_verify_locations(cafile='ca.pem') return HTTPSConnection( proxy_url.hostname, proxy_url.port, context=context, )
def execute_request(self): """"Execute the request, and get a response. Returns: an HttpResponse object from httplib """ if self.is_secure_connection: conn = HTTPSConnection(self.get_hostname(), timeout=self.timeout) else: conn = HTTPConnection(self.get_hostname(), timeout=self.timeout) conn.request(self.method, self.url, self.body, self.headers) response = conn.getresponse() return response
def retry_http_ntlm_sspi_auth(self, host, req, authenticatehdr, auth_header): url = req.full_url scheme, _, host, path = url.split('/', 3) h = HTTPConnection(host) if scheme == 'http:' else HTTPSConnection( host) headers = dict(req.unredirected_hdrs) headers.update( dict((k, v) for k, v in req.headers.items() if k not in headers)) headers["Connection"] = "Keep-Alive" headers[auth_header] = "NTLM " + self.get_auth_req() h.request(req.get_method(), req.selector, req.data, headers) response = h.getresponse() response.fp = None # keep-alive ntlmauth = response.headers.get(authenticatehdr) if ntlmauth is not None and ntlmauth.startswith('NTLM '): challenge = ntlmauth[5:] challenge_response = self.create_challenge_response(challenge) if challenge_response is None: _log.warning('Failed to authenticate using NTLM') return None headers["Connection"] = "Close" headers[auth_header] = "NTLM " + challenge_response h.request(req.get_method(), req.selector, req.data, headers) response = h.getresponse() return addinfourl(response, response.msg, req.get_full_url(), response.status)
def getBestServer(servers): """Perform a speedtest.net latency request to determine which speedtest.net server has the lowest latency """ results = {} for server in servers: cum = [] url = '%s/latency.txt' % os.path.dirname(server['url']) urlparts = urlparse(url) for i in range(0, 3): try: if urlparts[0] == 'https': h = HTTPSConnection(urlparts[1]) else: h = HTTPConnection(urlparts[1]) headers = {'User-Agent': user_agent} start = timeit.default_timer() h.request("GET", urlparts[2], headers=headers) r = h.getresponse() total = (timeit.default_timer() - start) except (HTTPError, URLError, socket.error): cum.append(3600) continue text = r.read(9) if int(r.status) == 200 and text == 'test=test'.encode(): cum.append(total) else: cum.append(3600) h.close() avg = round((sum(cum) / 6) * 1000, 3) results[avg] = server fastest = sorted(results.keys())[0] best = results[fastest] best['latency'] = fastest return best
class HessianProxy(object): def __init__(self, service_uri, credentials=None, key_file=None, cert_file=None, timeout=10, buffer_size=65535, error_factory=lambda x: x, overload=False, version=1): self.version = version self._headers = list() self._headers.append(( 'User-Agent', 'python-hessian/' + __version__, )) self._headers.append(( 'Content-Type', 'application/x-hessian', )) if sys.version_info < (2, 6): warn('HessianProxy timeout not enforceable before Python 2.6', RuntimeWarning, stacklevel=2) kwargs = {} else: kwargs = {'timeout': timeout} if six.PY2: kwargs['strict'] = True self._uri = urlparse(service_uri) if self._uri.scheme == 'http': self._client = HTTPConnection(self._uri.hostname, self._uri.port or 80, **kwargs) elif self._uri.scheme == 'https': self._client = HTTPSConnection(self._uri.hostname, self._uri.port or 443, key_file=key_file, cert_file=cert_file, **kwargs) else: raise NotImplementedError( "HessianProxy only supports http:// and https:// URIs") # autofill credentials if they were passed via url instead of kwargs if (self._uri.username and self._uri.password) and not credentials: credentials = (self._uri.username, self._uri.password) if credentials: auth = 'Basic ' + base64.b64encode(':'.join(credentials)) self._headers.append(('Authorization', auth)) self._buffer_size = buffer_size self._error_factory = error_factory self._overload = overload self._parser = Parser() class __RemoteMethod(object): # dark magic for autoloading methods def __init__(self, caller, method): self.__caller = caller self.__method = method def __call__(self, *args): return self.__caller(self.__method, args) def __getattr__(self, method): return self.__RemoteMethod(self, method) def __repr__(self): return "<pyhessian.client.HessianProxy(\"%s\")>" % ( self._uri.geturl(), ) def __str__(self): return self.__repr__() def __call__(self, method, args): try: self._client.putrequest('POST', self._uri.path) for header in self._headers: self._client.putheader(*header) request = encode_object( Call(method, args, overload=self._overload, version=self.version)) self._client.putheader("Content-Length", str(len(request))) self._client.endheaders() self._client.send(six.binary_type(request)) response = self._client.getresponse() if response.status != 200: raise ProtocolError(self._uri.geturl(), response.status, response.reason) length = response.getheader('Content-Length', -1) if length == '0': raise ProtocolError(self._uri.geturl(), 'FATAL:', 'Server sent zero-length response') reply = self._parser.parse_stream( BufferedReader(response, buffer_size=self._buffer_size)) self._client.close() if isinstance(reply.value, Fault): raise self._error_factory(reply.value) else: return reply.value except: self._client.close() raise
def _send_pushover(self, msg, title, sound=None, user_key=None, api_key=None, priority=None): """ Send a pushover notification to the address provided. :param msg: The message to send (unicode) :param title: The title of the message :param sound: The notification sound to use :param user_key: The pushover user id to send the message to (or to subscribe with) :param api_key: The pushover api key to use :param priority: The pushover priority to use :return: True if the message succeeded, False otherwise """ if user_key is None: user_key = app.PUSHOVER_USERKEY if api_key is None: api_key = app.PUSHOVER_APIKEY if sound is None: sound = app.PUSHOVER_SOUND if priority is None: priority = app.PUSHOVER_PRIORITY # build up the URL and parameters msg = msg.strip() # default args args = { 'token': api_key, 'user': user_key, 'title': title.encode('utf-8'), 'message': msg.encode('utf-8'), 'timestamp': int(time.time()), 'retry': 60, 'expire': 3600, 'priority': priority, } # If sound is not default, add it. if sound != 'default': args['sound'] = sound if app.PUSHOVER_DEVICE: args['device'] = ','.join(app.PUSHOVER_DEVICE) log.debug('PUSHOVER: Sending notice with details: title="{0}" message="{1}", priority={2}, sound={3}', title, msg, priority, sound) conn = HTTPSConnection('api.pushover.net:443') conn.request('POST', '/1/messages.json', urlencode(args), {'Content-type': 'application/x-www-form-urlencoded'}) conn_resp = conn.getresponse() if conn_resp.status == 200: log.info('Pushover notification successful.') return True # HTTP status 404 if the provided email address isn't a Pushover user. elif conn_resp.status == 404: log.warning('Username is wrong/not a pushover email. Pushover will send an email to it') return False # For HTTP status code 401's, it is because you are passing in either an invalid token, or the user has not added your service. elif conn_resp.status == 401: # HTTP status 401 if the user doesn't have the service added subscribe_note = self._send_pushover(msg, title, sound=sound, user_key=user_key, api_key=api_key) if subscribe_note: log.debug('Subscription sent') return True else: log.error('Subscription could not be sent') # If you receive an HTTP status code of 400, it is because you failed to send the proper parameters elif conn_resp.status == 400: log.error('Wrong keys sent to pushover') return False # If you receive a HTTP status code of 429, it is because the message limit has been reached (free limit is 7,500) elif conn_resp.status == 429: log.error('Pushover API message limit reached - try a different API key') return False # Something else has gone wrong... who knows what's really happening else: log.error('Pushover notification failed. HTTP response code: {0}', conn_resp.status) return False
def indirect_caller(): conn = HTTPSConnection('example.com') indirect_callee(conn)
def __init__(self, uri, basepath=None): self.basepath = basepath self.mimetype = None self.file = None self.data = None self.uri = None self.local = None self.tmp_file = None uri = uri or str() uri = uri.encode('utf-8') log.debug("FileObject %r, Basepath: %r", uri, basepath) # Data URI if isinstance(uri, binary_type): uri = uri.decode('utf-8') if uri.startswith("data:"): m = _rx_datauri.match(uri) self.mimetype = m.group("mime") self.data = base64.decodestring(m.group("data")) else: # Check if we have an external scheme if basepath and not urlparse(uri).scheme: urlParts = urlparse(basepath) else: urlParts = urlparse(uri) log.debug("URLParts: %r", urlParts) if urlParts.scheme == 'file': if basepath and uri.startswith('/'): uri = urljoin(basepath, uri[1:]) urlResponse = urlopen(uri) self.mimetype = urlResponse.info().get( "Content-Type", '').split(";")[0] self.uri = urlResponse.geturl() self.file = urlResponse # Drive letters have len==1 but we are looking # for things like http: elif urlParts.scheme in ('http', 'https'): # External data if basepath: uri = urljoin(basepath, uri) #path = urlsplit(url)[2] #mimetype = getMimeType(path) # Using HTTPLIB parts = urlsplit(uri[uri.find("//"):]) server, path = parts.netloc, parts.path if uri.startswith("https://"): conn = HTTPSConnection(server) else: conn = HTTPConnection(server) conn.request("GET", path) r1 = conn.getresponse() # log.debug("HTTP %r %r %r %r", server, path, uri, r1) if (r1.status, r1.reason) == (200, "OK"): self.mimetype = r1.getheader( "Content-Type", '').split(";")[0] self.uri = uri if r1.getheader("content-encoding") == "gzip": import gzip self.file = gzip.GzipFile(mode="rb", fileobj=BytesIO(r1.read())) else: self.file = r1 else: try: urlResponse = urlopen(uri) except HTTPError: return self.mimetype = urlResponse.info().get( "Content-Type", '').split(";")[0] self.uri = urlResponse.geturl() self.file = urlResponse else: # Local data if basepath: uri = os.path.normpath(os.path.join(basepath, uri)) if os.path.isfile(uri): self.uri = uri self.local = uri self.setMimeTypeByName(uri) self.file = open(uri, "rb")
# At this stage, the SDK granted the permission to start transferring the # disk, and the user should choose its preferred tool for doing it - # regardless of the SDK. In this example, we will use Python's # httplib.HTTPSConnection for transferring the data. proxy_url = urlparse(transfer.proxy_url) context = ssl.create_default_context() # Note that ovirt-imageio-proxy by default checks the certificates, # so if you don't have your CA certificate of the engine in the system, # you need to pass it to HTTPSConnection. context.load_verify_locations(cafile='ca.pem') proxy_connection = HTTPSConnection( proxy_url.hostname, proxy_url.port, context=context, ) # Find the disk entry among the OVA entries for ova_entry in ova_entries[1:]: if ova_entry.name == props['fileRef']: disk_entry = ova_entry break print("Uploading disk %s..." % disk_entry.name) disk_file = ova_file.extractfile(disk_entry) image_size = disk_entry.size # Send the request head. Note the following: #
def __init__(self, host, **kwargs): self._ca_certs_file = kwargs.pop('ca_certs_file', None) self._check_hostname = kwargs.pop('check_hostname', True) HTTPSConnection.__init__(self, host, **kwargs)
import datetime counter = [0] def handle_event(e, p, counter): # print(e) if counter[0] % 1000 == 0: print('processed ' + str(counter[0]) + ' objects') counter[0] += 1 if __name__ == '__main__': url = '/prust/wikipedia-movie-data/master/movies.json' endpoint = HTTPSConnection('raw.githubusercontent.com', '443', context=ssl._create_unverified_context()) try: ts1 = datetime.datetime.now().timestamp() print('start time=' + str(ts1)) endpoint.request('GET', url) response = endpoint.getresponse() tokenizer = JsonTokenizer(response, 'ISO-8859-1', 65536) JsonDecoder()\ .tokenizer(tokenizer)\ .root_class_name('Data')\ .event_handler(lambda e, p: handle_event(e, p, counter))\ .predicate('genres') \ .with_snake_cased_props()\
def _sendPushover(self, msg, title, sound=None, userKey=None, apiKey=None): """ Sends a pushover notification to the address provided msg: The message to send (unicode) title: The title of the message sound: The notification sound to use userKey: The pushover user id to send the message to (or to subscribe with) apiKey: The pushover api key to use returns: True if the message succeeded, False otherwise """ if userKey is None: userKey = app.PUSHOVER_USERKEY if apiKey is None: apiKey = app.PUSHOVER_APIKEY if sound is None: sound = app.PUSHOVER_SOUND log.debug(u'Pushover API KEY in use: {0}', apiKey) # build up the URL and parameters msg = msg.strip() # send the request to pushover try: if app.PUSHOVER_SOUND != 'default': args = { 'token': apiKey, 'user': userKey, 'title': title.encode('utf-8'), 'message': msg.encode('utf-8'), 'timestamp': int(time.time()), 'retry': 60, 'expire': 3600, 'sound': sound, } else: # sound is default, so don't send it args = { 'token': apiKey, 'user': userKey, 'title': title.encode('utf-8'), 'message': msg.encode('utf-8'), 'timestamp': int(time.time()), 'retry': 60, 'expire': 3600, } if app.PUSHOVER_DEVICE: args['device'] = ','.join(app.PUSHOVER_DEVICE) conn = HTTPSConnection('api.pushover.net:443') conn.request('POST', '/1/messages.json', urlencode(args), {'Content-type': 'application/x-www-form-urlencoded'}) except HTTPError as e: # if we get an error back that doesn't have an error code then who knows what's really happening if not hasattr(e, 'code'): log.error(u'Pushover notification failed. {}', ex(e)) return False else: log.error(u'Pushover notification failed. Error code: {0}', e.code) # HTTP status 404 if the provided email address isn't a Pushover user. if e.code == 404: log.warning( u'Username is wrong/not a pushover email. Pushover will send an email to it' ) return False # For HTTP status code 401's, it is because you are passing in either an invalid token, or the user has not added your service. elif e.code == 401: # HTTP status 401 if the user doesn't have the service added subscribeNote = self._sendPushover(msg, title, sound=sound, userKey=userKey, apiKey=apiKey) if subscribeNote: log.debug(u'Subscription sent') return True else: log.error(u'Subscription could not be sent') return False # If you receive an HTTP status code of 400, it is because you failed to send the proper parameters elif e.code == 400: log.error(u'Wrong data sent to pushover') return False # If you receive a HTTP status code of 429, it is because the message limit has been reached (free limit is 7,500) elif e.code == 429: log.error( u'Pushover API message limit reached - try a different API key' ) return False log.info(u'Pushover notification successful.') return True
def __init__(self, url, key=None, secret=None, timeout=5, context=None): """ Create a new Crossbar.io push client. The only mandatory argument is the Push service endpoint of the Crossbar.io instance to push to. For signed pushes, provide authentication key and secret. If those are not given, unsigned pushes are performed. :param url: URL of the HTTP bridge of Crossbar.io (e.g. http://example.com:8080/push). :type url: str :param key: Optional key to use for signing requests. :type key: str :param secret: When using signed request, the secret corresponding to key. :type secret: str :param timeout: Timeout for requests. :type timeout: int :param context: If the HTTP bridge is running on HTTPS (that is securely over TLS), then the context provides the SSL settings the client should use (e.g. the certificate chain against which to verify the server certificate). This parameter is only available on Python 2.7.9+ and Python 3 (otherwise the parameter is silently ignored!). See: https://docs.python.org/2/library/ssl.html#ssl.SSLContext :type context: obj or None """ if six.PY2: if type(url) == str: url = six.u(url) if type(key) == str: key = six.u(key) if type(secret) == str: secret = six.u(secret) assert (type(url) == six.text_type) assert ((key and secret) or (not key and not secret)) assert (key is None or type(key) == six.text_type) assert (secret is None or type(secret) == six.text_type) assert (type(timeout) == int) if _HAS_SSL and _HAS_SSL_CLIENT_CONTEXT: assert (context is None or isinstance(context, ssl.SSLContext)) self._seq = 1 self._key = key self._secret = secret self._endpoint = _parse_url(url) self._endpoint['headers'] = { "Content-type": "application/json", "User-agent": "crossbarconnect-python" } if self._endpoint['secure']: if not _HAS_SSL: raise Exception( "Bridge URL is using HTTPS, but Python SSL module is missing" ) if _HAS_SSL_CLIENT_CONTEXT: self._connection = HTTPSConnection(self._endpoint['host'], self._endpoint['port'], timeout=timeout, context=context) else: self._connection = HTTPSConnection(self._endpoint['host'], self._endpoint['port'], timeout=timeout) else: self._connection = HTTPConnection(self._endpoint['host'], self._endpoint['port'], timeout=timeout)
def retry_using_http_NTLM_auth(self, req, auth_header_field, realm, headers): user, pw = self.passwd.find_user_password(realm, req.get_full_url()) if pw is not None: user_parts = user.split('\\', 1) if len(user_parts) == 1: UserName = user_parts[0] DomainName = '' type1_flags = ntlm.NTLM_TYPE1_FLAGS & ~ntlm.NTLM_NegotiateOemDomainSupplied else: DomainName = user_parts[0].upper() UserName = user_parts[1] type1_flags = ntlm.NTLM_TYPE1_FLAGS # ntlm secures a socket, so we must use the same socket for the complete handshake headers = dict(req.headers) headers.update(req.unredirected_hdrs) auth = 'NTLM %s' % ntlm.create_NTLM_NEGOTIATE_MESSAGE(user, type1_flags).decode("utf-8") if req.headers.get(self.auth_header, None) == auth: return None headers[self.auth_header] = auth host = req.host if not host: raise urllib.request.URLError('no host given') h = None if req.get_full_url().startswith('https://'): h = HTTPSConnection(host) # will parse host:port else: h = HTTPConnection(host) # will parse host:port h.set_debuglevel(self._debuglevel) # we must keep the connection because NTLM authenticates the connection, not single requests headers["Connection"] = "Keep-Alive" headers = dict((name.title(), val) for name, val in headers.items()) # For some reason, six doesn't do this translation correctly # TODO rsanders low - find bug in six & fix it try: selector = req.selector except AttributeError: selector = req.get_selector() h.request(req.get_method(), selector, req.data, headers) r = h.getresponse() r.begin() r._safe_read(int(r.getheader('content-length'))) if r.getheader('set-cookie'): # this is important for some web applications that store authentication-related info in cookies (it took a long time to figure out) headers['Cookie'] = r.getheader('set-cookie') r.fp = None # remove the reference to the socket, so that it can not be closed by the response object (we want to keep the socket open) auth_header_value = r.getheader(auth_header_field, None) # some Exchange servers send two WWW-Authenticate headers, one with the NTLM challenge # and another with the 'Negotiate' keyword - make sure we operate on the right one m = re.match('(NTLM [A-Za-z0-9+\-/=]+)', auth_header_value) if m: auth_header_value, = m.groups() (ServerChallenge, NegotiateFlags) = ntlm.parse_NTLM_CHALLENGE_MESSAGE(auth_header_value[5:]) auth = 'NTLM %s' % ntlm.create_NTLM_AUTHENTICATE_MESSAGE(ServerChallenge, UserName, DomainName, pw, NegotiateFlags).decode("utf-8") headers[self.auth_header] = auth headers["Connection"] = "Close" headers = dict((name.title(), val) for name, val in headers.items()) try: h.request(req.get_method(), selector, req.data, headers) # none of the configured handlers are triggered, for example redirect-responses are not handled! response = h.getresponse() def notimplemented(): raise NotImplementedError response.readline = notimplemented infourl = urllib.response.addinfourl(response, response.msg, req.get_full_url()) infourl.code = response.status infourl.msg = response.reason return infourl except socket.error as err: raise urllib.error.URLError(err) else: return None
'Authorization': transfer.signed_ticket, } # At this stage, the SDK granted the permission to start transferring the disk, and the # user should choose its preferred tool for doing it - regardless of the SDK. # In this example, we will use Python's httplib.HTTPSConnection for transferring the data. proxy_url = urlparse(transfer.proxy_url) context = ssl.create_default_context() # Note that ovirt-imageio-proxy by default checks the certificates, so if you don't have # your CA certificate of the engine in the system, you need to pass it to HTTPSConnection. context.load_verify_locations(cafile='ca.pem') proxy_connection = HTTPSConnection( proxy_url.hostname, proxy_url.port, context=context, ) print("Downloading image...") try: # Send the request proxy_connection.request( 'GET', proxy_url.path, headers=transfer_headers, ) # Get response r = proxy_connection.getresponse()
def apihit(host,conntype,authtoken,queryurl,reqbody, proxy_host=None, proxy_port=None): retdata = '' if proxy_host and proxy_port: connection = HTTPSConnection(proxy_host, proxy_port) connection.set_tunnel(host, 443) else: connection = HTTPSConnection(host) tokenheader = {"Authorization": 'Bearer ' + authtoken, "Content-type": "application/json", "Accept": "text/plain"} if conntype == "GET": connection.request(conntype, queryurl, '', tokenheader) else: connection.request(conntype, queryurl, json.dumps(reqbody), tokenheader) response = connection.getresponse() respbody = response.read().decode('ascii', 'ignore') try: jsondata = respbody.decode() retdata = json.loads(jsondata) except AttributeError: retdata = json.loads(respbody) except: raise connection.close() return retdata
def get_auth_token(host, clientid, clientsecret, proxy_host=None, proxy_port=None): queryurl = '/oauth/access_token' if proxy_host and proxy_port: connection = HTTPSConnection(proxy_host, proxy_port) connection.set_tunnel(host, 443) else: connection = HTTPSConnection(host) authtoken = base64.b64encode(six.b('{0}:{1}'.format(clientid, clientsecret))) authstring = b"Basic %s" % (authtoken,) header = {"Authorization": authstring} params = urlencode({'grant_type': 'client_credentials'}) connection.request("POST", queryurl, params, header) response = connection.getresponse() jsondata = bytes(response.read()).decode('utf-8') data = json.loads(str(jsondata)) try: key = data['access_token'] except KeyError: print("We're having trouble getting a session token. Please check your API key.") print("Error output: ") print(data) sys.exit() finally: connection.close() return key
class HessianProxy(object): def __init__(self, service_uri, credentials=None, key_file=None, cert_file=None, timeout=10, buffer_size=65535, error_factory=lambda x: x, overload=False, version=1): self.version = version self._headers = list() self._headers.append(('User-Agent', 'python-hessian/' + __version__,)) self._headers.append(('Content-Type', 'application/x-hessian',)) if sys.version_info < (2, 6): warn('HessianProxy timeout not enforceable before Python 2.6', RuntimeWarning, stacklevel=2) kwargs = {} else: kwargs = {'timeout': timeout} if six.PY2: kwargs['strict'] = True self._uri = urlparse(service_uri) if self._uri.scheme == 'http': self._client = HTTPConnection(self._uri.hostname, self._uri.port or 80, **kwargs) elif self._uri.scheme == 'https': self._client = HTTPSConnection(self._uri.hostname, self._uri.port or 443, key_file=key_file, cert_file=cert_file, **kwargs) else: raise NotImplementedError("HessianProxy only supports http:// and https:// URIs") # autofill credentials if they were passed via url instead of kwargs if (self._uri.username and self._uri.password) and not credentials: credentials = (self._uri.username, self._uri.password) if credentials: auth = 'Basic ' + base64.b64encode(':'.join(credentials)) self._headers.append(('Authorization', auth)) self._buffer_size = buffer_size self._error_factory = error_factory self._overload = overload self._parser = Parser() class __RemoteMethod(object): # dark magic for autoloading methods def __init__(self, caller, method): self.__caller = caller self.__method = method def __call__(self, *args): return self.__caller(self.__method, args) def __getattr__(self, method): return self.__RemoteMethod(self, method) def __repr__(self): return "<pyhessian.client.HessianProxy(\"%s\")>" % (self._uri.geturl(),) def __str__(self): return self.__repr__() def __call__(self, method, args): try: self._client.putrequest('POST', self._uri.path) for header in self._headers: self._client.putheader(*header) request = encode_object(Call(method, args, overload=self._overload, version=self.version)) self._client.putheader("Content-Length", str(len(request))) self._client.endheaders() self._client.send(six.binary_type(request)) response = self._client.getresponse() if response.status != 200: raise ProtocolError(self._uri.geturl(), response.status, response.reason) length = response.getheader('Content-Length', -1) if length == '0': raise ProtocolError(self._uri.geturl(), 'FATAL:', 'Server sent zero-length response') reply = self._parser.parse_stream(BufferedReader(response, buffer_size=self._buffer_size)) self._client.close() if isinstance(reply.value, Fault): raise self._error_factory(reply.value) else: return reply.value except: self._client.close() raise
def _send_pushover(self, msg, title, sound=None, user_key=None, api_key=None, priority=None): """ Send a pushover notification to the address provided. :param msg: The message to send (unicode) :param title: The title of the message :param sound: The notification sound to use :param user_key: The pushover user id to send the message to (or to subscribe with) :param api_key: The pushover api key to use :param priority: The pushover priority to use :return: True if the message succeeded, False otherwise """ if user_key is None: user_key = app.PUSHOVER_USERKEY if api_key is None: api_key = app.PUSHOVER_APIKEY if sound is None: sound = app.PUSHOVER_SOUND if priority is None: priority = app.PUSHOVER_PRIORITY # build up the URL and parameters msg = msg.strip() # default args args = { 'token': api_key, 'user': user_key, 'title': title.encode('utf-8'), 'message': msg.encode('utf-8'), 'timestamp': int(time.time()), 'retry': 60, 'expire': 3600, 'priority': priority, } # If sound is not default, add it. if sound != 'default': args['sound'] = sound if app.PUSHOVER_DEVICE: args['device'] = ','.join(app.PUSHOVER_DEVICE) log.debug( 'PUSHOVER: Sending notice with details: title="{0}" message="{1}", priority={2}, sound={3}', args['title'], args['message'], priority, sound) conn = HTTPSConnection('api.pushover.net:443') conn.request('POST', '/1/messages.json', urlencode(args), {'Content-type': 'application/x-www-form-urlencoded'}) conn_resp = conn.getresponse() if conn_resp.status == 200: log.info('Pushover notification successful.') return True # HTTP status 404 if the provided email address isn't a Pushover user. elif conn_resp.status == 404: log.warning( 'Username is wrong/not a pushover email. Pushover will send an email to it' ) return False # For HTTP status code 401's, it is because you are passing in either an invalid token, or the user has not added your service. elif conn_resp.status == 401: # HTTP status 401 if the user doesn't have the service added subscribe_note = self._send_pushover(msg, title, sound=sound, user_key=user_key, api_key=api_key) if subscribe_note: log.debug('Subscription sent') return True else: log.error('Subscription could not be sent') # If you receive an HTTP status code of 400, it is because you failed to send the proper parameters elif conn_resp.status == 400: log.error('Wrong keys sent to pushover') return False # If you receive a HTTP status code of 429, it is because the message limit has been reached (free limit is 7,500) elif conn_resp.status == 429: log.error( 'Pushover API message limit reached - try a different API key') return False # Something else has gone wrong... who knows what's really happening else: log.error('Pushover notification failed. HTTP response code: {0}', conn_resp.status) return False
def __init__(self, uri, basepath=None): self.basepath = basepath self.mimetype = None self.file = None self.data = None self.uri = None self.local = None self.tmp_file = None uri = uri or str() uri = uri.encode('utf-8') log.debug("FileObject %r, Basepath: %r", uri, basepath) # Data URI if isinstance(uri, binary_type): uri = uri.decode('utf-8') if uri.startswith("data:"): m = _rx_datauri.match(uri) self.mimetype = m.group("mime") self.data = base64.decodestring(m.group("data")) else: # Check if we have an external scheme if basepath and not urlparse(uri).scheme: urlParts = urlparse(basepath) else: urlParts = urlparse(uri) log.debug("URLParts: %r", urlParts) if urlParts.scheme == 'file': if basepath and uri.startswith('/'): uri = urljoin(basepath, uri[1:]) urlResponse = urlopen(uri) self.mimetype = urlResponse.info().get("Content-Type", '').split(";")[0] self.uri = urlResponse.geturl() self.file = urlResponse # Drive letters have len==1 but we are looking # for things like http: elif urlParts.scheme in ('http', 'https'): # External data if basepath: uri = urljoin(basepath, uri) #path = urlsplit(url)[2] #mimetype = getMimeType(path) # Using HTTPLIB parts = urlsplit(uri[uri.find("//"):]) server, path = parts.netloc, parts.path if uri.startswith("https://"): conn = HTTPSConnection(server) else: conn = HTTPConnection(server) conn.request("GET", path) r1 = conn.getresponse() # log.debug("HTTP %r %r %r %r", server, path, uri, r1) if (r1.status, r1.reason) == (200, "OK"): self.mimetype = r1.getheader("Content-Type", '').split(";")[0] self.uri = uri if r1.getheader("content-encoding") == "gzip": import gzip self.file = gzip.GzipFile(mode="rb", fileobj=BytesIO(r1.read())) else: self.file = r1 else: try: urlResponse = urlopen(uri) except HTTPError: return self.mimetype = urlResponse.info().get("Content-Type", '').split(";")[0] self.uri = urlResponse.geturl() self.file = urlResponse else: # Local data if basepath: uri = os.path.normpath(os.path.join(basepath, uri)) if os.path.isfile(uri): self.uri = uri self.local = uri self.setMimeTypeByName(uri) self.file = open(uri, "rb")
def retry_using_http_NTLM_auth(self, req, auth_header_field, realm, headers): user, pw = self.passwd.find_user_password(realm, req.get_full_url()) if pw is not None: user_parts = user.split('\\', 1) if len(user_parts) == 1: UserName = user_parts[0] DomainName = '' type1_flags = ntlm.NTLM_TYPE1_FLAGS & ~ntlm.NTLM_NegotiateOemDomainSupplied else: DomainName = user_parts[0].upper() UserName = user_parts[1] type1_flags = ntlm.NTLM_TYPE1_FLAGS # ntlm secures a socket, so we must use the same socket for the complete handshake headers = dict(req.headers) headers.update(req.unredirected_hdrs) auth = 'NTLM %s' % ntlm.create_NTLM_NEGOTIATE_MESSAGE(user, type1_flags) if req.headers.get(self.auth_header, None) == auth: return None headers[self.auth_header] = auth host = req.host if not host: raise urllib.request.URLError('no host given') h = None if req.get_full_url().startswith('https://'): h = HTTPSConnection(host) # will parse host:port else: h = HTTPConnection(host) # will parse host:port h.set_debuglevel(self._debuglevel) # we must keep the connection because NTLM authenticates the connection, not single requests headers["Connection"] = "Keep-Alive" headers = dict((name.title(), val) for name, val in headers.items()) # For some reason, six doesn't do this translation correctly # TODO rsanders low - find bug in six & fix it try: selector = req.selector except AttributeError: selector = req.get_selector() h.request(req.get_method(), selector, req.data, headers) r = h.getresponse() r.begin() r._safe_read(int(r.getheader('content-length'))) if r.getheader('set-cookie'): # this is important for some web applications that store authentication-related info in cookies (it took a long time to figure out) headers['Cookie'] = r.getheader('set-cookie') r.fp = None # remove the reference to the socket, so that it can not be closed by the response object (we want to keep the socket open) auth_header_value = r.getheader(auth_header_field, None) # some Exchange servers send two WWW-Authenticate headers, one with the NTLM challenge # and another with the 'Negotiate' keyword - make sure we operate on the right one m = re.match('(NTLM [A-Za-z0-9+\-/=]+)', auth_header_value) if m: auth_header_value, = m.groups() (ServerChallenge, NegotiateFlags) = ntlm.parse_NTLM_CHALLENGE_MESSAGE(auth_header_value[5:]) auth = 'NTLM %s' % ntlm.create_NTLM_AUTHENTICATE_MESSAGE(ServerChallenge, UserName, DomainName, pw, NegotiateFlags) headers[self.auth_header] = auth headers["Connection"] = "Close" headers = dict((name.title(), val) for name, val in headers.items()) try: h.request(req.get_method(), selector, req.data, headers) # none of the configured handlers are triggered, for example redirect-responses are not handled! response = h.getresponse() def notimplemented(): raise NotImplementedError response.readline = notimplemented infourl = urllib.response.addinfourl(response, response.msg, req.get_full_url()) infourl.code = response.status infourl.msg = response.reason return infourl except socket.error as err: raise urllib.error.URLError(err) else: return None
def fetch_response(self, request): """Executes request and fetchs service response""" connection = HTTPSConnection(self.SERVER_URL) connection.request(request.method.upper(), request.to_url()) return connection.getresponse().read()
def _sendPushover(self, msg, title, sound=None, userKey=None, apiKey=None): """ Sends a pushover notification to the address provided msg: The message to send (unicode) title: The title of the message sound: The notification sound to use userKey: The pushover user id to send the message to (or to subscribe with) apiKey: The pushover api key to use returns: True if the message succeeded, False otherwise """ if userKey is None: userKey = sickbeard.PUSHOVER_USERKEY if apiKey is None: apiKey = sickbeard.PUSHOVER_APIKEY if sound is None: sound = sickbeard.PUSHOVER_SOUND logger.log(u"Pushover API KEY in use: " + apiKey, logger.DEBUG) # build up the URL and parameters msg = msg.strip() # send the request to pushover try: if sickbeard.PUSHOVER_SOUND != "default": args = { "token": apiKey, "user": userKey, "title": title.encode('utf-8'), "message": msg.encode('utf-8'), "timestamp": int(time.time()), "retry": 60, "expire": 3600, "sound": sound, } else: # sound is default, so don't send it args = { "token": apiKey, "user": userKey, "title": title.encode('utf-8'), "message": msg.encode('utf-8'), "timestamp": int(time.time()), "retry": 60, "expire": 3600, } if sickbeard.PUSHOVER_DEVICE: args["device"] = sickbeard.PUSHOVER_DEVICE conn = HTTPSConnection("api.pushover.net:443") conn.request("POST", "/1/messages.json", urlencode(args), {"Content-type": "application/x-www-form-urlencoded"}) except HTTPError as e: # if we get an error back that doesn't have an error code then who knows what's really happening if not hasattr(e, 'code'): logger.log(u"Pushover notification failed." + ex(e), logger.ERROR) return False else: logger.log(u"Pushover notification failed. Error code: " + str(e.code), logger.ERROR) # HTTP status 404 if the provided email address isn't a Pushover user. if e.code == 404: logger.log(u"Username is wrong/not a pushover email. Pushover will send an email to it", logger.WARNING) return False # For HTTP status code 401's, it is because you are passing in either an invalid token, or the user has not added your service. elif e.code == 401: # HTTP status 401 if the user doesn't have the service added subscribeNote = self._sendPushover(msg, title, sound=sound, userKey=userKey, apiKey=apiKey) if subscribeNote: logger.log(u"Subscription sent", logger.DEBUG) return True else: logger.log(u"Subscription could not be sent", logger.ERROR) return False # If you receive an HTTP status code of 400, it is because you failed to send the proper parameters elif e.code == 400: logger.log(u"Wrong data sent to pushover", logger.ERROR) return False # If you receive a HTTP status code of 429, it is because the message limit has been reached (free limit is 7,500) elif e.code == 429: logger.log(u"Pushover API message limit reached - try a different API key", logger.ERROR) return False logger.log(u"Pushover notification successful.", logger.INFO) return True
class TestLicenseClient(object): def __init__(self, parser=None): # type: (Optional[ArgumentParser]) -> None """ Class constructor for the test license client and HTMLParser """ self.log = logging.getLogger("License_Client") self.setup_logging() self.parser = parser self.license_server_url = 'license.univention.de' self.license_filename = 'ValidTest.license' self.connection = None # type: Optional[HTTPSConnection] self.server_username = '******' self.server_password = '' self.secret_file = '/etc/license.secret' self.cookie = '' self.license_shop = 'testing' self.license_params = { "kundeUnternehmen": "Univention", "kundeEmail": "*****@*****.**", "BaseDN": "", "EndDate": "", "Servers": 50, "Support": 0, "PremiumSupport": 0, "Users": 50, "ManagedClients": 50, "CorporateClients": 50, "VirtualDesktopUsers": 0, "VirtualDesktopClients": 0, "Type": "UCS", } # type: Dict[str, Any] def setup_logging(self): # type: () -> None """ Creates and configures the logger with an INFO level """ self.log.setLevel(logging.INFO) ch = logging.StreamHandler() ch.setLevel(logging.DEBUG) ch.setFormatter( logging.Formatter( "%(asctime)s - %(name)s - %(levelname)s - %(message)s")) self.log.addHandler(ch) def create_connection(self): # type: () -> None """ Creates a HTTPS connection instance on a default port (443) to the 'self.license_server_url' """ self.log.debug("In 'create_connection'") self.connection = HTTPSConnection(self.license_server_url) def close_connection(self): # type: () -> None """ Closes the license server connection if the connection instance was created """ self.log.debug("In 'close_connection'") if self.connection: try: self.connection.close() except HTTPException as exc: self.log.exception( "An HTTP Exception occurred while closing the connection: '%s'", exc) def get_server_password(self, secret_file='/etc/license.secret'): # type: (str) -> None """ Opens and reads the 'secret_file'. Saves the result to a 'self.server_password' """ self.log.debug("In 'get_server_password': secret_file='%s'", secret_file) if not path.exists(secret_file): self.log.critical( "The '%s' secret file does not exist, cannot proceed without password", secret_file) raise CredentialsMissing("The '%s' secret file does not exist" % secret_file) try: with open(secret_file, 'r') as password: self.server_password = password.read() except (IOError, ValueError) as exc: self.log.exception( "Failed to get the password from the '%s', an error occurred: %r", secret_file, exc) exit(1) if not self.server_password: self.log.critical( "The password to access the license service cannot be empty") exit(1) def get_cookie(self): # type: () -> None """ Makes a POST request with 'self.server_username' and 'self.server_password' into login forms and saves the cookie from the response received. """ self.log.debug("In 'get_cookie' method") body = { "username": self.server_username, "password": self.server_password, } headers = { "Content-type": "application/x-www-form-urlencoded", "Accept": "text/plain", } response = self.make_post_request('/shop/%s/' % (self.license_shop, ), urlencode(body), headers) self.log.debug( "The response status is '%s', reason is '%s', headers are '%s'", response.status, response.reason, response.getheaders()) self.cookie = response.getheader('set-cookie') if not ('sessionid' in self.cookie): self.log.critical( "The 'sessionid' field was not found in the received cookie: '%s'", self.cookie) exit(1) # extracting only the 'sessionid' part of the cookie: self.cookie = self.cookie[:self.cookie.find(';')] self.log.debug("The cookie is: '%s'", self.cookie) # reading the response to avoid 'ResponseNotReady' exception later: response.read() def make_post_request(self, url, body, headers): # type: (str, str, Dict[str, str]) -> HTTPResponse """ Makes a POST request with the given 'url', 'body', 'headers' and returns the response """ self.log.debug( "In 'make_post_request' method: url='%s', body='%s', headers='%s'", url, body, headers) assert self.connection try: self.connection.request("POST", url, body, headers) return self.connection.getresponse() except HTTPException as exc: self.log.exception( "An HTTP Exception occurred while making '%s' POST request: '%s'", url, exc) exit(1) def make_get_request(self, url, headers): # type: (str, Dict[str, str]) -> HTTPResponse """ Makes a GET request with the given 'url', 'headers' and returns the response """ self.log.debug("In 'make_get_request' method: url='%s', headers='%s'", url, headers) assert self.connection try: self.connection.request("GET", url, headers=headers) return self.connection.getresponse() except HTTPException as exc: self.log.exception( "An HTTP Exception occurred while making '%s' GET request: '%s'", url, exc) exit(1) def get_the_license(self, body): # type: (str) -> None """ Processes the given 'body' with HTMLParser to find the link to a created license file and downloads the license after. """ self.log.debug("In 'get_the_license' method: body='%s'", body) parser = ShopParser(self.log) parser.feed(body) # process the response 'body' for license link if not parser.link_to_license: self.log.critical( "The link to the license file was not found in the body from server: '%s'", body) exit(1) self.download_license_file(parser.link_to_license) def order_a_license(self): # type: () -> str """ Makes a POST request with encoded 'self.license_params' as a body to order a new license. Returns the response body. """ self.log.debug("In 'order_a_license' method") body = self.license_params headers = { "Cookie": self.cookie, "Content-type": "application/x-www-form-urlencoded", } response = self.make_post_request( '/shop/%s/order' % (self.license_shop, ), urlencode(body), headers) assert response.status == 202 return self.get_body(response) def get_body(self, response): # type: (HTTPResponse) -> str self.log.debug( "The response status is '%s', reason is '%s', headers are '%s'", response.status, response.reason, response.getheaders()) content_type = response.getheader('Content-Type') mimetype, options = cgi.parse_header(content_type) encoding = options.get("charset", "ascii") return response.read().decode(encoding, "replace") def download_license_file(self, link_to_license): # type: (str) -> None """ Downloads the license located at `filename` and saves it to the file with a 'self.license_filename' """ self.log.debug("In 'download_license_file' method") headers = { "Cookie": self.cookie, "Accept": "text/plain", } response = self.make_get_request( '/shop/%s/%s' % (self.license_shop, link_to_license), headers) try: body = self.get_body(response) with open(self.license_filename, 'w') as license_file: license_file.write(body) self.log.info("The license was written to file '%s'", self.license_filename) except (IOError, ValueError) as exc: self.log.exception( "An error happened while writing the downloaded license to a file '%s': '%s'", self.license_filename, exc) exit(1) def check_date_format(self): # type: () -> None """ Checks if the 'EndDate' format is correct. """ try: if self.license_params['EndDate'] != 'unlimited': datetime.strptime(self.license_params['EndDate'], '%d.%m.%Y') except ValueError as exc: self.log.exception( "The 'EndDate' for the license has a wrong format, supported format is 'dd.mm.yyyy': %r", exc) exit(1) def update_with_parsed_args(self, args): # type: (Dict[str, Any]) -> None """ Updates the loglevel and license filename settings if given among the parsed arguments. Merges parsed data with default license parameters. """ log_level = args.pop('LogLevel') if log_level: numeric_level = getattr(logging, log_level.upper(), None) if isinstance(numeric_level, int): self.log.setLevel(numeric_level) else: self.log.info( "The LogLevel was not changed, unknown '%s' log level given", log_level) self.license_shop = args.pop('shop') self.server_username = args.pop('username') self.secret_file = args.pop('secret_file') license_file = args.pop('FileName') if license_file: self.license_filename = license_file self.log.debug("The filename for the license will be '%s'", self.license_filename) # merging parsed args with the default values: self.license_params.update( (key, val) for key, val in args.items() if val is not None) self.log.info("Requested license parameters are: '%s'", self.license_params) def process_cmd_arguments(self): # type: () -> None """ Populates self.parser class with positional and optional arguments and processes the user input, checks the date format and than merges it with the default values in the 'self.license_params' dictionary """ self.log.debug("In 'process_cmd_arguments' method") assert self.parser self.parser.add_argument("BaseDN", help="A base DN for the license") self.parser.add_argument( "EndDate", help= "The date till which the license will be valid (max 1 year from now)" ) self.parser.add_argument( "-f", "--FileName", help="The filename to be used for the issued license", default="ValidTest.license") self.parser.add_argument( "-s", "--Servers", type=int, help="Max amount of servers allowed with the license", default=50) self.parser.add_argument( "-u", "--Users", type=int, help="Max amount of users allowed with the license", default=50) self.parser.add_argument( "-mc", "--ManagedClients", type=int, help="Max amount of managed clients allowed with the license", default=50) self.parser.add_argument( "-cc", "--CorporateClients", type=int, help="Max amount of corporate clients allowed with the license", default=50) self.parser.add_argument("-ll", "--LogLevel", help="Logging level", choices=("INFO", "DEBUG", "ERROR", "CRITICAL"), default="INFO") self.parser.add_argument("--shop", help="The shop", default=self.license_shop) self.parser.add_argument("--username", help="username", default=self.server_username) self.parser.add_argument("--secret-file", help="password file", default=self.secret_file) opts = self.parser.parse_args() args = vars(opts) # converting Namespace to a dictionary self.log.debug("Parsed arguments are: '%s'", args) self.update_with_parsed_args(args) def main(self, base_dn="", end_date="", server_url="", license_file=""): # type: (str, str, str, str) -> None """ A method to order and download a test license from the license server. 'base_dn' and 'end_date' should be provided if argument parser is not used. 'server_url' is an optional argument for the license shop server. 'license_file' is an optional argument for the license filename. """ self.log.debug( "In 'main' method: server_url='%s', license_file='%s', base_dn='%s', end_date='%s'", server_url, license_file, base_dn, end_date) if self.parser: self.process_cmd_arguments() elif base_dn and end_date: self.license_params['BaseDN'] = base_dn self.license_params['EndDate'] = end_date else: self.log.error( "The 'BaseDN' or/and 'EndDate' were not provided for the license to create" ) exit(1) self.check_date_format() if server_url: self.license_server_url = server_url if license_file: self.license_filename = license_file self.get_server_password(self.secret_file) try: self.create_connection() self.get_cookie() self.get_the_license(self.order_a_license()) finally: self.close_connection()
def _update_index(self, url=None): """A helper function that ensures that self._index is up-to-date. If the index is older than self.INDEX_TIMEOUT, then download it again.""" # Check if the index is aleady up-to-date. If so, do nothing. if not (self._index is None or url is not None or time.time()-self._index_timestamp > self.INDEX_TIMEOUT): return # If a URL was specified, then update our URL. self._url = url or self._url source = self._source assert source == 'google' or source == 'mirror' # Download the index file. if source == 'google': host = "www.googleapis.com" conn = HTTPSConnection(host) conn.request("GET", "/storage/v1/b/{}/o".format(self._url)) r1 = conn.getresponse() data = r1.read() elif source == 'mirror': index_url = path.join(self._url, 'index.json') data = urlopen(index_url).read() if six.PY3: data = data.decode('utf-8') data = loads(data) objs = data["items"] self._index_timestamp = time.time() # Build a dictionary of packages. packages = [] for p in objs: P = Package.fromcsobj(p) packages.append(P) self._packages = dict((p.id, p) for p in packages) # Build language collections. langs = defaultdict(lambda: []) for k in self._packages: package = self._packages[k] langs[package.language].append(package) tasks = defaultdict(lambda: []) for k in self._packages: package = self._packages[k] tasks[package.task].append(package) collections = [] for lang in langs: children = langs[lang] name1 = Locale(lang).getDisplayLanguage() try: name2 = isoLangs[lang]['name'] except: name2 = None if name1 and name1 != lang: name = name1 elif name2: name = name2 else: name = lang id = "{}{}".format(Downloader.LANG_PREFIX, lang) name = "{:<20} packages and models".format(name) c = Collection(id=id, name=name, children=children) collections.append(c) for task in tasks: children = tasks[task] id = "{}{}".format(Downloader.TASK_PREFIX, task) c = Collection(id=id, name=task, children=children) collections.append(c) self._collections = dict((c.id, c) for c in collections) # Replace identifiers with actual children in collection.children. for collection in self._collections.values(): for i, child_id in enumerate(collection.children): if child_id in self._packages: collection.children[i] = self._packages[child_id] if child_id in self._collections: collection.children[i] = self._collections[child_id] # Fill in collection.packages for each collection. for collection in self._collections.values(): packages = {} queue = [collection] for child in queue: if isinstance(child, Collection): queue.extend(child.children) else: packages[child.id] = child collection.packages = packages.values() # Flush the status cache self._status_cache.clear()
def _sendPushover(self, msg, title, sound=None, userKey=None, apiKey=None): """ Sends a pushover notification to the address provided msg: The message to send (unicode) title: The title of the message sound: The notification sound to use userKey: The pushover user id to send the message to (or to subscribe with) apiKey: The pushover api key to use returns: True if the message succeeded, False otherwise """ if userKey is None: userKey = app.PUSHOVER_USERKEY if apiKey is None: apiKey = app.PUSHOVER_APIKEY if sound is None: sound = app.PUSHOVER_SOUND logger.log(u"Pushover API KEY in use: " + apiKey, logger.DEBUG) # build up the URL and parameters msg = msg.strip() # send the request to pushover try: if app.PUSHOVER_SOUND != "default": args = { "token": apiKey, "user": userKey, "title": title.encode('utf-8'), "message": msg.encode('utf-8'), "timestamp": int(time.time()), "retry": 60, "expire": 3600, "sound": sound, } else: # sound is default, so don't send it args = { "token": apiKey, "user": userKey, "title": title.encode('utf-8'), "message": msg.encode('utf-8'), "timestamp": int(time.time()), "retry": 60, "expire": 3600, } if app.PUSHOVER_DEVICE: args["device"] = app.PUSHOVER_DEVICE conn = HTTPSConnection("api.pushover.net:443") conn.request("POST", "/1/messages.json", urlencode(args), {"Content-type": "application/x-www-form-urlencoded"}) except HTTPError as e: # if we get an error back that doesn't have an error code then who knows what's really happening if not hasattr(e, 'code'): logger.log(u"Pushover notification failed." + ex(e), logger.ERROR) return False else: logger.log(u"Pushover notification failed. Error code: " + str(e.code), logger.ERROR) # HTTP status 404 if the provided email address isn't a Pushover user. if e.code == 404: logger.log(u"Username is wrong/not a pushover email. Pushover will send an email to it", logger.WARNING) return False # For HTTP status code 401's, it is because you are passing in either an invalid token, or the user has not added your service. elif e.code == 401: # HTTP status 401 if the user doesn't have the service added subscribeNote = self._sendPushover(msg, title, sound=sound, userKey=userKey, apiKey=apiKey) if subscribeNote: logger.log(u"Subscription sent", logger.DEBUG) return True else: logger.log(u"Subscription could not be sent", logger.ERROR) return False # If you receive an HTTP status code of 400, it is because you failed to send the proper parameters elif e.code == 400: logger.log(u"Wrong data sent to pushover", logger.ERROR) return False # If you receive a HTTP status code of 429, it is because the message limit has been reached (free limit is 7,500) elif e.code == 429: logger.log(u"Pushover API message limit reached - try a different API key", logger.ERROR) return False logger.log(u"Pushover notification successful.", logger.INFO) return True
def _update_index(self, url=None): """A helper function that ensures that self._index is up-to-date. If the index is older than self.INDEX_TIMEOUT, then download it again.""" # Check if the index is aleady up-to-date. If so, do nothing. if not (self._index is None or url is not None or time.time()-self._index_timestamp > self.INDEX_TIMEOUT): return # If a URL was specified, then update our URL. self._url = url or self._url # Download the index file. host = "www.googleapis.com" conn = HTTPSConnection(host) conn.request("GET", "/storage/v1/b/{}/o".format(self._url)) r1 = conn.getresponse() data = r1.read() if six.PY3: data = data.decode('utf-8') data = loads(data) objs = data["items"] self._index_timestamp = time.time() # Build a dictionary of packages. packages = [] for p in objs: P = Package.fromcsobj(p) packages.append(P) self._packages = dict((p.id, p) for p in packages) # Build language collections. langs = defaultdict(lambda: []) for k in self._packages: package = self._packages[k] langs[package.language].append(package) tasks = defaultdict(lambda: []) for k in self._packages: package = self._packages[k] tasks[package.task].append(package) collections = [] for lang in langs: children = langs[lang] name1 = Locale(lang).getDisplayLanguage() try: name2 = isoLangs[lang]['name'] except: name2 = None if name1 and name1 != lang: name = name1 elif name2: name = name2 else: name = lang id = "{}{}".format(Downloader.LANG_PREFIX, lang) name = "{:<20} packages and models".format(name) c = Collection(id=id, name=name, children=children) collections.append(c) for task in tasks: children = tasks[task] id = "{}{}".format(Downloader.TASK_PREFIX, task) c = Collection(id=id, name=task, children=children) collections.append(c) self._collections = dict((c.id, c) for c in collections) # Replace identifiers with actual children in collection.children. for collection in self._collections.values(): for i, child_id in enumerate(collection.children): if child_id in self._packages: collection.children[i] = self._packages[child_id] if child_id in self._collections: collection.children[i] = self._collections[child_id] # Fill in collection.packages for each collection. for collection in self._collections.values(): packages = {} queue = [collection] for child in queue: if isinstance(child, Collection): queue.extend(child.children) else: packages[child.id] = child collection.packages = packages.values() # Flush the status cache self._status_cache.clear()