def open_url(url, proxy, resp_code, sslContext): if isinstance(url, request.Request): logging.info('opener: opening "{0}"'.format(url.full_url)) else: logging.info('opener: opening "{0}"'.format(url)) opener = request.OpenerDirector() if proxy: # not doing 'is not None', so empty strings don't try and proxy # have a proxy option to take it from the envrionment vars opener.add_handler( request.ProxyHandler({ 'http': proxy, 'https': proxy })) else: opener.add_handler(request.ProxyHandler({})) opener.add_handler(request.HTTPHandler()) opener.add_handler(PackratHandler()) if hasattr(http.client, 'HTTPSConnection'): opener.add_handler(request.HTTPSHandler()) # context=sslContext opener.add_handler(PackratsHandler()) # context=sslContext opener.add_handler(request.FileHandler()) opener.add_handler(request.FTPHandler()) opener.add_handler(request.UnknownHandler()) try: resp = opener.open(url, timeout=WEB_HANDLE_TIMEOUT) except request.HTTPError as e: raise FileRetrieveException('HTTPError "{0}"'.format(e)) except request.URLError as e: if isinstance(e.reason, socket.timeout): raise FileRetrieveException( 'Request Timeout after {0} seconds'.format(WEB_HANDLE_TIMEOUT)) raise FileRetrieveException( 'URLError "{0}" for "{1}" via "{2}"'.format( e, url.full_url, proxy)) except socket.timeout: raise FileRetrieveException( 'Request Timeout after {0} seconds'.format(WEB_HANDLE_TIMEOUT)) except socket.error as e: raise FileRetrieveException('Socket Error "{0}"'.format(e)) if resp.code is not None: # FileHandler, FTPHandler do not have a response code if resp.code == 404: raise FileRetrieveException('File "{0}" not Found'.format(url)) if resp.code != resp_code: raise FileRetrieveException('Invalid Response code "{0}"'.format( resp.code)) return resp
def __init__( self, host, root_path, proxy=None, verify_ssl=True, retry_event=None ): # retry_event should be an Event Object, use to cancel retry loops, if the event get's set the retry loop will throw the most recent Exception it ignored super().__init__() self.retry_event = retry_event if not host.startswith(('http:', 'https:')): raise ValueError('hostname must start with http(s):') if host[-1] == '/': raise ValueError('hostname must not end with "/"') self.proxy = proxy self.host = host logging.debug( 'cinp: new client host: "{0}", root_path: "{1}", via: "{2}"'. format(self.host, root_path, self.proxy)) self.uri = URI(root_path) self.opener = request.OpenerDirector() if self.proxy: # not doing 'is not None', so empty strings don't try and proxy # have a proxy option to take it from the envrionment vars self.opener.add_handler( request.ProxyHandler({ 'http': self.proxy, 'https': self.proxy })) else: self.opener.add_handler(request.ProxyHandler({})) self.opener.add_handler(request.HTTPHandler()) if hasattr(http.client, 'HTTPSConnection'): if not verify_ssl: self.opener.add_handler( request.HTTPSHandler( context=ssl._create_unverified_context())) else: self.opener.add_handler(request.HTTPSHandler()) self.opener.add_handler(request.UnknownHandler()) self.opener.addheaders = [ ('User-Agent', 'python CInP client {0}'.format(__CLIENT_VERSION__)), ('Accepts', 'application/json'), ('Accept-Charset', 'utf-8'), ('CInP-Version', __CINP_VERSION__) ]
def setUp(self): super(HttpCase, self).setUp() if self.registry_test_mode: self.registry.enter_test_mode() self.addCleanup(self.registry.leave_test_mode) # setup a magic session_id that will be rollbacked self.session = odoo.http.root.session_store.new() self.session_id = self.session.sid self.session.db = get_db_name() odoo.http.root.session_store.save(self.session) # setup an url opener helper self.opener = urllib2.OpenerDirector() self.opener.add_handler(urllib2.UnknownHandler()) self.opener.add_handler(urllib2.HTTPHandler()) self.opener.add_handler(urllib2.HTTPSHandler()) self.opener.add_handler(urllib2.HTTPCookieProcessor()) self.opener.add_handler(RedirectHandler()) self.opener.addheaders.append(('Cookie', 'session_id=%s' % self.session_id))
def _downloadFiles(self, wrk_dir, lease, host, header_map, proxy): ovf_files = [] opener = request.OpenerDirector() if proxy: # not doing 'is not None', so empty strings don't try and proxy # have a proxy option to take it from the envrionment vars opener.add_handler( request.ProxyHandler({ 'http': proxy, 'https': proxy })) else: opener.add_handler(request.ProxyHandler({})) opener.add_handler(request.HTTPHandler()) if hasattr(http.client, 'HTTPSConnection'): opener.add_handler( request.HTTPSHandler()) # context=self.sslContext opener.add_handler(request.UnknownHandler()) logging.debug('OVAExportHandler: Starting file downloads(s)...') for device in lease.info.deviceUrl: url = device.url.replace('*', host) if not device.targetId: logging.debug( 'ExportLease: No targetId for "{0}", skipping...'.format( url)) continue logging.debug('OVAExportHandler: Downloading "{0}"...'.format( device.url)) req = request.Request(url, headers=header_map, method='GET') resp = opener.open(req, timeout=DOWNLOAD_FILE_TIMEOUT) try: content_length = int(resp.headers['content-length']) except TypeError: # ESX dosen't supply contect-length? content_length = '<unknwon>' file_hash = hashlib.sha256() local_file = open(os.path.join(wrk_dir, device.targetId), 'wb') buff = resp.read(4096 * 1024) cp = datetime.utcnow() while buff: if datetime.utcnow() > cp: cp = datetime.utcnow() + timedelta( seconds=PROGRESS_INTERVAL) logging.debug( 'OVAExportHandler: download at {0} of {1}'.format( local_file.tell(), content_length)) local_file.write(buff) file_hash.update(buff) buff = resp.read(4096 * 1024) ovf_file = vim.OvfManager.OvfFile() ovf_file.deviceId = device.key ovf_file.path = device.targetId ovf_file.size = local_file.tell() ovf_files.append((ovf_file, file_hash.hexdigest())) local_file.close() return ovf_files