def get_page(self, url): # http://peak.telecommunity.com/DevCenter/EasyInstall#package-index-api scheme, netloc, path, _, _, _ = compat.urlparse(url) if scheme == 'file' and os.path.isdir(url2pathname(path)): url = compat.urljoin(ensure_slash(url), 'index.html') # The `retrieve()` method follows any eventual redirects, so the # initial url might be different from the final one try: response, final_url = self.opener.retrieve(url) except urllib3.exceptions.MaxRetryError: return content_type = response.headers.get('content-type', '') if locators.HTML_CONTENT_TYPE.match(content_type): data = response.data encoding = response.headers.get('content-encoding') if encoding: decoder = self.decoders[encoding] # fail if not found data = decoder(data) encoding = 'utf-8' m = locators.CHARSET.search(content_type) if m: encoding = m.group(1) try: data = data.decode(encoding) except UnicodeError: data = data.decode('latin-1') # fallback return locators.Page(data, final_url)
def get_page(self, url): # http://peak.telecommunity.com/DevCenter/EasyInstall#package-index-api scheme, netloc, path, _, _, _ = compat.urlparse(url) if scheme == 'file' and os.path.isdir(url2pathname(path)): url = compat.urljoin(ensure_slash(url), 'index.html') # The `retrieve()` method follows any eventual redirects, so the # initial url might be different from the final one try: response, final_url = http_retrieve(self.opener, url) except urllib3.exceptions.MaxRetryError: return content_type = response.headers.get('content-type', '') if locators.HTML_CONTENT_TYPE.match(content_type): data = response.data encoding = response.headers.get('content-encoding') if encoding: decoder = self.decoders[encoding] # fail if not found data = decoder(data) encoding = 'utf-8' m = locators.CHARSET.search(content_type) if m: encoding = m.group(1) try: data = data.decode(encoding) except UnicodeError: data = data.decode('latin-1') # fallback return locators.Page(data, final_url)
def _get_project(self, name): # Retrieve the info url = compat.urljoin(self.url, 'api/' + name) try: response, _ = self.opener.retrieve(url) except urllib3.exceptions.MaxRetryError: return None if response.status == 200: data = json.loads(response.data) return dict((v['version'], self._get_distribution(v)) for v in data) else: self.requirements_not_found.append(name)
def _get_project(self, name): # Retrieve the info url = compat.urljoin(self.url, 'api/' + name) try: response, _ = http_retrieve(self.opener, url) except urllib3.exceptions.MaxRetryError: return None if response.status == 200: data = json.loads(response.data) return dict( (v['version'], self._get_distribution(v)) for v in data) else: self.requirements_not_found.append(name)
def handle(self, requester, data): # Preparing the url to PUT the file wheel = data.get('wheel') server = data.get('server') file_name = os.path.basename(wheel) url = compat.urljoin(server, 'p/{0}'.format(file_name)) # Sending the file to the server. Both `method` and `url` parameters # for calling `request_encode_body()` must be `str()` instances, not # unicode. contents = io.open(wheel, 'rb').read() self.opener.request_encode_body( b'PUT', bytes(url), {file_name: (file_name, contents)}, headers=get_auth_info_from_url(url)) return {'upload_url': url, 'requirement': data['requirement']}
def handle(self, requester, data): # Preparing the url to PUT the file wheel = data.get('wheel') server = data.get('server') file_name = os.path.basename(wheel) url = compat.urljoin(server, 'p/{0}'.format(file_name)) # Sending the file to the server. Both `method` and `url` parameters # for calling `request_encode_body()` must be `str()` instances, not # unicode. contents = io.open(wheel, 'rb').read() self.opener.request_encode_body(b'PUT', bytes(url), {file_name: (file_name, contents)}, headers=get_auth_info_from_url(url)) return {'upload_url': url, 'requirement': data['requirement']}
def _get_project(self, name): # It sounds lame, but we're trying to match requirements with more than # one word separated with either `_` or `-`. Notice that we prefer # hyphens cause there is currently way more packages using hyphens than # underscores in pypi.p.o. Let's wait for the best here. options = [name] if '-' in name or '_' in name: options = (name.replace('_', '-'), name.replace('-', '_')) # Iterate over all the possible names a package can have. for package_name in options: url = compat.urljoin(self.base_url, '{0}/'.format( compat.quote(package_name))) found = self._fetch(url, package_name) if found: return found
def _get_project(self, name): # It sounds lame, but we're trying to match requirements with more than # one word separated with either `_` or `-`. Notice that we prefer # hyphens cause there is currently way more packages using hyphens than # underscores in pypi.p.o. Let's wait for the best here. options = [name] if '-' in name or '_' in name: options = (name.replace('_', '-'), name.replace('-', '_')) # Iterate over all the possible names a package can have. for package_name in options: url = compat.urljoin(self.base_url, '{0}/'.format(compat.quote(package_name))) found = self._fetch(url, package_name) if found: return found
def http_retrieve(pool, url, attempt=0): if attempt >= REDIRECT_LIMIT: raise TooManyRedirects('Too many redirects') # Params to be passed to request. The `preload_content` must be set to # False, otherwise `read()` wont honor `decode_content`. params = { 'headers': util.get_auth_info_from_url(url), 'preload_content': False, 'redirect': False, } # Request the url and ensure we've reached the final location response = pool.request('GET', url, **params) if 'location' in response.headers: location = response.headers['location'] if location.startswith('/'): url = compat.urljoin(url, location) else: url = location return http_retrieve(pool, url, attempt=attempt + 1) return response, url
def get_distribution_names(self): return json.loads( self.opener.retrieve( compat.urljoin(self.url, 'api'))[0].data)
def get_distribution_names(self): return json.loads( http_retrieve(self.opener, compat.urljoin(self.url, 'api'))[0].data)