def get_page(self, url):
# http://peak.telecommunity.com/DevCenter/EasyInstall#package-index-api
scheme, netloc, path, _, _, _ = compat.urlparse(url)
if scheme == 'file' and os.path.isdir(url2pathname(path)):
url = compat.urljoin(ensure_slash(url), 'index.html')
# The `retrieve()` method follows any eventual redirects, so the
# initial url might be different from the final one
try:
response, final_url = self.opener.retrieve(url)
except urllib3.exceptions.MaxRetryError:
return
content_type = response.headers.get('content-type', '')
if locators.HTML_CONTENT_TYPE.match(content_type):
data = response.data
encoding = response.headers.get('content-encoding')
if encoding:
decoder = self.decoders[encoding] # fail if not found
data = decoder(data)
encoding = 'utf-8'
m = locators.CHARSET.search(content_type)
if m:
encoding = m.group(1)
try:
data = data.decode(encoding)
except UnicodeError:
data = data.decode('latin-1') # fallback
return locators.Page(data, final_url)
def get_page(self, url):
# http://peak.telecommunity.com/DevCenter/EasyInstall#package-index-api
scheme, netloc, path, _, _, _ = compat.urlparse(url)
if scheme == 'file' and os.path.isdir(url2pathname(path)):
url = compat.urljoin(ensure_slash(url), 'index.html')
# The `retrieve()` method follows any eventual redirects, so the
# initial url might be different from the final one
try:
response, final_url = http_retrieve(self.opener, url)
except urllib3.exceptions.MaxRetryError:
return
content_type = response.headers.get('content-type', '')
if locators.HTML_CONTENT_TYPE.match(content_type):
data = response.data
encoding = response.headers.get('content-encoding')
if encoding:
decoder = self.decoders[encoding] # fail if not found
data = decoder(data)
encoding = 'utf-8'
m = locators.CHARSET.search(content_type)
if m:
encoding = m.group(1)
try:
data = data.decode(encoding)
except UnicodeError:
data = data.decode('latin-1') # fallback
return locators.Page(data, final_url)
def _get_project(self, name):
# Retrieve the info
url = compat.urljoin(self.url, 'api/' + name)
try:
response, _ = self.opener.retrieve(url)
except urllib3.exceptions.MaxRetryError:
return None
if response.status == 200:
data = json.loads(response.data)
return dict((v['version'], self._get_distribution(v)) for v in data)
else:
self.requirements_not_found.append(name)
def _get_project(self, name):
# Retrieve the info
url = compat.urljoin(self.url, 'api/' + name)
try:
response, _ = http_retrieve(self.opener, url)
except urllib3.exceptions.MaxRetryError:
return None
if response.status == 200:
data = json.loads(response.data)
return dict(
(v['version'], self._get_distribution(v)) for v in data)
else:
self.requirements_not_found.append(name)
def handle(self, requester, data):
# Preparing the url to PUT the file
wheel = data.get('wheel')
server = data.get('server')
file_name = os.path.basename(wheel)
url = compat.urljoin(server, 'p/{0}'.format(file_name))
# Sending the file to the server. Both `method` and `url` parameters
# for calling `request_encode_body()` must be `str()` instances, not
# unicode.
contents = io.open(wheel, 'rb').read()
self.opener.request_encode_body(
b'PUT', bytes(url), {file_name: (file_name, contents)},
headers=get_auth_info_from_url(url))
return {'upload_url': url, 'requirement': data['requirement']}
def handle(self, requester, data):
# Preparing the url to PUT the file
wheel = data.get('wheel')
server = data.get('server')
file_name = os.path.basename(wheel)
url = compat.urljoin(server, 'p/{0}'.format(file_name))
# Sending the file to the server. Both `method` and `url` parameters
# for calling `request_encode_body()` must be `str()` instances, not
# unicode.
contents = io.open(wheel, 'rb').read()
self.opener.request_encode_body(b'PUT',
bytes(url),
{file_name: (file_name, contents)},
headers=get_auth_info_from_url(url))
return {'upload_url': url, 'requirement': data['requirement']}
def _get_project(self, name):
# It sounds lame, but we're trying to match requirements with more than
# one word separated with either `_` or `-`. Notice that we prefer
# hyphens cause there is currently way more packages using hyphens than
# underscores in pypi.p.o. Let's wait for the best here.
options = [name]
if '-' in name or '_' in name:
options = (name.replace('_', '-'), name.replace('-', '_'))
# Iterate over all the possible names a package can have.
for package_name in options:
url = compat.urljoin(self.base_url, '{0}/'.format(
compat.quote(package_name)))
found = self._fetch(url, package_name)
if found:
return found
def _get_project(self, name):
# It sounds lame, but we're trying to match requirements with more than
# one word separated with either `_` or `-`. Notice that we prefer
# hyphens cause there is currently way more packages using hyphens than
# underscores in pypi.p.o. Let's wait for the best here.
options = [name]
if '-' in name or '_' in name:
options = (name.replace('_', '-'), name.replace('-', '_'))
# Iterate over all the possible names a package can have.
for package_name in options:
url = compat.urljoin(self.base_url,
'{0}/'.format(compat.quote(package_name)))
found = self._fetch(url, package_name)
if found:
return found
def http_retrieve(pool, url, attempt=0):
if attempt >= REDIRECT_LIMIT:
raise TooManyRedirects('Too many redirects')
# Params to be passed to request. The `preload_content` must be set to
# False, otherwise `read()` wont honor `decode_content`.
params = {
'headers': util.get_auth_info_from_url(url),
'preload_content': False,
'redirect': False,
}
# Request the url and ensure we've reached the final location
response = pool.request('GET', url, **params)
if 'location' in response.headers:
location = response.headers['location']
if location.startswith('/'):
url = compat.urljoin(url, location)
else:
url = location
return http_retrieve(pool, url, attempt=attempt + 1)
return response, url
def http_retrieve(pool, url, attempt=0):
if attempt >= REDIRECT_LIMIT:
raise TooManyRedirects('Too many redirects')
# Params to be passed to request. The `preload_content` must be set to
# False, otherwise `read()` wont honor `decode_content`.
params = {
'headers': util.get_auth_info_from_url(url),
'preload_content': False,
'redirect': False,
}
# Request the url and ensure we've reached the final location
response = pool.request('GET', url, **params)
if 'location' in response.headers:
location = response.headers['location']
if location.startswith('/'):
url = compat.urljoin(url, location)
else:
url = location
return http_retrieve(pool, url, attempt=attempt + 1)
return response, url
def get_distribution_names(self):
return json.loads(
self.opener.retrieve(
compat.urljoin(self.url, 'api'))[0].data)
def get_distribution_names(self):
return json.loads(
http_retrieve(self.opener, compat.urljoin(self.url,
'api'))[0].data)
