def index(): path = get_package_path(request.form['name']) if not exists(path): makedirs(path) if request.files: file = request.files['content'] filename = secure_filename(file.filename) file.save(join(path, filename)) with open(join(path, filename + '.sha256'), 'w') as sha256_file: sha256_file.write(request.form['sha256_digest']) return 'Registered'
def index(): path = get_package_path(request.form['name']) if not exists(path): makedirs(path) if request.files: file = request.files['content'] filename = secure_filename(file.filename) file.save(join(path, filename)) with open(join(path, filename + '.md5'), 'w') as md5_file: md5_file.write(request.form['md5_digest']) return 'Registered'
def simple_package(package_name): ''' Given a package name, returns all the versions for downloading that package. If the package doesn't exists, then it will call PyPi (CheeseShop). But if the package exists in the local path, then it will get all the versions for the local package. This will take into account if the egg is private or if it is a normal egg that was uploaded to PyPi. This is important to take into account the version of the eggs. For example, a proyect requires request==1.0.4 and another package uses request==1.0.3. Then the instalation of the second package will fail because it wasn't downloaded an the **request** folder only has the 1.0.4 version. To solve this problem, the system uses 2 different kinds of eggs: * private eggs: are the eggs that you uploaded to the private repo. * normal eggs: are the eggs that are downloaded from pypi. So the normal eggs will always get the simple page from the pypi repo, will the private eggs will always be read from the filesystem. :param package_name: the name of the egg package. This is only the name of the package with the version or anything else. :return: a template with all the links to download the packages. ''' app.logger.debug('Requesting index for: %s', package_name) package_folder = get_package_path(package_name) if (is_private(package_name) or ( exists(package_name) and app.config['SHOULD_USE_EXISTING'])): app.logger.debug('Found information of package: %s in local repository', package_name) package_versions = [] template_data = dict( source_letter=package_name[0], package_name=package_name, versions=package_versions ) for filename in listdir(package_folder): if not filename.endswith('.md5'): # I only read .md5 files so I skip this egg (or tar, # or zip) file continue with open(join(package_folder, filename)) as md5_file: md5 = md5_file.read(-1) # remove .md5 extension name = filename[:-4] data = VersionData(name, md5, None) package_versions.append(data) return render_template('simple_package.html', **template_data) else: app.logger.debug('Didnt found package: %s in local repository. ' 'Using proxy.', package_name) url = app.config['PYPI_URL'] + 'simple/%s/' % package_name response = get(url) if response.status_code != 200: app.logger.warning('Error while getting proxy info for: %s' 'Errors details: %s', package_name, response.text) abort(response.status_code) if response.history: app.logger.debug('The url was redirected') # in this case, the request was redirect, so I should also # take into account this change. For example, this happens # when requesting flask-bcrypt and on Pypi the request is # redirected to Flask-Bcrypt package_name = urlparse.urlparse(response.url).path package_name = package_name.replace('/simple/', '') package_name = package_name.replace('/', '') content = response.content external_links = set() # contains the list of pacges whih where checked because # on the link they had the information of visited_download_pages = set() soup = BeautifulSoup(content) package_versions = [] for panchor in soup.find_all('a'): if panchor.get('rel') and panchor.get('rel')[0] == 'homepage': # skip getting information on the project homepage continue href = panchor.get('href') app.logger.debug('Found the link: %s', panchor.get('href')) if href.startswith('../../packages/'): # then the package is hosted on pypi. pk_name = basename(href) pk_name, md5_data = pk_name.split('#md5=') pk_name = pk_name.replace('#md5=', '') data = VersionData(pk_name, md5_data, None) package_versions.append(data) continue parsed = urlparse.urlparse(href) if parsed.hostname: # then the package had a full path to the file if parsed.hostname == 'pypi.python.org': # then it is hosted on the pypi server, so I change # it to make it a relative url pk_name = basename(parsed.path) if '#md5=' in parsed.path: pk_name, md5_data = pk_name.split('#md5=') pk_name = pk_name.replace('#md5=', '') else: md5_data = '' data = VersionData(pk_name, md5_data, None) package_versions.append(data) else: # the python package is hosted on another server # that isn't pypi. The packages that doesn't have # rel=download, then they are links to some pages if panchor.get('rel') and panchor.get('rel')[0] == 'download': if url_is_egg_file(parsed.path): external_links.add(href) else: # href point to an external page where the links # to download the package will be found if href not in visited_download_pages: visited_download_pages.add(href) external_links.update(find_external_links(href)) # after collecting all external links, we insert them in the html page for external_url in external_links: package_version = basename(external_url) existing_value = filter(lambda pv: pv.name == package_version, package_versions) external_link = urllib.urlencode({'remote': external_url}) if existing_value: package_versions.remove(existing_value[0]) # check that the package version doens't override the one that # already exists on pypi existing_data = filter(lambda v: v.name == package_version, package_versions) if existing_data: continue data = VersionData(package_version, '', external_link) package_versions.append(data) template_data = dict( source_letter=package_name[0], package_name=package_name, versions=package_versions ) return render_template('simple_package.html', **template_data)
def simple_package(package_name): ''' Given a package name, returns all the versions for downloading that package. If the package doesn't exists, then it will call PyPi (CheeseShop). But if the package exists in the local path, then it will get all the versions for the local package. This will take into account if the egg is private or if it is a normal egg that was uploaded to PyPi. This is important to take into account the version of the eggs. For example, a proyect requires request==1.0.4 and another package uses request==1.0.3. Then the instalation of the second package will fail because it wasn't downloaded an the **request** folder only has the 1.0.4 version. To solve this problem, the system uses 2 different kinds of eggs: * private eggs: are the eggs that you uploaded to the private repo. * normal eggs: are the eggs that are downloaded from pypi. So the normal eggs will always get the simple page from the pypi repo, will the private eggs will always be read from the filesystem. :param package_name: the name of the egg package. This is only the name of the package with the version or anything else. :return: a template with all the links to download the packages. ''' app.logger.debug('Requesting index for: %s', package_name) package_folder = get_package_path(package_name) if (is_private(package_name) or ( exists(package_name) and app.config['SHOULD_USE_EXISTING'])): app.logger.debug('Found information of package: %s in local repository', package_name) package_versions = [] template_data = dict( source_letter=package_name[0], package_name=package_name, versions=package_versions ) for filename in listdir(package_folder): if not filename.endswith('.md5'): # I only read .md5 files so I skip this egg (or tar, # or zip) file continue with open(join(package_folder, filename)) as md5_file: md5 = md5_file.read(-1) # remove .md5 extension name = filename[:-4] data = VersionData(name, md5) package_versions.append(data) return render_template('simple_package.html', **template_data) else: app.logger.debug('Didnt found package: %s in local repository. ' 'Using proxy.', package_name) url = app.config['PYPI_URL'] + 'simple/%s' % package_name response = get(url) if response.status_code != 200: app.logger.warning('Error while getting proxy info for: %s' 'Errors details: %s', package_name, response.text) abort(response.status_code) content = response.content p = PyQuery(content) external_links = set() for anchor in p("a"): panchor = PyQuery(anchor) href = panchor.attr('href') # robin-jarry: modified the href to ../../packages/ # so that it works also for non-source packages (.egg, .exe and .msi) parsed = urlparse.urlparse(href) if parsed.hostname: # the link is to an external server. if parsed.hostname == 'pypi.python.org': # we remove the hostname to make the URL relative panchor.attr('href', parsed.path) else: if panchor.attr('rel') == 'download': if url_is_egg_file(parsed.path): # href points to a filename external_links.add('<a href="%s">%s</a>' % (href, basename(parsed.path))) else: # href points to an external page where we will find # links to package files external_links.update(find_external_links(href)) # what ever happens, we remove the link for now # we'll add the external_links after that we found after panchor.remove() else: # local link to pypi.python.org if not href.startswith('../../packages/'): # ignore anything else than package links panchor.remove() # after collecting all external links, we insert them in the html page for link in external_links: plink = PyQuery(link) href = plink.attr('href') plink.attr('href', convert_to_internal_url(href, package_name, basename(href))) p('a').after(plink) content = p.outerHtml() return content
def package(package_type, letter, package_name, package_file): ''' Downloads the egg :param str package_type: the nature of the package. For example: 'source' or '2.7' :param str letter: the first char of the package name. For example: D :param str package_name: the name of the package. For example: Django :param str package_file: the name of the package and it's version. For example: Django-1.5.0.tar.gz ''' egg_filename = join(get_base_path(), package_name, package_file) url = request.args.get('remote') if request.method == 'HEAD': # in this case the content type of the file is what is # required if not exists(egg_filename): pypi_response = head(url) return _respond(pypi_response.content, pypi_response.headers['content-type']) else: mimetype = magic.from_file(egg_filename, mime=True) return _respond('', mimetype) app.logger.debug('Downloading: %s', package_file) if exists(egg_filename): app.logger.debug('Found local file in repository for: %s', package_file) # if the file exists, then use the local file. path = get_package_path(package_name) path = join(path, package_file) with open(path, 'rb') as egg: content = egg.read(-1) mimetype = magic.from_file(egg_filename, mime=True) return _respond(content, mimetype) else: # Downloads the egg from pypi and saves it locally, then # it will return it. package_path = get_package_path(package_name) app.logger.debug('Starting to download: %s using the url: %s', package_file, url) pypi_response = get(url) app.logger.debug('Finished downloading package: %s', package_file) if pypi_response.status_code != 200: app.logger.warning('Error respose while downloading for proxy: %s' 'Response details: %s', package_file, pypi_response.text) abort(pypi_response.status_code) if not exists(package_path): makedirs(package_path) with open(egg_filename, 'w') as egg_file: egg_file.write(pypi_response.content) with open(egg_filename) as egg_file: filecontent = egg_file.read(-1) mimetype = magic.from_file(egg_filename, mime=True) with open(egg_filename + '.md5', 'w') as md5_output: md5 = get_md5_for_content(filecontent) md5_output.write(md5) return _respond(filecontent, mimetype)
def package(package_type, letter, package_name, package_file): """ Downloads the egg :param str package_type: the nature of the package. For example: 'source' or '2.7' :param str letter: the first char of the package name. For example: D :param str package_name: the name of the package. For example: Django :param str package_file: the name of the package and it's version. For example: Django-1.5.0.tar.gz """ egg_filename = join(get_base_path(), package_name, package_file) url = unquote(request.args.get('remote')) if request.method == 'HEAD': # in this case the content type of the file is what is # required if not exists(egg_filename): pypi_response = head(url) return _respond(pypi_response.content, pypi_response.headers['content-type']) else: mimetype = magic.from_file(egg_filename, mime=True) return _respond('', mimetype) app.logger.debug('Downloading: %s', package_file) if exists(egg_filename): app.logger.debug('Found local file in repository for: %s', package_file) # if the file exists, then use the local file. path = get_package_path(package_name) path = join(path, package_file) with open(path, 'rb') as egg: content = egg.read() mimetype = magic.from_file(egg_filename, mime=True) return _respond(content, mimetype) else: # Downloads the egg from pypi and saves it locally, then # it will return it. package_path = get_package_path(package_name) app.logger.debug('Starting to download: %s using the url: %s', package_file, url) pypi_response = get(url) app.logger.debug('Finished downloading package: %s', package_file) if pypi_response.status_code != 200: app.logger.warning( 'Error respose while downloading for proxy: %s' 'Response details: %s', package_file, pypi_response.text) abort(pypi_response.status_code) if not exists(package_path): makedirs(package_path) with open(egg_filename, 'wb') as egg_file: egg_file.write(pypi_response.content) with open(egg_filename, 'rb') as egg_file: filecontent = egg_file.read() mimetype = magic.from_file(egg_filename, mime=True) with open(egg_filename + '.sha256', 'w') as sha256_output: sha256 = get_sha256_for_content(filecontent) sha256_output.write(sha256) return _respond(filecontent, mimetype)
def simple_package(package_name): ''' Given a package name, returns all the versions for downloading that package. If the package doesn't exists, then it will call PyPI (CheeseShop). But if the package exists in the local path, then it will get all the versions for the local package. This will take into account if the egg is private or if it is a normal egg that was uploaded to PyPI. This is important to take into account the version of the eggs. For example, a project requires requests==1.0.4 and another package uses requests==1.0.3. Then the instalation of the second package will fail because it wasn't downloaded and the **requests** folder only has the 1.0.4 version. To solve this problem, the system uses 2 different kinds of eggs: * private eggs: are the eggs that you uploaded to the private repo. * normal eggs: are the eggs that are downloaded from PyPI. So the normal eggs will always get the simple page from the PyPI repo, will the private eggs will always be read from the filesystem. :param package_name: the name of the egg package. This is only the name of the package with the version or anything else. :return: a template with all the links to download the packages. ''' app.logger.debug('Requesting index for: %s', package_name) package_folder = get_package_path(package_name) if (is_private(package_name) or (exists(package_folder) and app.config['SHOULD_USE_EXISTING'])): app.logger.debug( 'Found information of package: %s in local repository', package_name) package_versions = [] template_data = dict(source_letter=package_name[0], package_name=package_name, versions=package_versions) for filename in listdir(package_folder): if not filename.endswith('.md5'): # I only read .md5 files so I skip this egg (or tar, # or zip) file continue with open(join(package_folder, filename)) as md5_file: md5 = md5_file.read(-1) # remove .md5 extension name = filename[:-4] data = VersionData('source', name, md5, None) package_versions.append(data) return render_template('simple_package.html', **template_data) else: app.logger.debug( 'Didnt found package: %s in local repository. ' 'Using proxy.', package_name) url = app.config['PYPI_URL'] + 'simple/%s/' % package_name response = get(url) if response.status_code != 200: app.logger.warning( 'Error while getting proxy info for: %s' 'Errors details: %s', package_name, response.text) abort(response.status_code) if response.history: app.logger.debug('The url was redirected') # in this case, the request was redirect, so I should also # take into account this change. For example, this happens # when requesting flask-bcrypt and on Pypi the request is # redirected to Flask-Bcrypt package_name = urlparse.urlparse(response.url).path package_name = package_name.replace('/simple/', '') package_name = package_name.replace('/', '') content = response.content external_links = set() # contains the list of pacges whih where checked because # on the link they had the information of visited_download_pages = set() soup = BeautifulSoup(content) package_versions = [] for panchor in soup.find_all('a'): if panchor.get('rel') and panchor.get('rel')[0] == 'homepage': # skip getting information on the project homepage continue href = panchor.get('href') app.logger.debug('Found the link: %s', href) if href.startswith('../../packages/'): # then the package is hosted on PyPI. pk_name = basename(href) pk_name, md5_data = pk_name.split('#md5=') pk_name = pk_name.replace('#md5=', '') # remove md5 part to make the url shorter. split_data = urlparse.urlsplit(href) absolute_url = urlparse.urljoin(url, split_data.path) external_link = urllib.urlencode({'remote': absolute_url}) pk_type = href.split('/')[3] data = VersionData(pk_type, pk_name, md5_data, external_link) package_versions.append(data) app.logger.debug('pk_type: %s', pk_type) continue parsed = urlparse.urlparse(href) if parsed.hostname: # then the package had a full path to the file if parsed.hostname == 'pypi.python.org': # then it is hosted on the PyPI server, so I change # it to make it a relative url pk_name = basename(parsed.path) if '#md5=' in parsed.path: pk_name, md5_data = pk_name.split('#md5=') pk_name = pk_name.replace('#md5=', '') else: md5_data = '' absolute_url = urlparse.urljoin(url, parsed.path) external_link = urllib.urlencode({'remote': absolute_url}) pk_type = parsed.path.split('/')[2] data = VersionData(pk_type, pk_name, md5_data, external_link) package_versions.append(data) else: # the python package is hosted on another server # that isn't PyPI. The packages that don't have # rel=download are links to some pages if panchor.get('rel') and panchor.get( 'rel')[0] == 'download': if url_is_egg_file(parsed.path): external_links.add(href) else: # href points to an external page where the links # to download the package will be found if href not in visited_download_pages: visited_download_pages.add(href) external_links.update( find_external_links(href)) # after collecting all external links, we insert them in the html page for external_url in external_links: package_version = basename(external_url) existing_value = filter(lambda pv: pv.name == package_version, package_versions) if existing_value: # if the package already exists on PyPI, then # use its version instead of using the one that is # hosted on a remote server continue external_link = urllib.urlencode({'remote': external_url}) data = VersionData('source', package_version, '', external_link) package_versions.append(data) package_versions.sort(key=lambda v: v.name) template_data = dict(source_letter=package_name[0], package_name=package_name, versions=package_versions) return render_template('simple_package.html', **template_data)