def __init__(self):
self.package_cache = PackageCache()
self.utils = self.package_cache.utils
try:
self.package_cache.init_downloaded_packages()
except Exception as e:
logger.info("caught %s when reading downloaded_packages_json_file",
e)
def __init__(self):
self.package_cache = PackageCache()
self.utils = self.package_cache.utils
try:
self.package_cache.init_downloaded_packages()
except Exception as e:
logger.info("caught %s when reading downloaded_packages_json_file", e)
class Getter(object):
def __init__(self):
self.package_cache = PackageCache()
self.utils = self.package_cache.utils
try:
self.package_cache.init_downloaded_packages()
except Exception as e:
logger.info("caught %s when reading downloaded_packages_json_file", e)
def list_tmp_pip_build_dirs(self):
tmp_pip_build_dirs = []
tmpdir = tempfile.gettempdir()
for subdir in os.listdir(tmpdir):
if re.match("^pip-.*-build$", subdir):
tmp_pip_build_dirs.append("%s/%s" % (tmpdir, subdir))
for subdir in os.listdir(self.package_cache.dumps_dir):
tmp_pip_build_dirs.append("%s/%s" % (self.package_cache.dumps_dir, subdir))
return tmp_pip_build_dirs
def clean_pip_build_dirs(self):
for tmp_pip_build_dir in self.list_tmp_pip_build_dirs():
shutil.rmtree(tmp_pip_build_dir)
def _download_package_from_url(self, url):
logger.info("Getting package %s ...", url)
self.clean_pip_build_dirs()
success = False
args = shlex.split('pip install --no-install --force-reinstall --ignore-installed --upgrade "%s" -b "%s"' % (url, self.package_cache.dumps_dir))
p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=open(os.devnull))
stdout = p.stdout.readlines()
downloaded_names = None
for line in stdout:
match = re.match("^Successfully downloaded (.*)", line)
if match:
downloaded_names = match.groups()[0].split(' ')
else:
continue
success = True
break
if not success:
logger.info('Error when downloading %s : %s', url, ''.join(stdout))
self.package_cache.skipped_packages.append(url)
if downloaded_names:
return self.find_downloaded_packages(downloaded_names)
def _post_download(self, downloaded_packages):
self.package_cache.create_downloaded_packages_json_file(downloaded_packages)
if self.package_cache.skipped_packages:
logger.error('%s = \n%s ', self.package_cache.skipped_packages_urls_filename, self.package_cache.skipped_packages)
skipped_packages_file = open(self.package_cache.skipped_packages_urls_filename, 'w')
skipped_packages_file.write(''.join(self.package_cache.skipped_packages)+'\n')
skipped_packages_file.close()
self.clean_pip_build_dirs()
def download_package_from_url(self, urls):
url = urls[0]
print 'download_package_from_url =', url
downloaded_packages = self._download_package_from_url(url)
self._post_download([downloaded_packages])
def get_packages_from_lists_of_urls(self, url_files):
all_downloaded_packages = self.package_cache.all_downloaded_packages
for url in self.get_urls_from_filenames(url_files):
# download from url
downloaded_packages = self._download_package_from_url(url)
if downloaded_packages and downloaded_packages not in all_downloaded_packages:
all_downloaded_packages.append(downloaded_packages)
self._post_download(all_downloaded_packages)
def find_downloaded_packages(self, downloaded_names):
packages = {}
for source_dir in self.list_tmp_pip_build_dirs():
pkg_name = self.package_cache.get_pkg_name(source_dir)
pkg_version = self.package_cache.get_pkg_version(source_dir)
pkg_fullname = self.package_cache.get_pkg_fullname(source_dir)
normalised_pkg_name = self.normalize_pkg_name(pkg_name)
version_dir = '%s/%s' % (self.package_cache.versions_dir, pkg_fullname)
if pkg_fullname and pkg_fullname not in os.listdir(self.package_cache.versions_dir):
shutil.move(source_dir, version_dir)
packages[normalised_pkg_name] = (pkg_name, pkg_version)
package_name = self.normalize_pkg_name(downloaded_names[0])
if package_name in packages:
package = [packages[package_name]]
else:
self.package_cache.skipped_packages.append(package_name)
return
deps = []
for dep in downloaded_names[1:]:
logger.info("found dependency %s", dep)
deps.append(packages[self.normalize_pkg_name(dep)])
if deps:
package.append(deps)
return package
def get_urls_from_filenames(self,filenames):
urls_from_filenames=[]
for filename in filenames:
try:
fileIN = open(filename, "r")
except Exception as e:
logger.error("Opening %s : %s", filename, e)
continue
for url in fileIN.readlines():
url = url.strip()
if not url:
continue
if url.startswith('#'):
continue
urls_from_filenames.append(url)
return urls_from_filenames
def normalize_pkg_name(self, name):
return name.lower().replace('-','_')
def list_downloaded_versions(self):
print "Downloaded packages :"
for version_dir in os.listdir(self.package_cache.versions_dir):
print version_dir
print
def list_skipped_packages(self):
print "Skipped packages :"
print open(self.package_cache.skipped_packages_urls_filename).read()
print
class Getter(object):
def __init__(self):
self.package_cache = PackageCache()
self.utils = self.package_cache.utils
try:
self.package_cache.init_downloaded_packages()
except Exception as e:
logger.info("caught %s when reading downloaded_packages_json_file",
e)
def list_tmp_pip_build_dirs(self):
tmp_pip_build_dirs = []
tmpdir = tempfile.gettempdir()
for subdir in os.listdir(tmpdir):
if re.match("^pip-.*-build$", subdir):
tmp_pip_build_dirs.append("%s/%s" % (tmpdir, subdir))
for subdir in os.listdir(self.package_cache.dumps_dir):
tmp_pip_build_dirs.append("%s/%s" %
(self.package_cache.dumps_dir, subdir))
return tmp_pip_build_dirs
def clean_pip_build_dirs(self):
for tmp_pip_build_dir in self.list_tmp_pip_build_dirs():
shutil.rmtree(tmp_pip_build_dir)
def _download_package_from_url(self, url):
logger.info("Getting package %s ...", url)
self.clean_pip_build_dirs()
success = False
args = shlex.split(
'pip install --no-install --force-reinstall --ignore-installed --upgrade "%s" -b "%s"'
% (url, self.package_cache.dumps_dir))
p = subprocess.Popen(args,
stdout=subprocess.PIPE,
stderr=open(os.devnull))
stdout = p.stdout.readlines()
downloaded_names = None
for line in stdout:
match = re.match("^Successfully downloaded (.*)", line)
if match:
downloaded_names = match.groups()[0].split(' ')
else:
continue
success = True
break
if not success:
logger.info('Error when downloading %s : %s', url, ''.join(stdout))
self.package_cache.skipped_packages.append(url)
if downloaded_names:
return self.find_downloaded_packages(downloaded_names)
def _post_download(self, downloaded_packages):
self.package_cache.create_downloaded_packages_json_file(
downloaded_packages)
if self.package_cache.skipped_packages:
logger.error('%s = \n%s ',
self.package_cache.skipped_packages_urls_filename,
self.package_cache.skipped_packages)
skipped_packages_file = open(
self.package_cache.skipped_packages_urls_filename, 'w')
skipped_packages_file.write(
''.join(self.package_cache.skipped_packages) + '\n')
skipped_packages_file.close()
self.clean_pip_build_dirs()
def download_package_from_url(self, urls):
url = urls[0]
print 'download_package_from_url =', url
downloaded_packages = self._download_package_from_url(url)
self._post_download([downloaded_packages])
def get_packages_from_lists_of_urls(self, url_files):
all_downloaded_packages = self.package_cache.all_downloaded_packages
for url in self.get_urls_from_filenames(url_files):
# download from url
downloaded_packages = self._download_package_from_url(url)
if downloaded_packages and downloaded_packages not in all_downloaded_packages:
all_downloaded_packages.append(downloaded_packages)
self._post_download(all_downloaded_packages)
def find_downloaded_packages(self, downloaded_names):
packages = {}
for source_dir in self.list_tmp_pip_build_dirs():
pkg_name = self.package_cache.get_pkg_name(source_dir)
pkg_version = self.package_cache.get_pkg_version(source_dir)
pkg_fullname = self.package_cache.get_pkg_fullname(source_dir)
normalised_pkg_name = self.normalize_pkg_name(pkg_name)
version_dir = '%s/%s' % (self.package_cache.versions_dir,
pkg_fullname)
if pkg_fullname and pkg_fullname not in os.listdir(
self.package_cache.versions_dir):
shutil.move(source_dir, version_dir)
packages[normalised_pkg_name] = (pkg_name, pkg_version)
package_name = self.normalize_pkg_name(downloaded_names[0])
if package_name in packages:
package = [packages[package_name]]
else:
self.package_cache.skipped_packages.append(package_name)
return
deps = []
for dep in downloaded_names[1:]:
logger.info("found dependency %s", dep)
deps.append(packages[self.normalize_pkg_name(dep)])
if deps:
package.append(deps)
return package
def get_urls_from_filenames(self, filenames):
urls_from_filenames = []
for filename in filenames:
try:
fileIN = open(filename, "r")
except Exception as e:
logger.error("Opening %s : %s", filename, e)
continue
for url in fileIN.readlines():
url = url.strip()
if not url:
continue
if url.startswith('#'):
continue
urls_from_filenames.append(url)
return urls_from_filenames
def normalize_pkg_name(self, name):
return name.lower().replace('-', '_')
def list_downloaded_versions(self):
print "Downloaded packages :"
for version_dir in os.listdir(self.package_cache.versions_dir):
print version_dir
print
def list_skipped_packages(self):
print "Skipped packages :"
print open(self.package_cache.skipped_packages_urls_filename).read()
print
