def download_egg(eggname, dir):
"""Download an egg to a specific place
:param eggname: name of egg to download
:param dir: destination directory
:return: local path
"""
logger.info("Downloading %s"%eggname)
return pi.download(eggname, dir)
def fetch_requirement(req, dest_dir, force_download):
from setuptools.package_index import PackageIndex # @Reimport
from pkg_resources import working_set # @Reimport # NOQA
i = PackageIndex()
if force_download:
[i.remove(i[req.key][0]) for _ in xrange(len(i[req.key]))]
d = i.download(req, dest_dir)
else:
d = i.fetch_distribution(req, dest_dir, force_scan=True)
d = getattr(d, 'location', d) if d else ''
return (d if d else working_set.resolve([req])[0].location)
class Command(LabelCommand):
option_list = LabelCommand.option_list + (make_option(
"-o",
"--owner",
help="add packages as OWNER",
metavar="OWNER",
default=None), )
help = """Add one or more packages to the repository. Each argument can
be a package name or a URL to an archive or egg. Package names honour
the same rules as easy_install with regard to indicating versions etc.
If a version of the package exists, but is older than what we want to install,
the owner remains the same.
For new packages there needs to be an owner. If the --owner option is present
we use that value. If not, we try to match the maintainer of the package, form
the metadata, with a user in out database, based on the If it's a new package
and the maintainer emailmatches someone in our user list, we use that. If not,
the package can not be
added"""
def __init__(self, *args, **kwargs):
self.pypi = PackageIndex()
LabelCommand.__init__(self, *args, **kwargs)
def handle_label(self, label, **options):
with tempdir() as tmp:
path = self.pypi.download(label, tmp)
if path:
self._save_package(path, options["owner"])
else:
print "Could not add %s. Not found." % label
def _save_package(self, path, ownerid):
meta = self._get_meta(path)
try:
# can't use get_or_create as that demands there be an owner
project = Project.objects.get(name=meta.name)
isnewproject = False
except Project.DoesNotExist:
project = Project(name=meta.name)
isnewproject = True
release = project.get_release(meta.version)
if not isnewproject and release and release.version == meta.version:
print "%s-%s already added" % (meta.name, meta.version)
return
# algorithm as follows: If owner is given, try to grab user with that
# username from db. If doesn't exist, bail. If no owner set look at
# mail address from metadata and try to get that user. If it exists
# use it. If not, bail.
owner = None
if ownerid:
try:
if "@" in ownerid:
owner = User.objects.get(email=ownerid)
else:
owner = User.objects.get(username=ownerid)
except User.DoesNotExist:
pass
else:
try:
owner = User.objects.get(email=meta.author_email)
except User.DoesNotExist:
pass
if not owner:
print "No owner defined. Use --owner to force one"
return
# at this point we have metadata and an owner, can safely add it.
project.owner = owner
# Some packages don't have proper licence, seems to be a problem
# with setup.py upload. Use "UNKNOWN"
project.license = meta.license or "Unknown"
project.metadata_version = meta.metadata_version
project.author = meta.author
project.home_page = meta.home_page
project.download_url = meta.download_url
project.summary = meta.summary
project.description = meta.description
project.author_email = meta.author_email
project.save()
for classifier in meta.classifiers:
project.classifiers.add(
Classifier.objects.get_or_create(name=classifier)[0])
release = Release()
release.version = meta.version
release.project = project
filename = os.path.basename(path)
file = File(open(path, "rb"))
release.distribution.save(filename, file)
release.save()
print "%s-%s added" % (meta.name, meta.version)
def _get_meta(self, path):
data = pkginfo.get_metadata(path)
if data:
return data
else:
print "Couldn't get metadata from %s. Not added to chishop" % os.path.basename(
path)
return None
class Command(LabelCommand):
option_list = LabelCommand.option_list + (
make_option("-o", "--owner", help="add packages as OWNER", metavar="OWNER", default=None),
)
help = """Add one or more packages to the repository. Each argument can
be a package name or a URL to an archive or egg. Package names honour
the same rules as easy_install with regard to indicating versions etc.
If a version of the package exists, but is older than what we want to install,
the owner remains the same.
For new packages there needs to be an owner. If the --owner option is present
we use that value. If not, we try to match the maintainer of the package, form
the metadata, with a user in out database, based on the If it's a new package
and the maintainer emailmatches someone in our user list, we use that. If not,
the package can not be
added"""
def __init__(self, *args, **kwargs):
self.pypi = PackageIndex()
LabelCommand.__init__(self, *args, **kwargs)
def handle_label(self, label, **options):
with tempdir() as tmp:
path = self.pypi.download(label, tmp)
if path:
self._save_package(path, options["owner"])
else:
print "Could not add %s. Not found." % label
def _save_package(self, path, ownerid):
meta = self._get_meta(path)
try:
# can't use get_or_create as that demands there be an owner
project = Project.objects.get(name=meta.name)
isnewproject = False
except Project.DoesNotExist:
project = Project(name=meta.name)
isnewproject = True
release = project.get_release(meta.version)
if not isnewproject and release and release.version == meta.version:
print "%s-%s already added" % (meta.name, meta.version)
return
# algorithm as follows: If owner is given, try to grab user with that
# username from db. If doesn't exist, bail. If no owner set look at
# mail address from metadata and try to get that user. If it exists
# use it. If not, bail.
owner = None
if ownerid:
try:
if "@" in ownerid:
owner = User.objects.get(email=ownerid)
else:
owner = User.objects.get(username=ownerid)
except User.DoesNotExist:
pass
else:
try:
owner = User.objects.get(email=meta.author_email)
except User.DoesNotExist:
pass
if not owner:
print "No owner defined. Use --owner to force one"
return
# at this point we have metadata and an owner, can safely add it.
project.owner = owner
# Some packages don't have proper licence, seems to be a problem
# with setup.py upload. Use "UNKNOWN"
project.license = meta.license or "Unknown"
project.metadata_version = meta.metadata_version
project.author = meta.author
project.home_page = meta.home_page
project.download_url = meta.download_url
project.summary = meta.summary
project.description = meta.description
project.author_email = meta.author_email
project.save()
for classifier in meta.classifiers:
project.classifiers.add(Classifier.objects.get_or_create(name=classifier)[0])
release = Release()
release.version = meta.version
release.project = project
filename = os.path.basename(path)
file = File(open(path, "rb"))
release.distribution.save(filename, file)
release.save()
print "%s-%s added" % (meta.name, meta.version)
def _get_meta(self, path):
data = pkginfo.get_metadata(path)
if data:
return data
else:
print "Couldn't get metadata from %s. Not added to chishop" % os.path.basename(path)
return None
class Command(LabelCommand):
option_list = LabelCommand.option_list + (
make_option("-o", "--owner", help="add packages as OWNER",
metavar="OWNER", default=None),
)
help = """Add one or more packages to the repository. Each argument can
be a package name or a URL to an archive or egg. Package names honour
the same rules as easy_install with regard to indicating versions etc.
If a version of the package exists, but is older than what we want to install,
the owner remains the same.
For new packages there needs to be an owner. If the --owner option is present
we use that value. If not, we try to match the maintainer of the package, form
the metadata, with a user in out database, based on the If it's a new package
and the maintainer emailmatches someone in our user list, we use that. If not,
the package can not be
added"""
def __init__(self, *args, **kwargs):
self.pypi = PackageIndex()
LabelCommand.__init__(self, *args, **kwargs)
def handle_label(self, label, **options):
with tempdir() as tmp:
path = self.pypi.download(label, tmp)
if path:
self._save_package(path, options["owner"])
else:
print "Could not add %s. Not found." % label
def _save_package(self, path, ownerid):
meta = self._get_meta(path)
try:
# can't use get_or_create as that demands there be an owner
package = Package.objects.get(name=meta.name)
isnewpackage = False
except Package.DoesNotExist:
package = Package(name=meta.name)
isnewpackage = True
release = package.get_release(meta.version)
if not isnewpackage and release and release.version == meta.version:
print "%s-%s already added" % (meta.name, meta.version)
return
# algorithm as follows: If owner is given, try to grab user with that
# username from db. If doesn't exist, bail. If no owner set look at
# mail address from metadata and try to get that user. If it exists
# use it. If not, bail.
owner = None
if ownerid:
try:
if "@" in ownerid:
owner = User.objects.get(email=ownerid)
else:
owner = User.objects.get(username=ownerid)
except User.DoesNotExist:
pass
else:
try:
owner = User.objects.get(email=meta.author_email)
except User.DoesNotExist:
pass
if not owner:
print "No owner defined. Use --owner to force one"
return
# at this point we have metadata and an owner, can safely add it.
package.save()
package.owners.add(owner)
package.maintainers.add(owner)
for classifier in meta.classifiers:
package.classifiers.add(
Classifier.objects.get_or_create(name=classifier)[0])
release = Release()
release.version = meta.version
release.package = package
release.metadata_version = meta.metadata_version
package_info = MultiValueDict()
package_info.update(meta.__dict__)
release.package_info = package_info
release.save()
file = File(open(path, "rb"))
if isinstance(meta, pkginfo.SDist):
dist = 'sdist'
elif meta.filename.endswith('.rmp') or meta.filename.endswith('.srmp'):
dist = 'bdist_rpm'
elif meta.filename.endswith('.exe'):
dist = 'bdist_wininst'
elif meta.filename.endswith('.egg'):
dist = 'bdist_egg'
elif meta.filename.endswith('.dmg'):
dist = 'bdist_dmg'
else:
dist = 'bdist_dumb'
release.distributions.create(content=file, uploader=owner, filetype=dist)
print "%s-%s added" % (meta.name, meta.version)
def _get_meta(self, path):
data = pkginfo.get_metadata(path)
if data:
return data
else:
print "Couldn't get metadata from %s. Not added to chishop" % os.path.basename(path)
return None
class InstallGcloudCommand(Command):
""" Install Google Cloud SDK"""
def initialize_options(self):
self.win_gcloud_url = None
self.win_gcloud_installer = None
self.nix_gcloud_url = None
self.silent = None
self.curl = None
self.bash = None
self.package_index = None
def finalize_options(self):
if platform.system() != "Windows":
self.curl = which('curl')
self.bash = which('bash')
self.gcloud_url = "https://sdk.cloud.google.com"
self.silent = "--disable-prompts"
else:
self.silent = "/S"
self.gcloud_url = "https://dl.google.com/dl/cloudsdk/channels/" \
"rapid/GoogleCloudSDKInstaller.exe"
self.package_index = PackageIndex()
# Copied from setuptools.command.easy_install.easy_install
@contextlib.contextmanager
def _tmpdir(self):
tmpdir = tempfile.mkdtemp(prefix=six.u("install_gcloud-"))
try:
# cast to str as workaround for #709 and #710 and #712
yield str(tmpdir)
finally:
os.path.exists(tmpdir) and rmtree(rmtree_safe(tmpdir))
def run(self):
warn_msg = "Please install the Google Cloud SDK manually:\n\t" \
"https://cloud.google.com/sdk/downloads"
if platform.system() == "Windows":
with self._tmpdir() as tmpdir:
gcloud_install_cmd = \
self.package_index.download(self.gcloud_url, tmpdir)
try:
output = subprocess.check_output([gcloud_install_cmd,
self.silent],
stderr=subprocess.STDOUT)
log.info(output.decode('utf-8'))
except subprocess.CalledProcessError as cpe:
log.warn(u' '.join(cpe.cmd) + u":\n\t" +
cpe.output.decode('utf-8'))
elif self.curl is not None and self.bash is not None:
try:
script = subprocess.check_output([self.curl, "-s", "-S",
self.gcloud_url],
stderr=subprocess.STDOUT)
if script:
with self._tmpdir() as tmpdir:
gcloud_install_cmd = os.path.join(tmpdir,
'installer.sh')
with open(gcloud_install_cmd, 'w') as gcloud_install_fd:
gcloud_install_fd.write(script.decode('utf-8'))
output = subprocess.check_output([self.bash,
gcloud_install_cmd,
self.silent],
stderr=subprocess.STDOUT)
log.info(output.decode('utf-8'))
else:
log.warn("Unable to download installer script. " + warn_msg)
except subprocess.CalledProcessError as cpe:
log.warn(u' '.join(cpe.cmd) + u":\n\t" +
cpe.output.decode('utf-8'))
else:
log.warn("Unable to find curl and/or bash. " + warn_msg)
def get_inputs(self):
return []
def get_outputs(self):
return []
class Command(LabelCommand):
option_list = LabelCommand.option_list + (make_option(
"-o",
"--owner",
help="add packages as OWNER",
metavar="OWNER",
default=None), )
help = """Add one or more packages to the repository. Each argument can
be a package name or a URL to an archive or egg. Package names honour
the same rules as easy_install with regard to indicating versions etc.
If a version of the package exists, but is older than what we want to install,
the owner remains the same.
For new packages there needs to be an owner. If the --owner option is present
we use that value. If not, we try to match the maintainer of the package, form
the metadata, with a user in out database, based on the If it's a new package
and the maintainer emailmatches someone in our user list, we use that. If not,
the package can not be
added"""
def __init__(self, *args, **kwargs):
self.pypi = PackageIndex()
LabelCommand.__init__(self, *args, **kwargs)
def handle_label(self, label, **options):
with tempdir() as tmp:
path = self.pypi.download(label, tmp)
if path:
self._save_package(path, options["owner"])
else:
print "Could not add %s. Not found." % label
def _save_package(self, path, ownerid):
meta = self._get_meta(path)
try:
# can't use get_or_create as that demands there be an owner
package = Package.objects.get(name=meta.name)
isnewpackage = False
except Package.DoesNotExist:
package = Package(name=meta.name)
isnewpackage = True
release = package.get_release(meta.version)
if not isnewpackage and release and release.version == meta.version:
print "%s-%s already added" % (meta.name, meta.version)
return
# algorithm as follows: If owner is given, try to grab user with that
# username from db. If doesn't exist, bail. If no owner set look at
# mail address from metadata and try to get that user. If it exists
# use it. If not, bail.
owner = None
if ownerid:
try:
if "@" in ownerid:
owner = User.objects.get(email=ownerid)
else:
owner = User.objects.get(username=ownerid)
except User.DoesNotExist:
pass
else:
try:
owner = User.objects.get(email=meta.author_email)
except User.DoesNotExist:
pass
if not owner:
print "No owner defined. Use --owner to force one"
return
# at this point we have metadata and an owner, can safely add it.
package.save()
package.owners.add(owner)
package.maintainers.add(owner)
for classifier in meta.classifiers:
package.classifiers.add(
Classifier.objects.get_or_create(name=classifier)[0])
release = Release()
release.version = meta.version
release.package = package
release.metadata_version = meta.metadata_version
package_info = MultiValueDict()
package_info.update(meta.__dict__)
release.package_info = package_info
release.save()
file = File(open(path, "rb"))
if isinstance(meta, pkginfo.SDist):
dist = 'sdist'
elif meta.filename.endswith('.rmp') or meta.filename.endswith('.srmp'):
dist = 'bdist_rpm'
elif meta.filename.endswith('.exe'):
dist = 'bdist_wininst'
elif meta.filename.endswith('.egg'):
dist = 'bdist_egg'
elif meta.filename.endswith('.dmg'):
dist = 'bdist_dmg'
else:
dist = 'bdist_dumb'
release.distributions.create(content=file,
uploader=owner,
filetype=dist)
print "%s-%s added" % (meta.name, meta.version)
def _get_meta(self, path):
data = pkginfo.get_metadata(path)
if data:
return data
else:
print "Couldn't get metadata from %s. Not added to chishop" % os.path.basename(
path)
return None
class Command(LabelCommand):
option_list = LabelCommand.option_list + (
make_option("-o", "--owner", help="foce package owner to OWNER", metavar="OWNER", default=None),
make_option(
"-f",
"--fallback-owner",
dest="fallback_owner",
help="default packages owner if not present",
metavar="OWNER",
default=None,
),
make_option(
"-i", "--index", dest="index", help="Pypi site url", metavar="URL", default="http://pypi.python.org/simple"
),
make_option(
"--visibility",
type="choice",
choices=map(str, dict(Package.VISIBILITY).keys()),
help="set package visibility; %s" % r", ".join("%s=%s" % (a, b) for a, b in Package.VISIBILITY),
metavar="VISIBILITY",
default=Package.VISIBLE_ALL,
),
)
help = """Add one or more packages to the repository. Each argument can
be a package name or a URL to an archive or egg. Package names honour
the same rules as easy_install with regard to indicating versions etc.
If a version of the package exists, but is older than what we want to install,
the owner remains the same.
For new packages there needs to be an owner. If the --owner option is present
we use that value. If not, we try to match the maintainer of the package, form
the metadata, with a user in out database, based on the If it's a new package
and the maintainer emailmatches someone in our user list, we use that. If not,
the package can not be
added"""
def __init__(self, *args, **kwargs):
LabelCommand.__init__(self, *args, **kwargs)
def handle_label(self, label, **options):
with tempdir() as tmp:
self.pypi = PackageIndex(options["index"])
path = self.pypi.download(label, tmp)
if path:
self._save_package(path, **options)
else:
print "Could not add %s. Not found." % label
@transaction.commit_on_success
def _save_package(self, path, **kwargs):
force_ownerid = kwargs["owner"]
fallback_ownerid = kwargs["fallback_owner"]
visibility = kwargs["visibility"]
if fallback_ownerid:
try:
fallback_owner = User.objects.get(username=fallback_ownerid)
except User.DoesNotExist:
print "Fallback owner does not exists"
return
meta = self._get_meta(path)
try:
# can't use get_or_create as that demands there be an owner
package = Package.objects.get(name=meta.name)
isnewpackage = False
except Package.DoesNotExist:
package = Package(name=meta.name, visibility=visibility)
isnewpackage = True
release = package.get_release(meta.version)
if not isnewpackage and release and release.version == meta.version:
print "%s-%s already added" % (meta.name, meta.version)
return
# algorithm as follows: If owner is given, try to grab user with that
# username from db. If doesn't exist, bail. If no owner set look at
# mail address from metadata and try to get that user. If it exists
# use it. If not, bail.
owner = None
if force_ownerid:
try:
owner = User.objects.get(username=force_ownerid)
except User.DoesNotExist:
print "Forced owner does not exists."
return
else:
try:
owner = User.objects.get(email=meta.author_email)
except User.DoesNotExist:
owner = fallback_owner
if not owner:
print "No owner defined. Use --owner to force one"
return
# at this point we have metadata and an owner, can safely add it.
package.save()
package.owners.add(owner)
package.maintainers.add(owner)
for classifier in meta.classifiers:
package.classifiers.add(Classifier.objects.get_or_create(name=classifier)[0])
release = Release()
release.version = meta.version
release.package = package
release.metadata_version = meta.metadata_version
package_info = MultiValueDict()
package_info.update(meta.__dict__)
release.package_info = package_info
release.save()
try:
file = File(open(path, "rb"))
if isinstance(meta, pkginfo.SDist):
dist = "sdist"
elif meta.filename.endswith(".rmp") or meta.filename.endswith(".srmp"):
dist = "bdist_rpm"
elif meta.filename.endswith(".exe"):
dist = "bdist_wininst"
elif meta.filename.endswith(".egg"):
dist = "bdist_egg"
elif meta.filename.endswith(".dmg"):
dist = "bdist_dmg"
else:
dist = "bdist_dumb"
release.distributions.create(content=file, uploader=owner, filetype=get_distribution_type(dist))
print "%s-%s added" % (meta.name, meta.version)
except FileExistsError as e:
print e
transaction.rollback()
def _get_meta(self, path):
data = pkginfo.get_metadata(path)
if data:
return data
else:
print "Couldn't get metadata from %s. Not added to chishop" % os.path.basename(path)
return None
