Example #1
0
def download_egg(eggname, dir):
    """Download an egg to a specific place
    
    :param eggname: name of egg to download
    :param dir: destination directory
    :return: local path
    """
    logger.info("Downloading %s"%eggname)
    return pi.download(eggname, dir)
Example #2
0
 def fetch_requirement(req, dest_dir, force_download):
     from setuptools.package_index import PackageIndex  # @Reimport
     from pkg_resources import working_set  # @Reimport  # NOQA
     i = PackageIndex()
     if force_download:
         [i.remove(i[req.key][0]) for _ in xrange(len(i[req.key]))]
         d = i.download(req, dest_dir)
     else:
         d = i.fetch_distribution(req, dest_dir, force_scan=True)
     d = getattr(d, 'location', d) if d else ''
     return (d if d else working_set.resolve([req])[0].location)
Example #3
0
class Command(LabelCommand):
    option_list = LabelCommand.option_list + (make_option(
        "-o",
        "--owner",
        help="add packages as OWNER",
        metavar="OWNER",
        default=None), )
    help = """Add one or more packages to the repository. Each argument can
be a package name or a URL to an archive or egg. Package names honour
the same rules as easy_install with regard to indicating versions etc.

If a version of the package exists, but is older than what we want to install,
the owner remains the same.

For new packages there needs to be an owner. If the --owner option is present
we use that value. If not, we try to match the maintainer of the package, form
the metadata, with a user in out database, based on the If it's a new package
and the maintainer emailmatches someone in our user list, we use that. If not,
the package can not be
added"""

    def __init__(self, *args, **kwargs):
        self.pypi = PackageIndex()
        LabelCommand.__init__(self, *args, **kwargs)

    def handle_label(self, label, **options):
        with tempdir() as tmp:
            path = self.pypi.download(label, tmp)
            if path:
                self._save_package(path, options["owner"])
            else:
                print "Could not add %s. Not found." % label

    def _save_package(self, path, ownerid):
        meta = self._get_meta(path)

        try:
            # can't use get_or_create as that demands there be an owner
            project = Project.objects.get(name=meta.name)
            isnewproject = False
        except Project.DoesNotExist:
            project = Project(name=meta.name)
            isnewproject = True

        release = project.get_release(meta.version)
        if not isnewproject and release and release.version == meta.version:
            print "%s-%s already added" % (meta.name, meta.version)
            return

        # algorithm as follows: If owner is given, try to grab user with that
        # username from db. If doesn't exist, bail. If no owner set look at
        # mail address from metadata and try to get that user. If it exists
        # use it. If not, bail.
        owner = None

        if ownerid:
            try:
                if "@" in ownerid:
                    owner = User.objects.get(email=ownerid)
                else:
                    owner = User.objects.get(username=ownerid)
            except User.DoesNotExist:
                pass
        else:
            try:
                owner = User.objects.get(email=meta.author_email)
            except User.DoesNotExist:
                pass

        if not owner:
            print "No owner defined. Use --owner to force one"
            return

        # at this point we have metadata and an owner, can safely add it.

        project.owner = owner
        # Some packages don't have proper licence, seems to be a problem
        # with setup.py upload. Use "UNKNOWN"
        project.license = meta.license or "Unknown"
        project.metadata_version = meta.metadata_version
        project.author = meta.author
        project.home_page = meta.home_page
        project.download_url = meta.download_url
        project.summary = meta.summary
        project.description = meta.description
        project.author_email = meta.author_email

        project.save()

        for classifier in meta.classifiers:
            project.classifiers.add(
                Classifier.objects.get_or_create(name=classifier)[0])

        release = Release()
        release.version = meta.version
        release.project = project
        filename = os.path.basename(path)

        file = File(open(path, "rb"))
        release.distribution.save(filename, file)
        release.save()
        print "%s-%s added" % (meta.name, meta.version)

    def _get_meta(self, path):
        data = pkginfo.get_metadata(path)
        if data:
            return data
        else:
            print "Couldn't get metadata from %s. Not added to chishop" % os.path.basename(
                path)
            return None
Example #4
0
class Command(LabelCommand):
    option_list = LabelCommand.option_list + (
        make_option("-o", "--owner", help="add packages as OWNER", metavar="OWNER", default=None),
    )
    help = """Add one or more packages to the repository. Each argument can
be a package name or a URL to an archive or egg. Package names honour
the same rules as easy_install with regard to indicating versions etc.

If a version of the package exists, but is older than what we want to install,
the owner remains the same.

For new packages there needs to be an owner. If the --owner option is present
we use that value. If not, we try to match the maintainer of the package, form
the metadata, with a user in out database, based on the If it's a new package
and the maintainer emailmatches someone in our user list, we use that. If not,
the package can not be
added"""

    def __init__(self, *args, **kwargs):
        self.pypi = PackageIndex()
        LabelCommand.__init__(self, *args, **kwargs)

    def handle_label(self, label, **options):
        with tempdir() as tmp:
            path = self.pypi.download(label, tmp)
            if path:
                self._save_package(path, options["owner"])
            else:
                print "Could not add %s. Not found." % label

    def _save_package(self, path, ownerid):
        meta = self._get_meta(path)

        try:
            # can't use get_or_create as that demands there be an owner
            project = Project.objects.get(name=meta.name)
            isnewproject = False
        except Project.DoesNotExist:
            project = Project(name=meta.name)
            isnewproject = True

        release = project.get_release(meta.version)
        if not isnewproject and release and release.version == meta.version:
            print "%s-%s already added" % (meta.name, meta.version)
            return

        # algorithm as follows: If owner is given, try to grab user with that
        # username from db. If doesn't exist, bail. If no owner set look at
        # mail address from metadata and try to get that user. If it exists
        # use it. If not, bail.
        owner = None

        if ownerid:
            try:
                if "@" in ownerid:
                    owner = User.objects.get(email=ownerid)
                else:
                    owner = User.objects.get(username=ownerid)
            except User.DoesNotExist:
                pass
        else:
            try:
                owner = User.objects.get(email=meta.author_email)
            except User.DoesNotExist:
                pass

        if not owner:
            print "No owner defined. Use --owner to force one"
            return

        # at this point we have metadata and an owner, can safely add it.

        project.owner = owner
        # Some packages don't have proper licence, seems to be a problem
        # with setup.py upload. Use "UNKNOWN"
        project.license = meta.license or "Unknown"
        project.metadata_version = meta.metadata_version
        project.author = meta.author
        project.home_page = meta.home_page
        project.download_url = meta.download_url
        project.summary = meta.summary
        project.description = meta.description
        project.author_email = meta.author_email

        project.save()

        for classifier in meta.classifiers:
            project.classifiers.add(Classifier.objects.get_or_create(name=classifier)[0])

        release = Release()
        release.version = meta.version
        release.project = project
        filename = os.path.basename(path)

        file = File(open(path, "rb"))
        release.distribution.save(filename, file)
        release.save()
        print "%s-%s added" % (meta.name, meta.version)

    def _get_meta(self, path):
        data = pkginfo.get_metadata(path)
        if data:
            return data
        else:
            print "Couldn't get metadata from %s. Not added to chishop" % os.path.basename(path)
            return None
Example #5
0
class Command(LabelCommand):
    option_list = LabelCommand.option_list + (
            make_option("-o", "--owner", help="add packages as OWNER",
                        metavar="OWNER", default=None),
        )
    help = """Add one or more packages to the repository. Each argument can
be a package name or a URL to an archive or egg. Package names honour
the same rules as easy_install with regard to indicating versions etc.

If a version of the package exists, but is older than what we want to install,
the owner remains the same.

For new packages there needs to be an owner. If the --owner option is present
we use that value. If not, we try to match the maintainer of the package, form
the metadata, with a user in out database, based on the If it's a new package
and the maintainer emailmatches someone in our user list, we use that. If not,
the package can not be
added"""

    def __init__(self, *args, **kwargs):
        self.pypi = PackageIndex()
        LabelCommand.__init__(self, *args, **kwargs)

    def handle_label(self, label, **options):
        with tempdir() as tmp:
            path = self.pypi.download(label, tmp)
            if path:
                self._save_package(path, options["owner"])
            else:
                print "Could not add %s. Not found." % label

    def _save_package(self, path, ownerid):
        meta = self._get_meta(path)

        try:
            # can't use get_or_create as that demands there be an owner
            package = Package.objects.get(name=meta.name)
            isnewpackage = False
        except Package.DoesNotExist:
            package = Package(name=meta.name)
            isnewpackage = True

        release = package.get_release(meta.version)
        if not isnewpackage and release and release.version == meta.version:
            print "%s-%s already added" % (meta.name, meta.version)
            return

        # algorithm as follows: If owner is given, try to grab user with that
        # username from db. If doesn't exist, bail. If no owner set look at
        # mail address from metadata and try to get that user. If it exists
        # use it. If not, bail.
        owner = None

        if ownerid:
            try:
                if "@" in ownerid:
                    owner = User.objects.get(email=ownerid)
                else:
                    owner = User.objects.get(username=ownerid)
            except User.DoesNotExist:
                pass
        else:
            try:
                owner = User.objects.get(email=meta.author_email)
            except User.DoesNotExist:
                pass

        if not owner:
            print "No owner defined. Use --owner to force one"
            return

        # at this point we have metadata and an owner, can safely add it.
        package.save()

        package.owners.add(owner)
        package.maintainers.add(owner)

        for classifier in meta.classifiers:
            package.classifiers.add(
                    Classifier.objects.get_or_create(name=classifier)[0])

        release = Release()
        release.version = meta.version
        release.package = package
        release.metadata_version = meta.metadata_version
        package_info = MultiValueDict()
        package_info.update(meta.__dict__)
        release.package_info = package_info
        release.save()

        file = File(open(path, "rb"))
        if isinstance(meta, pkginfo.SDist):
            dist = 'sdist'
        elif meta.filename.endswith('.rmp') or meta.filename.endswith('.srmp'):
            dist = 'bdist_rpm'
        elif meta.filename.endswith('.exe'):
            dist = 'bdist_wininst'
        elif meta.filename.endswith('.egg'):
            dist = 'bdist_egg'
        elif meta.filename.endswith('.dmg'):
            dist = 'bdist_dmg'
        else:
            dist = 'bdist_dumb'
        release.distributions.create(content=file, uploader=owner, filetype=dist)
        print "%s-%s added" % (meta.name, meta.version)

    def _get_meta(self, path):
        data = pkginfo.get_metadata(path)
        if data:
            return data
        else:
            print "Couldn't get metadata from %s. Not added to chishop" % os.path.basename(path)
            return None
Example #6
0
class InstallGcloudCommand(Command):
    """ Install Google Cloud SDK"""
    def initialize_options(self):
        self.win_gcloud_url = None
        self.win_gcloud_installer = None
        self.nix_gcloud_url = None
        self.silent = None
        self.curl = None
        self.bash = None
        self.package_index = None

    def finalize_options(self):

        if platform.system() != "Windows":
            self.curl = which('curl')
            self.bash = which('bash')
            self.gcloud_url = "https://sdk.cloud.google.com"
            self.silent = "--disable-prompts"
        else:
            self.silent = "/S"
            self.gcloud_url = "https://dl.google.com/dl/cloudsdk/channels/" \
                              "rapid/GoogleCloudSDKInstaller.exe"
        self.package_index = PackageIndex()

    # Copied from setuptools.command.easy_install.easy_install
    @contextlib.contextmanager
    def _tmpdir(self):
        tmpdir = tempfile.mkdtemp(prefix=six.u("install_gcloud-"))
        try:
            # cast to str as workaround for #709 and #710 and #712
            yield str(tmpdir)
        finally:
            os.path.exists(tmpdir) and rmtree(rmtree_safe(tmpdir))

    def run(self):
        warn_msg = "Please install the Google Cloud SDK manually:\n\t" \
                   "https://cloud.google.com/sdk/downloads"
        if platform.system() == "Windows":
            with self._tmpdir() as tmpdir:
                gcloud_install_cmd = \
                       self.package_index.download(self.gcloud_url, tmpdir)
                try:
                    output = subprocess.check_output([gcloud_install_cmd,
                                                      self.silent],
                                                     stderr=subprocess.STDOUT)
                    log.info(output.decode('utf-8'))
                except subprocess.CalledProcessError as cpe:
                    log.warn(u' '.join(cpe.cmd) + u":\n\t" +
                             cpe.output.decode('utf-8'))

        elif self.curl is not None and self.bash is not None:
            try:
                script = subprocess.check_output([self.curl, "-s", "-S",
                                                  self.gcloud_url],
                                                 stderr=subprocess.STDOUT)
                if script:
                    with self._tmpdir() as tmpdir:
                        gcloud_install_cmd = os.path.join(tmpdir,
                                                          'installer.sh')
                        with open(gcloud_install_cmd, 'w') as gcloud_install_fd:
                            gcloud_install_fd.write(script.decode('utf-8'))
                        output = subprocess.check_output([self.bash,
                                                          gcloud_install_cmd,
                                                          self.silent],
                                                         stderr=subprocess.STDOUT)
                        log.info(output.decode('utf-8'))
                else:
                    log.warn("Unable to download installer script. " + warn_msg)
            except subprocess.CalledProcessError as cpe:
                log.warn(u' '.join(cpe.cmd) + u":\n\t" +
                         cpe.output.decode('utf-8'))
        else:
            log.warn("Unable to find curl and/or bash. " + warn_msg)

    def get_inputs(self):
        return []

    def get_outputs(self):
        return []
Example #7
0
class Command(LabelCommand):
    option_list = LabelCommand.option_list + (make_option(
        "-o",
        "--owner",
        help="add packages as OWNER",
        metavar="OWNER",
        default=None), )
    help = """Add one or more packages to the repository. Each argument can
be a package name or a URL to an archive or egg. Package names honour
the same rules as easy_install with regard to indicating versions etc.

If a version of the package exists, but is older than what we want to install,
the owner remains the same.

For new packages there needs to be an owner. If the --owner option is present
we use that value. If not, we try to match the maintainer of the package, form
the metadata, with a user in out database, based on the If it's a new package
and the maintainer emailmatches someone in our user list, we use that. If not,
the package can not be
added"""

    def __init__(self, *args, **kwargs):
        self.pypi = PackageIndex()
        LabelCommand.__init__(self, *args, **kwargs)

    def handle_label(self, label, **options):
        with tempdir() as tmp:
            path = self.pypi.download(label, tmp)
            if path:
                self._save_package(path, options["owner"])
            else:
                print "Could not add %s. Not found." % label

    def _save_package(self, path, ownerid):
        meta = self._get_meta(path)

        try:
            # can't use get_or_create as that demands there be an owner
            package = Package.objects.get(name=meta.name)
            isnewpackage = False
        except Package.DoesNotExist:
            package = Package(name=meta.name)
            isnewpackage = True

        release = package.get_release(meta.version)
        if not isnewpackage and release and release.version == meta.version:
            print "%s-%s already added" % (meta.name, meta.version)
            return

        # algorithm as follows: If owner is given, try to grab user with that
        # username from db. If doesn't exist, bail. If no owner set look at
        # mail address from metadata and try to get that user. If it exists
        # use it. If not, bail.
        owner = None

        if ownerid:
            try:
                if "@" in ownerid:
                    owner = User.objects.get(email=ownerid)
                else:
                    owner = User.objects.get(username=ownerid)
            except User.DoesNotExist:
                pass
        else:
            try:
                owner = User.objects.get(email=meta.author_email)
            except User.DoesNotExist:
                pass

        if not owner:
            print "No owner defined. Use --owner to force one"
            return

        # at this point we have metadata and an owner, can safely add it.
        package.save()

        package.owners.add(owner)
        package.maintainers.add(owner)

        for classifier in meta.classifiers:
            package.classifiers.add(
                Classifier.objects.get_or_create(name=classifier)[0])

        release = Release()
        release.version = meta.version
        release.package = package
        release.metadata_version = meta.metadata_version
        package_info = MultiValueDict()
        package_info.update(meta.__dict__)
        release.package_info = package_info
        release.save()

        file = File(open(path, "rb"))
        if isinstance(meta, pkginfo.SDist):
            dist = 'sdist'
        elif meta.filename.endswith('.rmp') or meta.filename.endswith('.srmp'):
            dist = 'bdist_rpm'
        elif meta.filename.endswith('.exe'):
            dist = 'bdist_wininst'
        elif meta.filename.endswith('.egg'):
            dist = 'bdist_egg'
        elif meta.filename.endswith('.dmg'):
            dist = 'bdist_dmg'
        else:
            dist = 'bdist_dumb'
        release.distributions.create(content=file,
                                     uploader=owner,
                                     filetype=dist)
        print "%s-%s added" % (meta.name, meta.version)

    def _get_meta(self, path):
        data = pkginfo.get_metadata(path)
        if data:
            return data
        else:
            print "Couldn't get metadata from %s. Not added to chishop" % os.path.basename(
                path)
            return None
Example #8
0
File: ppadd.py Project: saxix/pyppi
class Command(LabelCommand):
    option_list = LabelCommand.option_list + (
        make_option("-o", "--owner", help="foce package owner to OWNER", metavar="OWNER", default=None),
        make_option(
            "-f",
            "--fallback-owner",
            dest="fallback_owner",
            help="default packages owner if not present",
            metavar="OWNER",
            default=None,
        ),
        make_option(
            "-i", "--index", dest="index", help="Pypi site url", metavar="URL", default="http://pypi.python.org/simple"
        ),
        make_option(
            "--visibility",
            type="choice",
            choices=map(str, dict(Package.VISIBILITY).keys()),
            help="set package visibility; %s" % r", ".join("%s=%s" % (a, b) for a, b in Package.VISIBILITY),
            metavar="VISIBILITY",
            default=Package.VISIBLE_ALL,
        ),
    )
    help = """Add one or more packages to the repository. Each argument can
be a package name or a URL to an archive or egg. Package names honour
the same rules as easy_install with regard to indicating versions etc.

If a version of the package exists, but is older than what we want to install,
the owner remains the same.

For new packages there needs to be an owner. If the --owner option is present
we use that value. If not, we try to match the maintainer of the package, form
the metadata, with a user in out database, based on the If it's a new package
and the maintainer emailmatches someone in our user list, we use that. If not,
the package can not be
added"""

    def __init__(self, *args, **kwargs):
        LabelCommand.__init__(self, *args, **kwargs)

    def handle_label(self, label, **options):
        with tempdir() as tmp:
            self.pypi = PackageIndex(options["index"])
            path = self.pypi.download(label, tmp)
            if path:
                self._save_package(path, **options)
            else:
                print "Could not add %s. Not found." % label

    @transaction.commit_on_success
    def _save_package(self, path, **kwargs):
        force_ownerid = kwargs["owner"]
        fallback_ownerid = kwargs["fallback_owner"]
        visibility = kwargs["visibility"]
        if fallback_ownerid:
            try:
                fallback_owner = User.objects.get(username=fallback_ownerid)
            except User.DoesNotExist:
                print "Fallback owner does not exists"
                return

        meta = self._get_meta(path)
        try:
            # can't use get_or_create as that demands there be an owner
            package = Package.objects.get(name=meta.name)
            isnewpackage = False
        except Package.DoesNotExist:
            package = Package(name=meta.name, visibility=visibility)
            isnewpackage = True

        release = package.get_release(meta.version)
        if not isnewpackage and release and release.version == meta.version:
            print "%s-%s already added" % (meta.name, meta.version)
            return

        # algorithm as follows: If owner is given, try to grab user with that
        # username from db. If doesn't exist, bail. If no owner set look at
        # mail address from metadata and try to get that user. If it exists
        # use it. If not, bail.
        owner = None

        if force_ownerid:
            try:
                owner = User.objects.get(username=force_ownerid)
            except User.DoesNotExist:
                print "Forced owner does not exists."
                return
        else:
            try:
                owner = User.objects.get(email=meta.author_email)
            except User.DoesNotExist:
                owner = fallback_owner

        if not owner:
            print "No owner defined. Use --owner to force one"
            return

        # at this point we have metadata and an owner, can safely add it.
        package.save()

        package.owners.add(owner)
        package.maintainers.add(owner)

        for classifier in meta.classifiers:
            package.classifiers.add(Classifier.objects.get_or_create(name=classifier)[0])

        release = Release()
        release.version = meta.version
        release.package = package
        release.metadata_version = meta.metadata_version
        package_info = MultiValueDict()
        package_info.update(meta.__dict__)
        release.package_info = package_info
        release.save()
        try:
            file = File(open(path, "rb"))
            if isinstance(meta, pkginfo.SDist):
                dist = "sdist"
            elif meta.filename.endswith(".rmp") or meta.filename.endswith(".srmp"):
                dist = "bdist_rpm"
            elif meta.filename.endswith(".exe"):
                dist = "bdist_wininst"
            elif meta.filename.endswith(".egg"):
                dist = "bdist_egg"
            elif meta.filename.endswith(".dmg"):
                dist = "bdist_dmg"
            else:
                dist = "bdist_dumb"
            release.distributions.create(content=file, uploader=owner, filetype=get_distribution_type(dist))

            print "%s-%s added" % (meta.name, meta.version)
        except FileExistsError as e:
            print e
            transaction.rollback()

    def _get_meta(self, path):
        data = pkginfo.get_metadata(path)
        if data:
            return data
        else:
            print "Couldn't get metadata from %s. Not added to chishop" % os.path.basename(path)
            return None