Пример #1
0
    def wrapper(*args, **kwargs):
        new_args = []
        for arg in args:
            if isinstance(arg, str):
                # when a json was supplied, we deserialize it to dict
                try:
                    new_args.append(json.loads(arg))
                except:
                    # regular string from a user
                    new_args.append(arg)
            else:
                new_args.append(arg)

            if FileId.is_file_id(new_args[-1]):
                new_args[-1] = FileId(new_args[-1])

        for key, value in kwargs.iteritems():
            if isinstance(value, str):
                # when a json was supplied, we deserialize it to dict
                try:
                    kwargs[key] = json.loads(value)
                except:
                    pass

            if FileId.is_file_id(kwargs[key]):
                kwargs[key] = FileId(value)

        return func(*tuple(new_args), **kwargs)
Пример #2
0
    def wrapper(*args, **kwargs):
        new_args = []
        for arg in args:
            if isinstance(arg, str):
                # when a json was supplied, we deserialize it to dict
                try:
                    new_args.append(json.loads(arg))
                except:
                    # regular string from a user
                    new_args.append(arg)
            else:
                new_args.append(arg)

            if FileId.is_file_id(new_args[-1]):
                new_args[-1] = FileId(new_args[-1])

        for key, value in kwargs.iteritems():
            if isinstance(value, str):
                # when a json was supplied, we deserialize it to dict
                try:
                    kwargs[key] = json.loads(value)
                except:
                    pass

            if FileId.is_file_id(kwargs[key]):
                kwargs[key] = FileId(value)

        return func(*tuple(new_args), **kwargs)
Пример #3
0
    def spec_get(self, package_name, branch=None, commit=None):
        '''
        Get specfile of a package packaged in Fedora
        @param package_name: package name
        @param branch: branch (e.g. "f23", ...); if omitted "rawhide" is used
        @param commit: fedpkg git commit; if omitted, the latest commit is used
        @return: specfile file id
        '''
        ret = ServiceResult()

        # prevent from accessing suspicious files
        package_name = os.path.basename(package_name)

        if branch is None or branch == "rawhide":
            branch = "master"

        # we have to ensure that such package/branch/commit exist
        with self.get_lock(package_name):
            path = self._git_tree_prepare(package_name, branch, commit)
            ident = "%s/%s/%s/%s.spec" % (
                package_name, branch, self._commit2ident(commit), package_name)
            ret.result = FileId.construct(self,
                                          ident,
                                          path=os.path.join(
                                              path, "%s.spec" % package_name))
        return ret
Пример #4
0
    def tarball_get(self, upstream_url, commit):
        '''
        Retrieve a tarball from upstream, try to detect upstream provider
        @param upstream_url: an upstream url
        @param commit: commit of upstream file
        @return: tarball file id
        '''
        res = ServiceResult()

        if upstream_url.startswith('https://github.com'):
            m = re.search('https://github.com/([a-z]+)/([a-z-_]+)/?', upstream_url)
            if m is None:
                raise ValueError("Expected URL in form 'https://github.com/<USER>/<REPO>/, got %s", (upstream_url,))
            tarball_url = self._get_github_tarball_url(m.group(1), m.group(2), commit)
            filename = self._get_github_file_name(m.group(1), m.group(2), commit)
        elif upstream_url.startswith('https://bitbucket.org'):
            m = re.search('https://bitbucket.org/([a-z]+)/([a-z-_]+)/?', upstream_url)
            if m is None:
                raise ValueError("Expected URL in form 'https://bitbucket.org/<USER>/<REPO>/, got %s", (upstream_url,))
            tarball_url = self._get_bitbucket_tarball_url(m.group(1), m.group(2), commit)
            filename = self._get_bitbucket_file_name(m.group(1), m.group(2), commit)
        else:
            raise NotImplementedError("Unknown upstream provider %s" % (upstream_url,))

        with self.get_lock(filename):
            if self.dircache.is_available(filename):
                res.result = FileId.construct(self, self.dircache.get_file_path(filename))
            else:
                res.result = self._download_tarball(tarball_url, filename)

        return res
Пример #5
0
    def spec_patch_get(self, package_name, patch_name, branch=None, commit=None):
        """
        Get file id of a downstream patch of a package packaged in Fedora
        @param package_name: package name
        @param patch_name: name of the patch file
        @param branch: branch (e.g. "f23", ...); if omitted, "rawhide" is used
        @param commit: fedpkg git commit; if omitted, the latest commit is used
        @return: file id of the patch
        """
        ret = ServiceResult()
        ret.result = []

        # prevent from accessing suspicious files
        package_name = os.path.basename(package_name)
        patch_name = os.path.basename(patch_name)

        if branch is None or branch == "rawhide":
            branch = "master"

        with self.get_lock(package_name):
            path = self._git_tree_prepare(package_name, branch, commit)

            patch_path = os.path.join(path, patch_name)
            if not os.path.isfile(patch_path):
                raise ValueError(
                    "There is not patch %s for package %s, branch %s and commit %s"
                    % (patch_name, package_name, branch, commit)
                )

            ident = ("%s/%s/%s/%s" % (package_name, branch, self._commit2ident(commit), patch_name),)
            ret.result = FileId.construct(self, ident, path=patch_path)

        return ret
Пример #6
0
    def _download_tarball(self, tarball_url, filename):
        log.debug("Downloading from %s" % (tarball_url,))
        response = urllib2.urlopen(tarball_url)
        blob = response.read()
        h = blob_hash(blob)

        self.dircache.store(blob, filename)

        return FileId.construct(self, self.dircache.get_file_path(filename), hash_ = h)
Пример #7
0
    def _common_get(self, url, filename):
        ret = ServiceResult()

        with self.get_lock(filename):
            if self.dircache.is_available(filename):
                ret.result = FileId.construct(self, self.dircache.get_file_path(filename))
            elif remote_exists(url):
                log.debug("Downloading from %s" % (url,))
                response = urllib2.urlopen(url)
                blob = response.read()
                h = blob_hash(blob)

                self.dircache.store(blob, filename)

                ret.result = FileId.construct(self, self.dircache.get_file_path(filename), hash_ = h)
            else:
                raise KeyError("Desired file '%s' does not exist ( %s )" % (filename, url))

        ret.meta['origin'] = url
        return ret
Пример #8
0
    def scm_store(self, repo_url, commit=None, branch=None):
        '''
        Store a SCM repo
        @param repo_url: repo URL
        @param commit: commit hash; if None, the latest is used
        @param branch: branch; if None, "master" is used
        @return:
        '''

        ret = ServiceResult()

        if not branch:
            branch = "master"

        if commit:
            commit = commit[:7]

        dirname = self._get_dirname(repo_url, commit, branch)
        filename = self._get_filename(dirname)
        dst_path = self.dircache.get_location(dirname)

        with self.get_lock(dirname):
            if not self.dircache.is_available(filename):
                repo = GitCmd.git_clone_repo(repo_url, dst_path)
                repo.git_checkout(branch)
                if commit:
                    repo.git_checkout(commit)
                else:
                    commit = repo.git_rev_parse_head(dst_path)[:7]

                # if user did not supplied commit, we have to check it explicitly
                filename_old = filename
                filename = self._get_filename(
                    self._get_dirname(repo_url, commit, branch))
                # we have to move it so it will be available with specified commit and branch
                if filename_old != filename:
                    shutil.move(filename_old, filename)

                if not self.dircache.is_available(filename):
                    # if user did not supplied commit, we have to pack the repo
                    self._pack_repo(dirname, filename)
                shutil.rmtree(dst_path)

                if not self.dircache.is_available(filename):
                    self.dircache.register(filename)

        ret.result = FileId.construct(self,
                                      self.dircache.get_file_path(filename))
        ret.meta = {'origin': repo_url}
        return ret
Пример #9
0
    def scm_store(self, repo_url, commit=None, branch=None):
        '''
        Store a SCM repo
        @param repo_url: repo URL
        @param commit: commit hash; if None, the latest is used
        @param branch: branch; if None, "master" is used
        @return:
        '''

        ret = ServiceResult()

        if not branch:
            branch = "master"

        if commit:
            commit = commit[:7]

        dirname = self._get_dirname(repo_url, commit, branch)
        filename = self._get_filename(dirname)
        dst_path = self.dircache.get_location(dirname)

        with self.get_lock(dirname):
            if not self.dircache.is_available(filename):
                repo = GitCmd.git_clone_repo(repo_url, dst_path)
                repo.git_checkout(branch)
                if commit:
                    repo.git_checkout(commit)
                else:
                    commit = repo.git_rev_parse_head(dst_path)[:7]

                # if user did not supplied commit, we have to check it explicitly
                filename_old = filename
                filename = self._get_filename(self._get_dirname(repo_url, commit, branch))
                # we have to move it so it will be available with specified commit and branch
                if filename_old != filename:
                    shutil.move(filename_old, filename)

                if not self.dircache.is_available(filename):
                    # if user did not supplied commit, we have to pack the repo
                    self._pack_repo(dirname, filename)
                shutil.rmtree(dst_path)

                if not self.dircache.is_available(filename):
                    self.dircache.register(filename)

        ret.result = FileId.construct(self, self.dircache.get_file_path(filename))
        ret.meta = {'origin': repo_url}
        return ret
Пример #10
0
    def upload(self, blob):
        '''
        Upload file to the system
        @param blob: a file content to be store
        @return: file id
        '''
        res = ServiceResult()
        log.info("uploading")
        h = blob_hash(blob)
        dst = os.path.join(self.upload_dir, h)

        with self.get_lock():
            with open(dst, 'wb') as f:
                f.write(blob)

        creation_time = datetime_parse(time.ctime(os.path.getctime(dst)))
        # TODO: remove, use dircache instead
        valid_until = creation_time + self.file_lifetime

        res.result = FileId.construct(self, dst, h)
        return res
Пример #11
0
    def spec_get(self, package_name, branch=None, commit=None):
        """
        Get specfile of a package packaged in Fedora
        @param package_name: package name
        @param branch: branch (e.g. "f23", ...); if omitted "rawhide" is used
        @param commit: fedpkg git commit; if omitted, the latest commit is used
        @return: specfile file id
        """
        ret = ServiceResult()

        # prevent from accessing suspicious files
        package_name = os.path.basename(package_name)

        if branch is None or branch == "rawhide":
            branch = "master"

        # we have to ensure that such package/branch/commit exist
        with self.get_lock(package_name):
            path = self._git_tree_prepare(package_name, branch, commit)
            ident = "%s/%s/%s/%s.spec" % (package_name, branch, self._commit2ident(commit), package_name)
            ret.result = FileId.construct(self, ident, path=os.path.join(path, "%s.spec" % package_name))
        return ret
Пример #12
0
    def spec_patch_get(self,
                       package_name,
                       patch_name,
                       branch=None,
                       commit=None):
        '''
        Get file id of a downstream patch of a package packaged in Fedora
        @param package_name: package name
        @param patch_name: name of the patch file
        @param branch: branch (e.g. "f23", ...); if omitted, "rawhide" is used
        @param commit: fedpkg git commit; if omitted, the latest commit is used
        @return: file id of the patch
        '''
        ret = ServiceResult()
        ret.result = []

        # prevent from accessing suspicious files
        package_name = os.path.basename(package_name)
        patch_name = os.path.basename(patch_name)

        if branch is None or branch == "rawhide":
            branch = "master"

        with self.get_lock(package_name):
            path = self._git_tree_prepare(package_name, branch, commit)

            patch_path = os.path.join(path, patch_name)
            if not os.path.isfile(patch_path):
                raise ValueError(
                    "There is not patch %s for package %s, branch %s and commit %s"
                    % (patch_name, package_name, branch, commit))

            ident = "%s/%s/%s/%s" % (package_name, branch,
                                     self._commit2ident(commit), patch_name),
            ret.result = FileId.construct(self, ident, path=patch_path)

        return ret