Пример #1
0
        identifier = file_id.get_identifier()
        identifier = identifier.split('/')

        if len(identifier) != 4:
            raise ValueError("Unknown file to be accessed")

        package_name = os.path.basename(identifier[0])
        branch = os.path.basename(identifier[1])
        commit = self._ident2commit(os.path.basename(identifier[2]))
        f = os.path.basename(identifier[3])

        if not f.endswith('.spec') and not f.endswith('.patch'):
            raise ValueError("Unknown file to be accessed")

        path = os.path.join(self.pkg_dir, package_name)
        file_path = os.path.join(path, f)

        log.debug("downloading '%s'" % (file_path, ))

        with self.get_lock(package_name):
            self._git_tree_prepare(package_name, branch, commit)

            with open(file_path, 'rb') as f:
                content = f.read()

        return content


if __name__ == "__main__":
    ServiceEnvelope.serve(SpecStorageService)
Пример #2
0
    def license_compatible(self, license1, license2):
        '''
        Say if two licenses are compatible
        @param license1: a first license
        @param license2: a first license
        @return: True if licenses are compatible
        '''
        return "TODO"

    @action
    def license_abbreviation(self, license):
        '''
        Get an abbrevation for a license
        @param license: a license to abbreviate
        @return: abbreviated license
        '''
        return "TODO"

    @action
    def license_summarize(self, licenses):
        '''
        Try to get the most suitable license which would cover all of the license
        @param licenses: a list of licenses
        @return: abbreviated license
        '''
        return "TODO"


if __name__ == "__main__":
    ServiceEnvelope.serve(LicenseService)
Пример #3
0
                if not self.dircache.is_available(filename):
                    # if user did not supplied commit, we have to pack the repo
                    self._pack_repo(dirname, filename)
                shutil.rmtree(dst_path)

                if not self.dircache.is_available(filename):
                    self.dircache.register(filename)

        ret.result = FileId.construct(self,
                                      self.dircache.get_file_path(filename))
        ret.meta = {'origin': repo_url}
        return ret

    @action
    def download(self, file_id):
        '''
        Download a file
        @param file_id: a file to be downloaded
        @return: file content
        '''
        filename = os.path.basename(file_id.get_identifier())

        with self.get_lock(filename):
            content = self.dircache.retrieve(filename)

        return content


if __name__ == "__main__":
    ServiceEnvelope.serve(ScmStorageService)
Пример #4
0
        cursor = self.deps_package.find({'package': package, 'distro': distro}, filtering)
        for item in cursor:
            if item['distro'] not in ret:
                ret.result.append(item['distro'])

        return ret

    @action
    def deps_package(self, package, version, distro):
        '''
        Dependencies of the given project in specified commit
        @param package: package name
        @param version: package version
        @param distro: distribution
        @return: list of dependendencies of package with analysis metadata
        '''
        ret = ServiceResult()
        ret.result = []

        filtering = {'version': 0, '_id': 0, 'package': 0}

        cursor = self.deps_package.find({'package': package, 'version': version, 'distro': distro}, filtering)
        for item in cursor:
            ret.result.append({'deps': item['deps'], 'meta': item['meta']})

        return ret

if __name__ == "__main__":
    ServiceEnvelope.serve(DepsStorageService)
Пример #5
0
            try:
                ret.result = gofedlib.api(src_path)
            except:
                exc_info = sys.exc_info()
                ret.meta['error'] = [ str(exc_info[0]), str(exc_info[1]), str(exc_info[2])]
            finally:
                ret.meta['language'] = 'golang'
                ret.meta['tool'] = 'gofedlib'
        else:
            raise NotImplementedError()

        return ret

    @action
    def api_diff(self, api1, api2):
        '''
        Make a diff of APIs
        @param api1: the first API
        @param api2: the second API
        @return: list of API differences
        '''
        ret = ServiceResult()

        ret.result = apidiff(api1, api2)
        ret.meta = {'language': 'golang', 'tool': 'gofedlib'}

        return ret

if __name__ == "__main__":
    ServiceEnvelope.serve(ApiService)
Пример #6
0
        identifier = file_id.get_identifier()
        identifier = identifier.split("/")

        if len(identifier) != 4:
            raise ValueError("Unknown file to be accessed")

        package_name = os.path.basename(identifier[0])
        branch = os.path.basename(identifier[1])
        commit = self._ident2commit(os.path.basename(identifier[2]))
        f = os.path.basename(identifier[3])

        if not f.endswith(".spec") and not f.endswith(".patch"):
            raise ValueError("Unknown file to be accessed")

        path = os.path.join(self.pkg_dir, package_name)
        file_path = os.path.join(path, f)

        log.debug("downloading '%s'" % (file_path,))

        with self.get_lock(package_name):
            self._git_tree_prepare(package_name, branch, commit)

            with open(file_path, "rb") as f:
                content = f.read()

        return content


if __name__ == "__main__":
    ServiceEnvelope.serve(SpecStorageService)
Пример #7
0
# ####################################################################

import os
from common.helpers.output import log
from common.service.storageService import StorageService
from common.service.serviceEnvelope import ServiceEnvelope
from common.service.action import action

DEFAULT_PACKAGES_TREE_DIR = "packages/"


class PackageStorageService(StorageService):
    '''  Fedora Git packages provider '''
    @action
    def package_get(self, package_name):
        '''
        Needs design
        '''
        return "TODO"

    @action
    def package_get_commit(self, package_name, commit):
        '''
        Needs design
        '''
        return "TODO"


if __name__ == "__main__":
    ServiceEnvelope.serve(PackageStorageService)
Пример #8
0
from common.service.computationalService import ComputationalService
from common.service.serviceEnvelope import ServiceEnvelope
from common.service.action import action


class GoTestService(ComputationalService):
    ''' Golang tests '''
    @action
    def gotest_run(self, file_id):
        '''
        Run a golang test on a file
        @param file_id: a file id referencing a file with tests and sources to be tested
        @return: test statistics
        '''
        # This will be implemented using a custom tool, which needs to be designed
        # and implemented
        #  * run each test inside a docker instance (?)
        #    * a user can specify on which Fedora version
        #  * as simple tool as possible
        #  * design a configuration file, which will say how to prepare the
        #  environment for tests
        #    * it could be a parameter for gotest_run() or it can be a part of
        #    file (e.g. in root dir of fedora package git repo)
        #  * needs brainstorming...
        #  * ...
        return "TODO"


if __name__ == "__main__":
    ServiceEnvelope.serve(GoTestService)
Пример #9
0
import sys
from common.helpers.utils import json_pretty_format
from common.service.computationalService import StorageService
from common.service.serviceEnvelope import ServiceEnvelope


class Storage0001Service(StorageService):
    def exposed_storage0001(self):
        return "foo"


if __name__ == "__main__":
    ServiceEnvelope.serve(Storage0001Service)
Пример #10
0
            ret.meta['language'] = 'golang'
            ret.meta['tool'] = 'gofedlib'

        return ret

    @action
    def deps_diff(self, deps1, deps2, opts=None):
        '''
        Make a diff of dependencies
        @param deps1: the first dependency list
        @param deps2: the second dependency list
        @param opts: additional analysis opts
        @return: list of dependency differences
        '''
        default_opts = {'language': 'detect', 'tool': 'default'}
        ret = ServiceResult()

        if opts is None:
            opts = default_opts
        else:
            default_opts.update(opts)
            opts = default_opts

        # TODO: implement deps difference
        raise NotImplementedError("Currently not implemented")

        return ret

if __name__ == "__main__":
    ServiceEnvelope.serve(DepsService)
Пример #11
0
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
# ####################################################################

import os
from common.helpers.output import log
from common.service.computationalService import ComputationalService
from common.service.serviceEnvelope import ServiceEnvelope
from common.service.action import action


class GithubService(ComputationalService):
    ''' Github project analysis and statistics '''
    @action
    def github_liveness(self, url):
        '''
        Analyse a project liveness on github
        @param url: github project url
        @return: liveness score (needs to be designed)
        '''
        return "TODO"


if __name__ == "__main__":
    ServiceEnvelope.serve(GithubService)
Пример #12
0
                    str(exc_info[0]),
                    str(exc_info[1]),
                    str(exc_info[2])
                ]
            finally:
                ret.meta['language'] = 'golang'
                ret.meta['tool'] = 'gofedlib'
        else:
            raise NotImplementedError()

        return ret

    @action
    def api_diff(self, api1, api2):
        '''
        Make a diff of APIs
        @param api1: the first API
        @param api2: the second API
        @return: list of API differences
        '''
        ret = ServiceResult()

        ret.result = apidiff(api1, api2)
        ret.meta = {'language': 'golang', 'tool': 'gofedlib'}

        return ret


if __name__ == "__main__":
    ServiceEnvelope.serve(ApiService)
Пример #13
0
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
# ####################################################################

import os
from common.helpers.output import log
from common.service.computationalService import ComputationalService
from common.service.serviceEnvelope import ServiceEnvelope
from common.service.action import action


class GithubService(ComputationalService):
    ''' Github project analysis and statistics '''

    @action
    def github_liveness(self, url):
        '''
        Analyse a project liveness on github
        @param url: github project url
        @return: liveness score (needs to be designed)
        '''
        return "TODO"

if __name__ == "__main__":
    ServiceEnvelope.serve(GithubService)
Пример #14
0
            t = GoTranslator(self.mappings_json)
            ret.result = t.pkgname2upstream(package_name)

        return ret

    @action
    def goland_package_listing(self):
        '''
        List of all available golang packages packaged in fedora
        @return: packages packaged in fedora
        '''
        ret = ServiceResult()

        def data_cached():
            return self.packages['packages'] is not None and (
                time() - self.packages['updated'] < self.update_interval)

        if not data_cached():
            with self.get_lock(self._fedora_pkgdb_packages_list):
                if not data_cached():
                    self.packages[
                        'packages'] = self._fedora_pkgdb_packages_list()
                    self.packages['updated'] = time()

        ret.result = self.packages['packages']
        return ret


if __name__ == "__main__":
    ServiceEnvelope.serve(GolandService)
Пример #15
0
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
# ####################################################################

import os
from common.helpers.output import log
from common.service.computationalService import ComputationalService
from common.service.serviceEnvelope import ServiceEnvelope
from common.service.action import action


class RpmService(ComputationalService):
    ''' RPM and Source RPM related analysis '''
    @action
    def rpm_lint(self, file_id):
        '''
        Run rpmlint on given rpm
        @param file_id: id of a file to run rpmlint on
        @return: rpmlint output
        '''
        return "TODO"


if __name__ == "__main__":
    ServiceEnvelope.serve(RpmService)
Пример #16
0
class Computational0001Service(ComputationalService):
    ''' This is an example of a service '''

    @classmethod
    def signal_startup(cls, config):
        print("got startup signal")
        print json_pretty_format(config)

    @classmethod
    def signal_termination(cls):
        print("got termination signal")

    def signal_connect(self):
        print("got connect signal")

    def signal_disconnect(self):
        print("got disconnect signal")

    def signal_process(self):
        print("got process signal")

    def signal_processed(self):
        print("got processed signal")

    def exposed_computational0001(self, project, commit):
        print("inside action1")
        return project + commit

if __name__ == "__main__":
    ServiceEnvelope.serve(Computational0001Service)
Пример #17
0
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
# ####################################################################

import os
from common.helpers.output import log
from common.service.computationalService import ComputationalService
from common.service.serviceEnvelope import ServiceEnvelope
from common.service.action import action


class RpmService(ComputationalService):
    ''' RPM and Source RPM related analysis '''

    @action
    def rpm_lint(self, file_id):
        '''
        Run rpmlint on given rpm
        @param file_id: id of a file to run rpmlint on
        @return: rpmlint output
        '''
        return "TODO"

if __name__ == "__main__":
    ServiceEnvelope.serve(RpmService)
Пример #18
0
        ret = ServiceResult()

        with self.get_lock(self.mappings_json):
            t = GoTranslator(self.mappings_json)
            ret.result = t.pkgname2upstream(package_name)

        return ret

    @action
    def goland_package_listing(self):
        '''
        List of all available golang packages packaged in fedora
        @return: packages packaged in fedora
        '''
        ret = ServiceResult()

        def data_cached():
            return self.packages['packages'] is not None and (time() - self.packages['updated'] < self.update_interval)

        if not data_cached():
            with self.get_lock(self._fedora_pkgdb_packages_list):
                if not data_cached():
                    self.packages['packages'] = self._fedora_pkgdb_packages_list()
                    self.packages['updated'] = time()

        ret.result = self.packages['packages']
        return ret

if __name__ == "__main__":
    ServiceEnvelope.serve(GolandService)
Пример #19
0
        return ret

    @action
    def deps_package(self, package, version, distro):
        '''
        Dependencies of the given project in specified commit
        @param package: package name
        @param version: package version
        @param distro: distribution
        @return: list of dependendencies of package with analysis metadata
        '''
        ret = ServiceResult()
        ret.result = []

        filtering = {'version': 0, '_id': 0, 'package': 0}

        cursor = self.deps_package.find(
            {
                'package': package,
                'version': version,
                'distro': distro
            }, filtering)
        for item in cursor:
            ret.result.append({'deps': item['deps'], 'meta': item['meta']})

        return ret


if __name__ == "__main__":
    ServiceEnvelope.serve(DepsStorageService)
Пример #20
0
        creation_time = datetime_parse(time.ctime(os.path.getctime(dst)))
        # TODO: remove, use dircache instead
        valid_until = creation_time + self.file_lifetime

        res.result = FileId.construct(self, dst, h)
        return res

    @action
    def download(self, file_id):
        '''
        Download a file
        @param file_id: a file to be downloaded
        @return: file content
        '''
        # avoid getting files from the local system
        if not file_id.get_service_name() == self.get_service_name():
            raise ValueError("File not from this service")

        filename = os.path.basename(file_id.get_raw()['identifier'])
        file_path = os.path.join(self.upload_dir, filename)

        with self.get_lock():
            log.info("downloading '%s'" % str(file_path))
            with open(file_path, 'rb') as f:
                content = f.read()

        return content

if __name__ == "__main__":
    ServiceEnvelope.serve(FileStorageService)
Пример #21
0
class Computational0001Service(ComputationalService):
    ''' This is an example of a service '''
    @classmethod
    def signal_startup(cls, config):
        print("got startup signal")
        print json_pretty_format(config)

    @classmethod
    def signal_termination(cls):
        print("got termination signal")

    def signal_connect(self):
        print("got connect signal")

    def signal_disconnect(self):
        print("got disconnect signal")

    def signal_process(self):
        print("got process signal")

    def signal_processed(self):
        print("got processed signal")

    def exposed_computational0001(self, project, commit):
        print("inside action1")
        return project + commit


if __name__ == "__main__":
    ServiceEnvelope.serve(Computational0001Service)
Пример #22
0
    @action
    def license_compatible(self, license1, license2):
        '''
        Say if two licenses are compatible
        @param license1: a first license
        @param license2: a first license
        @return: True if licenses are compatible
        '''
        return "TODO"

    @action
    def license_abbreviation(self, license):
        '''
        Get an abbrevation for a license
        @param license: a license to abbreviate
        @return: abbreviated license
        '''
        return "TODO"

    @action
    def license_summarize(self, licenses):
        '''
        Try to get the most suitable license which would cover all of the license
        @param licenses: a list of licenses
        @return: abbreviated license
        '''
        return "TODO"

if __name__ == "__main__":
    ServiceEnvelope.serve(LicenseService)
Пример #23
0
            tarball_url = self._get_bitbucket_tarball_url(m.group(1), m.group(2), commit)
            filename = self._get_bitbucket_file_name(m.group(1), m.group(2), commit)
        else:
            raise NotImplementedError("Unknown upstream provider %s" % (upstream_url,))

        with self.get_lock(filename):
            if self.dircache.is_available(filename):
                res.result = FileId.construct(self, self.dircache.get_file_path(filename))
            else:
                res.result = self._download_tarball(tarball_url, filename)

        return res

    @action
    def download(self, file_id):
        '''
        Retrieve file stored in the service
        @param file_id: id of the file that will be downloaded
        @return: file
        '''
        filename = os.path.basename(file_id.get_identifier())

        with self.get_lock(filename):
            content = self.dircache.retrieve(filename)

        return content


if __name__ == "__main__":
    ServiceEnvelope.serve(TarballStorageService)
Пример #24
0
            ret.meta['tool'] = 'gofedlib'

        return ret

    @action
    def deps_diff(self, deps1, deps2, opts=None):
        '''
        Make a diff of dependencies
        @param deps1: the first dependency list
        @param deps2: the second dependency list
        @param opts: additional analysis opts
        @return: list of dependency differences
        '''
        default_opts = {'language': 'detect', 'tool': 'default'}
        ret = ServiceResult()

        if opts is None:
            opts = default_opts
        else:
            default_opts.update(opts)
            opts = default_opts

        # TODO: implement deps difference
        raise NotImplementedError("Currently not implemented")

        return ret


if __name__ == "__main__":
    ServiceEnvelope.serve(DepsService)
Пример #25
0
# ####################################################################

import os
from common.helpers.output import log
from common.service.storageService import StorageService
from common.service.serviceEnvelope import ServiceEnvelope
from common.service.action import action

DEFAULT_PACKAGES_TREE_DIR = "packages/"


class PackageStorageService(StorageService):
    '''  Fedora Git packages provider '''

    @action
    def package_get(self, package_name):
        '''
        Needs design
        '''
        return "TODO"

    @action
    def package_get_commit(self, package_name, commit):
        '''
        Needs design
        '''
        return "TODO"

if __name__ == "__main__":
    ServiceEnvelope.serve(PackageStorageService)
Пример #26
0
from common.service.computationalService import ComputationalService
from common.service.serviceEnvelope import ServiceEnvelope
from common.service.action import action


class GoTestService(ComputationalService):
    ''' Golang tests '''

    @action
    def gotest_run(self, file_id):
        '''
        Run a golang test on a file
        @param file_id: a file id referencing a file with tests and sources to be tested
        @return: test statistics
        '''
        # This will be implemented using a custom tool, which needs to be designed
        # and implemented
        #  * run each test inside a docker instance (?)
        #    * a user can specify on which Fedora version
        #  * as simple tool as possible
        #  * design a configuration file, which will say how to prepare the
        #  environment for tests
        #    * it could be a parameter for gotest_run() or it can be a part of
        #    file (e.g. in root dir of fedora package git repo)
        #  * needs brainstorming...
        #  * ...
        return "TODO"

if __name__ == "__main__":
    ServiceEnvelope.serve(GoTestService)
Пример #27
0
                    shutil.move(filename_old, filename)

                if not self.dircache.is_available(filename):
                    # if user did not supplied commit, we have to pack the repo
                    self._pack_repo(dirname, filename)
                shutil.rmtree(dst_path)

                if not self.dircache.is_available(filename):
                    self.dircache.register(filename)

        ret.result = FileId.construct(self, self.dircache.get_file_path(filename))
        ret.meta = {'origin': repo_url}
        return ret

    @action
    def download(self, file_id):
        '''
        Download a file
        @param file_id: a file to be downloaded
        @return: file content
        '''
        filename = os.path.basename(file_id.get_identifier())

        with self.get_lock(filename):
            content = self.dircache.retrieve(filename)

        return content

if __name__ == "__main__":
    ServiceEnvelope.serve(ScmStorageService)
Пример #28
0
    @action
    def api_store_package(self, package, version, release, distro, api, meta):
        '''
        Store API of a package
        @param package: package name to store
        @param version: version of package
        @param release: a package release
        @param distro: distribution
        @param api: exported api
        @param meta: metadata from analysis
        '''
        ret = ServiceResult()
        item = {
            'package': package,
            'version': version,
            'release': release,
            'distro': distro,
            'api': api,
            'meta': meta
        }

        self.api_package.insert(item)

        ret.result = True
        return ret

if __name__ == "__main__":
    ServiceEnvelope.serve(ApiSaveStorageService)

Пример #29
0
        @param branch: repo branch
        @return: list of scm commits (abbreviated hash, author, author email, author time, subject)
        '''
        ret = ServiceResult()

        if branch is not None and branch != "master":
            raise NotImplementedError("Handling different branch than master is not implement")

        dirname = self._get_clone_dir_name(repo_url)
        dst_path = self.dircache.get_location(dirname)

        with self.get_lock(dirname):
            if self.dircache.is_available(dirname):
                repo_type = self._scm_pull(dst_path)
                self.dircache.mark_used(dirname)
            else:
                repo_type = self._scm_clone(repo_url, dst_path)
                self.dircache.register(dirname)

            if repo_type == REPO_TYPE_GIT:
                ret.result = GitCmd.git_log_repo(dst_path, max_depth, since_date)
            elif repo_type == REPO_TYPE_MERCURIAL:
                ret.result = self._hg_log(dst_path, max_depth, since_date)
            else:
                raise ValueError("Internal Error: Unhandled repo type")

        return ret

if __name__ == "__main__":
    ServiceEnvelope.serve(ScmService)
Пример #30
0
        @param file_id: a file id of a specfile/src.rpm stored in the system
        @return: list of requires per package
        '''
        ret = ServiceResult()

        input_path = self._prepare_file(file_id)
        output = self._specker_call(SpecFileRenderer.requires_show, input_path)
        ret.result = self._parse_specker_output(output)
        ret.meta = {'tool': 'specker'}

        return ret

    @action
    def spec_buildrequires(self, file_id):
        '''
        Get all buildrequires for a package
        @param specfile_id: a file id of a specfile/src.rpm stored in the system
        @return: list of buildrequires per package
        '''
        ret = ServiceResult()

        input_path = self._prepare_file(file_id)
        output = self._specker_call(SpecFileRenderer.buildrequires_show, input_path)
        ret.result = self._parse_specker_output(output)
        ret.meta = {'tool': 'specker'}

        return ret

if __name__ == "__main__":
    ServiceEnvelope.serve(SpecService)
Пример #31
0
import sys
from common.helpers.utils import json_pretty_format
from common.service.computationalService import StorageService
from common.service.serviceEnvelope import ServiceEnvelope


class Storage0001Service(StorageService):

    def exposed_storage0001(self):
        return "foo"

if __name__ == "__main__":
    ServiceEnvelope.serve(Storage0001Service)
Пример #32
0
    def rpm_src_get(self, package_name, version, release, distro):
        '''
        Get a source RPM file id
        @param package_name:
        @param version:
        @param release:
        @param distro:
        @return:
        '''
        url = self._rpm_src_url(package_name, version, release, distro)
        filename = self._rpm_src_filename(package_name, version, release, distro)
        return self._common_get(url, filename)

    @action
    def download(self, file_id):
        '''
        Retrieve file stored in the service
        @param file_id: id of the file that will be downloaded
        @return: file content
        '''
        filename = os.path.basename(file_id.get_identifier())

        with self.get_lock(filename):
            content = self.dircache.retrieve(filename)

        return content

if __name__ == "__main__":
    ServiceEnvelope.serve(RpmStorageService)