identifier = file_id.get_identifier() identifier = identifier.split('/') if len(identifier) != 4: raise ValueError("Unknown file to be accessed") package_name = os.path.basename(identifier[0]) branch = os.path.basename(identifier[1]) commit = self._ident2commit(os.path.basename(identifier[2])) f = os.path.basename(identifier[3]) if not f.endswith('.spec') and not f.endswith('.patch'): raise ValueError("Unknown file to be accessed") path = os.path.join(self.pkg_dir, package_name) file_path = os.path.join(path, f) log.debug("downloading '%s'" % (file_path, )) with self.get_lock(package_name): self._git_tree_prepare(package_name, branch, commit) with open(file_path, 'rb') as f: content = f.read() return content if __name__ == "__main__": ServiceEnvelope.serve(SpecStorageService)
def license_compatible(self, license1, license2): ''' Say if two licenses are compatible @param license1: a first license @param license2: a first license @return: True if licenses are compatible ''' return "TODO" @action def license_abbreviation(self, license): ''' Get an abbrevation for a license @param license: a license to abbreviate @return: abbreviated license ''' return "TODO" @action def license_summarize(self, licenses): ''' Try to get the most suitable license which would cover all of the license @param licenses: a list of licenses @return: abbreviated license ''' return "TODO" if __name__ == "__main__": ServiceEnvelope.serve(LicenseService)
if not self.dircache.is_available(filename): # if user did not supplied commit, we have to pack the repo self._pack_repo(dirname, filename) shutil.rmtree(dst_path) if not self.dircache.is_available(filename): self.dircache.register(filename) ret.result = FileId.construct(self, self.dircache.get_file_path(filename)) ret.meta = {'origin': repo_url} return ret @action def download(self, file_id): ''' Download a file @param file_id: a file to be downloaded @return: file content ''' filename = os.path.basename(file_id.get_identifier()) with self.get_lock(filename): content = self.dircache.retrieve(filename) return content if __name__ == "__main__": ServiceEnvelope.serve(ScmStorageService)
cursor = self.deps_package.find({'package': package, 'distro': distro}, filtering) for item in cursor: if item['distro'] not in ret: ret.result.append(item['distro']) return ret @action def deps_package(self, package, version, distro): ''' Dependencies of the given project in specified commit @param package: package name @param version: package version @param distro: distribution @return: list of dependendencies of package with analysis metadata ''' ret = ServiceResult() ret.result = [] filtering = {'version': 0, '_id': 0, 'package': 0} cursor = self.deps_package.find({'package': package, 'version': version, 'distro': distro}, filtering) for item in cursor: ret.result.append({'deps': item['deps'], 'meta': item['meta']}) return ret if __name__ == "__main__": ServiceEnvelope.serve(DepsStorageService)
try: ret.result = gofedlib.api(src_path) except: exc_info = sys.exc_info() ret.meta['error'] = [ str(exc_info[0]), str(exc_info[1]), str(exc_info[2])] finally: ret.meta['language'] = 'golang' ret.meta['tool'] = 'gofedlib' else: raise NotImplementedError() return ret @action def api_diff(self, api1, api2): ''' Make a diff of APIs @param api1: the first API @param api2: the second API @return: list of API differences ''' ret = ServiceResult() ret.result = apidiff(api1, api2) ret.meta = {'language': 'golang', 'tool': 'gofedlib'} return ret if __name__ == "__main__": ServiceEnvelope.serve(ApiService)
identifier = file_id.get_identifier() identifier = identifier.split("/") if len(identifier) != 4: raise ValueError("Unknown file to be accessed") package_name = os.path.basename(identifier[0]) branch = os.path.basename(identifier[1]) commit = self._ident2commit(os.path.basename(identifier[2])) f = os.path.basename(identifier[3]) if not f.endswith(".spec") and not f.endswith(".patch"): raise ValueError("Unknown file to be accessed") path = os.path.join(self.pkg_dir, package_name) file_path = os.path.join(path, f) log.debug("downloading '%s'" % (file_path,)) with self.get_lock(package_name): self._git_tree_prepare(package_name, branch, commit) with open(file_path, "rb") as f: content = f.read() return content if __name__ == "__main__": ServiceEnvelope.serve(SpecStorageService)
# #################################################################### import os from common.helpers.output import log from common.service.storageService import StorageService from common.service.serviceEnvelope import ServiceEnvelope from common.service.action import action DEFAULT_PACKAGES_TREE_DIR = "packages/" class PackageStorageService(StorageService): ''' Fedora Git packages provider ''' @action def package_get(self, package_name): ''' Needs design ''' return "TODO" @action def package_get_commit(self, package_name, commit): ''' Needs design ''' return "TODO" if __name__ == "__main__": ServiceEnvelope.serve(PackageStorageService)
from common.service.computationalService import ComputationalService from common.service.serviceEnvelope import ServiceEnvelope from common.service.action import action class GoTestService(ComputationalService): ''' Golang tests ''' @action def gotest_run(self, file_id): ''' Run a golang test on a file @param file_id: a file id referencing a file with tests and sources to be tested @return: test statistics ''' # This will be implemented using a custom tool, which needs to be designed # and implemented # * run each test inside a docker instance (?) # * a user can specify on which Fedora version # * as simple tool as possible # * design a configuration file, which will say how to prepare the # environment for tests # * it could be a parameter for gotest_run() or it can be a part of # file (e.g. in root dir of fedora package git repo) # * needs brainstorming... # * ... return "TODO" if __name__ == "__main__": ServiceEnvelope.serve(GoTestService)
import sys from common.helpers.utils import json_pretty_format from common.service.computationalService import StorageService from common.service.serviceEnvelope import ServiceEnvelope class Storage0001Service(StorageService): def exposed_storage0001(self): return "foo" if __name__ == "__main__": ServiceEnvelope.serve(Storage0001Service)
ret.meta['language'] = 'golang' ret.meta['tool'] = 'gofedlib' return ret @action def deps_diff(self, deps1, deps2, opts=None): ''' Make a diff of dependencies @param deps1: the first dependency list @param deps2: the second dependency list @param opts: additional analysis opts @return: list of dependency differences ''' default_opts = {'language': 'detect', 'tool': 'default'} ret = ServiceResult() if opts is None: opts = default_opts else: default_opts.update(opts) opts = default_opts # TODO: implement deps difference raise NotImplementedError("Currently not implemented") return ret if __name__ == "__main__": ServiceEnvelope.serve(DepsService)
# but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. # #################################################################### import os from common.helpers.output import log from common.service.computationalService import ComputationalService from common.service.serviceEnvelope import ServiceEnvelope from common.service.action import action class GithubService(ComputationalService): ''' Github project analysis and statistics ''' @action def github_liveness(self, url): ''' Analyse a project liveness on github @param url: github project url @return: liveness score (needs to be designed) ''' return "TODO" if __name__ == "__main__": ServiceEnvelope.serve(GithubService)
str(exc_info[0]), str(exc_info[1]), str(exc_info[2]) ] finally: ret.meta['language'] = 'golang' ret.meta['tool'] = 'gofedlib' else: raise NotImplementedError() return ret @action def api_diff(self, api1, api2): ''' Make a diff of APIs @param api1: the first API @param api2: the second API @return: list of API differences ''' ret = ServiceResult() ret.result = apidiff(api1, api2) ret.meta = {'language': 'golang', 'tool': 'gofedlib'} return ret if __name__ == "__main__": ServiceEnvelope.serve(ApiService)
t = GoTranslator(self.mappings_json) ret.result = t.pkgname2upstream(package_name) return ret @action def goland_package_listing(self): ''' List of all available golang packages packaged in fedora @return: packages packaged in fedora ''' ret = ServiceResult() def data_cached(): return self.packages['packages'] is not None and ( time() - self.packages['updated'] < self.update_interval) if not data_cached(): with self.get_lock(self._fedora_pkgdb_packages_list): if not data_cached(): self.packages[ 'packages'] = self._fedora_pkgdb_packages_list() self.packages['updated'] = time() ret.result = self.packages['packages'] return ret if __name__ == "__main__": ServiceEnvelope.serve(GolandService)
# but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. # #################################################################### import os from common.helpers.output import log from common.service.computationalService import ComputationalService from common.service.serviceEnvelope import ServiceEnvelope from common.service.action import action class RpmService(ComputationalService): ''' RPM and Source RPM related analysis ''' @action def rpm_lint(self, file_id): ''' Run rpmlint on given rpm @param file_id: id of a file to run rpmlint on @return: rpmlint output ''' return "TODO" if __name__ == "__main__": ServiceEnvelope.serve(RpmService)
class Computational0001Service(ComputationalService): ''' This is an example of a service ''' @classmethod def signal_startup(cls, config): print("got startup signal") print json_pretty_format(config) @classmethod def signal_termination(cls): print("got termination signal") def signal_connect(self): print("got connect signal") def signal_disconnect(self): print("got disconnect signal") def signal_process(self): print("got process signal") def signal_processed(self): print("got processed signal") def exposed_computational0001(self, project, commit): print("inside action1") return project + commit if __name__ == "__main__": ServiceEnvelope.serve(Computational0001Service)
ret = ServiceResult() with self.get_lock(self.mappings_json): t = GoTranslator(self.mappings_json) ret.result = t.pkgname2upstream(package_name) return ret @action def goland_package_listing(self): ''' List of all available golang packages packaged in fedora @return: packages packaged in fedora ''' ret = ServiceResult() def data_cached(): return self.packages['packages'] is not None and (time() - self.packages['updated'] < self.update_interval) if not data_cached(): with self.get_lock(self._fedora_pkgdb_packages_list): if not data_cached(): self.packages['packages'] = self._fedora_pkgdb_packages_list() self.packages['updated'] = time() ret.result = self.packages['packages'] return ret if __name__ == "__main__": ServiceEnvelope.serve(GolandService)
return ret @action def deps_package(self, package, version, distro): ''' Dependencies of the given project in specified commit @param package: package name @param version: package version @param distro: distribution @return: list of dependendencies of package with analysis metadata ''' ret = ServiceResult() ret.result = [] filtering = {'version': 0, '_id': 0, 'package': 0} cursor = self.deps_package.find( { 'package': package, 'version': version, 'distro': distro }, filtering) for item in cursor: ret.result.append({'deps': item['deps'], 'meta': item['meta']}) return ret if __name__ == "__main__": ServiceEnvelope.serve(DepsStorageService)
creation_time = datetime_parse(time.ctime(os.path.getctime(dst))) # TODO: remove, use dircache instead valid_until = creation_time + self.file_lifetime res.result = FileId.construct(self, dst, h) return res @action def download(self, file_id): ''' Download a file @param file_id: a file to be downloaded @return: file content ''' # avoid getting files from the local system if not file_id.get_service_name() == self.get_service_name(): raise ValueError("File not from this service") filename = os.path.basename(file_id.get_raw()['identifier']) file_path = os.path.join(self.upload_dir, filename) with self.get_lock(): log.info("downloading '%s'" % str(file_path)) with open(file_path, 'rb') as f: content = f.read() return content if __name__ == "__main__": ServiceEnvelope.serve(FileStorageService)
@action def license_compatible(self, license1, license2): ''' Say if two licenses are compatible @param license1: a first license @param license2: a first license @return: True if licenses are compatible ''' return "TODO" @action def license_abbreviation(self, license): ''' Get an abbrevation for a license @param license: a license to abbreviate @return: abbreviated license ''' return "TODO" @action def license_summarize(self, licenses): ''' Try to get the most suitable license which would cover all of the license @param licenses: a list of licenses @return: abbreviated license ''' return "TODO" if __name__ == "__main__": ServiceEnvelope.serve(LicenseService)
tarball_url = self._get_bitbucket_tarball_url(m.group(1), m.group(2), commit) filename = self._get_bitbucket_file_name(m.group(1), m.group(2), commit) else: raise NotImplementedError("Unknown upstream provider %s" % (upstream_url,)) with self.get_lock(filename): if self.dircache.is_available(filename): res.result = FileId.construct(self, self.dircache.get_file_path(filename)) else: res.result = self._download_tarball(tarball_url, filename) return res @action def download(self, file_id): ''' Retrieve file stored in the service @param file_id: id of the file that will be downloaded @return: file ''' filename = os.path.basename(file_id.get_identifier()) with self.get_lock(filename): content = self.dircache.retrieve(filename) return content if __name__ == "__main__": ServiceEnvelope.serve(TarballStorageService)
ret.meta['tool'] = 'gofedlib' return ret @action def deps_diff(self, deps1, deps2, opts=None): ''' Make a diff of dependencies @param deps1: the first dependency list @param deps2: the second dependency list @param opts: additional analysis opts @return: list of dependency differences ''' default_opts = {'language': 'detect', 'tool': 'default'} ret = ServiceResult() if opts is None: opts = default_opts else: default_opts.update(opts) opts = default_opts # TODO: implement deps difference raise NotImplementedError("Currently not implemented") return ret if __name__ == "__main__": ServiceEnvelope.serve(DepsService)
shutil.move(filename_old, filename) if not self.dircache.is_available(filename): # if user did not supplied commit, we have to pack the repo self._pack_repo(dirname, filename) shutil.rmtree(dst_path) if not self.dircache.is_available(filename): self.dircache.register(filename) ret.result = FileId.construct(self, self.dircache.get_file_path(filename)) ret.meta = {'origin': repo_url} return ret @action def download(self, file_id): ''' Download a file @param file_id: a file to be downloaded @return: file content ''' filename = os.path.basename(file_id.get_identifier()) with self.get_lock(filename): content = self.dircache.retrieve(filename) return content if __name__ == "__main__": ServiceEnvelope.serve(ScmStorageService)
@action def api_store_package(self, package, version, release, distro, api, meta): ''' Store API of a package @param package: package name to store @param version: version of package @param release: a package release @param distro: distribution @param api: exported api @param meta: metadata from analysis ''' ret = ServiceResult() item = { 'package': package, 'version': version, 'release': release, 'distro': distro, 'api': api, 'meta': meta } self.api_package.insert(item) ret.result = True return ret if __name__ == "__main__": ServiceEnvelope.serve(ApiSaveStorageService)
@param branch: repo branch @return: list of scm commits (abbreviated hash, author, author email, author time, subject) ''' ret = ServiceResult() if branch is not None and branch != "master": raise NotImplementedError("Handling different branch than master is not implement") dirname = self._get_clone_dir_name(repo_url) dst_path = self.dircache.get_location(dirname) with self.get_lock(dirname): if self.dircache.is_available(dirname): repo_type = self._scm_pull(dst_path) self.dircache.mark_used(dirname) else: repo_type = self._scm_clone(repo_url, dst_path) self.dircache.register(dirname) if repo_type == REPO_TYPE_GIT: ret.result = GitCmd.git_log_repo(dst_path, max_depth, since_date) elif repo_type == REPO_TYPE_MERCURIAL: ret.result = self._hg_log(dst_path, max_depth, since_date) else: raise ValueError("Internal Error: Unhandled repo type") return ret if __name__ == "__main__": ServiceEnvelope.serve(ScmService)
@param file_id: a file id of a specfile/src.rpm stored in the system @return: list of requires per package ''' ret = ServiceResult() input_path = self._prepare_file(file_id) output = self._specker_call(SpecFileRenderer.requires_show, input_path) ret.result = self._parse_specker_output(output) ret.meta = {'tool': 'specker'} return ret @action def spec_buildrequires(self, file_id): ''' Get all buildrequires for a package @param specfile_id: a file id of a specfile/src.rpm stored in the system @return: list of buildrequires per package ''' ret = ServiceResult() input_path = self._prepare_file(file_id) output = self._specker_call(SpecFileRenderer.buildrequires_show, input_path) ret.result = self._parse_specker_output(output) ret.meta = {'tool': 'specker'} return ret if __name__ == "__main__": ServiceEnvelope.serve(SpecService)
def rpm_src_get(self, package_name, version, release, distro): ''' Get a source RPM file id @param package_name: @param version: @param release: @param distro: @return: ''' url = self._rpm_src_url(package_name, version, release, distro) filename = self._rpm_src_filename(package_name, version, release, distro) return self._common_get(url, filename) @action def download(self, file_id): ''' Retrieve file stored in the service @param file_id: id of the file that will be downloaded @return: file content ''' filename = os.path.basename(file_id.get_identifier()) with self.get_lock(filename): content = self.dircache.retrieve(filename) return content if __name__ == "__main__": ServiceEnvelope.serve(RpmStorageService)