Example #1
0
    def main(self):
        with self.get_system() as system:

            golang_pkgs = system.async_call.goland_package_listing()
            stored_projects = system.async_call.deps_project_listing()

            for pkg in golang_pkgs.result:

                if not pkg['name'].startswith('golang-github-'):
                    log.warning("Skipping %s" % pkg['name'])
                    # TODO: remove once support for mercurial and full package->upstream translation will be available
                    continue

                try:
                    raise ValueError("value error")
                    print("Inspecting '%s'" % pkg['name'])
                    upstream_url = system.async_call.golang_package2upstream(
                        pkg['name'])

                    if pkg['name'] in stored_projects.result:
                        stored_commits = system.async_call.deps_project_commit_listing(
                            pkg['name'])
                    else:
                        stored_commits = None

                    scm_log = system.async_call.scm_log(
                        upstream_url.result,
                        max_depth=self.max_depth,
                        since_date=self.since_date)

                    for commit in tqdm(scm_log.result):
                        log.debug("Commit %s project %s" %
                                  (commit['hash'], pkg['name']))
                        if not stored_commits or commit not in stored_commits.result:
                            file_id = system.async_call.scm_store(
                                upstream_url.result, commit['hash'])
                            deps = system.async_call.deps_analysis(
                                file_id.result)
                            system.async_call.deps_store_project(
                                pkg['name'], commit['hash'], commit['time'],
                                deps.result, deps.meta)
                except:
                    exc_info = sys.exc_info()
                    if self.skip_errors:
                        log.error(exc_info[2].print_exc())
                    else:
                        raise exc_info
Example #2
0
    def main(self):
        with self.get_system() as system:

            golang_pkgs = system.async_call.goland_package_listing()
            stored_projects = system.async_call.deps_project_listing()

            for pkg in golang_pkgs.result:

                if not pkg['name'].startswith('golang-github-'):
                    log.warning("Skipping %s" % pkg['name'])
                    # TODO: remove once support for mercurial and full package->upstream translation will be available
                    continue

                try:
                    raise ValueError("value error")
                    print("Inspecting '%s'" % pkg['name'])
                    upstream_url = system.async_call.golang_package2upstream(pkg['name'])

                    if pkg['name'] in stored_projects.result:
                        stored_commits = system.async_call.deps_project_commit_listing(pkg['name'])
                    else:
                        stored_commits = None

                    scm_log = system.async_call.scm_log(upstream_url.result,
                                                        max_depth=self.max_depth,
                                                        since_date=self.since_date)

                    for commit in tqdm(scm_log.result):
                        log.debug("Commit %s project %s" % (commit['hash'], pkg['name']))
                        if not stored_commits or commit not in stored_commits.result:
                            file_id = system.async_call.scm_store(upstream_url.result, commit['hash'])
                            deps = system.async_call.deps_analysis(file_id.result)
                            system.async_call.deps_store_project(pkg['name'], commit['hash'], commit['time'],
                                                                 deps.result, deps.meta)
                except:
                    exc_info = sys.exc_info()
                    if self.skip_errors:
                        log.error(exc_info[2].print_exc())
                    else:
                        raise exc_info
Example #3
0
    def main(self):
        try:
            self.conf
        except Exception:
            log.error("config option is mandatory, see '--help'")
            sys.exit(1)

        self.conf = config2dict(self.conf)

        cls = ServiceEnvelope.SERVICE_CLASS
        logfile = self.logfile if self.logfile != '-' else None
        verbose = not self.quiet
        name = cls.get_service_name()
        version = cls.get_service_version()

        log.init(logfile, verbose, name)
        log.critical("Service starting at %s, version is %s" %
                     (get_time_str(), version))

        service_cls = ServiceEnvelope.SERVICE_CLASS
        service_cls.on_startup(self.conf, self.system_json)

        ServiceEnvelope.worker_pid = os.fork()

        if ServiceEnvelope.worker_pid == 0:
            self.run_worker_process()
        else:
            signal.signal(signal.SIGINT, ServiceEnvelope.signal_handler)
            signal.signal(signal.SIGTERM, ServiceEnvelope.signal_handler)

            try:
                while os.waitpid(ServiceEnvelope.worker_pid, 0):
                    pass
            except Exception:
                pass

            # finish logging and ensure that all messages are flushed
            log.critical("Service terminated at %s" % get_time_str())
            log.close()