def main(params): ''' Main program. Loads the configuration, pulls the repositories and sends notification mails. ''' logging.info("GitNotify - Push/Commit notification script") logging.info("(c) 2013 by Manuel Peuster <*****@*****.**>") c = config.Config.load_config(path=params.config_file) if c is None: logging.error("No configuration file loaded. Exiting.") exit(1) for repo_config in c["repositories"]: logging.info("Checking repository: %s (at: %s)", repo_config["name"], repo_config["url"]) gr = repository.GitRepository(repo_config["name"], repo_config["url"]) gr.clone() gr.pull() commit_list = gr.get_commits(branch=repo_config["branch"], limit=repo_config["limit"]) ch = repository.CommitHistory() filtered_commit_list = ch.filter_commits_to_notify(commit_list) if len(filtered_commit_list) > 0: n = notify.Notifier(c, repo_config) if n.send_notifications(filtered_commit_list): ch.add_notified_commits(filtered_commit_list) logging.info("Notifications sent: %i", len(filtered_commit_list)) else: logging.error("Error while sending notifications.") else: logging.info("No new commits. No notifications are generated.")
def Run(): register_stack_trace_dump() notifier = notify.Notifier(config.get("job-manager")) notifier.start() create_log() while True: update_file_modification_time("job_manager") with manager_iteration_histogram.labels("job_manager").time(): try: config["racks"] = k8sUtils.get_node_labels("rack") config["skus"] = k8sUtils.get_node_labels("sku") except Exception as e: logging.exception("get node labels failed") try: dataHandler = DataHandler() pendingJobs = dataHandler.GetPendingJobs() TakeJobActions(pendingJobs) pendingJobs = dataHandler.GetPendingJobs() logging.info("Updating status for %d jobs" % len(pendingJobs)) for job in pendingJobs: try: logging.info("Processing job: %s, status: %s" % (job["jobId"], job["jobStatus"])) if job["jobStatus"] == "killing": KillJob(job["jobId"], "killed") elif job["jobStatus"] == "pausing": KillJob(job["jobId"], "paused") elif job["jobStatus"] == "scheduling" or job[ "jobStatus"] == "running": UpdateJobStatus(job, notifier) elif job["jobStatus"] == "unapproved": ApproveJob(job) except Exception as e: logging.warning(e, exc_info=True) except Exception as e: logging.warning("Process job failed!", exc_info=True) finally: try: dataHandler.Close() except: pass time.sleep(1)
def Run(redis_port, target_status): register_stack_trace_dump() process_name = "job_manager_" + target_status create_log(process_name=process_name) notifier = notify.Notifier(config.get("job-manager")) notifier.start() launcher_type = config.get("job-manager", {}).get("launcher", "python") if launcher_type == "python": launcher = PythonLauncher() elif launcher_type == "controller": launcher = LauncherStub() else: logger.error("unknown launcher_type %s", launcher_type) sys.exit(2) launcher.start() redis_conn = redis.StrictRedis(host="localhost", port=redis_port, db=0) while True: update_file_modification_time(process_name) with manager_iteration_histogram.labels(process_name).time(): try: config["racks"] = k8sUtils.get_node_labels("rack") config["skus"] = k8sUtils.get_node_labels("sku") except Exception as e: logger.exception("get node labels failed") try: launcher.wait_tasks_done( ) # wait for tasks from previous batch done data_handler = DataHandler() if target_status == "queued": jobs = data_handler.GetJobList( "all", "all", num=None, status="queued,scheduling,running") take_job_actions(data_handler, redis_conn, launcher, jobs) else: jobs = data_handler.GetJobList("all", "all", num=None, status=target_status) logger.info("Updating status for %d %s jobs", len(jobs), target_status) for job in jobs: logger.info("Processing job: %s, status: %s" % (job["jobId"], job["jobStatus"])) if job["jobStatus"] == "killing": launcher.kill_job(job["jobId"], "killed") elif job["jobStatus"] == "pausing": launcher.kill_job(job["jobId"], "paused") elif job["jobStatus"] == "running": UpdateJobStatus(redis_conn, launcher, job, notifier, dataHandlerOri=data_handler) elif job["jobStatus"] == "scheduling": UpdateJobStatus(redis_conn, launcher, job, notifier, dataHandlerOri=data_handler) elif job["jobStatus"] == "unapproved": ApproveJob(redis_conn, job, dataHandlerOri=data_handler) else: logger.error("unknown job status %s for job %s", job["jobStatus"], job["jobId"]) except Exception as e: logger.exception("Process jobs failed!") finally: try: data_handler.Close() except: pass time.sleep(1)