def schedule_run(self, commit: str, branch: str, job_name: str = "test"): self.project_config.git.main_repo.set_fake_head(commit=commit, branch=branch) context = dict(**self.project_config.context) init_job = self.project_config.jobs[0] if init_job.db_broken_count() >= init_job.retries: logger.warning( f"Skipping job '{init_job}' since it already has {init_job.db_broken_count()} broken builds" ) yield 0 else: test_job = self.project_config.get(job_name) for schedule_document, schedule_index in self._process_job( test_job, context): scheduled_already = DBStats.get_schedule_repetitions( schedule_index) schedule_document.details.repetitions -= scheduled_already if schedule_document.details.repetitions > 0: Mongo().col_scheduler.insert(schedule_document) logger.debug( f"Inserted {schedule_document.details.repetitions} requests for the job:\n{test_job.pretty_index}" ) yield schedule_document.details.repetitions else: logger.debug( f"Already scheduled {scheduled_already} runs, which is more than enough:\n{test_job.pretty_index}" )
def save_index_info(job, context: Dict): """ :type job: cihpc.config.types.project_config_job.ProjectConfigJob """ index = get_index(job, context) index_info = create_index_info(index, job) try: Mongo().col_index_info.insert(index_info) except Exception as e: logger.error(f"Could not save info to db: {e}")
def test(): _current_dir = Path(__file__).absolute().parent args = parse_worker_args([ "--cfg", str(_current_dir), "--vars", str(_current_dir) + "/user/desired_variables.yaml", "--rnd=true" ]) # force commit args.commit = "e88a8335648b08843aa58062c2488120c458d737" args.commit = "ec5aa26c2c53802e4f5c1d73c0baa1b756640715" args.commit = "25bf49f2c16f5cc3d167c5022398942dc2371c4d" args.commit = "61dd0ec01b450342a1eb2037a63f6f47d3071f65" # read yaml G.init(args) logger.info(f"Using workdir: {G.project_work_dir}") config = get_project_config(args) Mongo.set_default_project(config.name) config.execute()
def execute(self, context: Dict): logger.info(f"Collecting artifacts...") extra = configure_recursive(self.extra, context) files = list() for file in self.files: path = Path(configure(file, context)).absolute() match = glob.glob(str(path), recursive=True) logger.info(f"searching path {str(path)}, found: {match}") files.extend(match) collector_type = _collectors[self.module] collector = collector_type(context, extra) for file in files: try: reports = collector.process_file(Path(file)) except Exception as e: logger.info( f"Failed to obtain timers from {file} using {collector_type}: {e}" ) logger.debug(Path(file).read_text()) continue # convert to ColTimer timers = [ColTimer(**report) for report in reports] if not timers: logger.info(f"Did not found any timers in {file}") continue if self.save_to_db: # save to db try: Mongo().col_timers.insert_many(timers) except Exception as e: logger.error(f"Failed to save data to db: {e}") for timer in timers: yield timer.index
from cihpc.shared.utils import data_util ASCENDING = 1 """Ascending sort order.""" DESCENDING = -1 """Descending sort order.""" if __name__ == '__main__': args = parse_scheduler_args() # read yaml G.init(args) logger.info(f"Using workdir: {G.project_work_dir}") project_config = get_project_config(args) Mongo.set_default_project(project_config.name) cursor = Mongo().col_scheduler.find( TimerIndex( project=project_config.name, status=ColScheduleStatus.NotProcessed, ), alter_cursor=lambda x: x.sort("_id", DESCENDING)) scheduler_items = list(cursor) total = len(scheduler_items) for i, schedule in enumerate(scheduler_items): logger.info(f"Schedule {i+1:d}/{total:2d} starting") _id = schedule.id index = schedule.index
import sys MIN_PYTHON = 3, 6 if sys.version_info < MIN_PYTHON: sys.exit("Error: Python %s.%s or later is required.\n" % MIN_PYTHON) from loguru import logger from cihpc.config import get_project_config from cihpc.parsers.main import parse_scheduler_args from cihpc.repo.repo_util import RepoUtil from cihpc.shared.db.mongo_db import Mongo from cihpc.shared.g import G if __name__ == '__main__': args = parse_scheduler_args() # read yaml G.init(args) logger.info(f"Using workdir: {G.project_work_dir}") project_config = get_project_config(args) Mongo.set_default_project(project_config.name) project_config.initialize(with_git=False) repo = RepoUtil(dir=G.project_work_dir / '.cihpc' / '.repo', url=project_config.git.main_repo.url, project_config=project_config) repo.to_latest() repo.schedule_runs(max_per_branch=args.per_branch)
def extract_info(self, per_branch, max_age, single_branch=None): logger.info("obtaining commit details") branches = single_branch if single_branch else get_active_branches( self.repo, max_age) info: Dict[Commit, List[str]] = defaultdict(list) documents = list() for branch in branches: branch_head = None if isinstance(branch, str) else branch.head branch_name = branch if not branch_head else str(branch.head) branch_full = f"origin/{branch_name}" if not branch_name.startswith( "origin/") else branch_name branch_short = branch_full[7:] for commit in iter_revision(self.repo, branch_head or branch_full, limit=per_branch, first_parent=False): info[commit].append(branch_short) for commit, branches in info.items(): doc = ColRepoInfo() doc.author = commit.author.name doc.email = commit.author.email doc.commit = commit.hexsha doc.branches = branches doc.branch = None if len(branches) > 1 else branches[0] doc.authored_datetime = commit.authored_datetime doc.committed_datetime = commit.committed_datetime doc.message = commit.message doc.distance = -1 doc.parents = [c.hexsha for c in commit.parents] documents.append(doc) logger.info("comparing changes in db") # to_be_updated = [doc.commit for doc in documents] # rexisting_cmts = [x.commit for x in Mongo().col_repo_info.find({}, {"commit": 1})] results = Mongo().col_repo_info.find( {"commit": in_list([doc.commit for doc in documents])}, ["commit"], raw=True) logger.info("traversing parents") documents = self._update_edges(documents) existing = [r.commit for r in results] filtered = [d for d in documents if d.commit not in existing] logger.info( f"inspected total of {len(documents)} commits, {len(filtered)} new ones" ) if filtered: Mongo().col_repo_info.insert_many(filtered) else: logger.info(f"no new commits to add...") logger.info("updating commit parents and children") changes = list(Mongo().col_repo_info.batch_update( documents, lambda x: dict(commit=x.commit), lambda x: dict(parents=x.parents, children=x.children), )) logger.info(f"updated {len(changes)} parents and children")
def _try_execute(func, default): try: return func(Mongo()) except Exception as e: logger.warning(f"Mongo command failed: {e}") return default