def get_index(job, context: Dict) -> TimerIndex:
"""
:type job: cihpc.config.types.project_config_job.ProjectConfigJob
"""
extra = job.collect.extra.copy(
) if job.collect else _default_job_index.copy()
extra = {k: v for k, v in extra.items() if k in _default_job_index}
return TimerIndex(**configure_recursive(extra, context, Missing.NONE))
def default_index(self) -> Dict:
try:
from cihpc.shared.db.timer_index import TimerIndex
index = TimerIndex(**job_util.get_index(
self,
{'job': self, **self.project_config.context}
))
index["branch"] = None
return index
except:
return dict()
def _process_job(cls, job: ProjectConfigJob, context: Dict):
for sub_job, extra_context, variation in job.expand(context):
rest = dict(job=sub_job)
new_context = {**extra_context, **variation, **rest}
schedule_index = TimerIndex(**job_util.get_index(job, new_context))
ok_count, broken_count = DBStats.get_run_count(schedule_index)
# TODO specify repetitions
schedule_details = ColScheduleDetails(priority=0, repetitions=7)
schedule_document = ColSchedule(
index=schedule_index,
details=schedule_details,
status=ColScheduleStatus.NotProcessed,
worker=None)
yield schedule_document, schedule_index
from pymongo import MongoClient
from cihpc.shared.db.cols.col_index_info import ColIndexInfo
from cihpc.shared.db.cols.col_index_stat import ColIndexStat
from cihpc.shared.db.cols.col_repo_info import ColRepoInfo
from cihpc.shared.db.cols.col_schedule import ColSchedule
from cihpc.shared.db.cols.col_timers import ColTimer
from cihpc.shared.db.models import IdEntity
from cihpc.shared.db.timer_index import TimerIndex
from cihpc.shared.errors.config_error import ConfigError
from cihpc.shared.g import G
AllColTypes = Union[IdEntity, ColIndexInfo, ColSchedule, ColTimer, ColIndexStat, ColRepoInfo]
MatchType = Union[TimerIndex, Dict]
_timer_index_keys = list(TimerIndex().keys())
def in_list(items):
return {"$in": items}
class MongoCollection:
def __init__(self, collection: pymongo.collection.Collection, factory: Type[IdEntity]):
self._collection = collection
self._factory = factory
self.name = collection.name
def find(self, index: MatchType, projection: List = None, raw = False, alter_cursor: callable = None, **kwargs) -> List[AllColTypes]:
match = dict()
for k, v in index.items():
DESCENDING = -1
"""Descending sort order."""
if __name__ == '__main__':
args = parse_scheduler_args()
# read yaml
G.init(args)
logger.info(f"Using workdir: {G.project_work_dir}")
project_config = get_project_config(args)
Mongo.set_default_project(project_config.name)
cursor = Mongo().col_scheduler.find(
TimerIndex(
project=project_config.name,
status=ColScheduleStatus.NotProcessed,
),
alter_cursor=lambda x: x.sort("_id", DESCENDING))
scheduler_items = list(cursor)
total = len(scheduler_items)
for i, schedule in enumerate(scheduler_items):
logger.info(f"Schedule {i+1:d}/{total:2d} starting")
_id = schedule.id
index = schedule.index
repetitions = schedule.details.repetitions
project, job_name, commit, branch = data_util.pop(
index, "project", "job", "commit", "branch")
variables = {
k: data_util.ensure_list(v)
def __init__(self, **kwargs):
self.index = TimerIndex.from_dict(kwargs.pop('index', {}))
self.details = ColScheduleDetails.from_dict(kwargs.pop('details', {}))
self.status = kwargs.pop("status", None)
self.worker = kwargs.pop("worker", None)
super().__init__(**kwargs)
def __init__(self, **kwargs):
self.index = TimerIndex.from_dict(kwargs.pop('index', {}))
self.stat = ColIndexStatComputed.from_dict(kwargs.pop('stat', {}))
super().__init__(**kwargs)
def __init__(self, **kwargs):
self.index = TimerIndex.from_dict(kwargs.pop('index', {}))
self.system = ColTimerSystem.from_dict(kwargs.pop('system', {}))
self.problem = ColTimerProblem.from_dict(kwargs.pop('problem', {}))
self.result = ColTimerResult.from_dict(kwargs.pop('result', {}))
super().__init__(**kwargs)
def __init__(self, **kwargs):
self.index = TimerIndex.from_dict(kwargs.pop('index', {}))
self.run = IndexInfoRun.from_dict(kwargs.pop('run', {}))
super().__init__(**kwargs)