def sync_jobs(self): log_entries = self.logwatcher.nexts() for le in log_entries: if le.id in self.entries_cid: # in dict keys entry = self.entries_cid[le.id] else: entry = ScheduleEntry(condor_id=le.id) self.entries.append(entry) self.entries_cid[le.id] = entry print "++Job", le.id entry.log[le.event] = le.timestamp if le.event == LogKey.execute: entry.status = EntryStatus.executing elif le.event == LogKey.job_terminated: entry.status = EntryStatus.completed wf_id, dag_job_id, slot = condor_history(le.id) job = next( (j for j in self.workflow.jobs if j.dag_job_id == dag_job_id and j.wf_id == wf_id), None) if job: entry.job = job entry.host = next( (m for m in self.machines if m.condor_slot == slot), self.machines[0]) print "--Job", le.id, dag_job_id, entry.host.condor_slot
def __init__(self, vm_limit, azure_config, skip_setup, local): self.vm_limit = vm_limit # user input self.budget = 0 self.timestamp = datetime.now() self.cost_pred = 0 self.wf_end = None self.jobs_terminated = False self.last_resched = None self.workflow = Workflow() self.logwatcher = LogWatcher() self.schedule = Schedule() manager = Machine() manager.status = MachineStatus.manager manager.condor_slot = 'manager' self.machines = [manager] boot_entry = ScheduleEntry(Job('boot', None), manager, self.timestamp, self.timestamp) boot_entry.real_start = self.timestamp boot_entry.real_end = self.timestamp boot_entry.status = EntryStatus.completed self.schedule.add_entry_host(boot_entry, manager) self.local = local if azure_config and not local: hostname = socket.gethostname() self.exp = AzureExperiment(azure_config, skip_setup=skip_setup, name=hostname) self.master_addr = socket.gethostbyname(hostname) self.user = azure_config.admin_username else: self.exp = self.master_addr = self.user = None
def sync_machines(self): slots = condor_slots() for s in slots: if s not in [m.condor_slot for m in self.machines]: machine = Machine() machine.status = MachineStatus.running machine.condor_slot = s boot_job = Job('boot', None) boot_entry = ScheduleEntry(boot_job, machine, None, None) boot_entry.log[LogKey.real_start] = self.creation_timestamp boot_entry.log[LogKey.real_end] = self.timestamp boot_entry.status = EntryStatus.completed self.entries.append(boot_entry) self.machines.append(machine) print "++Machine", s
def sync_jobs(self): log_entries = self.logwatcher.nexts() for le in log_entries: if le.id in self.entries_cid: # in dict keys entry = self.entries_cid[le.id] else: entry = ScheduleEntry(condor_id=le.id) self.entries.append(entry) self.entries_cid[le.id] = entry print "++Job", le.id entry.log[le.event] = le.timestamp if le.event == LogKey.execute: entry.status = EntryStatus.executing elif le.event == LogKey.job_terminated: entry.status = EntryStatus.completed wf_id, dag_job_id, slot = condor_history(le.id) job = next((j for j in self.workflow.jobs if j.dag_job_id == dag_job_id and j.wf_id == wf_id), None) if job: entry.job = job entry.host = next((m for m in self.machines if m.condor_slot == slot), self.machines[0]) print "--Job", le.id, dag_job_id, entry.host.condor_slot
def __init__(self): self.workflow = Workflow() self.creation_timestamp = self.timestamp = datetime.now() self.logwatcher = LogWatcher() manager = Machine() manager.status = MachineStatus.manager manager.condor_slot = 'local' self.machines = [manager] boot_entry = ScheduleEntry(Job('boot', None), manager, None, None) boot_entry.real_start = self.timestamp boot_entry.real_end = self.timestamp boot_entry.status = EntryStatus.completed self.entries = [boot_entry] self.entries_cid = {}