def sync_jobs(self):
log_entries = self.logwatcher.nexts()
for le in log_entries:
if le.id in self.entries_cid: # in dict keys
entry = self.entries_cid[le.id]
else:
entry = ScheduleEntry(condor_id=le.id)
self.entries.append(entry)
self.entries_cid[le.id] = entry
print "++Job", le.id
entry.log[le.event] = le.timestamp
if le.event == LogKey.execute:
entry.status = EntryStatus.executing
elif le.event == LogKey.job_terminated:
entry.status = EntryStatus.completed
wf_id, dag_job_id, slot = condor_history(le.id)
job = next(
(j for j in self.workflow.jobs
if j.dag_job_id == dag_job_id and j.wf_id == wf_id), None)
if job:
entry.job = job
entry.host = next(
(m for m in self.machines if m.condor_slot == slot),
self.machines[0])
print "--Job", le.id, dag_job_id, entry.host.condor_slot
def __init__(self, vm_limit, azure_config, skip_setup, local):
self.vm_limit = vm_limit # user input
self.budget = 0
self.timestamp = datetime.now()
self.cost_pred = 0
self.wf_end = None
self.jobs_terminated = False
self.last_resched = None
self.workflow = Workflow()
self.logwatcher = LogWatcher()
self.schedule = Schedule()
manager = Machine()
manager.status = MachineStatus.manager
manager.condor_slot = 'manager'
self.machines = [manager]
boot_entry = ScheduleEntry(Job('boot', None), manager, self.timestamp, self.timestamp)
boot_entry.real_start = self.timestamp
boot_entry.real_end = self.timestamp
boot_entry.status = EntryStatus.completed
self.schedule.add_entry_host(boot_entry, manager)
self.local = local
if azure_config and not local:
hostname = socket.gethostname()
self.exp = AzureExperiment(azure_config, skip_setup=skip_setup, name=hostname)
self.master_addr = socket.gethostbyname(hostname)
self.user = azure_config.admin_username
else:
self.exp = self.master_addr = self.user = None
def sync_machines(self):
slots = condor_slots()
for s in slots:
if s not in [m.condor_slot for m in self.machines]:
machine = Machine()
machine.status = MachineStatus.running
machine.condor_slot = s
boot_job = Job('boot', None)
boot_entry = ScheduleEntry(boot_job, machine, None, None)
boot_entry.log[LogKey.real_start] = self.creation_timestamp
boot_entry.log[LogKey.real_end] = self.timestamp
boot_entry.status = EntryStatus.completed
self.entries.append(boot_entry)
self.machines.append(machine)
print "++Machine", s
def sync_jobs(self):
log_entries = self.logwatcher.nexts()
for le in log_entries:
if le.id in self.entries_cid: # in dict keys
entry = self.entries_cid[le.id]
else:
entry = ScheduleEntry(condor_id=le.id)
self.entries.append(entry)
self.entries_cid[le.id] = entry
print "++Job", le.id
entry.log[le.event] = le.timestamp
if le.event == LogKey.execute:
entry.status = EntryStatus.executing
elif le.event == LogKey.job_terminated:
entry.status = EntryStatus.completed
wf_id, dag_job_id, slot = condor_history(le.id)
job = next((j for j in self.workflow.jobs if j.dag_job_id == dag_job_id and j.wf_id == wf_id), None)
if job:
entry.job = job
entry.host = next((m for m in self.machines if m.condor_slot == slot), self.machines[0])
print "--Job", le.id, dag_job_id, entry.host.condor_slot
def sync_machines(self):
slots = condor_slots()
for s in slots:
if s not in [m.condor_slot for m in self.machines]:
machine = Machine()
machine.status = MachineStatus.running
machine.condor_slot = s
boot_job = Job('boot', None)
boot_entry = ScheduleEntry(boot_job, machine, None, None)
boot_entry.log[LogKey.real_start] = self.creation_timestamp
boot_entry.log[LogKey.real_end] = self.timestamp
boot_entry.status = EntryStatus.completed
self.entries.append(boot_entry)
self.machines.append(machine)
print "++Machine", s
def __init__(self):
self.workflow = Workflow()
self.creation_timestamp = self.timestamp = datetime.now()
self.logwatcher = LogWatcher()
manager = Machine()
manager.status = MachineStatus.manager
manager.condor_slot = 'local'
self.machines = [manager]
boot_entry = ScheduleEntry(Job('boot', None), manager, None, None)
boot_entry.real_start = self.timestamp
boot_entry.real_end = self.timestamp
boot_entry.status = EntryStatus.completed
self.entries = [boot_entry]
self.entries_cid = {}
def __init__(self):
self.workflow = Workflow()
self.creation_timestamp = self.timestamp = datetime.now()
self.logwatcher = LogWatcher()
manager = Machine()
manager.status = MachineStatus.manager
manager.condor_slot = 'local'
self.machines = [manager]
boot_entry = ScheduleEntry(Job('boot', None), manager, None, None)
boot_entry.real_start = self.timestamp
boot_entry.real_end = self.timestamp
boot_entry.status = EntryStatus.completed
self.entries = [boot_entry]
self.entries_cid = {}