def main():
global free_slots
global machines
global slots
global args
parser = argparse.ArgumentParser(description='Run AWCY scheduler daemon.')
parser.add_argument('-machineconf')
parser.add_argument('-port',default=4000)
parser.add_argument('-awsgroup', default='AOM Test')
parser.add_argument('-max-machines', default=3, type=int)
args = parser.parse_args()
if args.machineconf:
machineconf = json.load(open(args.machineconf, 'r'))
for m in machineconf:
machines.append(sshslot.Machine(m['host'],m['user'],m['cores'],m['work_root'],str(m['port']),m['media_path']))
for machine in machines:
slots.extend(machine.get_slots())
free_slots.extend(slots)
app = tornado.web.Application(
[
(r"/work_list.json", WorkListHandler),
(r"/run_status.json", RunStatusHandler),
(r"/machine_usage.json", MachineUsageHandler),
(r"/free_slots.json", FreeSlotsHandler),
(r"/submit", RunSubmitHandler),
(r"/cancel", CancelHandler),
(r"/execute_tick",ExecuteTick)
],
static_path=os.path.join(os.path.dirname(__file__), "static"),
xsrf_cookies=True,
debug=False,
)
app.listen(args.port)
ioloop = tornado.ioloop.IOLoop.current()
if not args.machineconf:
machine_thread = threading.Thread(target=machine_allocator,daemon=True)
machine_thread.start()
scheduler_tick()
ioloop.start()
#so up to 18 jobs, use 1 machine, then up to 64 use 2, etc...
num_instances_to_use = (31 + total_num_of_jobs) // 18
#...but lock AWS to a max number of instances
max_num_instances_to_use = int(args.machines)
if num_instances_to_use > max_num_instances_to_use:
rd_print('Ideally, we should use', num_instances_to_use,
'instances, but the max is', max_num_instances_to_use, '.')
num_instances_to_use = max_num_instances_to_use
machines = []
if args.machineconf:
machineconf = json.load(open(args.machineconf, 'r'))
for m in machineconf:
machines.append(sshslot.Machine(m['host'], m['user'], m['cores']))
else:
while not machines:
machines = awsremote.get_machines(num_instances_to_use, aws_group_name)
slots = []
#set up our instances and their free job slots
for machine in machines:
machine.setup(args.codec)
slots.extend(machine.get_slots())
#Make a list of the bits of work we need to do.
#We pack the stack ordered by filesize ASC, quality ASC (aka. -v DESC)
#so we pop the hardest encodes first,
#for more efficient use of the AWS machines' time.
num_instances_to_use = (31 + total_num_of_jobs) // 18
#...but lock AWS to a max number of instances
max_num_instances_to_use = int(args.machines)
if num_instances_to_use > max_num_instances_to_use:
rd_print(None, 'Ideally, we should use', num_instances_to_use,
'instances, but the max is', max_num_instances_to_use, '.')
num_instances_to_use = max_num_instances_to_use
machines = []
if args.machineconf:
machineconf = json.load(open(args.machineconf, 'r'))
for m in machineconf:
machines.append(
sshslot.Machine(m['host'], m['user'], m['cores'], m['work_root'],
str(m['port']), m['media_path']))
else:
while not machines:
machines = awsremote.get_machines(num_instances_to_use, aws_group_name)
slots = []
#set up our instances and their free job slots
for machine in machines:
slots.extend(machine.get_slots())
if len(slots) < 1:
rd_print(None, 'All AWS machines are down.')
sys.exit(1)
work_done = scheduler.run(work_items, slots)