def assign_tasks_bulk(inputs_dict, user_task_master):
# get arguments back from dictory. This is tedious.
# there should be a better way to deal with things like that
args_ifile = inputs_dict[def_config.bulk_assign_dict.ifile]
args_proj = inputs_dict[def_config.bulk_assign_dict.proj]
args_exfun = inputs_dict[def_config.bulk_assign_dict.exfun]
args_walltime = inputs_dict[def_config.bulk_assign_dict.walltime]
args_tasks_id = inputs_dict[def_config.bulk_assign_dict.tasks_id]
# ***** initialize variables here *****
Walltime = args_walltime
TasksID = args_tasks_id
if args_exfun !=None:
ExtnFunction = args_exfun
else:
ExtnFunction = 'echo'
# initialization the task master.
task_master = user_task_master(args_ifile,args_proj)
task_master.extract_tasks()
# fetch servers ip from a file
# if serverfile is None, use localhost instead
workers = checkIP.get_total_workers(inputs_dict[def_config.bulk_assign_dict.serverfile])
# filter the servers,
workers = checkIP.check_servers(workers,args_proj)
if len(workers) == 0:
print('No server available for the proj %s right now. might try later'%args_proj)
exit()
task_master.chunk_tasks(len(workers))
# ****** Initialization End here ************
# prepare a worker list to store the essential infomation
for worker in workers:
#worker = {}
#worker[def_config.worker.worker_ip] = avail_ip
#worker[def_config.worker.worker_port] = def_config.default_port
worker[def_config.worker.client_tasks_id] = TasksID
#workers.append(worker)
# ***** Actual assigning tasks here ********
# for each avail ip assign it a task list
for idx in range(len(workers)):
# First, check the server again
sock = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
server_address = (workers[idx][def_config.worker.worker_ip]
,workers[idx][def_config.worker.worker_port])
sock.connect(server_address)
try:
#
msg_json = PROTOC.gen_req_stat_msg(args_proj)
PROTOC.send_msg(sock,msg_json)
reply_json = PROTOC.recv_msg(sock)
reply = PROTOC.parse(reply_json)
print('Received : %s'%reply)
finally:
sock.close()
# Second, send tasks as request to the avail server
if(reply[PROTOC.attr.server_stat] == SSTAT.status.ready):
# recreate the socket
sock = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
server_address = (workers[idx][def_config.worker.worker_ip]
,workers[idx][def_config.worker.worker_port])
sock.connect(server_address)
try:
# essential fix project path and output path
task_master.project_path = reply[PROTOC.attr.ab_projs_dir] + '/' + args_proj
task_master.output_dir = task_master.project_path + '/Output'
sub_task_list = task_master.get_idx_chunk(idx,ExtnFunction)
print('%s get %d tasks '%(workers[idx][def_config.worker.worker_ip],len( sub_task_list ) ))
# Exchange information
req_json = PROTOC.gen_req_task_msg(TasksID, sub_task_list,Walltime)
PROTOC.send_msg(sock,req_json)
reply_json = PROTOC.recv_msg(sock)
reply = PROTOC.parse(reply_json)
# if the back message contain a task_log_idx that is not 0
# means server accepted the tasks.
if( reply[PROTOC.attr.client_tasks_id] == TasksID
and reply[PROTOC.attr.task_log_idx]!= 0 ):
task_log_idx = reply[PROTOC.attr.task_log_idx]
print('%s starts doing tasks as number %d task in its log'
%(workers[idx][def_config.worker.worker_ip],task_log_idx))
# Record these infomation somewhere
workers[idx][def_config.worker.task_log_idx] = task_log_idx
else:
print('%s refuse to work on sub_task_list. Tasks below need redo:'%workers[idx][def_config.worker.worker_ip])
logging.warning('TaskID back is wrong. %s refuse to work on these tasks. Tasks below need redo.',workers[idx][def_config.worker.worker_ip])
for task in sub_task_list:
print(task)
logging.warning(task)
finally:
sock.close()
else:
# If the server is not ready for tasks. show user the undo tasks
print('%s is now busy. cant assign tasks'%workers[idx][def_config.worker.worker_ip])
logging.debug('%s is busy for tasks id - %s',workers[idx][def_config.worker.worker_ip],TasksID)
logging.warning('these tasks need redo: ')
sub_task_list = task_master.get_idx_chunk(idx,ExtnFunction)
for task in sub_task_list:
logging.warning(task)
save_workers2record(args_ifile,workers,TasksID)
return workers
print 'args.serverfile is '+ args.serverfile + ' :'
with open(args.serverfile,'r') as serverF:
for line in serverF:
print line
print('The \'None\' means we will use default value.')
tip = raw_input("\npress any key to continue;\n")
print('Ready to run \n Workers = checkIP.fetch_workers(serverfile) '
+'\n checkIP.check_servers(Workers,\'test\')' )
print('\nIf you finished the content above and ready to run '
+ 'the checkIP.py.')
tip = raw_input("\npress any key to continue;\n")
print('=====================checkIP.py=================================')
Workers = checkIP.fetch_workers(args.serverfile)
Workers = checkIP.check_servers(Workers,'test')
print('=====================checkIP.py=================================')
tip = raw_input("\npress any key to continue;\n")
print('As can be seen, check_servers take two arguments, first one is servers info, second'
+ ' one is the name of project, \'test\' is a fault project that used for test')
print('\nOK, we got the available workers now. If you didnot get any available workers,'
+'please check if your local network blocked some ports.')
print('Now we can assign tasks for them.\n'
+'In this demo, we will run a demo.py in server side to process serval tasks.')
print('demo.py will take two arguments, one is number of seconds to elapse. second one is'
+' the location of /Output directory\n')
print('demo.py -t runningtime -o output_directory')
print('First, put all the tasks in a file, in this demo, tasklist is '+ args.ifile+':')
with open(args.ifile,'r') as ifile: