def build_workflow(wf, img, cfg_file, serial_number):
location_list = dict()
task_list = list()
workflow = dict()
for task_const in wf.task_list:
task_obj = task.get_task(task_const[TASK_ID])
task_data = _build_task(task_obj)
if task_obj.id == BOOTSTRAP_CONFIG_ID:
loc = _get_server_location()
update_handler(task_data, BOOTSTRAP_CONF_SCRIPT)
task_data[PARAMETERS] = _update_config_params(cfg_file,
serial_number)
elif task_obj.id == BOOTSTRAP_IMAGE_ID:
loc = _get_server_location()
update_handler(task_data, BOOTSTRAP_IMAGE_SCRIPT)
task_data[PARAMETERS] = _update_image_params(img,
serial_number)
else:
loc = _get_server_location(task_obj)
task_data[PARAMETERS] = task_const[PARAMETERS]
location_list.update({task_data[LOCATION]: loc})
task_list.append(task_data)
workflow.update({TASK: task_list})
workflow.update({LOCATION: location_list})
return workflow
def build_workflow(wf, img, cfg_file, serial_number):
location_list = dict()
task_list = list()
workflow = dict()
for task_const in wf.task_list:
task_obj = task.get_task(task_const[TASK_ID])
task_data = _build_task(task_obj)
if task_obj.id == BOOTSTRAP_CONFIG_ID:
loc = _get_server_location()
update_handler(task_data, BOOTSTRAP_CONF_SCRIPT)
task_data[PARAMETERS] = _update_config_params(
cfg_file, serial_number)
elif task_obj.id == BOOTSTRAP_IMAGE_ID:
loc = _get_server_location()
update_handler(task_data, BOOTSTRAP_IMAGE_SCRIPT)
task_data[PARAMETERS] = _update_image_params(img, serial_number)
else:
loc = _get_server_location(task_obj)
task_data[PARAMETERS] = task_const[PARAMETERS]
location_list.update({task_data[LOCATION]: loc})
task_list.append(task_data)
workflow.update({TASK: task_list})
workflow.update({LOCATION: location_list})
return workflow
def handle_request(req):
if req.startswith("REQ_TASK"):
return str(task.get_task())
elif req.startswith("RETURN"):
cmd, id, result = req.split(":")
return task.receive_results(id, result)
return "error_invalid"
# time.sleep(random.randint(1,3))
from task import get_task
from db import RedisClient
r = RedisClient()
queue_len = int(r.queue_len)
def download_many(cc_list):
print('download_many')
workers = min(30, len(cc_list))
with futures.ThreadPoolExecutor(workers) as executor:
executor.map(get_task, cc_list)
[get_task(single) for x in range(10000)]
# B0755HH6NG
# 323091
# B074N42NXL
# 323091
# B01LFZVEFS
# 323090
# B01M6A8KHR
# 323088
# B0759DMF4K
# 323085
# B01KKW0ZXE
def get_task(id):
task_obj = task.get_task(id)
serializer = TaskSerializer(task_obj)
return serializer.data