def main(process_num): host_name = socket.gethostname() if not config.task_table_path: task_table_path = os.path.join(config.data_dir, 'task_table.pkl') else: task_table_path = config.task_table_path global_task_table = TaskManager.load_task_table(task_table_path) local_task_table = global_task_table.ix[global_task_table.node_name == host_name] launcher_script = SourceFileLoader('launcher_script', os.path.join(config.script_dir, 'launcher_script.py')).load_module() executor_script = SourceFileLoader('executor_script', os.path.join(config.script_dir, 'executor_script.py')).load_module() collector_script = SourceFileLoader('collector_script', os.path.join(config.script_dir, 'collector_script.py')).load_module() launch_func = launcher_script.QLaunchFunctor(*config.launch_arguments) execute_func = executor_script.ExecuteFunctor(*config.execute_arguments) collect_func = collector_script.CollectFunctor(*config.collect_arguments) processor = Processor(local_task_table, launch_func, execute_func, collect_func, process_num, config.input_queue_length) processor.process()
from launcher_collector_scheme import Processor import time import pickle import executor_script import multiprocessing as mp import concurrent.futures from data_parser import DataParser from task_manager import TaskManager import config import socket from importlib.machinery import SourceFileLoader import os.path dp = DataParser(config.data_dir, config.initial_time, config.time_step, config.max_chunk_size) global_task_table = TaskManager.load_task_table(config.task_table_path) local_task_table = global_task_table.ix[global_task_table.node_name == socket.gethostname()] launcher_script = SourceFileLoader('launcher_script', os.path.join(config.script_dir, 'launcher_script.py')).load_module() launch_func = launcher_script.QLaunchFunctor(*config.launch_arguments)