def main(process_num):
host_name = socket.gethostname()
if not config.task_table_path:
task_table_path = os.path.join(config.data_dir, 'task_table.pkl')
else:
task_table_path = config.task_table_path
global_task_table = TaskManager.load_task_table(task_table_path)
local_task_table = global_task_table.ix[global_task_table.node_name == host_name]
launcher_script = SourceFileLoader('launcher_script',
os.path.join(config.script_dir, 'launcher_script.py')).load_module()
executor_script = SourceFileLoader('executor_script',
os.path.join(config.script_dir, 'executor_script.py')).load_module()
collector_script = SourceFileLoader('collector_script',
os.path.join(config.script_dir, 'collector_script.py')).load_module()
launch_func = launcher_script.QLaunchFunctor(*config.launch_arguments)
execute_func = executor_script.ExecuteFunctor(*config.execute_arguments)
collect_func = collector_script.CollectFunctor(*config.collect_arguments)
processor = Processor(local_task_table, launch_func, execute_func, collect_func, process_num,
config.input_queue_length)
processor.process()
from launcher_collector_scheme import Processor
import time
import pickle
import executor_script
import multiprocessing as mp
import concurrent.futures
from data_parser import DataParser
from task_manager import TaskManager
import config
import socket
from importlib.machinery import SourceFileLoader
import os.path
dp = DataParser(config.data_dir, config.initial_time, config.time_step, config.max_chunk_size)
global_task_table = TaskManager.load_task_table(config.task_table_path)
local_task_table = global_task_table.ix[global_task_table.node_name == socket.gethostname()]
launcher_script = SourceFileLoader('launcher_script',
os.path.join(config.script_dir, 'launcher_script.py')).load_module()
launch_func = launcher_script.QLaunchFunctor(*config.launch_arguments)
