def __init__(self):
master = Master(20)
for i in range(1, 101):
t = Task()
t.setId(i)
t.setName("任务" + str(i))
t.setPrice(random()*1000)
master.submit(t)
master.execute()
start = time.time()
res = master.get_results()
while True:
if master.is_complete():
end = time.time() - start
print(master.get_results())
break
class Driver(object):
def __init__(self, host_name, port, worker_port, alive_port, master_port,
membList, dfs, messageInterval, result_file, num_threads,
undirected):
self.host_name = host_name
self.host = socket.gethostbyname(host_name)
self.port = port
self.worker_port = worker_port
self.alive_port = alive_port
self.master_port = master_port
self.membList = membList
self.dfs = dfs
self.message_input = 'User has already inputted'
self.message_output = 'I am done with processing file'
self.message_fail = 'Some worker failed'
self.message_congrats = 'Congrats! you are the new master'
self.messageInterval = messageInterval
self.result_file = result_file
self.worker_num_threads = num_threads
self.is_undirected = undirected
self.client_ip = None
self.role = 'unknown'
self.master = None # make sense only if role == 'master'
self.filename_pair = [None, None]
self.app_file = -1
self.app_args = -1
self.dup_check = {}
def drive(self):
newstdin = os.fdopen(os.dup(sys.stdin.fileno()))
queue = Queue()
# a monitor receive message, check and response, also multicase failure message
self.server_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.server_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.server_sock.bind((self.host, self.port))
self.server_sock.listen(5)
self.input_task = Process(target=self.get_input,
args=(newstdin, queue))
self.input_task.daemon = True
self.input_task.start()
self.server_task = Process(target=self.background_server,
args=(queue, ))
self.server_task.start()
self.alive_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.alive_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.alive_sock.bind((self.host, self.alive_port))
self.alive_sock.listen(10)
self.alive_task = Process(target=self.Im_alive)
self.alive_task.daemon = True
self.alive_task.start()
self.app_file, self.app_args, self.filename_pair, self.role, self.client_ip, self.masters_workers, self.is_undirected = queue.get(
)
if (self.role == 'client'):
self.start_as_client()
elif (self.role == 'master'):
self.start_as_master()
elif (self.role == 'worker'):
self.start_as_worker()
elif (self.role == 'standby'):
self.start_as_standby()
# assert no one fails during input time
def get_input(self, newstdin, queue):
sys.stdin = newstdin
print 'Files to choose from: '
os.system('rm -f checkpt_file_*')
os.system('rm -f file_piece_*')
os.system('ls')
print
self.input_ready = False
while (not self.input_ready):
self.input_ready = True
try:
argv = raw_input(
'Input graph_file app_file app_args, or enter help: '
).strip().split()
graph_file, app_file = argv[:2]
app_args = argv[2:]
except:
print 'Example input: com-amazon.ungraph.txt pr_vertex.py 20'
self.input_ready = False
continue
for filename in (graph_file, app_file):
if not os.path.exists(filename):
print 'File {} does not exist'.format(filename)
self.input_ready = False
if len(app_file) < 4 or app_file[-3:] != '.py':
print 'Application file must be a python file'
self.input_ready = False
queue.put((app_file, app_args, (graph_file, self.result_file),
'client', self.host, None, self.is_undirected))
def background_server(self, queue):
conn, addr = self.server_sock.accept()
rmtHost = socket.gethostbyaddr(addr[0])[0]
message = receive_all_decrypted(conn) # the instruction
if message == self.message_input:
self.input_task.terminate()
print
self.app_file, _ = receive_all_to_target(conn,
self.messageInterval)
self.app_args = receive_all_decrypted(conn)
self.masters_workers = receive_all_decrypted(conn)
self.is_undirected = receive_all_decrypted(conn)
if self.host in self.masters_workers[0:2]:
self.role = 'master'
self.filename_pair[0], _ = receive_all_to_target(
conn, self.messageInterval)
self.filename_pair[1] = receive_all_decrypted(conn)
if self.host == self.masters_workers[1]:
self.role = 'standby'
print 'I am the standby master!'
else:
self.role = 'worker'
queue.put(
(self.app_file, self.app_args, self.filename_pair, self.role,
addr[0], self.masters_workers, self.is_undirected))
return
elif message == self.message_output: # for client and standby
if self.role != 'standby': # a hack since self.role not updated in this process
filename, _ = receive_all_to_target(conn, self.messageInterval)
assert (filename == self.result_file)
print 'Task done, result is published to {}'.format(filename)
elif message == self.message_congrats:
print('Hehe, now it is my turn')
self.role = 'master'
queue.put(self.role)
def Im_alive(self):
while True:
self.alive_sock.accept()
def start_as_client(self):
print 'I am the client!'
sleep(0.5)
real_members = [
host.split('_')[0] for host in sorted(self.membList.keys())
]
print 'All members: {}'.format(real_members)
self.masters_workers = [
socket.gethostbyname(host) for host in real_members
if host != self.host_name
]
for host in self.masters_workers:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((host, self.port))
send_all_encrypted(sock, self.message_input)
send_all_from_file(sock, self.app_file, self.messageInterval)
send_all_encrypted(sock, self.app_args)
send_all_encrypted(sock, self.masters_workers)
send_all_encrypted(sock, self.is_undirected)
if host in self.masters_workers[0:2]:
send_all_from_file(sock, self.filename_pair[0],
self.messageInterval)
send_all_encrypted(sock, self.filename_pair[1])
def init_master(self, standby):
self.master = Master(
self.filename_pair, self.masters_workers, self.host_name,
(self.master_port, self.worker_port, self.port),
(self.client_ip, self.message_output, self.message_fail), self.dfs,
standby)
def start_as_master(self):
#self.master = Master
print 'I am the master!'
self.init_master(False)
self.master.execute()
def start_as_worker(self):
print 'I am the worker!'
self.worker = Worker(self.app_file, self.host_name,
(self.master_port, self.worker_port),
self.masters_workers, self.app_args, self.dfs,
self.worker_num_threads, self.is_undirected)
self.worker.start_main_server()
def start_as_standby(self):
# wait for either master fail or receiving a finished signal
queue = Queue()
self.server_task = Process(target=self.background_server,
args=(queue, ))
self.server_task.start()
self.role = queue.get()
if (self.role == 'master'):
self.masters_workers[0:2] = self.masters_workers[1::-1]
self.init_master(True)
self.master.execute()
def really_failed(self, failed_process):
try:
failed_process = failed_process.split('_')[0]
failed_ip = socket.gethostbyname(failed_process)
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((failed_ip, self.alive_port))
return False
except socket.error, e:
return True
class MapReduce:
def __get_input_splits(self, input_path):
'''
Internal function to optionally partition (if single file passed) or retrieve a list of partitioned input files
'''
if isinstance(input_path, str):
if ospath.isdir(input_path):
input_files = [f'{input_path}/{f}' for f in listdir(input_path)]
n = len(input_files)
if n == 0:
raise ValueError(
"Expected 1 or more files in the directory")
elif n < self.M:
print(f"Insufficient input files. Setting mappers to {n}")
self.M = n
# TODO: Can be changed to split all files in the directory evenly for each mapper
elif ospath.isfile(input_path):
SPLIT_DIR = f"{self.TMP_DIR}/input"
# Create input_splits directory
pathlibpath(ospath.dirname(
f'{SPLIT_DIR}/')).mkdir(parents=True, exist_ok=True)
with open(input_path, 'r') as reader:
mappers_used = set()
n = 0
line = reader.readline()
while len(line) != 0:
if not line.endswith('\n'):
line += '\n'
mapper_id = n % self.M
mappers_used.add(mapper_id)
with open(f'{SPLIT_DIR}/{mapper_id}.txt', 'a') as writer:
writer.write(line)
line = reader.readline()
n += 1
if len(mappers_used) < self.M:
print(
f"Insufficient input lines. Setting mappers to {mappers_used}")
self.M = mappers_used
input_files = [
f'{SPLIT_DIR}/{i}.txt' for i in range(self.M)]
else:
raise ValueError("Expected a file or directory path")
else:
raise ValueError(
"Type mismatch. Expected a file or directory path")
return input_files
def __init__(self, n_mappers, n_reducers, input_path, map_func, reduce_func, kill_idx=-1):
'''
Acts as an intermediate between Master and user program
Responsible for spawning the Master worker
Args:
1. n_mappers/n_reducers - number of wrokers for each phase
2. input_path - disk location of the input file
3. map_func - handle for UDF map function
4. reduce_func - handle for UDF map function
5. kill_idx - specifies the worker to be killed; used to simulate fault tolerance when >= 0
'''
self.job_id = f'{int(time())}'
if ospath.isdir(f'./tmp/{self.job_id}'):
i = 1
while ospath.isdir(f'./tmp/{self.job_id}-{i}'):
i += 1
self.job_id += f'-{i}'
#get the number of cpu cores
n_processes = mp.cpu_count()
#adjust num_mappers and num_reducers according to num_cpu_cores
self.M, self.R = min(n_mappers, n_processes), min(n_reducers, n_processes)
print(f"Starting Job ID #{self.job_id} (M={self.M}, R={self.R})\n")
# Create a temp directory for intermediate files
self.TMP_DIR = f'./tmp/{self.job_id}'
# Create an output directory for final reducer files
self.OUT_DIR = f'./output/{self.job_id}'
pathlibpath(ospath.dirname(
f'{self.TMP_DIR}/')).mkdir(parents=True, exist_ok=True)
# Get the list of input file paths; split data into intermediate files, if required
input_files = self.__get_input_splits(input_path)
# Initialize master
self.master = Master(self.M, self.R, input_files, self.TMP_DIR, self.OUT_DIR)
# Transfer control to master and execute MapReduce
self.master.execute(map_func, reduce_func, kill_idx=kill_idx)
# Prints output directory and returns the path
print("\n{} output file(s) written to {}/\n".format(self.R, ospath.abspath(self.OUT_DIR)))
def read_output(self):
'''
Returns the output of the MapReduce task to the user program
'''
output = []
for fname in listdir(self.OUT_DIR):
with open(ospath.join(self.OUT_DIR, fname), 'r') as reader:
output += reader.readlines()
return output
