def create_slave_threads(self):
threads = []
# allocate a host to each slave
allocated_hosts = {}
local_count = reduce(lambda count, data: count+1 if data.get('local') else count, self.slaves_data, 0)
sample = random.sample(self.hostlist, len(self.slaves_data)-local_count)
for data in self.slaves_data:
if data.get('local', False):
allocated_hosts[data.get('tag')] = None
else:
allocated_hosts[data.get('tag')] = sample.pop()
#setup threads
for data in self.slaves_data:
tag = data.get('tag')
slave = Slave(self.access_details, allocated_hosts[tag], tag)
# shared data, everybody should know where everyone else is
slave.accum['metadata']['hosts'] = allocated_hosts
threads.append(slave)
return threads
def run(self):
print "Test: starting emulation"
# Single shuffle (one-to-many)
num_transfers = 1
size = 1024*1024*10
transfers = self.genShuffleTransfers(nodes[0], nodes, num_transfers,size)
# Parallel shuffle (many-to-many)
# num_transfers = 1
# size = 1024
# transfers = []
# for mapper in nodes:
# transfers += self.genShuffleTransfers(mapper, nodes, num_transfers,size)
# start master
if host in master:
m = Master(host, nodes)
m.start()
# start slaves
if host in nodes:
s = Slave(host, master, nodes, transfers)
s.start()
if host in nodes:
s.join()
if host in master:
m.join()
outfile = open("./output/done.json", 'w')
json.dump(m.result, outfile, indent=4, sort_keys=True)
outfile.close()
return False
def run_slave(url):
slave = Slave(url)
while True:
task, status_code = slave.fetch_next_task()
if status_code != HTTP_404_NOT_FOUND:
slave.process_task(task)
print()
else:
print()
print('PID {}: No tasks found! Sleeping for 2 second...'
.format(os.getpid()))
time.sleep(2)
def setUp(self):
"""
var setups
"""
self.master_config = '../../docs/config/my_master.json'
self.slave_config = '../../docs/config/my_slave.json'
self.base = None
self.conn = None
self.ins = None
self.base = BaseDB()
self.base.load_config(self.slave_config)
self.conn, db, t = self.base.connect_server()
self.ins = Slave(self.conn)
def __init__(self, slaves):
self.total_workers = slaves
self.slaves_available = []
self.slaves_unavailable = []
self.request_log = []
for x in range(slaves):
self.slaves_available.append(Slave("ip" + str(x + 1), "192.168.0." + str(101 + x)))
def enlarge(self):
super(Master, self).enlarge()
if self.__slave is not None:
return
# first, we collect everything that is necessary to perform
# the relocation - the header builder and the target directory
# builder.
# note that both need not be present from the beginning. they
# just might not have been seen yet because of indeterminisms
# in the order in which builders are called, or the respective
# directory entries might physically not exist because they
# are subject to be created.
if self.__header_builder is None:
self.__header_builder = self.parentbuilder().find_entry_builder(
[self.__filename])
if self.__header_builder is None:
self.force_enlarge()
self.__num_retries += 1
if self.__num_retries > 20:
raise Error(self.__error_prefix() + ': ' +
self.__filename + ' not seen after 20 rounds')
pass
elif not isinstance(self.__header_builder, HeaderBuilder):
raise Error(self.__error_prefix() + ': not a header file (' +
str(source_header_builder) + ')')
pass
if self.__target_directory_builder is None:
self.__target_directory_builder = self.package().rootbuilder(
).find_entry_builder(self.__directory)
if self.__target_directory_builder is None:
self.force_enlarge()
self.__num_retries += 1
if self.__num_retries > 20:
raise Error(self.__error_prefix() + ': ' +
'/'.join(self.__directory) +
' not seen after 20 rounds')
pass
elif not isinstance(self.__target_directory_builder,
DirectoryBuilder):
raise Error(self.__error_prefix() +
': target not a directory (' +
str(self.__target_directory_builder) + ')')
pass
# once we have everything at hand, create a slave in the
# target directory and tell our header builder to shut up.
if self.__header_builder is None or self.__target_directory_builder is None:
return
self.__slave = Slave(header_builder=self.__header_builder)
self.__target_directory_builder.add_builder(self.__slave)
self.__header_builder.disable_dependency_info()
pass
def parse_slave(config_path):
"""Returns a dictionary of `Slave` objects."""
config = configparser.ConfigParser()
config.read(config_path)
return {
name: Slave.factory(config[name])
for name in config.sections()
}
class SlaveTest(unittest.TestCase):
def setUp(self):
self.slaveObj = Slave('192.168.1.2', '/home/jboss_home', 'userName',
'passWord', 'hostUserName', 'hostPassword')
def test_get_slave_ip(self):
self.assertEquals('192.168.1.2', self.slaveObj.get_slave_ip())
def test_get_jboss_home(self):
self.assertEquals('/home/jboss_home', self.slaveObj.get_jboss_home())
def test_get_user_name(self):
self.assertEquals('userName', self.slaveObj.get_user_name())
def test_get_password(self):
self.assertEquals('passWord', self.slaveObj.get_pass_word())
def test_get_host_user_name(self):
self.assertEquals('hostUserName', self.slaveObj.get_host_user_name())
def test_base64_encode_of_a_password(self):
self.assertEquals('cGFzc1dvcmQ=',
self.slaveObj.get_base64_encode_of_password())
def test_get_host_pasword(self):
self.assertEquals('hostPassword', self.slaveObj.get_host_password())
def main():
parser = argparse.ArgumentParser(
description='Galatea slave servers that manage the chats and the AI')
parser.add_argument("-p",
"--port",
help="sets port of slave",
default=24833)
args = parser.parse_args()
Slave(args.port, L("logger.log", 1))
class TestSlave(unittest.TestCase):
def setUp(self):
self.slave = Slave(1)
@patch('slave.requests.get')
def test_postmanEchoRequest_success(self, mock_get):
mock_get.return_value.status_code = 200
self.slave.delay = 1
result = self.slave.postmanEchoRequest()
self.assertTrue(mock_get.called)
self.assertEqual(result, 0)
@patch('slave.requests.get')
def test_postmanEchoRequest_(self, mock_get):
mock_get.return_value.status_code = 400
self.slave.delay = 1
result = self.slave.postmanEchoRequest()
self.assertTrue(mock_get.called)
self.assertEqual(result, 1)
def get_bridges_files(self, standalone):
content = []
if not standalone:
if self.client:
self.client.send_json(rfc.create_request('get_bridges_files', None))
res = self.client.recv_json()
if res and res['success']:
for key in res['result']:
if not res['result'][key].has_key('error'):
content.extend(res['result'][key]['content'])
return files
else:
# if standalone no Slave but make use of slave static functions
return open(Slave.bridges_files(), 'r')
def run(self):
while True:
# poll for devices
r = requests.get(self.URI)
json_data = r.text
#f = open("devices.json", "r")
#json_data = f.read()
#f.close()
deviceList_json = json.loads(json_data)["_embedded"]["devices"]
devListNew = []
for deviceEntry in deviceList_json:
entry = (deviceEntry["id"], deviceEntry["samplingRate"], deviceEntry["longitude"], deviceEntry["latitude"])
devListNew.append(entry)
for device in devListNew:
if device not in self.devListAct:
self.devListAct.append( device )
newSlave = Slave(self.URI, device[0], device[1])
self.deviceMap[device[0]] = newSlave
neuron.updateBias(len(self.devListAct))
newSlave.start()
# create new slave
for device in self.devListAct:
if device not in devListNew:
removedDevice = self.deviceMap[device[0]]
removedDevice.stop()
self.devListAct.remove(device)
neuron.updateBias(len(self.devListAct))
# shut down slave
#print(self.devListAct)
time.sleep(1)
def get_interfaces_files(self, standalone):
files = []
if not standalone:
if self.client:
self.client.send_json(rfc.create_request('get_interfaces_files', None))
res = self.client.recv_json()
if res and res['success']:
for key in res['result']:
if not res['result'][key].has_key('error'):
files.append({'name': key, 'content': res['result'][key]['content']})
return files
else:
for if_file in Slave.interface_files():
files.append({'name': if_file, 'content': open(if_file, 'r')})
return files
def get_bridges_files(self, standalone):
files = []
if not standalone:
pass
# if self.client:
# self.client.send_json(rfc.create_request('get_bridges_files', None))
# res = self.client.recv_json()
# if res and res['success']:
# for key in res['result']:
# if not res['result'][key].has_key('error'):
# files.append({'name': key, 'content': res['result'][key]['content']})
# return files
else:
#if standalone no Slave but make use of slave static functions
# /!\ for now only use brctl conf file without creating file object nor normalizing
# might need to change this later to generalize
#for bridge_file in Slave.bridges_files():
# files.append({'name': bridge_file, 'content': open(bridge_file, 'r')})
return open(Slave.bridges_files(), 'r')
def launch(max_size=None, verbose=0):
"""
Launch all machines
Params:
:max_size -- int: max_size of each machine
:verbose -- int: level of verbose
Return:
:manager -- Manager: an instance of the memory manager
"""
rank = MPI.COMM_WORLD.Get_rank()
if (rank == 0):
return Manager()
elif rank == 1:
Master(max_size).run(verbose)
else:
Slave(rank, max_size).run(verbose)
exit(0)
class TestSlave(unittest.TestCase):
"""
Tests for class Slave, Jez I'm getting tired of writting
this shit
"""
def setUp(self):
"""
var setups
"""
self.master_config = '../../docs/config/my_master.json'
self.slave_config = '../../docs/config/my_slave.json'
self.base = None
self.conn = None
self.ins = None
self.base = BaseDB()
self.base.load_config(self.slave_config)
self.conn, db, t = self.base.connect_server()
self.ins = Slave(self.conn)
def test_slave(self):
self.assertIsNotNone(self.ins.get_job())
def run(self):
print "Test: starting emulation"
# Single shuffle (one-to-many)
num_transfers = 1
size = 1024 * 1024 * 10
transfers = self.genShuffleTransfers(nodes[0], nodes, num_transfers,
size)
# Parallel shuffle (many-to-many)
# num_transfers = 1
# size = 1024
# transfers = []
# for mapper in nodes:
# transfers += self.genShuffleTransfers(mapper, nodes, num_transfers,size)
# start master
if host in master:
m = Master(host, nodes)
m.start()
# start slaves
if host in nodes:
s = Slave(host, master, nodes, transfers)
s.start()
if host in nodes:
s.join()
if host in master:
m.join()
outfile = open("./output/done.json", 'w')
json.dump(m.result, outfile, indent=4, sort_keys=True)
outfile.close()
return False
#!/usr/bin/env python
from master import Master
from slave import Slave
import atom
import networkx as nx
from matplotlib import pyplot as plt
task = 1000000000
num_of_slaves = 20
# create slaves
slaves = []
for i in range(num_of_slaves):
slaves.append(Slave(i))
# create master
master = Master(task, slaves)
nodes = [master]
nodes.extend(slaves)
# create topology
cluster = atom.Atom21(nodes)
nx.draw(cluster)
plt.show()
for node in nodes:
node.init(cluster, nodes)
node.print_routing_table()
def setUp(self):
self.slaveObj = Slave('192.168.1.2', '/home/jboss_home', 'userName',
'passWord', 'hostUserName', 'hostPassword')
def add_slave(self, name):
self.slaves.append(Slave(name))
#!/usr/bin/env python
from mpi4py import MPI
from master import Master
from slave import Slave
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
size = comm.Get_size()
assert size > 1
# slaves on ranks [1,size)
if rank:
slave = Slave()
slave.run()
# master on rank 0
else:
options = set(['archive'])
master = Master(options)
master.run()
from slave import Slave
from config import CONFIG_SLAVES
import os
import logging
logging.basicConfig(level=logging.INFO)
if not CONFIG_SLAVES or len(CONFIG_SLAVES) == 0:
raise Exception("No configured accounts in `config.py` file")
export_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)),
'exports')
processes = []
for config in CONFIG_SLAVES:
slave = Slave(ig_login=config['ig_login'],
ig_password=config['ig_password'],
comments=config['comments'],
hashtags=config['hashtags'],
export_dir=export_dir)
processes.append(slave)
slave.start()
def monaco():
'''
The management daemon for Monaco
'''
# Set up top-level console logger
consolehandler = logging.StreamHandler()
consolehandler.setLevel(logging.ERROR)
consolehandler.setFormatter(
logging.Formatter('%(name)s - %(levelname)s - %(message)s'))
logger = logging.getLogger('monaco')
logger.setLevel(logging.DEBUG)
logger.addHandler(consolehandler)
# Load config
config.initLoggers()
# Install reload handler
def USR1_handler(signum, frame):
''' SIGUSR1 handler '''
logger.info('Received SIGUSR1, reloading Monaco...')
# this kills the current process tree and re-executes the
# original call arguments in the same PID (good for upstart)
os.execl(sys.executable, *([sys.executable] + sys.argv))
signal.signal(signal.SIGUSR1, USR1_handler)
# Setup scheduling
sched = Scheduler(daemon=True)
atexit.register(lambda: sched.shutdown(wait=False))
sched.start()
# Config and start slave thread to maintain app state
slave = Slave()
slave.update_subs()
#slave.start() starting in apsched inherits daemon=True
@sched.interval_schedule(seconds=1)
def maintain_slave():
'''
This periodically ensures that the slave is subscribed to all the
channels it should be. Again, this should be redundant
'''
if not hasattr(maintain_slave, 'started'):
maintain_slave.started = True
slave.start()
logger.info('Slave thread started')
if not slave.is_alive():
logger.info('Slave thread died')
slave.start()
# lifeRaft - Use Raft algorithm to maintain management leader
liferaft = LifeRaft()
# Start master/monitor thread
master = Master(liferaft)
master.start()
# maintain mgmt redis role from LifeRaft
r = redis.StrictRedis(port=config.config['mgmt_port'])
mastercli = None
while True:
time.sleep(1)
try:
master_tup = liferaft.value
if not master_tup:
logger.warn("Couldn't learn master value!")
else:
logger.debug('MGMT DB maintaining: %s', repr(master_tup))
if master_tup[0] == config.config['IP']:
if r.info()['role'] == 'master':
# We're the master, and properly configured
continue
# Else set to master, rewrite config to persist
logger.debug('Promoting self to master')
r.slaveof()
r.config_rewrite()
else:
if r.info()['role'] == 'slave' and r.info(
)['master_host'] == master_tup[0]:
# We're a slave and properly configured to the master
continue
for idx in xrange(3):
if idx != 0:
# Decongest on retry
time.sleep(0.5)
try:
# Try to get a connection to the mgmt master and verify it thinks it's master
if not mastercli:
mastercli = redis.StrictRedis(
host=master_tup[0],
port=config.config['mgmt_port'])
if mastercli.info()['role'] != 'master':
# If it doesn't think it's master, delete client and continue to retry
del mastercli
continue
break
except Exception:
continue
else:
# We didn't break
logger.debug(
'Assigned master (%s) has not yet assumed role',
master_tup[0])
continue
# Set slave to proper master
logger.debug('Assigning self as slaveof %s:6379',
master_tup[0])
r.slaveof(host=master_tup[0],
port=config.config['mgmt_port'])
except Exception:
try:
r = redis.StrictRedis(port=config.config['mgmt_port'])
except Exception:
logger.exception("Can't connect to mgmt db!")
def monaco():
'''
The management daemon for Monaco
'''
# Set up top-level console logger
consolehandler = logging.StreamHandler()
consolehandler.setLevel(logging.ERROR)
consolehandler.setFormatter(logging.Formatter('%(name)s - %(levelname)s - %(message)s'))
logger = logging.getLogger('monaco')
logger.setLevel(logging.DEBUG)
logger.addHandler(consolehandler)
# Load config
config.initLoggers()
# Install reload handler
def USR1_handler(signum, frame):
''' SIGUSR1 handler '''
logger.info('Received SIGUSR1, reloading Monaco...')
# this kills the current process tree and re-executes the
# original call arguments in the same PID (good for upstart)
os.execl(sys.executable, *([sys.executable]+sys.argv))
signal.signal(signal.SIGUSR1, USR1_handler)
# Setup scheduling
sched = Scheduler(daemon=True)
atexit.register(lambda: sched.shutdown(wait=False))
sched.start()
# Config and start slave thread to maintain app state
slave = Slave()
slave.update_subs()
#slave.start() starting in apsched inherits daemon=True
@sched.interval_schedule(seconds=1)
def maintain_slave():
'''
This periodically ensures that the slave is subscribed to all the
channels it should be. Again, this should be redundant
'''
if not hasattr(maintain_slave, 'started'):
maintain_slave.started = True
slave.start()
logger.info('Slave thread started')
if not slave.is_alive():
logger.info('Slave thread died')
slave.start()
# lifeRaft - Use Raft algorithm to maintain management leader
liferaft = LifeRaft()
# Start master/monitor thread
master = Master(liferaft)
master.start()
# maintain mgmt redis role from LifeRaft
r = redis.StrictRedis(port=config.config['mgmt_port'])
mastercli = None
while True:
time.sleep(1)
try:
master_tup = liferaft.value
if not master_tup:
logger.warn("Couldn't learn master value!")
else:
logger.debug('MGMT DB maintaining: %s', repr(master_tup))
if master_tup[0] == config.config['IP']:
if r.info()['role'] == 'master':
# We're the master, and properly configured
continue
# Else set to master, rewrite config to persist
logger.debug('Promoting self to master')
r.slaveof()
r.config_rewrite()
else:
if r.info()['role'] == 'slave' and r.info()['master_host'] == master_tup[0]:
# We're a slave and properly configured to the master
continue
for idx in xrange(3):
if idx != 0:
# Decongest on retry
time.sleep(0.5)
try:
# Try to get a connection to the mgmt master and verify it thinks it's master
if not mastercli:
mastercli = redis.StrictRedis(host=master_tup[0], port=config.config['mgmt_port'])
if mastercli.info()['role'] != 'master':
# If it doesn't think it's master, delete client and continue to retry
del mastercli
continue
break
except Exception:
continue
else:
# We didn't break
logger.debug('Assigned master (%s) has not yet assumed role', master_tup[0])
continue
# Set slave to proper master
logger.debug('Assigning self as slaveof %s:6379', master_tup[0])
r.slaveof(host=master_tup[0], port=config.config['mgmt_port'])
except Exception:
try:
r = redis.StrictRedis(port=config.config['mgmt_port'])
except Exception:
logger.exception("Can't connect to mgmt db!")
def init(max_size=100):
if MPI.COMM_WORLD.Get_rank() == 0: # Master
return Master(max_size)
# Any slave
Slave().run()
def setUp(self):
self.slave = Slave(1)
def initialize_slave(logging, sdfs_m):
slave = Slave(logging, sdfs_m)
return slave
def parse_args(args):
parser = argparse.ArgumentParser()
parser.add_argument("--master", action="store_true")
parser.add_argument("--slave", action="store_true")
parser.add_argument("port", type=int)
return parser.parse_args(args)
if __name__ == '__main__':
args = parse_args(sys.argv[1:])
if not (args.master or args.slave):
print('wrong args')
if args.master:
print(f'starting master on {get_ip()}:{args.port}')
master = Master(args.port)
master.set_size(1200, 750)
addr, port = input('slave port? ').split(':')
master.set_slave((addr, int(port)))
master.run()
if args.slave:
print(f'starting slave on {get_ip()}:{args.port}')
slave = Slave(args.port)
addr, port = input('master port? ').split(':')
slave.set_master((addr, int(port)))
slave.run()
def factory_slave(self):
slave = Slave(self, [12, 34], self.config)
return slave
from threading import Thread
def run_tcp_server(tcp_obj):
server.register_instance(tcp_obj)
server.serve_forever()
if __name__ == '__main__':
logging.basicConfig(filename='mp3.log',level=logging.INFO, filemode='w')
os.system("rm sdfs/*")
console = logging.StreamHandler()
console.setLevel(logging.DEBUG)
logging.getLogger('').addHandler(console)
sdfs_master = SDFS_Master()
slave = Slave(logging, sdfs_master)
udpserver = UDPServer(slave)
udpserver.run_server()
slave.run()
cli = CLI(slave, logging)
cli.run()
slave.init_join()
tcpserver = TCPServer(slave, sdfs_master, logging)
run_tcp_server(tcpserver)
