def test_Distribute():
#initilalize master node
CNN = [{
"l_type": "conv",
"kernel": "W1",
"hparams": {
"stride": 1,
"pad": 0
}
}, {
"l_type": "max",
"hparams": {
"stride": 1,
"f": 2
}
}]
nodes = [{"ip": "localhost", "port": 9998}]
edge = {"ip": "localhost", "port": 9000}
image = np.array([1, 2])
master_node = Master(CNN, nodes, edge, image)
np.random.seed(1)
X1 = np.random.randn(1, 3, 3, 1)
out = vecConv(X1[0, :, :, :], kernels["W1"], {"stride": 1, "pad": 0})
out2 = master_node.thread_Compute(X1, CNN[0])
np.testing.assert_array_equal(out2, out)
def main():
parser = argparse.ArgumentParser(description="Python Phone")
parser.add_argument("--srv", dest="srv", action="store_true",
help="Start the server")
parser.add_argument("--call", dest="hostname",
help="Call a hostname")
parser.add_argument("--dev", dest="devices", action="store_true",
help="List devices")
args = parser.parse_args()
master = Master()
if args.devices:
p = pyaudio.PyAudio()
i = 0
while True:
try:
print "{0}: {1}".format(i, p.get_device_info_by_index(i)['name'])
i += 1
except:
break
if args.srv:
master.serve()
elif args.hostname:
master.call(args.hostname, Config.port)
else:
print "Please specify either --srv or --call."
def run_job(self, job):
"""TODO: Docstring for run_job.
:job: TODO
:returns: TODO
"""
self._log.info("running job: {}".format(job.id))
job.priority = self._safe_priority(job.priority)
job.save()
queue = job.queue
if queue is None or queue == "":
queue = self.AMQP_JOB_QUEUE
handler = JobHandler(job, queue, self)
self._job_handlers[str(job.id)] = handler
with self._job_queue_lock:
job_priority_queue = self._job_amqp_queues.setdefault(queue, PQ())
# items are fetched by lowest priority value first, so we need to
# invert the priorities
job_priority_queue.put(((1000 - job.priority), handler))
Master.instance().update_status(queues=self._get_queues())
def run_job(self, job):
"""TODO: Docstring for run_job.
:job: TODO
:returns: TODO
"""
self._log.info("running job: {}".format(job.id))
job.priority = self._safe_priority(job.priority)
job.save()
queue = job.queue
if queue is None or queue == "":
queue = self.AMQP_JOB_QUEUE
handler = JobHandler(job, queue, self)
self._job_handlers[str(job.id)] = handler
with self._job_queue_lock:
job_priority_queue = self._job_amqp_queues.setdefault(queue, PQ())
# items are fetched by lowest priority value first, so we need to
# invert the priorities
job_priority_queue.put(((1000-job.priority), handler))
Master.instance().update_status(queues=self._get_queues())
def stop_job(self, job):
"""This is intended to be called once a job has been completed
(not cancelled, but completed)
"""
self._log.info("stopping job: {}".format(job.id))
if str(job.id) in self._job_handlers:
with self._job_queue_lock:
handler = self._job_handlers[str(job.id)]
queue = self._job_amqp_queues[handler.queue_name]
new_queue = []
for priority, handler in queue.queue:
if handler.job.id == job.id:
continue
new_queue.append((priority, handler))
queue.queue = new_queue
Master.instance().update_status(queues=self._get_queues())
AmqpManager.instance().queue_msg(json.dumps(
dict(type="cancel", job=str(job.id))),
"",
exchange=Master.AMQP_BROADCAST_XCHG)
job.reload()
job.status = {"name": "finished"}
job.timestamps["finished"] = time.time()
job.save()
self._log.info("stopped job: {}".format(job.id))
self._cleanup_job(job)
def run(self):
print "Test: starting emulation"
# Single shuffle (one-to-many)
num_transfers = 1
size = 1024*1024*10
transfers = self.genShuffleTransfers(nodes[0], nodes, num_transfers,size)
# Parallel shuffle (many-to-many)
# num_transfers = 1
# size = 1024
# transfers = []
# for mapper in nodes:
# transfers += self.genShuffleTransfers(mapper, nodes, num_transfers,size)
# start master
if host in master:
m = Master(host, nodes)
m.start()
# start slaves
if host in nodes:
s = Slave(host, master, nodes, transfers)
s.start()
if host in nodes:
s.join()
if host in master:
m.join()
outfile = open("./output/done.json", 'w')
json.dump(m.result, outfile, indent=4, sort_keys=True)
outfile.close()
return False
class TestSlave(unittest.TestCase):
def setUp(self):
self.master = Master()
# Mock async calls
def __AsyncMock(self, *args, **kwargs):
m = unittest.mock.MagicMock(*args, **kwargs)
async def mock_coro(*args, **kwargs):
return m(*args, **kwargs)
mock_coro.mock = m
return mock_coro
@parameterized.expand([
(0,),
(1,),
])
def test_RequestFromSlave(self, mock_result):
self.master.executeComandAsync = self.__AsyncMock(
return_value=mock_result)
self.master.requestFromSlave()
if mock_result == 0:
for slave in self.master.slave_dict:
self.assertEqual(
self.master.slave_dict[slave].exit_code, mock_result)
def _handle_configure(self, id_, image):
"""Configure the image (spin it up, let the user muck around in it, commit all changes
back into the original image)
"""
child_snapshots = master.models.Image.objects(base_image=image.id)
if len(child_snapshots) > 0:
self._log.warn("ERROR! ILLEGAL OPERATION! I WILL NOT MODIFY AN IMAGE WITH {} DEPENDENT SNAPSHOTS!".format(
len(child_snapshots)
))
image.status = {"name": "ready"}
return
vagrantfile = image.status.setdefault("vagrantfile", None)
user_interaction = image.status.setdefault("user_interaction", False)
vnc_info = self._vm_manager.configure_image(
str(image.id),
vagrantfile = vagrantfile,
user_interaction = user_interaction,
on_success = self._set_image_ready,
kvm = image.status["kvm"]
)
self._log.debug("got vnc info from configure image: {!r}".format(vnc_info))
from master import Master
Master.instance().update_status(vms=self._get_running_vms())
image = master.models.Image.objects(id=image.id)[0]
if user_interaction:
image.status = {
"name": "configuring",
"vnc": vnc_info
}
image.save()
def _handle_create(self, id_, image):
"""Handle creating a new VM based on an existing VM
"""
self._log.info("creating an image")
base = image.base_image
dest_name = image.name
vagrantfile = image.status.setdefault("vagrantfile", None)
user_interaction = image.status.setdefault("user_interaction", False)
vnc_info = self._vm_manager.create_image(
vagrantfile,
base_name = str(base.id),
dest_name = str(image.id),
user_interaction = user_interaction,
on_success = self._set_image_ready
)
from master import Master
Master.instance().update_status(vms=self._get_running_vms())
image = master.models.Image.objects(id=image.id)[0]
if user_interaction:
image.status = {
"name": "configuring",
"vnc": vnc_info
}
image.save()
def _handle_configure(self, id_, image):
"""Configure the image (spin it up, let the user muck around in it, commit all changes
back into the original image)
"""
vagrantfile = image.status.setdefault("vagrantfile", None)
user_interaction = image.status.setdefault("user_interaction", False)
vnc_info = self._vm_manager.configure_image(
str(image.id),
vagrantfile = vagrantfile,
user_interaction = user_interaction,
on_success = self._set_image_ready,
kvm = image.status["kvm"]
)
self._log.debug("got vnc info from configure image: {!r}".format(vnc_info))
from master import Master
Master.instance().update_status(vms=self._get_running_vms())
image = master.models.Image.objects(id=image.id)[0]
if user_interaction:
image.status = {
"name": "configuring",
"vnc": vnc_info
}
image.save()
def get_results(width, height, color, material, num_of_masters):
if material == 'parquet':
floor = ParquetFloor(width, height, color)
master = Master(num_of_masters)
return floor.price() + master.price_of_masters
elif material == 'wooden':
floor = WoodenFloor(width, height, color)
master = Master(num_of_masters)
return floor.price() + master.price_of_masters
def main():
master = Master(800, 800, img_size)
setup_road_network(master)
try:
master.run_simulation()
except KeyboardInterrupt:
pass
def main():
master = Master(800, 800, img_size)
setup_road_network(master)
try:
master.run_simulation()
except KeyboardInterrupt:
pass
def test(seed):
random.seed(seed)
master = Master()
numbers = set()
for i in range(90):
number = master.next_number()
assert type(number) is int
assert 0 < number < 91
assert number not in numbers
numbers.add(number)
def fetch_estate():
database = torndb.Connection(**dbutil.get_mysql_config())
urls = []
for result in database.query('select distinct communityId from house'):
estate_id = result.communityId
urls.append('http://www.iwjw.com/estate/%s/' % estate_id)
master = Master(rest_period=5, result_model='html', result_dir='../iwjw/estate')
fetcher = Fetcher(processor=ps.Processor_hn())
master.add_fetchers(fetcher)
master.start(urls)
def __init__(self, **kwargs):
super(Tag, self).__init__(**kwargs)
Master.__init__(self)
self.fullscreen = False
self.w = pypixel.WIDTH
self.h = pypixel.HEIGHT - conf.bar_height
self.y_offset = conf.bar_height
self.bar_hidden = False
def __init__(self, k):
# Pointers to root, and solution nodes; initially None
self.root = None
self.solution = None
self.M = Master(k)
# List of values already visited and discovered
self.visited = []
self.bfs_disc_val = []
# Queue of nodes to visit in BFS
self.bfs_queue = []
def main():
pool = Pool(size=100)
greenlets = []
master = Master(MASTER_SERVER)
for appid in APPIDS:
for addr in master.get_servers(appid=appid):
greenlets.append(pool.spawn(fetch_server, addr))
greenlets = [greenlet.get() for greenlet in greenlets]
servers = sorted(filter(None, greenlets),
key=lambda x: (x['gamedir'], x['hostname'].lower()))
with codecs.open('servers.json', 'w', 'utf8') as f:
f.write(json.dumps(servers))
def __init__(self, **kwargs):
super(Tag, self).__init__(**kwargs)
Master.__init__(self)
self.fullscreen = False
self.w = pypixel.WIDTH
self.h = pypixel.HEIGHT - conf.bar_height
self.y_offset = conf.bar_height
self.bar_hidden = False
def fetch_house_from_db():
print 'sales'
existed = set([f.replace('.html', '') for f in os.listdir('../iwjw/sale')])
master = Master(rest_period=5, result_model='html', result_dir='../iwjw/sale')
fetcher = Fetcher(processor=ps.Processor_hn())
master.add_fetchers(fetcher)
database = torndb.Connection(**dbutil.get_mysql_config())
sale_list = database.query('select houseId from house where type=1;')
sale_list = [result.houseId for result in sale_list if not result.houseId in existed]
sale_list = ['http://www.iwjw.com/sale/%s/' % hid for hid in sale_list]
master.start(sale_list)
database.close()
def _handle_iso_create(self, id_, image):
"""Handle creating a new VM from scratch using the iso provided in
the state object.
"""
self._log.info("creating new image using iso")
iso_id = bson.ObjectId(image.status["iso"])
iso_file = master.models.TmpFile.objects(id=iso_id)[0]
if iso_file.path.startswith("/"):
iso_file.path = iso_file.path[1:]
iso_path = os.path.join("/tmp/talus", iso_file.path)
if not os.path.exists(iso_path):
self._log.warn("cannot locate iso {!r} for image {!r} creation".format(
iso_file.path,
image.name,
))
iso_file.delete()
image.status = {
"name": "iso-create error",
}
image.save()
return
vnc_info = self._vm_manager.create_from_iso(
iso_path = iso_path,
#vagrantfile = image.status.setdefault("vagrantfile", None),
image_name = str(image.id),
username = image.username,
password = image.password,
on_success = self._set_image_ready,
)
from master import Master
Master.instance().update_status(vms=self._get_running_vms())
if os.path.exists(iso_file.path):
os.remove(iso_file.path)
iso_file.delete()
image.status = {
"name": "configuring",
"vnc": vnc_info,
}
image.save()
self._log.info("new VM is starting up with iso {!r}, ready for initial configuration\n {!r}".format(
os.path.basename(iso_path),
vnc_info,
))
def cancel_job(self, job):
"""Cancel the job ``job``
:job: The job object to cancel
:returns: None
"""
# TODO forcefully cancel the job (notify all slaves via amqp that
# this job.id needs to be forcefully cancelled
self._log.info("cancelling job: {}".format(job.id))
if str(job.id) in self._job_handlers:
with self._job_queue_lock:
handler = self._job_handlers[str(job.id)]
queue = self._job_amqp_queues[handler.queue_name]
new_queue = []
while queue.qsize() > 0:
priority,handler = queue.get()
# leave this one out (the one we're cancelling)
if handler.job.id == job.id:
continue
new_queue.append((priority, handler))
for item in new_queue:
queue.put(item)
Master.instance().update_status(queues=self._get_queues())
else:
self._log.debug("job to cancel ({}) not in job handlers, sending cancel message to amqp anyways".format(job.id))
AmqpManager.instance().queue_msg(
json.dumps(dict(
type = "cancel",
job = str(job.id)
)),
"",
exchange=Master.AMQP_BROADCAST_XCHG
)
job.reload()
job.status = {
"name": "cancelled"
}
job.timestamps["cancelled"] = time.time()
job.save()
self._log.info("cancelled job: {}".format(job.id))
self._cleanup_job(job)
def cancel_job(self, job):
"""Cancel the job ``job``
:job: The job object to cancel
:returns: None
"""
# TODO forcefully cancel the job (notify all slaves via amqp that
# this job.id needs to be forcefully cancelled
self._log.info("cancelling job: {}".format(job.id))
if str(job.id) in self._job_handlers:
with self._job_queue_lock:
handler = self._job_handlers[str(job.id)]
queue = self._job_amqp_queues[handler.queue_name]
new_queue = []
while queue.qsize() > 0:
priority,handler = queue.get()
# leave this one out (the one we're cancelling)
if handler.job.id == job.id:
continue
new_queue.append((priority, handler))
for item in new_queue:
queue.put(item)
Master.instance().update_status(queues=self._get_queues())
else:
self._log.debug("job to cancel ({}) not in job handlers, sending cancel message to amqp anyways".format(job.id))
AmqpManager.instance().queue_msg(
json.dumps(dict(
type = "cancel",
job = str(job.id)
)),
"",
exchange=Master.AMQP_BROADCAST_XCHG
)
job.reload()
job.status = {
"name": "cancelled"
}
job.timestamps["cancelled"] = time.time()
job.save()
self._log.info("cancelled job: {}".format(job.id))
self._cleanup_job(job)
def main():
my_state = InternalState()
init()
logging.info('start service ...')
try:
while (my_state.state is not STATE_MASTER):
client = Client()
my_state = client.run(4, my_state)
master = Master()
master.run(my_state)
except KeyboardInterrupt:
sys.exit()
def _handle_import(self, id_, image):
"""This is the initial step when importing an image from the API. The API
will insert a new Image document into the database with status["name"] set to
"importing"
"""
self._log.info("importing an image")
image_to_import = bson.ObjectId(image.status["tmpfile"])
tmp_file = master.models.TmpFile.objects(id=image_to_import)[0]
if tmp_file.path.startswith("/"):
tmp_file.path = tmp_file.path[1:]
image_path = os.path.join("/tmp/talus", tmp_file.path)
if not os.path.exists(image_path):
self._log.warn("Cannot import image: {!r}, image to import not found ({})".format(
image.name,
tmp_file.path
))
tmp_file.delete()
image.status = {
"name": "import_error"
}
image.save()
return
vnc_info = self._vm_manager.import_image(
image_path,
str(image.id), # image name
user_interaction = True,
username = image.username,
password = image.password,
on_success = self._set_image_ready
)
from master import Master
Master.instance().update_status(vms=self._get_running_vms())
if os.path.exists(tmp_file.path):
os.remove(tmp_file.path)
tmp_file.delete()
image.status = {
"name": "configuring",
"vnc": vnc_info
}
image.save()
self._log.info("image is imported and running, ready for initial configuration:\n\t{!r}".format(vnc_info))
def __init__ (self, url=None, net=None, fileName=None):
# Fuse.__init__ (self)
Master.__init__ (self, fileName=fileName)
self.debug (1, 'v: logging in %s' % fileName)
self.inodes= {}
self.policy= policies.WeightedUniform ()
navel= None
if url:
key= self.getNavelKey (url)
navel= self._peers.getNavel (url, key)
else:
while not navel:
navel= self.discover (net)
self.gossip (navel)
def fetch_house():
# print 'sales'
# master = Master(rest_period=5, result_model='html', result_dir='../iwjw/sale')
# fetcher = Fetcher(processor=ps.Processor_hn())
# master.add_fetchers(fetcher)
# sales = list(get_houses('../iwjw/sale_list', 'sale'))
# master.start(sales)
print 'rent'
master = Master(rest_period=5, result_model='html', result_dir='../iwjw/rent')
fetcher = Fetcher(processor=ps.Processor_hn())
master.add_fetchers(fetcher)
rents = list(get_houses('../iwjw/rent_list', 'chuzu'))
master.start(rents)
def __init__(self):
self.parse_options()
if self.mode == 'client':
client = Client()
client.standby()
elif self.mode == 'master':
master = Master()
master.start()
elif self.mode == 'server':
server = Server()
server.standby()
else:
BenchResultsParser()
def setUp(self):
"""
var setups
"""
self.master_config = '../../docs/config/my_master.json'
self.slave_config = '../../docs/config/my_slave.json'
self.base = None
self.conn = None
self.mast = None
self.base = BaseDB()
self.base.load_config(self.master_config)
self.conn, db, t = self.base.connect_server()
self.mast = Master(self.conn)
def load_crap(self):
# Loads all images, makes objects if applicable, adds any images and
# rects to all_sprites{}
# Now also loads all sounds at the very top!
self.sounds = {'gasp' : self.load_sound('gasp.ogg'),\
'ugh' : self.load_sound('ugh.ogg'),\
'ow' : self.load_sound('ow.ogg'),\
'ahh' : self.load_sound('ahh.ogg'),\
'cry' : self.load_sound('cry.ogg'),\
'whip' : self.load_sound('whip.ogg'),\
'music' : self.load_sound('music1.ogg'),\
'haha' : self.load_sound('haha.ogg')}
self.sounds['music'].play(-1)
self.all_sprites = {}
self.bg_img = pygame.image.load(\
path.join('data','background.png')).convert_alpha()
playerimg=pygame.image.load(path.join('data', 'player.png')).convert_alpha()
player_pos = [370,384] # Does not move?
self.all_sprites['player'] = [playerimg,player_pos]
self.master1 = Master('left', self.screen)
self.master2 = Master('right', self.screen)
self.all_sprites['master1'] = [self.master1.image, self.master1.rect]
self.all_sprites['master2'] = [self.master2.image, self.master2.rect]
big_bar = pygame.image.load(path.join('data','big_bar.png')).convert_alpha()
big_bar_pos = (400-250, 500) # 500 bottom? 10 top? Edit background for bot
self.all_sprites['big_bar'] = [big_bar, big_bar_pos]
self.bar = Bar(self.sounds) # Moving bar
self.all_sprites['moving_bar'] = [self.bar.image, self.bar.rect]
self.timer = Timer() #Clock so player knows how long they've gone
self.all_sprites['timer'] = [self.timer.image, self.timer.rect]
manliness = pygame.image.load(\
path.join('data','manliness.png')).convert_alpha()
manliness1pos = (65, 1)
manliness2pos = (100, 1)
self.all_sprites['man1'] = [manliness, manliness1pos]
self.all_sprites['man2'] = [manliness, manliness2pos]
self.blood = Blood(self.screen, player_pos)
self.all_sprites['blood'] = [self.blood.image, self.blood.rect]
def __init__(self):
self.parse_options()
if self.mode == 'client':
client = Client()
client.standby()
elif self.mode == 'master':
master = Master()
master.start()
elif self.mode == 'server':
server = Server()
server.standby()
else:
BenchResultsParser()
def run(name, ip, port, logging_level):
logger = init_logger(name, logging_level)
server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
# add write service to replica to handle database updates from master
write_service = WriteService(name, logger=logger)
search_pb2_grpc.add_DatabaseWriteServicer_to_server(write_service, server)
write_service = WriteService(name, logger=logger)
search_pb2_grpc.add_DatabaseWriteServicer_to_server(write_service, server)
# the dynamic replica need to query the backup hence doesn't need to know who the backup is
master = Master(name, ip, None, logging_level)
search_pb2_grpc.add_SearchServicer_to_server(master, server)
search_pb2_grpc.add_HealthCheckServicer_to_server(master, server)
search_pb2_grpc.add_ReplicaUpdateServicer_to_server(master, server)
search_pb2_grpc.add_ReplicaCreationServicer_to_server(master, server)
print("Starting replica " + name)
server.add_insecure_port('[::]:' + port)
server.start()
try:
while True:
time.sleep(_ONE_DAY_IN_SECONDS)
except KeyboardInterrupt:
master.logger.info("Shutting down server")
logging.shutdown()
server.stop(0)
def master_serve(server, own_ip, db_name, logging_level):
# NOTE: backup doesn't have a backup
# TODO: Sync with crawler and master metadata
master = Master(db_name, own_ip, None, logging_level)
search_pb2_grpc.add_SearchServicer_to_server(master, server)
search_pb2_grpc.add_HealthCheckServicer_to_server(master, server)
print("Starting master")
try:
thread.start_new_thread(updateReplicaAndBackup, (master, ))
except Exception as e:
print str(e)
master.logger.error("Cannot start new thread due to " + str(e))
try:
thread.start_new_thread(sendHeartbeatsToReplicas, (
db_name,
master,
))
except Exception as e:
print str(e)
master.logger.error("Cannot start new thread due to " + str(e))
try:
while True:
time.sleep(_ONE_DAY_IN_SECONDS)
except KeyboardInterrupt:
master.logger.info("Shutting down server")
logging.shutdown()
server.stop(0)
def refresh_pos():
# global明示しないとlocalになる
global master, order
master = Master()
order = Order(master.item_master)
# print(order.df)
return master, order
def run(master_server_ip, own_ip, crawler, logging_level, backup_port):
retries = 0
logger = init_logger('backup', logging_level)
server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
# add write service to backup server to handle database updates from crawler
write_service = WriteService('backup', logger=logger)
search_pb2_grpc.add_DatabaseWriteServicer_to_server(write_service, server)
master = Master("backup", own_ip, None, logging_level)
search_pb2_grpc.add_ReplicaUpdateServicer_to_server(master, server)
server.add_insecure_port('[::]:' + backup_port)
server.start()
try:
thread.start_new_thread(sendHeartBeatMessage, (
master_server_ip,
server,
master,
logger,
crawler,
logging_level,
))
except Exception as e:
print str(e)
logger.error("Cannot start new thread due to " + str(e))
try:
while True:
time.sleep(_ONE_DAY_IN_SECONDS)
except KeyboardInterrupt:
logger.info("Shutting down server")
logging.shutdown()
server.stop(0)
def fetch_list():
print 'sale_list'
master = Master(rest_period=5, result_model='html', result_dir='../iwjw/sale_list')
fetcher = Fetcher(processor=ps.ProcessorIwjw())
master.add_fetchers(fetcher)
urls = [line.split('#')[0].strip() for line in codecs.open('../district.id')]
urls = map(lambda x: 'http://www.iwjw.com/sale/shanghai/%sp1/' % x, urls)
master.start(urls)
print 'rent_list'
master = Master(rest_period=5, result_model='html', result_dir='../iwjw/rent_list')
fetcher = Fetcher(processor=ps.ProcessorIwjw())
master.add_fetchers(fetcher)
urls = [line.split('#')[0].strip() for line in codecs.open('../district.id')]
urls = map(lambda x: 'http://www.iwjw.com/chuzu/shanghai/%sp1/' % x, urls)
master.start(urls)
def run_local_tornado(self, port=8888):
self.compute_job_graph()
scheduler = NaiveScheduler(self.job_graph)
master = Master(scheduler, port)
success, results = master.run()
print success
print "here"
if not success:
failed_jobs = filter(lambda x: x.failed, self.jobs.values())
print "-"*80
for job in failed_jobs:
print "Job failed: {0}".format(job.id)
print self.job_spec(job)
print "Exception: "
print results[job.id]
print results[job.id].traceback
print "-"*80
return success
def __init__(self, amount):
self.masters = list()
for _ in range(0, amount):
mas = Master(int(random.random() * (self.MAX_SALARY -
self.MIN_SALARY + 1) + self.MIN_SALARY))
self.masters.append(mas)
self.queue = deque()
self.wentAway = 0
self.completedRequests = 0
def RELOCATE_HEADER(self, filename, directory):
if not type(filename) is str:
raise Error(
'RELOCATE_HEADER(): filename parameter must be a string')
if not type(directory) in (list, tuple):
raise Error(
'RELOCATE_HEADER(): directory parameter must be list or tuple')
self.__dirbuilder.add_builder(Master(filename, directory))
pass
def evo_main():
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
logging.basicConfig(level=logging.DEBUG, format=f'[%(asctime)s][rank {rank}] %(message)s')
pa = ProgramArguments(rank)
gpu_fix(pa.args.gpu_ram)
if rank == 0:
max_rank: int = comm.Get_size()
cute: Cute = Cute(pa)
master = Master(cute, comm, max_rank)
master.run()
else:
worker = Worker(rank, comm)
worker.run()
def main():
parser = ArgumentParser()
parser.add_argument('--cfg',
type=str,
required=True,
help="big daddy config file")
args = parser.parse_args()
with open(args.cfg) as f:
cfg = yaml.safe_load(f)
# print(rank, socket.gethostname())
rank = MPI.COMM_WORLD.Get_rank()
if rank == 0:
m = Master(cfg)
m.main()
else:
w = Worker(cfg)
w.main()
def init_master():
itr = RoundRobinIter()
cservers = [
ChunkServer(env=MemEnv()),
ChunkServer(env=MemEnv()),
ChunkServer(env=MemEnv())
]
m = Master(
chunk_server_iter=itr,
chunk_servers=cservers
)
return m
def getSolution(problem, processCount):
# single-process program
if processCount == 1:
solver = Serial(problem)
# multi-process program
else:
workerCount = processCount - 1
solver = Master(problem, workerCount)
# get solution using correct solver
return solver.solve()
def stop_job(self, job):
"""This is intended to be called once a job has been completed
(not cancelled, but completed)
"""
self._log.info("stopping job: {}".format(job.id))
if str(job.id) in self._job_handlers:
with self._job_queue_lock:
handler = self._job_handlers[str(job.id)]
queue = self._job_amqp_queues[handler.queue_name]
new_queue = []
for priority,handler in queue.queue:
if handler.job.id == job.id:
continue
new_queue.append((priority, handler))
queue.queue = new_queue
Master.instance().update_status(queues=self._get_queues())
AmqpManager.instance().queue_msg(
json.dumps(dict(
type = "cancel",
job = str(job.id)
)),
"",
exchange=Master.AMQP_BROADCAST_XCHG
)
job.reload()
job.status = {
"name": "finished"
}
job.timestamps["finished"] = time.time()
job.save()
self._log.info("stopped job: {}".format(job.id))
self._cleanup_job(job)
def main():
parser = argparse.ArgumentParser(description="Python Phone")
parser.add_argument("--srv",
dest="srv",
action="store_true",
help="Start the server")
parser.add_argument("--call", dest="hostname", help="Call a hostname")
parser.add_argument("--dev",
dest="devices",
action="store_true",
help="List devices")
args = parser.parse_args()
master = Master()
if args.devices:
p = pyaudio.PyAudio()
i = 0
while True:
try:
print "{0}: {1}".format(i,
p.get_device_info_by_index(i)['name'])
i += 1
except:
break
if args.srv:
master.serve()
elif args.hostname:
master.call(args.hostname, Config.port)
else:
print "Please specify either --srv or --call."
def main(mainfile, dataset_file, number_of_tree=3):
master = Master(dataset_file, max_depth=30, min_bag_size=2)
master.init_client()
print('~{ Random Forest Parameter }~')
print(' main JSON file: {}'.format(mainfile))
print(' dataset file: {}'.format(dataset_file))
print(' max depth: {}'.format(master.max_depth))
print(' min bag size: {}'.format(master.min_bag_size))
print(' will create {} tree(s):'.format(number_of_tree))
total_run_time = 0
file_list = []
# loop to create trees
for i in range(number_of_tree):
print('\nCreating tree {}...'.format(i))
t, tree_filename, time_use = master.create_tree()
file_list.append(tree_filename)
total_run_time += time_use
tree_information = {
'max_depth': master.max_depth,
'min_bag_size': master.min_bag_size,
'total_run_time': total_run_time,
'avg_run_time': total_run_time/(number_of_tree*1.0),
'file_list': file_list
}
with open(os.path.join(tree_root_folder, mainfile), 'w') as f:
json.dump(tree_information, f, indent=2)
print('\n{} Tree(s) creation successful'.format(number_of_tree))
print('Total run time: {} sec'.format(total_run_time))
print('Avg run time per tree: {} sec'.format(1.0*total_run_time/number_of_tree*1.0))
import logging logging.basicConfig() from apscheduler.scheduler import Scheduler from master import Master import signal m = Master() m.check() def snooze_signal(signum, frame): m.ring_snooze() signal.signal(signal.SIGUSR1, snooze_signal) def stop_signal(signum, frame): m.ring_stop() signal.signal(signal.SIGUSR2, stop_signal) scheduler = Scheduler(standalone=True) scheduler.add_interval_job(m.check, seconds=10) try: scheduler.start() except KeyboardInterrupt: pass
def doStuff():
while True:
print connectedDevices
time.sleep(4)
if __name__ == "__main__":
t = threading.Thread(target=app.run, kwargs={'debug': False, 'host':'0.0.0.0'})
t.daemon = True
t.start()
m = Master()
t = threading.Thread(target=manageButtons, kwargs={'master': m})
t.daemon = True
t.start()
getProgramName = m.getProgramName
m.runReal()
#doStuff()
def _on_worker_exited(self):
from master import Master
Master.instance().update_status(vms=self._get_running_vms())
def __init__(self, **kwargs):
super(Column, self).__init__(**kwargs)
Master.__init__(self)
self.x = None
self.w = None
from master import Master from slave import Slave import atom import networkx as nx from matplotlib import pyplot as plt task = 1000000000 num_of_slaves = 20 # create slaves slaves = [] for i in range(num_of_slaves): slaves.append(Slave(i)) # create master master = Master(task, slaves) nodes = [master] nodes.extend(slaves) # create topology cluster = atom.Atom21(nodes) nx.draw(cluster) plt.show() for node in nodes: node.init(cluster, nodes) node.print_routing_table() print "ATOM created"
def check_config(config):
"""
Check configurations in config.ini.
Args:
config: A second-level dict containing configurations. See also in
ConfigLoader module.
Returns:
Returns a string containing error message if there are errors in
configuration.
Otherwise returns a tuple consisting of three derived class.
(job_manager_class, slave_class, uploader_class)
job_manager_class: Derived class of BaseJobManager. It determines
file id and source of jobs.
slave_class: Derived class of BaseSlave. It retrieves resources
according to the source and sends it to uploader_class.
uploader_class: Derived class of BaseUploader, which will upload
resources to somewhere.
"""
import_libs()
derived_classes = {
"Local": (LocalFSJobManager, URLSlave, CloudImageV2Uploader),
"URLList": (URLListJobManager, URLSlave, CloudImageV2Uploader),
"Qiniu": (QiniuJobManager, URLSlave, CloudImageV2Uploader),
"Local_MD5": (LocalFSJobManager, URLSlave, MD5CloudImageV2Uploader),
}
# check config for base job manager
check_result = BaseJobManager.check_config(config)
if check_result:
return check_result
# check config for base slave
check_result = BaseSlave.check_config(config)
if check_result:
return check_result
# check config for base uploader
check_result = BaseUploader.check_config(config)
if check_result:
return check_reuslt
if config["migrateinfo"]["migrate.type"] not in derived_classes:
return "Error: Unsupported Migrateinfo.migrate.type %s" % config["migrateinfo"]["migrate.type"]
class_type = config["migrateinfo"]["migrate.type"]
job_manager_class = derived_classes[class_type][0]
slave_class = derived_classes[class_type][1]
uploader_class = derived_classes[class_type][2]
# check config for derived job manager and slave and uploader
check_result = job_manager_class.check_config(config)
if check_result:
return check_result
check_result = slave_class.check_config(config)
if check_result:
return check_result
check_result = uploader_class.check_config(config)
if check_result:
return check_result
# check config for master
check_result = Master.check_config(config)
if check_result:
return check_result
return (job_manager_class, slave_class, uploader_class)
config["paths"]["log_path"] = log_path
config["paths"]["job_db_path"] = os.path.join(log_path, "jobs.db")
config["paths"]["pid_path"] = os.path.join(log_path, "pid")
# check configurations
check_result = check_config(config)
if type(check_result) is str:
print(check_result)
exit(1)
else:
(job_manager_class, slave_class, uploader_class) = check_result
pid_path = config["paths"]["pid_path"]
with open(pid_path, "w") as pid:
pid.write(str(os.getpid()))
if task == 0:
# submit procedure
job_manager = job_manager_class(config)
job_manager.start()
print("New submitted: %d" % job_manager.new_submitted)
print("Submit failed: %d" % job_manager.submit_error)
print("Ignored: %d" % job_manager.ignore)
elif task == 1:
# upload procedure
master = Master(config, slave_class, uploader_class)
master.start()
if os.path.isfile(pid_path):
os.remove(pid_path)
from mpi4py import MPI
from master import Master
from worker import Worker
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
processor_name = MPI.Get_processor_name()
MEASURING = False
DEPTH = 7
MASTER_DEPTH = 2
WORKER_DEPTH = DEPTH - MASTER_DEPTH
if __name__ == '__main__':
print 'hello [%d] on %s' % (rank, processor_name)
comm.Barrier()
if rank == 0:
master = Master(MEASURING)
master.work()
else:
worker = Worker(WORKER_DEPTH)
worker.work()
# coding:utf8
import os
if os.name != "nt" and os.name != "posix":
from twisted.internet import epollreactor
epollreactor.install()
if __name__ == "__main__":
from master import Master
master = Master("config.json")
master.startMaster()
# master.startChildren()
def main():
master = Master()
master.setup()
master.setDisplay(startScreen.StartScreen())
