def test_peer(self):
# Spawn tracker and peers
tracker = self.host.spawn('tracker', Tracker)
p1 = self.host.spawn('peer1', Peer)
p2 = self.host.spawn('peer2', Peer)
p3 = self.host.spawn('peer3', Peer)
# Attach printer to peers
printer = self.host.spawn('printer', Printer)
p1.attach_printer(printer)
p2.attach_printer(printer)
p3.attach_printer(printer)
# Attach tracker to peers
p1.attach_tracker(tracker)
p2.attach_tracker(tracker)
p3.attach_tracker(tracker)
# Start intervals
tracker.init_start()
p1.init_start()
p2.init_start()
p3.init_start()
sleep(5)
self.assertEqual(set(['peer1', 'peer2', 'peer3']),
set(tracker.get_peers("file", True)))
sleep(18)
self.assertEqual([], tracker.get_peers("file"))
def waitMappers():
"""
Wait for all slaves to be done.
"""
print "\n\tEsperando a mappers..."
while num_mappers != registry.getMap() - 1:
sleep(1)
def test_make_push(self):
tracker = self.host.spawn('tracker', Tracker)
p1 = self.host.spawn('peer1', Peer)
p1.set_seed()
p2 = self.host.spawn('peer2', Peer)
p3 = self.host.spawn('peer3', Peer)
p1.attach_tracker(tracker)
p2.attach_tracker(tracker)
p3.attach_tracker(tracker)
# Attach printer to peers
printer = self.host.spawn('printer', Printer)
p1.attach_printer(printer)
p2.attach_printer(printer)
p3.attach_printer(printer)
# Start intervals
tracker.init_start()
p1.init_push()
p2.init_push()
p3.init_push()
sleep(30)
self.assertEqual(p1.get_data(), 'qwerty')
self.assertEqual(p2.get_data(), 'qwerty')
self.assertEqual(p3.get_data(), 'qwerty')
def receiveLamport(self, message, timeStampRecived, ide, total):
print message + ": " + str(timeStampRecived) + ":" + str(ide)
if not self.recivedMessages.get(timeStampRecived):
values = [" "] * total
values[ide - 1] = message
self.recivedMessages.update({timeStampRecived: values})
else:
values = self.recivedMessages.get(timeStampRecived)
values[ide - 1] = message
self.recivedMessages.update({timeStampRecived: values})
if self.timeStamp < timeStampRecived:
self.timeStamp = (timeStampRecived + 1)
else:
self.timeStamp = (self.timeStamp + 1)
self.members = self.tracker.get_members()
#enviar ACK del missatge rebut:
for member in self.members:
sleep(0.1)
#get USER PROXY & HOST PROXY
hostUser = self.members.get(member)
user = hostUser.lookup_url(member, User)
if member != self.url:
sleep(0.1)
user.reciveACK(message)
def multicast(self, message):
sleep(randint(1, 10) / 10)
self.members = self.tracker.get_members()
try:
self.timeStamp = self.sequencer.sequence()
print self.id + " SEND: " + str(message)
for member in self.members:
sleep(randint(1, 10) / 10)
#get USER PROXY & HOST PROXY
hostUser = self.members.get(member)
#print hostUser
user = hostUser.lookup_url(member, User)
user.receive(message, self.timeStamp)
except:
self.toSend.append(message)
if not self.bullyON:
trobat = False
for member in self.members:
hostUser = self.members.get(member)
user = hostUser.lookup_url(member, User)
if user != self.proxy:
if user.getBullyOn():
trobat = True
if not trobat:
self.bully(message)
def test_3wordcountingcompare(self):
#Set up reducer
self.reducer.setNumberOfMappers(2)
self.assertEqual(self.reducer.getNumberOfMappers(), 2)
#Get mapper 1
self.mapper1.map("WC", self.LOCALHOST + ":8000/0.part", self.reducer)
#Get mapper 2
self.mapper2.map("WC", self.LOCALHOST + ":8000/1.part", self.reducer)
#Get the reducer
sleep(3)
self.red_dic = self.reducer.getWC()
#Start a sequential map/reduce wit counting words
seqReducer = SeqReduce()
seqReducer.start()
seqMapper = SeqMap()
seqMapper.map("WC", "books/bible.txt", seqReducer)
sleep(2)
seqRed_dic = seqReducer.getWC()
#Assert each word has been counted the same times
for word, count in self.red_dic.items():
self.assertEqual(count, seqRed_dic[word])
def result(self):
resultado = collections.OrderedDict(sorted(self.dictionary.items()))
totalT = time() - self.timer
print resultado
print "\nElapsed time: %s" % totalT
sleep(1)
os.system("kill -9 $(ps -a|grep python|cut -f2 -d" + "'" + " " + "')")
def ping(self):
future = self.echo.say_something(future=True)
print 'pinging..'
future.add_callback('pong')
sleep(1)
print 'late callback:'
future.add_callback('pong')
def test_peer(self):
number_peers = 5
number_chunks = 9
protocol = "push-pull"
tracker = self.h.spawn('tracker', Tracker)
tracker.init_tracker()
self.peers = list()
sleep(2)
assistant = self.h.spawn('assistant', Assistant)
assistant.init_assistant(number_peers, number_chunks, protocol)
for i in range(number_peers):
self.peers.append(self.h.spawn('peer' + str(i), Peer))
self.peers[i].init_peer("hash1", number_chunks, protocol)
sleep(2)
self.peers[0].set_data({0: "A", 1: "S", 2: "D", 3: "F"})
self.peers[0].pull()
self.peers[1].push(0, "A")
self.peers[1].push_data()
self.peers[0].pull_data(0, self.peers[1])
shutdown()
def get_proxies(self, db):
future = db.get_proxies(future=True)
# future.add_callback('read_future', db)
print future.result()
return future
while not future.done:
sleep(0.1)
proxies = future.result()
return proxies
def create_hosts():
"""
Creation of master's Host and HTTP Server
"""
global host_master, registry
print "http://%s:1500/" % ip
host_master = create_host("http://%s:1500/" % ip)
print "-------------MAP REDUCE----------------\nHosts:\nListening Master Server at port 1500"
registry = host_master.spawn("regis", 'master/Registry')
os.chdir("../examples")
os.system("python -m SimpleHTTPServer &")
os.chdir("../project")
sleep(1)
def test_2countingwords(self):
#Set up reducer
self.reducer.setNumberOfMappers(2)
self.assertEqual(self.reducer.getNumberOfMappers(), 2)
self.assertEqual(self.reducer.getCW(), 0)
#Call mappers with counting words
self.mapper1.map("CW", self.LOCALHOST + ":8000/0.part", self.reducer)
self.mapper2.map("CW", self.LOCALHOST + ":8000/1.part", self.reducer)
sleep(2)
#Assert the result is correct
self.red_count = self.reducer.getCW()
self.assertEqual(self.red_count,
(self.mapper1.getCW() + self.mapper2.getCW()))
def multicastLamport(self, message):
#sleep(randint(1,10))
self.timeStamp = self.timeStamp + 1
self.members = self.tracker.get_members()
print "MESSAGE TO SEND: " + str(message)
for member in self.members:
#sleep(randint(1,10))
sleep(0.1)
#get USER PROXY & HOST PROXY
hostUser = self.members.get(member)
user = hostUser.lookup_url(member, User)
user.receiveLamport(message, self.timeStamp, self.identification,
len(self.members))
def test_4countingwordscompare(self):
#Set up reducer
self.reducer.setNumberOfMappers(2)
self.assertEqual(self.reducer.getNumberOfMappers(), 2)
self.assertEqual(self.reducer.getCW(), 0)
#Call mappers with counting words
self.mapper1.map("CW", self.LOCALHOST + ":8000/0.part", self.reducer)
self.mapper2.map("CW", self.LOCALHOST + ":8000/1.part", self.reducer)
#Start a sequential map/reduce wit counting words
seqReducer = SeqReduce()
seqReducer.start()
seqMapper = SeqMap()
seqMapper.map("CW", "books/bible.txt", seqReducer)
sleep(2)
seqRed_count = seqReducer.getCW()
sleep(2)
self.assertEqual(self.reducer.getCW(), seqRed_count)
def test_1wordcounting(self):
#Set up reducer
self.reducer.setNumberOfMappers(2)
self.assertEqual(self.reducer.getNumberOfMappers(), 2)
#Get mapper 1
self.mapper1.map("WC", self.LOCALHOST + ":8000/0.part", self.reducer)
sleep(1)
self.map1_dic = self.mapper1.getWC()
#Get mapper 2
self.mapper2.map("WC", self.LOCALHOST + ":8000/1.part", self.reducer)
sleep(1)
self.map2_dic = self.mapper2.getWC()
#Get the reducer
sleep(3)
self.red_dic = self.reducer.getWC()
#Assert each word has de same count
for word, count in self.red_dic.items():
if (self.map1_dic.get(word)):
count_map1 = self.map1_dic[word]
else:
count_map1 = 0
if (self.map2_dic.get(word)):
count_map2 = self.map2_dic[word]
else:
count_map2 = 0
count_final = count_map2 + count_map1
self.assertEqual(count, count_final)
def test_pull(self):
num_peers = 6
torrent_hash = 'file'
peers = []
time = 2 #is used to stop the intervals
option = 2
tracker = self.h.spawn('tracker', Tracker)
tracker._init_start(time)
for i in range(0, num_peers, 1):
peers.append(self.h.spawn("peer" + str(i), Peer))
seed = peers[0]
seed.be_seed()
for peer in peers:
peer.init_start(tracker, torrent_hash, time, option)
sleep(time)
for peer in peers:
self.assertIsNotNone(peer.get_data())
def test_tracker(self):
num_peers = 6
torrent_hash = 'file'
peers = []
time = 4 #is used to stop the intervals
expel_time = 1 # time to expel interval
tracker = self.h.spawn('tracker', Tracker)
tracker._init_start(time, expel_time)
for i in range(0, num_peers, 1):
peers.append(self.h.spawn("peer" + str(i), Peer))
for i in range(0, num_peers, 1):
peers[i].set_tracker(tracker)
for peer in peers:
peer.announce(torrent_hash)
sleep(2)
self.assertIsNotNone(tracker._get_peers(torrent_hash))
sleep(time)
self.assertEqual(tracker._get_announce_state(torrent_hash), {})
def pushGossip(self, torrent): #deliver
try:
self.torrentPeers = t.getPeers(torrent)
except Exception:
pass
self.chunks.append(sum(x is not None for x in self.torrentFile))
if self.torrentPeers != None:
if self.proxy in self.torrentPeers:
randomPeers = random.sample(self.torrentPeers, 2)
if self.proxy in randomPeers:
newPeers = copy.copy(self.torrentPeers)
newPeers.pop(self.proxy)
randomPeers = random.sample(newPeers, 2)
for peer in randomPeers:
sleep(0.2)
# PUSH - GOSSIP method:(self send chunk to random peer of randoms)
randomPosition = random.randint(0,
len(self.torrentFile) - 1)
peer.push(randomPosition, self.torrentFile[randomPosition])
def test_tracker(self):
try:
set_context()
except:
pass
try:
host = create_host()
except:
pass
# Spawn tracker and peers
tracker = host.spawn('tracker', Tracker)
p1 = host.spawn('peer1', Peer)
p2 = host.spawn('peer2', Peer)
p3 = host.spawn('peer3', Peer)
# Attach tracker to peers
p1.attach_tracker(tracker)
p2.attach_tracker(tracker)
p3.attach_tracker(tracker)
# Start intervals
tracker.init_start()
p1.announce_me()
sleep(0.5)
self.assertEqual(['peer1'], tracker.get_peers("file", True))
p2.announce_me()
sleep(0.5)
self.assertEqual(set(['peer1', 'peer2']),
set(tracker.get_peers("file", True)))
p3.announce_me()
sleep(0.5)
self.assertEqual(set(['peer1', 'peer2', 'peer3']),
set(tracker.get_peers("file", True)))
self.assertEqual([], tracker.get_peers("file1"))
shutdown()
def download(self, filename):
print 'downloading ' + filename
sleep(5)
return True
print i, e
def remote_server(self, web_server):
self.server = web_server
def download(self):
self.server.get_file('a1.txt', timeout=10)
print 'download finished'
if __name__ == "__main__":
set_context('green_thread')
# set_context()
host = create_host()
f1 = host.spawn('file1', File)
web = host.spawn('web1', Web)
sleep(1)
web.remote_server(f1)
load = host.spawn('wl1', Workload)
load.remote_server(web)
load2 = host.spawn('wl2', Workload)
load2.remote_server(web)
load.launch()
load2.download()
sleep(7)
shutdown()
def get_proxies(self):
return self.proxies
sleep(5)
return 'hi'
def say_something(self):
sleep(1)
return 'something'
def init(self):
sleep(5) # sleep implemented in order to avoid an update when execTime=0
self.interval1 = interval(self.host, 5, self.proxy, "update")
host = create_host('http://127.0.0.1:1679/')
clies = []
servs = []
for si in xrange(SERVERS):
serv = host.spawn('s' + str(si), Server)
servs.append(serv)
for i in xrange(CLIENTS):
c = host.spawn(str(si) + str(i), Connecter)
c.set_server(serv)
serv.register_client(c)
clies.append(c)
print si, 'online'
print "All nodes created. Starting..."
init = time()
for node in clies:
node.init_start()
try:
while not reduce(lambda x, y: x and y.end(timeout=None), servs, True):
sleep(0.01)
end = time()
print ((end - init) * 1000), ' ms.'
except TimeoutError:
print "Timeout"
shutdown()
def __init__(self):
self.greetings = ['hello', 'hi', 'hey', 'what`s up?']
def set_echo(self, echo):
self.echo = echo
def get_name(self):
return self.id
def say_hi(self):
for salute in self.greetings:
self.echo.echo(salute, self.proxy)
if __name__ == "__main__":
set_context()
h = create_host()
e1 = h.spawn('echo1', Echo)
bot = h.spawn('bot1', Bot)
bot2 = h.spawn('bot2', Bot)
bot.set_echo(e1) # Passing a proxy to a method marked as _ref
sleep(1) # Give time to host to lookup the first one
bot2.set_echo(e1)
bot.say_hi()
sleep(1)
e1.echo2('hello there!', [bot2])
e1.echo3('hello there!!', {'bot1': bot, 'bot2': bot2})
sleep(1)
shutdown()
def wait_a_lot(self):
sleep(2)
return 'ok'
def sleeping(self, t):
sleep(t)
return True
class Bot(object):
_tell = ['set_echo', 'ping', 'pong']
_ask = []
_ref = ['set_echo']
def set_echo(self, echo):
self.echo = echo
def ping(self):
future = self.echo.say_something(future=True)
future.add_callback('pong')
future.add_callback('pong')
print 'pinging..'
def pong(self, future):
msg = future.result()
print 'callback', msg
if __name__ == "__main__":
set_context()
h = create_host()
e1 = h.spawn('echo1', Echo)
bot = h.spawn('bot', Bot)
bot.set_echo(e1)
bot.ping()
sleep(2)
shutdown()
print(i, e)
def remote_server(self, web_server):
self.server = web_server
def download(self):
self.server.get_file('a1.txt', timeout=10)
print("download finished")
if __name__ == '__main__':
set_context('green_thread')
# set_context()
host = create_host()
f1 = host.spawn('file1', File)
web = host.spawn('web1', Web)
sleep(1)
web.remote_server(f1)
load = host.spawn('wl1', Workload)
load.remote_server(web)
load2 = host.spawn('wl2', Workload)
load2.remote_server(web)
load.launch()
load2.download()
sleep(7)
shutdown()
def sleeping(self, t):
sleep(t)
return True
_ask = []
_tell = ['hello', 'init_start', 'stop_interval']
# _ref = ['hello']
def init_start(self):
self.interval1 = interval(self.host, 1, self.proxy, "hello", "you")
later(5, self.proxy, "stop_interval")
def stop_interval(self):
print "stopping interval"
self.interval1.set()
def hello(self, msg):
print self.id, 'Hello', msg
if __name__ == "__main__":
N = 10 # 10000
set_context()
host = create_host()
registry = list()
for i in xrange(0, N):
registry.append(host.spawn(str(i), Registry))
for i in xrange(0, N):
registry[i].init_start()
sleep(8)
shutdown()
return x + y
def substract(self, x, y):
print 'subtract', x - y
def wait_a_lot(self):
sleep(2)
return 'ok'
if __name__ == "__main__":
set_context()
host = create_host('http://127.0.0.1:1679')
remote_host = host.lookup_url('http://127.0.0.1:1277/', Host)
print remote_host
server = remote_host.spawn('server', 's3_client/Server')
z = server.add(6, 7)
print z
server.substract(6, 5)
t = server.add(8, 7)
print t
try:
print server.wait_a_lot(timeout=1)
except TimeoutError, e:
print e
sleep(3)
shutdown()
msg = future.result()
print self.id, ': callback', msg
if __name__ == "__main__":
set_context()
# set_context('green_thread')
h = create_host()
e1 = h.spawn('echo1', Echo)
e1.echo('hello there !!')
bot = h.spawn('bot', Bot)
bot2 = h.spawn('bot2', Bot)
bot.set_echo()
bot.ping()
sleep(3)
# ask = e1.raise_something(future=True)
ask = e1.say_something(future=True)
print 'Future: ', ask
sleep(0.1)
if ask.done():
print 'Exception: ', ask.exception()
try:
print 'Result: ', ask.result(1)
except Exception, e:
print e
sleep(1)
shutdown()
def echoc(self, msg):
print msg
if __name__ == "__main__":
set_context('green_thread')
host = create_host('http://127.0.0.1:1679')
spk = host.spawn('echo', EchoC)
e1 = host.lookup_url('http://127.0.0.1:1277/echo1', 'Echo', 'server')
# print e1
h = host.lookup_url('http://127.0.0.1:1277/', Host)
# print h
e1.echo('HEY!') # TELL message
h.hello()
print h.say_hello(timeout=1), 'ASK message!' # ASK Message
f = h.say_hello(future=True)
e1.set_c(spk)
# e1.set_c(f)
# f.add_callback('echoc', e1)
print f.result(2), 'Future!'
# sleep(1)
# spk.set_callback(f)
sleep(4)
shutdown()
pong = SendPongMessage(self.proxy)
sender.send(pong)
self.pongCount = self.pongCount + 1
elif isinstance(msg, StopMessage):
self.proxy.stop()
else:
raise Exception("Unsupported message: " + msg)
if __name__ == "__main__":
set_context('green_thread')
host = create_host()
pong = host.spawn('pong', PongActor)
ping = host.spawn('ping', PingActor, [N, pong])
init = time()
msg = StartMessage()
ping.send(msg)
while pong.actor.is_alive():
sleep(0.1)
end = time()
print((end - init)), ' s.'
shutdown()
def echo(self, msg):
print msg
def say_something(self):
return 'something'
def raise_something(self):
raise Exception('raising something')
if __name__ == "__main__":
set_context()
# set_context('green_thread')
h = create_host()
e1 = h.spawn('echo1', Echo)
e1.echo('hello there !!')
# ask = e1.raise_something(future=True)
ask = e1.say_something(future=True)
print 'Future: ', ask
sleep(0.1)
if ask.done():
print 'Exception: ', ask.exception()
try:
print 'Result: ', ask.result(1)
except Exception, e:
print e
sleep(1)
shutdown()
class Bot(object):
_tell = ['set_echo', 'ping', 'pong']
_ask = []
def set_echo(self, echo):
self.echo = echo
def ping(self):
future = self.echo.say_something(future=True)
future.add_callback('pong')
future.add_callback('pong')
print 'pinging..'
def pong(self, future):
msg = future.result()
print 'callback', msg
if __name__ == "__main__":
set_context()
h = create_host()
e1 = h.spawn('echo1', Echo)
bot = h.spawn('bot', Bot)
bot.set_echo(e1)
bot.ping()
sleep(2)
shutdown()
# Para
# metros de entrada
N = 2
nom_fitxer = "200MB.txt"
word_count = False
os.system("python -m SimpleHTTPServer 8000 &"
) # Crear un servidor con el fichero
registry = master.spawn('registry', Registry)
wake_up_slaves(N)
print "Wait actors"
while registry.get_num_actors() < N:
sleep(0.2)
actors = registry.get_actors()
# print requests.get("http://0.0.0.0:8000/%s" % nom_fitxer).text # Printear el texto
list = split_file(N - 1)
# Reducer
reducer = actors[0].spawn(
"reducer", 'MapReduce/Reducer') # El actor n 0 sera el reducer
reducer.start(N - 1) # Quitamos el reducer del total
# Mappers
for i in range(1, N):
mapper = actors[i].spawn("mapper%s" % i, 'MapReduce/Mapper')
if i == 1:
mapper.start(0, list[i - 1], nom_fitxer, word_count, reducer)
else:
from pyactor.context import set_context, create_host, sleep, shutdown
class Echo(object):
_tell = ['echo', 'bye']
_ask = ['say_something']
def echo(self, msg):
print msg
def bye(self):
print 'bye'
def say_something(self):
return 'something'
if __name__ == "__main__":
set_context()
h = create_host()
e1 = h.spawn('echo1', Echo)
e = h.lookup('echo1')
print e.say_something()
ee = h.lookup_url('local://local:6666/echo1', Echo)
print ee.say_something()
sleep(1)
shutdown()
def say_something(self):
sleep(2)
return 'something'
def download(self, filename):
print(f"downloading {filename}")
sleep(5)
return True
nodes_h = {}
set_context('green_thread')
host = create_host()
# Create and initialize nodes
for i in range(len(nodes)):
nodes_h[i] = host.spawn(str(nodes[i]), Node)
nodes_h[i].init_node()
for i in range(len(nodes_h)):
j = 0 if i is 0 else i-1
try:
if nodes_h[i].join(nodes_h[j], timeout=20):
print 'JOIN OK', nodes_h[i].get_id()
else:
print 'Node %s fails' % str(i)
except Exception:
raise
else:
interval(host, 0.5, nodes_h[i], "update")
# Wait to give time to chord to fix its tables.
sleep(5)
found = nodes_h[0].find_successor(40)
print 'found', found.get_id()
shutdown()