def third():
print("= selftest 3 started")
R = Reactor()
a = Box("on x=10..12,y=10..12,z=10..12")
a.id = "Box1"
R += a
assert (R.size() == 27)
b = Box("on x=11..13,y=11..13,z=11..13")
b.id = "Box2"
R += b
check(R.size() == 27 + 19,
[R.size(), R.realcubes, [i.size() for i in R.realcubes]],
f=lambda: R.savefig())
c = Box("off x=9..11,y=9..11,z=9..11")
c.id = "Box3"
R += c
check(R.size() == 27 + 19 - 8,
[R.size(), R.realcubes, [i.size() for i in R.realcubes]],
f=lambda: R.savefig(colliding=True))
R += Box("on x=10..10,y=10..10,z=10..10")
assert (R.size() == 39)
print("= selftest 3 passed")
def __init__(self):
super(EventProcessor, self).__init__()
self.message_handler = MessageHandler()
#self.gps = GPS()
Message.init(self.message_handler.plugin.address())
Detector.start_plugins()
Reactor.add_plugin_events()
def __init__(self):
super(EventProcessor, self).__init__()
self.message_handler = MessageHandler()
self.gps = GPS()
Message.init()
Detector.start_plugins()
Reactor.add_plugin_events()
def test_get_conversation_new_with_reading_packet(self):
conversation_table = ConversationTable()
packet = packets.ReadRequestPacket('stub filename', 'stub mode')
reactor = Reactor('stub_socket', 'stub_router', conversation_table)
conversation = reactor.get_conversation('10.26.0.1', 3942, packet)
self.assertEqual(len(conversation_table), 1)
self.assertTrue(isinstance(conversation, TFTPConversation))
def __init__(self):
self._peer_id = self._gen_own_peer_id()
self.atorrents = []
self.reactor = Reactor(self)
listener = ListeningConnection(self, PORT)
self.reactor.add_reader(listener)
def __init__(self):
# state of various entities/room thingies
self.inv = []
self.rooms = make_rooms()
self.player_coords = [1, 1]
self.active_room = self.rooms[0]
self.hp = HP()
self.damage_rate = 0.2
self.time = 0
self.rads = 0
self.td = TimeDisplay()
self.map = None
self.time_txt = None
self.room_txt = None
self.health_txt = None
self.inventory_txt = None
self.radiation_exposure_txt = None
self.current_form = None
self.air_temp = None
self.set_map_pos = True
self.get_map_pos = False
self.last_time = 0
self.rad_mult = 1
self.equipment_txt = None
self.equipment = []
self.hinted = False
self.reactor = Reactor()
# Variable for end-state
self.good_end = None
def test_get_conversation_new_with_reading_packet(self):
conversation_table = ConversationTable()
packet = packets.ReadRequestPacket('stub filename', 'stub mode')
reactor = Reactor('stub_socket', 'stub_router', conversation_table)
conversation = reactor.get_conversation('10.26.0.1', 3942, packet)
self.assertEqual(len(conversation_table), 1)
self.assertTrue(isinstance(conversation, TFTPConversation))
class ConnectHandler(object):
def __init__(self, addr, requestor):
self._requestor = requestor
self._reactor = Reactor()
self._addr = addr
try:
self._socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self._socket.setblocking(0)
self._socket.connect(addr)
except socket.error:
err = self._socket.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR)
if err != 0 and err != errno.EINPROGRESS:
self._socket.close()
logger.debug("Connection failed: {0}\n".format(err))
requestor.connection_failed(addr)
return
self._reactor.register_for_write_events(self)
def stream(self):
return self._socket
def write_event(self):
self._reactor.unregister_for_write_events(self)
err = self._socket.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR)
if err == 0:
logger.info("Connection established with {0}"
.format(str(self._addr)))
self._requestor.connection_complete(self._addr, self._socket)
elif err != 0 and err != errno.EINPROGRESS:
logger.info("Connection establishment failed with {0}"
.format(str(self._addr)))
self._requestor.connection_failed(self._addr)
def add_atom(self, info, method='random_position', pbi = [0, 0, 0]):
"""Add later"""
from copy import deepcopy
from reactor import Reactor
if method == 'random_position':
position = self.shape.random_position_on_surface(self.cutoff_distance)
new_info = deepcopy(info)
new_info.extend(position)
for i in range(3):
new_info.append(str(pbi[i]))
Reactor.add_atom(self, new_info)
max_key = max(self.atoms)
temp_atoms = {}
temp_atoms[max_key] = self.atoms[max_key]
self._create_surface(temp_atoms)
# elif method == 'guess_position': not implemented in this version
# pass
elif method == 'known_position':
Reactor.add_atom(self, info)
max_key = max(self.atoms)
temp_atoms = {}
temp_atoms[max_key] = self.atoms[max_key]
self._create_surface(temp_atoms)
else:
raise RuntimeError("{0} is an invalid method.".format(method))
class ConnectHandler(object):
def __init__(self, addr, requestor):
self._requestor = requestor
self._reactor = Reactor()
self._addr = addr
try:
self._socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self._socket.setblocking(0)
self._socket.connect(addr)
except socket.error:
err = self._socket.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR)
if err != 0 and err != errno.EINPROGRESS:
self._socket.close()
logger.debug("Connection failed: {0}\n".format(err))
requestor.connection_failed(addr)
return
self._reactor.register_for_write_events(self)
def stream(self):
return self._socket
def write_event(self):
self._reactor.unregister_for_write_events(self)
err = self._socket.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR)
if err == 0:
logger.info("Connection established with {0}".format(
str(self._addr)))
self._requestor.connection_complete(self._addr, self._socket)
elif err != 0 and err != errno.EINPROGRESS:
logger.info("Connection establishment failed with {0}".format(
str(self._addr)))
self._requestor.connection_failed(self._addr)
class Emmer(object):
"""This is the wrapping class for the Emmer framework. It initializes
running services and also offers the client level interface.
"""
def __init__(self):
self.host = config.HOST
self.port = config.PORT
self.response_router = ResponseRouter()
self.conversation_table = ConversationTable()
self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
self.reactor = Reactor(self.sock, self.response_router,
self.conversation_table)
self.performer = Performer(self.sock, self.conversation_table,
config.RESEND_TIMEOUT,
config.RETRIES_BEFORE_GIVEUP)
def route_read(self, filename_pattern):
"""Adds a function with a filename pattern to the Emmer server. Upon a
read request, Emmer will run the action corresponding to the first
filename pattern to match the request's filename.
Use this function as a decorator on a function to add that function
as an action with which to handle a tftp conversation.
Args:
filename_pattern: a regex pattern to match filenames against.
"""
def decorator(action):
self.response_router.append_read_rule(filename_pattern, action)
return decorator
def route_write(self, filename_pattern):
"""Adds a function with a filename pattern to the Emmer server. Upon a
write request, Emmer will run the action corresponding to the first
filename pattern to match the request's filename.
Use this function as a decorator on a function to add that function
as an action with which to handle a tftp conversation.
Args:
filename_pattern: a regex pattern to match filenames against.
"""
def decorator(action):
self.response_router.append_write_rule(filename_pattern, action)
return decorator
def run(self):
"""Initiates the Emmer server. This includes:
* Listening on the given UDP host and port.
* Sending messages through the given port to reach out on timed out
tftp conversations.
"""
self.sock.bind((self.host, self.port))
print "TFTP Server running at %s:%s" % (self.host, self.port)
thread.start_new_thread(self.performer.run,
(config.PERFORMER_THREAD_INTERVAL, ))
self.reactor.run()
class Emmer(object):
"""This is the wrapping class for the Emmer framework. It initializes
running services and also offers the client level interface.
"""
def __init__(self):
self.host = config.HOST
self.port = config.PORT
self.response_router = ResponseRouter()
self.conversation_table = ConversationTable()
self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
self.reactor = Reactor(self.sock, self.response_router,
self.conversation_table)
self.performer = Performer(self.sock, self.conversation_table,
config.RESEND_TIMEOUT,
config.RETRIES_BEFORE_GIVEUP)
def route_read(self, filename_pattern):
"""Adds a function with a filename pattern to the Emmer server. Upon a
read request, Emmer will run the action corresponding to the first
filename pattern to match the request's filename.
Use this function as a decorator on a function to add that function
as an action with which to handle a tftp conversation.
Args:
filename_pattern: a regex pattern to match filenames against.
"""
def decorator(action):
self.response_router.append_read_rule(filename_pattern, action)
return decorator
def route_write(self, filename_pattern):
"""Adds a function with a filename pattern to the Emmer server. Upon a
write request, Emmer will run the action corresponding to the first
filename pattern to match the request's filename.
Use this function as a decorator on a function to add that function
as an action with which to handle a tftp conversation.
Args:
filename_pattern: a regex pattern to match filenames against.
"""
def decorator(action):
self.response_router.append_write_rule(filename_pattern, action)
return decorator
def run(self):
"""Initiates the Emmer server. This includes:
* Listening on the given UDP host and port.
* Sending messages through the given port to reach out on timed out
tftp conversations.
"""
self.sock.bind((self.host, self.port))
print "TFTP Server running at %s:%s" % (self.host, self.port)
thread.start_new_thread(self.performer.run,
(config.PERFORMER_THREAD_INTERVAL,))
self.reactor.run()
def download(name):
shared_mem = Queue.PriorityQueue()
if platform == "win32":
directory = 'C:\Users' + user + '\Torrents\\'
else:
directory = '/home/' + user + '/Torrents/'
Torrent = directory + name
peerMngr = PeerManager(Torrent)
bittorrentThread = Reactor(1, "Thread-1", peerMngr, shared_mem, debug=True)
bittorrentThread.run()
def test_get_conversation_old_with_data_packet(self):
conversation_table = ConversationTable()
packet = packets.DataPacket('stub block number', 'stub data')
old_conversation = TFTPConversation('10.26.0.1', 3942, 'stub_router')
conversation_table.add_conversation('10.26.0.1', 3942, old_conversation)
reactor = Reactor('stub_socket', 'stub_router', conversation_table)
conversation = reactor.get_conversation('10.26.0.1', 3942, packet)
self.assertEqual(len(conversation_table), 1)
self.assertTrue(isinstance(conversation, TFTPConversation))
self.assertEqual(conversation, old_conversation)
def delete_atoms(self, atom_keys): #finished #do you need to check if atoms are on surface
#can we assume atoms are on surface since we are working with a nano particle?
"""Add later"""
for key in atom_keys:
try:
self.surface.remove(key)
except ValueError:
continue
from reactor import Reactor
Reactor.delete_atoms(self, atom_keys)
def download(name):
shared_mem = Queue.PriorityQueue()
if platform == "win32":
directory = 'C:\Users' + user + '\Torrents\\'
else:
directory = '/home/'+ user +'/Torrents/'
Torrent = directory + name
peerMngr = PeerManager(Torrent)
bittorrentThread = Reactor(1, "Thread-1", peerMngr, shared_mem, debug=True)
bittorrentThread.run()
def test_get_conversation_old_with_data_packet(self):
conversation_table = ConversationTable()
packet = packets.DataPacket('stub block number', 'stub data')
old_conversation = TFTPConversation('10.26.0.1', 3942, 'stub_router')
conversation_table.add_conversation('10.26.0.1', 3942,
old_conversation)
reactor = Reactor('stub_socket', 'stub_router', conversation_table)
conversation = reactor.get_conversation('10.26.0.1', 3942, packet)
self.assertEqual(len(conversation_table), 1)
self.assertTrue(isinstance(conversation, TFTPConversation))
self.assertEqual(conversation, old_conversation)
class SocketReaderWriter(object):
def __init__(self, sock):
self._reactor = Reactor()
self._receiver = None
self._output = []
self._socket = sock
self._reactor.register_for_read_events(self)
def stream(self):
return self._socket
def stop(self):
self._reactor.unregister_for_read_events(self)
self._reactor.unregister_for_write_events(self)
def set_receiver(self, receiver):
self._receiver = receiver
def unset_receiver(self):
self._receiver = None
def read_event(self):
if self._receiver:
view, size = self._receiver.get_rx_buffer()
try:
num = self._socket.recv_into(view, size)
if num > 0:
self._receiver.rx_bytes(num)
else:
if self._receiver:
self._receiver.connection_lost()
except socket.error as err:
logger.info("Socket Error {}".format(err))
self._receiver.connection_lost()
def write_event(self):
if self._output == []:
self._reactor.unregister_for_write_events(self)
else:
while self._output != []:
num = self._socket.send(self._output[0])
if num == len(self._output[0]):
del self._output[0]
else:
self._output[0] = self._output[0][num:]
break
def tx_bytes(self, bytestr):
self._output.append(bytestr)
if len(self._output) == 1:
self._reactor.register_for_write_events(self)
def test_io_watcher():
reactor = Reactor()
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect(('localhost',8001))
def cb(a,b):
# should be reworked, this is a bit of a hammer
os.kill(os.getpid(), signal.SIGINT)
io_event = events.ReadEvent(s, cb)
reactor.install(io_event)
reactor.run()
def add_body_data(self, data): #start with this one
"""will implement later"""
old_new_keys = Body_data.add_atoms(self, data.atoms)
from reactor import Reactor
#the problem is in reactor change_atom_num
Reactor.change_atom_num(data, old_new_keys)
body_keywords = ["Angles", "Bonds", "Angle Coeffs", "Bond Coeffs",\
"Dihedral Coeffs", "Improper Coeffs", "Dihedrals", "Impropers",\
"Masses", "Pair Coeffs", "Velocities"]
for keyword in body_keywords:
Body_data.add_data(self, data.get_body_data(keyword), keyword)
def __init__(self):
self.host = config.HOST
self.port = config.PORT
self.response_router = ResponseRouter()
self.conversation_table = ConversationTable()
self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
self.reactor = Reactor(self.sock, self.response_router,
self.conversation_table)
self.performer = Performer(self.sock, self.conversation_table,
config.RESEND_TIMEOUT,
config.RETRIES_BEFORE_GIVEUP)
class SocketReaderWriter(object):
def __init__(self, sock):
self._reactor = Reactor()
self._receiver = None
self._output = []
self._socket = sock
self._reactor.register_for_read_events(self)
def stream(self):
return self._socket
def stop(self):
self._reactor.unregister_for_read_events(self)
self._reactor.unregister_for_write_events(self)
def set_receiver(self, receiver):
self._receiver = receiver
def unset_receiver(self):
self._receiver = None
def read_event(self):
if self._receiver:
view, size = self._receiver.get_rx_buffer()
try:
n = self._socket.recv_into(view, size)
if n > 0:
self._receiver.rx_bytes(n)
else:
if self._receiver:
self._receiver.connection_lost()
except socket.error as err:
logger.info("Socket Error {}".format(err))
self._receiver.connection_lost()
def write_event(self):
if self._output == []:
self._reactor.unregister_for_write_events(self)
else:
while self._output != []:
n = self._socket.send(self._output[0])
if n == len(self._output[0]):
del self._output[0]
else:
self._output[0] = self._output[0][n:]
break
def tx_bytes(self, bytestr):
self._output.append(bytestr)
if len(self._output) == 1:
self._reactor.register_for_write_events(self)
class Buttons:
Miss = Reactor.event()
Hit = Reactor.event()
def __init__(self, board):
self.board = board
self.reactor = board.reactor
self.time = self.board.time
self.tick_interval = 0.005
self.reactor.hook(Board.SwitchPress, self.press)
#self.board.hook(Board.SwitchRelease, self.release)
self.target(random.randint(0, self.board.switch_count() - 1))
self.wins = []
self.setup()
def setup(self):
ThreeWins(self.board)
def target(self, button):
assert button >= 0
assert button < self.board.switch_count()
self.clear()
self.__target = button
self.board.setPixelColorRGB(self.board.pixel(button), 0, 255, 0)
self.board.render()
def clear(self):
for x in range(self.board.switch_count()):
self.board.setPixelColorRGB(x, 0, 0, 0)
self.board.render()
def press(self, button, timestamp):
assert button >= 0
assert button < self.board.switch_count()
if button != self.__target:
self.missed(button, timestamp)
else:
self.hit(button, timestamp)
def missed(self, button, timestamp):
assert button >= 0
assert button < self.board.switch_count()
self.reactor.call(Buttons.Miss, button, timestamp)
self.board.setPixelColorRGB(self.board.pixel(button), 255, 0, 0)
self.board.render()
def hit(self, button, timestamp):
assert button >= 0
assert button < self.board.switch_count()
self.reactor.call(Buttons.Hit, button, timestamp)
self.target(random.randint(0, self.board.switch_count() - 1))
def start(self):
'''Opens ZeroMQ sockets, starts listening to PUB events and kicks off initial REQs'''
factory_kwargs = {'timeout': 60, 'safe': True, 'io_loop': self.io_loop}
pub_channel = salt.transport.client.AsyncPubChannel.factory(
self.opts, **factory_kwargs)
tok = pub_channel.auth.gen_token('salt')
yield pub_channel.connect()
req_channel = salt.transport.client.AsyncReqChannel.factory(
self.opts, **factory_kwargs)
reactor = Reactor(tok, req_channel, self.dump_path, self.opts)
pub_channel.on_recv(reactor.dispatch)
yield reactor.start()
def __init__(self, port=0, files = [], bootstrap_address = None):
self._logger = logging.getLogger("%s(%s)" % (self.__class__.__name__, hex(id(self))[:-1]))
# forwarding table: (message_id, payload_type) -> (connection_handler,
# expiration)
self.forwarding_table = {}
# flood/forward ignore table: message_id -> timestamp
# this table used to prevent loop in flood
# all message send out need to put their message_id and
# timestamp = time.time()+ttl*FIXED_EXPIRED_INTERVAL
self.ignore = {}
# create servent id
self.id = uuid.uuid4().bytes
self.log("id is %s" % self.id.encode('hex_codec'))
# calculate number of file and number of kilobyte shared
self._files = files
self.num_files = len(files)
self.num_kilobytes = 0
for f in files:
self.num_kilobytes += f.file_size
self.num_kilobytes /= 1000 # shrink the unit
# create Reactor class for socket management
self.reactor = Reactor(self, port)
# check if bootstrap_address is given
if bootstrap_address:
self.reactor.bootstrap_connect(bootstrap_address)
return
def __init__(self, root, bind_host, port=33348, control_sock_path="/tmp/tftp_control_sock"):
self.master_sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
self.master_sock.bind((bind_host, port))
self.bind_addr = (bind_host, port)
self.control_sock_path = control_sock_path
self.root = root
bind_host_ip = socket.gethostbyname(bind_host)
if bind_host_ip == '0.0.0.0' or bind_host_ip.startswith("127."):
self.my_addr = (get_my_ip(), port)
else:
self.my_addr = (bind_host_ip, port)
self.dload_count = 0
self.upload_count = 0
self.downloads = []
self.files = {} # name => FileInfo
self.servers = {} # addr => Server
self.server_ping_queue = set()
if os.path.exists(control_sock_path):
os.unlink(control_sock_path)
self.control_sock = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
self.control_sock.bind(control_sock_path)
self.reactor = Reactor()
self.reactor.register(self.control_sock, select.POLLIN,
self.on_control_cmd, None)
self.reactor.register(self.master_sock, select.POLLIN,
self.on_new_connection, None)
self.reactor.call_later(60, self.check_servers)
self.reactor.call_later(60, self.process_ping_queue)
def __init__(self):
self._peer_id = self._gen_own_peer_id()
self.atorrents = []
self.reactor = Reactor(self)
listener = ListeningConnection(self, PORT)
self.reactor.add_reader(listener)
def __init__(self, carbon_only=False, pruning_method=None, ignore_chirality=True, use_antimotifs=True, outstream=sys.stderr, reaction_database_fname="../rec/reaction_templates.dat"):
self.carbon_only = carbon_only
self.ignore_chirality = ignore_chirality
self.use_antimotifs = use_antimotifs
self.pruning_method = pruning_method
self.outstream = outstream
self.reaction_database_fname = reaction_database_fname
self.reactor = Reactor(carbon_only=self.carbon_only, ignore_chirality=self.ignore_chirality, use_antimotifs=self.use_antimotifs, reaction_database_fname=self.reaction_database_fname)
def __init__(self, config, data_store, telemetry, messagedispatcher):
self.messagedispatcher = messagedispatcher
self.reactor = Reactor(config, data_store, telemetry, messagedispatcher)
self.current_target = Point(
latitude = telemetry.get_location().latitude,
longitude = telemetry.get_location().longitude,
altitude = telemetry.get_location().altitude
)
self.c2_reactor = self.reactor.c2_reactor
def __init__(self,
carbon_only=False,
pruning_method=None,
ignore_chirality=True,
use_antimotifs=True,
outstream=sys.stderr,
reaction_database_fname="../rec/reaction_templates.dat"):
self.carbon_only = carbon_only
self.ignore_chirality = ignore_chirality
self.use_antimotifs = use_antimotifs
self.pruning_method = pruning_method
self.outstream = outstream
self.reaction_database_fname = reaction_database_fname
self.reactor = Reactor(
carbon_only=self.carbon_only,
ignore_chirality=self.ignore_chirality,
use_antimotifs=self.use_antimotifs,
reaction_database_fname=self.reaction_database_fname)
def __init__(self, addr, requestor):
self._requestor = requestor
self._reactor = Reactor()
self._addr = addr
try:
self._socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self._socket.setblocking(0)
self._socket.connect(addr)
except socket.error:
err = self._socket.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR)
if err != 0 and err != errno.EINPROGRESS:
self._socket.close()
logger.debug("Connection failed: {0}\n".format(err))
requestor.connection_failed(addr)
return
self._reactor.register_for_write_events(self)
def __init__(self, addr, server):
self._server = server
self._socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self._socket.setblocking(0)
self._socket.bind(addr)
self._socket.listen(5)
Reactor().register_for_read_events(self)
class Client(object):
def __init__(self):
self._peer_id = self._gen_own_peer_id()
self.atorrents = []
self.reactor = Reactor(self)
listener = ListeningConnection(self, PORT)
self.reactor.add_reader(listener)
@property
def peer_id(self):
return self._peer_id
def add_torrent(self, torrent_name):
torrent = ActiveTorrent(self, torrent_name, Strategy)
torrent.announce()
self.atorrents.append(torrent)
def add_torrent_peer(self, peer, info_hash):
info_hash_filter = lambda x: x.info_hash != info_hash
matching_torrent = filter(info_hash_filter, self.atorrents)
assert len(matching_torrent) <= 1
if not matching_torrent:
log.info('Invalid peer: no torrent found for info_hash %s' %
info_hash)
return
peer.atorrent = matching_torrent[0]
log.info('Adding peer %s to torrent %s' %
(repr(peer), matching_torrent[0]))
def receive_incoming_connection(self, sock, host, port):
peer = Peer(host, port, client=self, sock=sock)
self.reactor.add_reader_writer(peer)
def _gen_own_peer_id(self):
"""Return a 20 byte string to be used as a unique ID for this client"""
remain = 20 - len(PEER_ID)
seed_chars = string.ascii_lowercase + string.digits
seed = ''.join(random.choice(seed_chars) for x in range(remain))
return PEER_ID + seed
class Client(object):
def __init__(self):
self._peer_id = self._gen_own_peer_id()
self.atorrents = []
self.reactor = Reactor(self)
listener = ListeningConnection(self, PORT)
self.reactor.add_reader(listener)
@property
def peer_id(self):
return self._peer_id
def add_torrent(self, torrent_name):
torrent = ActiveTorrent(self, torrent_name, Strategy)
torrent.announce()
self.atorrents.append(torrent)
def add_torrent_peer(self, peer, info_hash):
info_hash_filter = lambda x: x.info_hash != info_hash
matching_torrent = filter(info_hash_filter, self.atorrents)
assert len(matching_torrent) <= 1
if not matching_torrent:
log.info('Invalid peer: no torrent found for info_hash %s' %
info_hash)
return
peer.atorrent = matching_torrent[0]
log.info('Adding peer %s to torrent %s' %
(repr(peer), matching_torrent[0]))
def receive_incoming_connection(self, sock, host, port):
peer = Peer(host, port, client=self, sock=sock)
self.reactor.add_reader_writer(peer)
def _gen_own_peer_id(self):
"""Return a 20 byte string to be used as a unique ID for this client"""
remain = 20 - len(PEER_ID)
seed_chars = string.ascii_lowercase + string.digits
seed = ''.join(random.choice(seed_chars) for x in range(remain))
return PEER_ID + seed
def __init__(self, client, filename, port, peer_id):
self._client = client
self._filename = filename
self._port = port
self._peer_id = peer_id
# _peers is a list of peers that the TorrentManager is trying
# to communicate with
self._peers = []
# _bitfields is a dictionary mapping peers to a bitfield of the pieces
# each has
self._bitfields = {}
try:
self._metainfo = Metainfo(filename)
except (IOError, ValueError) as err:
if isinstance(err, IOError):
message = err.strerror+' ('+filename+')'
else:
message = err.message+' ('+filename+')'
logger.error(message)
raise TorrentManagerError(message)
# _have is the bitfield for this torrent. It is initialized to reflect
# which pieces are already available on disk.
self._filemanager = FileManager(self._metainfo)
self._have = self._filemanager.have()
try:
self._tracker_proxy = TrackerProxy(self._metainfo, self._port,
self._peer_id)
except TrackerError as err:
logger.critical("Could not connect to tracker at {}"
.format(self._metainfo.announce))
logger.debug(" TrackerError: {}".format(err.message))
raise TorrentManagerError(err.message)
self._needed = {piece: (0, []) for piece
in list(self._have.findall('0b0'))}
self._interested = {}
self._requesting = {}
self._partial = []
self._reactor = Reactor()
self._reactor.schedule_timer(_TIMER_INTERVAL, self.timer_event)
self._tick = 1
print "Starting to serve torrent {} off of Cracker website...".format(filename)
self._connect_to_peers(20)
def __init__(self):
self.host = config.HOST
self.port = config.PORT
self.response_router = ResponseRouter()
self.conversation_table = ConversationTable()
self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
self.reactor = Reactor(self.sock, self.response_router,
self.conversation_table)
self.performer = Performer(self.sock, self.conversation_table,
config.RESEND_TIMEOUT,
config.RETRIES_BEFORE_GIVEUP)
def __init__(self, torrent):
self.torrent = torrent
self.torrent_state = 'random'
self.reactor = Reactor()
self.reactor_activated = False
self.peer_id = '-TZ-0000-00000000000'
self.peers = []
self.decode_torrent_and_setup_pieces()
self.handshake = self.build_handshake()
self.setup_tracker()
self.stitcher = Stitcher(self)
self.setup_peers()
def __init__(self, config, data_store, telemetry, messagedispatcher, communicator, detection):
self.uuid = config.get('DEFAULT', 'uuid')
self.debug = 'True' == config.get('DEFAULT', 'debug')
self.data_store = data_store
self.messagedispatcher = messagedispatcher
self.communicator = communicator
self.reactor = Reactor(config, data_store, telemetry, messagedispatcher, communicator, detection)
self.current_target = Point(
latitude = telemetry.get_location().latitude,
longitude = telemetry.get_location().longitude,
altitude = telemetry.get_location().altitude
)
def __init__(self):
self._peer_id = "-HS0001-" + str(int(time.time())).zfill(12)
self._torrents = {}
self._downloads = set()
for self._port in range(_PORT_FIRST, _PORT_LAST + 1):
try:
self._acceptor = Acceptor(("localhost", self._port), self)
break
except Exception as err:
logger.debug(err)
continue
else:
logger.critical(
("Could not find free port in range {}-{} to "
"accept connections").format(_PORT_FIRST, _PORT_LAST))
sys.exit(1)
logger.info("Listening on port {}".format(self._port))
Reactor().schedule_timer(.01, self.start_downloads)
Reactor().run()
def test_reactor():
reactor = Reactor()
step = 0
while 0 < reactor.status_percentage() < 1:
print(f"step {step:04d}")
print("\n".join(reactor.get_statuses()))
reactor.auto_changes()
step += 1
print("Final")
print("\n".join(reactor.get_statuses()))
def readinaTOR():
c=1
import time
t = time.time()
RR = Reactor()
for l in sys.stdin:
l = l.strip()
b = Box(l)
print(str(c)+": ("+str(len(RR.realcubes))+") "+str(b)+" "+str(int(time.time()-t)))
c+=1
RR = RR + b
return RR
def __init__(self, addr, requestor):
self._requestor = requestor
self._reactor = Reactor()
self._addr = addr
try:
self._socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self._socket.setblocking(0)
self._socket.connect(addr)
except socket.error:
err = self._socket.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR)
if err != 0 and err != errno.EINPROGRESS:
self._socket.close()
logger.debug("Connection failed: {0}\n".format(err))
requestor.connection_failed(addr)
return
self._reactor.register_for_write_events(self)
class Navigator:
def __init__(self, config, data_store, telemetry, messagedispatcher):
self.messagedispatcher = messagedispatcher
self.reactor = Reactor(config, data_store, telemetry, messagedispatcher)
self.current_target = Point(
latitude = telemetry.get_location().latitude,
longitude = telemetry.get_location().longitude,
altitude = telemetry.get_location().altitude
)
self.c2_reactor = self.reactor.c2_reactor
@asyncio.coroutine
def startup(self):
while True:
action = self.reactor.run()
if action is not None:
if action.has_move():
self.current_target = action.move
yield from asyncio.sleep(1)
def get_current_target(self):
return self.current_target
class Navigator:
def __init__(self, config, data_store, telemetry, messagedispatcher, communicator, detection):
self.uuid = config.get('DEFAULT', 'uuid')
self.debug = 'True' == config.get('DEFAULT', 'debug')
self.data_store = data_store
self.messagedispatcher = messagedispatcher
self.communicator = communicator
self.reactor = Reactor(config, data_store, telemetry, messagedispatcher, communicator, detection)
self.current_target = Point(
latitude = telemetry.get_location().latitude,
longitude = telemetry.get_location().longitude,
altitude = telemetry.get_location().altitude
)
@asyncio.coroutine
def startup(self):
while True:
action = self.reactor.run()
grid = self.data_store.get_grid_state()
if action is not None:
if action.has_move():
self.current_target = action.move
if self.current_target.altitude < 10:
self.current_target.altitude = 100
if action.has_claim_sector():
grid.set_state_for(action.claim_sector, datastore.SectorState.being_searched, self.uuid)
if action.has_complete_sector():
grid.set_state_for(action.complete_sector, datastore.SectorState.searched, self.uuid)
if (self.debug or self.current_target.altitude < 10) and action.has_move_info():
print(action.move_info)
if grid is not None:
yield from self.communicator.send_message(GridMesh(self.uuid, self.uuid, grid))
yield from asyncio.sleep(1)
def get_current_target(self):
return self.current_target
class TorrentManager(object):
def __init__(self, client, filename, port, peer_id):
self._client = client
self._filename = filename
self._port = port
self._peer_id = peer_id
# _peers is a list of peers that the TorrentManager is trying
# to communicate with
self._peers = []
# _bitfields is a dictionary mapping peers to a bitfield of the pieces
# each has
self._bitfields = {}
try:
self._metainfo = Metainfo(filename)
except (IOError, ValueError) as err:
if isinstance(err, IOError):
message = err.strerror+' ('+filename+')'
else:
message = err.message+' ('+filename+')'
logger.error(message)
raise TorrentManagerError(message)
# _have is the bitfield for this torrent. It is initialized to reflect
# which pieces are already available on disk.
self._filemanager = FileManager(self._metainfo)
self._have = self._filemanager.have()
try:
self._tracker_proxy = TrackerProxy(self._metainfo, self._port,
self._peer_id)
except TrackerError as err:
logger.critical("Could not connect to tracker at {}"
.format(self._metainfo.announce))
logger.debug(" TrackerError: {}".format(err.message))
raise TorrentManagerError(err.message)
self._needed = {piece: (0, []) for piece
in list(self._have.findall('0b0'))}
self._interested = {}
self._requesting = {}
self._partial = []
self._reactor = Reactor()
self._reactor.schedule_timer(_TIMER_INTERVAL, self.timer_event)
self._tick = 1
print "Starting to serve torrent {} off of Cracker website...".format(filename)
self._connect_to_peers(20)
def _connect_to_peers(self, n):
# Get addresses of n peers from the tracker and try to establish
# a connection with each
addrs = self._tracker_proxy.get_peers(n)
for addr in addrs:
peer = PeerProxy(self, self._peer_id, (addr['ip'], addr['port']),
info_hash=self._metainfo.info_hash)
self._peers.append(peer)
self._bitfields[peer] = BitArray(self._metainfo.num_pieces)
def _remove_peer(self, peer):
# Clean up references to the peer in various data structures
self._peers.remove(peer)
pieces = list(self._bitfields[peer].findall('0b1'))
for piece in pieces:
if piece in self._needed:
occurences, peers = self._needed[piece]
if peer in peers:
peers.remove(peer)
self._needed[piece] = (occurences-1, peers)
del self._bitfields[peer]
if peer in self._interested:
del self._interested[peer]
elif peer in self._requesting:
# If the peer is in the middle of downloading a piece, save
# the state in the partial list
index, offset, sha1, _, _ = self._requesting[peer]
self._partial.append((index, offset, sha1))
del self._requesting[peer]
def _rarest(self):
# Returns a list of tuples which includes a piece index sorted by
# the number of peers which have the piece in ascending order
return sorted([(occurences, peers, index)
for (index, (occurences, peers)) in self._needed.items()
if occurences != 0])
def _show_interest(self, peer):
if not peer.is_interested():
logger.debug("Expressing interest in peer {}"
.format(str(peer.addr())))
peer.interested()
if not peer.is_peer_choked():
self._request(peer)
def _check_interest(self, peer):
# If the peer is not already interested or requesting, identify a piece
# for it to download and show interest to the peer.
if peer not in self._interested and peer not in self._requesting:
# Compute the set of needed pieces which the peer has that are not
# already designated for another peer
needed = self._have.copy()
needed.invert()
of_interest = list((needed & self._bitfields[peer]).findall('0b1'))
dont_consider = [i for i, _, _, _ in self._interested.values()]
dont_consider.extend([i for i, _, _, _, _
in self._requesting.values()])
if len(of_interest) > 0:
for index, offset, sha1 in self._partial:
if index in of_interest:
self._partial.remove((index, offset, sha1))
self._interested[peer] = (index, offset, sha1,
self._tick)
self._show_interest(peer)
return
for _, _, index in self._rarest():
if index in of_interest and index not in dont_consider:
self._interested[peer] = (index, 0, hashlib.sha1(),
self._tick)
self._show_interest(peer)
return
if peer not in self._interested and peer.is_interested():
logger.debug("Expressing lack of interest in peer {}"
.format(str(peer.addr())))
peer.not_interested()
self._connect_to_peers(1)
def _request(self, peer):
if peer in self._interested:
index, offset, sha1, _ = self._interested[peer]
del self._interested[peer]
self._requesting[peer] = (index, offset, sha1, self._tick, 0)
index, received_bytes, _, _, _ = self._requesting[peer]
bytes_to_request = self._bytes_to_request(index, received_bytes)
logger.debug("Requesting pc: {} off: {} len: {} from {}"
.format(index, received_bytes, bytes_to_request,
str(peer.addr())))
peer.request(index, received_bytes, bytes_to_request)
def _is_last_piece(self, index):
return index == self._metainfo.num_pieces-1
def _length_of_last_piece(self):
return (self._metainfo.total_length -
(self._metainfo.num_pieces-1)*self._metainfo.piece_length)
def _length_of_piece(self, index):
if self._is_last_piece(index):
return self._length_of_last_piece()
else:
return self._metainfo.piece_length
def _in_last_block(self, index, offset):
if self._is_last_piece(index):
piece_length = self._length_of_last_piece()
else:
piece_length = self._metainfo.piece_length
return piece_length-offset < _BLOCK_SIZE
def _bytes_to_request(self, index, offset):
if not self._in_last_block(index, offset):
return _BLOCK_SIZE
else:
return self._length_of_piece(index) - offset
def info_hash(self):
return self._metainfo.info_hash
# PeerProxy callbacks
def get_bitfield(self):
return self._have
def peer_unconnected(self, peer):
logger.info("Peer {} is unconnected".format(str(peer.addr())))
self._remove_peer(peer)
self._connect_to_peers(1)
def peer_bitfield(self, peer, bitfield):
# Validate the bitfield
length = len(bitfield)
if (length < self._metainfo.num_pieces or
(length > self._metainfo.num_pieces and
bitfield[self._metainfo.num_pieces:length].any(1))):
logger.debug("Invalid bitfield from peer {}"
.format(str(peer.addr())))
peer.drop_connection()
self._remove_peer(peer)
self._connect_to_peers(1)
return
# Set the peer's bitfield and updated needed to reflect which pieces
# the peer has
logger.debug("Peer at {} sent bitfield".format(str(peer.addr())))
self._bitfields[peer] = bitfield[0:self._metainfo.num_pieces]
pieces = list(self._bitfields[peer].findall('0b1'))
for piece in pieces:
if piece in self._needed:
occurences, peers = self._needed[piece]
if peer not in peers:
peers.append(peer)
self._needed[piece] = (occurences+1, peers)
# Check whether there may be interest obtaining a piece from this peer
self._check_interest(peer)
def peer_has(self, peer, index):
# Update the peer's bitfield and needed to reflect the availability
# of the piece
logger.debug("Peer at {} has piece {}".format(str(peer.addr()), index))
if index < self._metainfo.num_pieces:
self._bitfields[peer][index] = 1
else:
raise IndexError
if index in self._needed:
occurences, peers = self._needed[index]
if peer not in peers:
peers.append(peer)
self._needed[index] = (occurences+1, peers)
# Check whether there may be interest obtaining a piece from this
# peer
self._check_interest(peer)
def peer_choked(self, peer):
logger.debug("Peer {} choked".format(str(peer.addr())))
if peer in self._interested:
del self._interested[peer]
elif peer in self._requesting:
# When choked in the middle of obtaining a piece, save the
# progress in the partial list
index, offset, sha1, _, _ = self._requesting[peer]
self._partial.append((index, offset, sha1))
del self._requesting[peer]
def peer_unchoked(self, peer):
logger.debug("Peer {} unchoked".format(str(peer.addr())))
if peer in self._interested:
self._request(peer)
def peer_sent_block(self, peer, index, begin, buf):
if peer not in self._requesting:
# If a peer is very slow in responding, a block could come after
# it has timed out. Just ignore the data at this point and
# ignore the slow peer
logger.debug("Received block from peer {} which has timed out"
.format(str(peer.addr())))
return
piece, received_bytes, sha1, _, _ = self._requesting[peer]
if piece == index and begin == received_bytes:
# When the next expected block is received, update the hash value
# and write the block to file
sha1.update(buf)
self._filemanager.write_block(index, begin, buf)
self._requesting[peer] = (piece, received_bytes + len(buf),
sha1, self._tick, 0)
if received_bytes + len(buf) < self._length_of_piece(index):
# Request the next block in the piece
self._request(peer)
else:
# On receipt of the last block in the piece, verify the hash
# and update the records to reflect receipt of the piece
if sha1.digest() == self._metainfo.piece_hash(index):
logger.info("Successfully got piece {} from {}"
.format(index, str(peer.addr())))
del self._needed[index]
percent = 100 * (1 - (len(self._needed) /
float(self._metainfo.num_pieces)))
print "{0}: Downloaded {1:1.4f}%".format(self._filename,
percent)
self._have[index] = 1
else:
logger.info("Unsuccessfully got piece {} from {}"
.format(index, str(peer.addr())))
del self._requesting[peer]
if self._needed != {}:
# Try to find another piece for this peer to get
self._check_interest(peer)
else:
logger.info("Successfully downloaded entire torrent {}"
.format(self._filename))
self._client.download_complete(self._filename)
def peer_interested(self, peer):
pass
def peer_not_interested(self, peer):
pass
def peer_request(self, peer, index, begin, length):
pass
def peer_canceled(self, peer, index, begin, length):
pass
# Reactor callback
def timer_event(self):
self._reactor.schedule_timer(_TIMER_INTERVAL, self.timer_event)
self._tick += 1
# For any peers that have been interested but unchoked for an
# excessive period of time, stop being interested, free up assigned
# piece and connect to another peer
for peer, (_, _, _, tick) in self._interested.items():
if tick + 4 == self._tick:
logger.debug("Timed out on interest for peer {}"
.format(str(peer.addr())))
peer.not_interested()
del self._interested[peer]
self._connect_to_peers(1)
# For any peer that has an outstanding request for an excessive period
# of time, resend the request message in case it got lost or is being
# ignored
for peer, (index, offset, sha1, tick, retries) \
in self._requesting.items():
if tick + 5 == self._tick:
logger.debug("Timed out on request for peer {}"
.format(str(peer.addr())))
if retries < _MAX_RETRIES:
self._requesting[peer] = (index, offset, sha1,
self._tick, retries+1)
self._request(peer)
else:
self._partial.append((index, offset, sha1))
del self._requesting[peer]
peer.not_interested()
self._connect_to_peers(1)
def react_external(self, event):
distance = self.compute_distance(event.location)
alert = Alert(event, distance)
Reactor.react(alert)
def react_internal(self, event):
alert = Alert(event)
Reactor.react(alert)
def run_main(listen_fd):
try:
ev_fd = Reactor()
ev_fd.register(listen_fd.fileno(), ev_fd.EV_IN | ev_fd.EV_DISCONNECTED)
except select.error, msg:
logger.error(msg)
class PathFinder:
def __init__(self,
carbon_only=False,
pruning_method=None,
ignore_chirality=True,
use_antimotifs=True,
outstream=sys.stderr,
reaction_database_fname="../rec/reaction_templates.dat"):
self.carbon_only = carbon_only
self.ignore_chirality = ignore_chirality
self.use_antimotifs = use_antimotifs
self.pruning_method = pruning_method
self.outstream = outstream
self.reaction_database_fname = reaction_database_fname
self.reactor = Reactor(
carbon_only=self.carbon_only,
ignore_chirality=self.ignore_chirality,
use_antimotifs=self.use_antimotifs,
reaction_database_fname=self.reaction_database_fname)
def balance_reaction(self, substrate, product):
""" Balances the reaction (by counting atoms)
"""
atom_gap = compound2graph(substrate).node_bag() - compounds2graph(
product).node_bag()
extra_bag = bag.Bag()
extra_bag['CO2'] = atom_gap['C']
extra_bag['H2O'] = atom_gap['O'] + atom_gap['N'] - 2 * atom_gap['C']
extra_bag['PO3'] = atom_gap['PO3']
for (atom, count) in atom_gap.itercounts():
if (not atom in ['C', 'O', 'N', 'PO3'] and count != 0):
raise Exception(
"cannot balance the number of '%s' atoms, between %s and %s"
% (atom, substrate, product))
for (metabolite, count) in extra_bag.itercounts():
if (count > 0):
product += (" + " + metabolite) * count
if (count < 0):
substrate += (" + " + metabolite) * (-count)
return (substrate, product)
def verify_hash(self, hash):
"""Returns True iff the hash passes a basic test (based on general requirements for pathways)
This method is used for pruning the search tree.
"""
if (
self.pruning_method == 'PP'
): # this method has the same assumptions as Melendez-Hevia's paper about the pentose phosephate cycle
for (nodes, bonds) in parse_hash(
hash): # check each of the molecules in the hash
node_bag = bag.Bag()
for atom in nodes:
(base_atom, valence, hydrogens, charge,
chirality) = parse_atom(atom)
node_bag[base_atom] += 1
if (node_bag['C']
in [1, 2]): # this is a 1 or 2 carbon sugar - invalid!
return False
elif (node_bag['C'] > 0 and node_bag['PO3']
== 0): # this is a unphosphorylated sugar - invalid!
return False
elif (node_bag['C'] == 0
): # this is not a sugar (might be PO3 or H2O) - valid!
pass
else: # this is a phosphorylated sugar with at least 3 carbons - valid!
pass
return True
def prune_product_list(self, prod_list):
unique_substrate_product_pairs = set([])
verified_list = []
count_failed_verification = 0
count_hash_duplications = 0
for (h_substrate, G_product, rid, mapping) in prod_list:
h_product = G_product.hash(ignore_chirality=self.ignore_chirality)
if (not self.verify_hash(h_product)):
count_failed_verification += 1
elif ((h_substrate, h_product) in unique_substrate_product_pairs):
count_hash_duplications += 1
else:
verified_list.append((h_substrate, G_product, rid, mapping))
unique_substrate_product_pairs.add((h_substrate, h_product))
return verified_list
def generate_new_compounds(self,
compounds,
write_progress_bar=True,
backward=False):
""" Produce a list of all the new products that can be derived from the given compounds
direction can be: "both", "forward", "backward"
"""
new_product_list = []
total_count = len(compounds)
if (write_progress_bar):
n_dots = 80
n_dots_written = 0
self.outstream.write("\t\t- [")
counter = 0
for (h, G) in compounds.iteritems():
if (write_progress_bar):
dots_to_write = (counter * n_dots /
total_count) - n_dots_written
self.outstream.write("." * dots_to_write)
n_dots_written += dots_to_write
counter += 1
for (G_product, rid,
mapping) in self.reactor.apply_all_reactions(G, backward):
new_product_list.append((h, G_product, rid, mapping))
if (write_progress_bar):
self.outstream.write("." * (n_dots - n_dots_written) + "]\n")
return self.prune_product_list(new_product_list)
def expand_tree(self,
compound_map,
set_of_processed_compounds,
reaction_tree=None,
backward=False):
""" Expands the tree of compounds by one level
* reaction_tree is a multi-map, where the keys are compound hashes, and the values are
lists if 3-tuples, containing (predecessor hash, reaction_id, reaction_mapping)
describing the reaction from the predecessor to the current compound (in the key).
* compound_map is a map from hashes to ChemGraphs, because we need the graph in order
to apply all reactions to it. We discard it in the next round of expand_tree to
save memory.
* set_of_processed_compounds is a set of all the hashes that have been processed,
i.e. entered the compound_map in an earlier stage. We need to know them in order
not to 'expand' the same compound twice. Note the it is common for both
substrate and product compound maps.
"""
new_compound_list = self.generate_new_compounds(compound_map, backward)
compound_map.clear()
for (h_predecessor, G, reaction_id, mapping) in new_compound_list:
h_compound = G.hash(ignore_chirality=self.ignore_chirality)
if (h_compound not in set_of_processed_compounds):
compound_map[h_compound] = G
set_of_processed_compounds.add(h_compound)
if (reaction_tree != None):
# add the reaction to the hash
if (not reaction_tree.has_key(h_compound)):
reaction_tree[h_compound] = []
reaction_tree[h_compound] += [(h_predecessor, reaction_id,
mapping)]
def reaction_DFS(self, reaction_tree, h, depth):
""" Returns all the pathways that lead from a seed to the given compound (h)
reaction_tree - is a dictionary mapping compounds to the reactions that create them
h - is a hash of the compound to be created
depth - will be the maximum number of reactions in the returned paths.
"""
if (
depth < 0
): # this means we exceeded the allowed depth, without reaching a seed, i.e. dead-end
return []
pathways = []
for (h_predecessor, rid, map) in reaction_tree[h]:
if (h_predecessor == None):
pathways += [
[h]
] # this means 'h' can be creating from nothing, i.e. it is a seed
else:
for pathway in self.reaction_DFS(reaction_tree, h_predecessor,
depth - 1):
pathways += [pathway + [(rid, map)]]
return pathways
def find_shortest_pathway(self,
substrates,
products,
max_levels=4,
stop_after_first_solution=False):
"""input is a list of substrates and a list of products
output is the shortest path between any of the substrates to any of the products
"""
# reaction_tree is a dictionary mapping each compound (represented by its hash) to a list,
# the first value is the hash of the same compound with the ignore-attributes flag on
# the second value in the list is the depth of the compound in the tree
# the following members in the list are (predecessor, reaction) pairs, i.e.
# predecessor - the substrate in the reaction to create this product
# reaction - the reaction for creating the product from the substrate
if (max_levels < 1):
raise Exception("max_levels must be at least 1")
# a map containing only the new compounds (from both trees), mapping hashes to ChemGraphs
# in order to save memory, only hashes of old compounds are saved, and the ChemGraphs discarded
original_compound_map = {}
set_of_processed_compounds = set()
substrate_reaction_tree = {}
product_reaction_tree = {}
current_substrate_map = {}
current_product_map = {}
for G in substrates:
G_temp = G.clone()
if (self.ignore_chirality):
G_temp.reset_chiralities()
h = G_temp.hash(ignore_chirality=self.ignore_chirality)
substrate_reaction_tree[h] = [(None, -1, [])]
original_compound_map[h] = G_temp
current_substrate_map[h] = G_temp
print >> self.outstream, "Substrate: " + h
for G in products:
G_temp = G.clone()
if (self.ignore_chirality):
G_temp.reset_chiralities()
h = G_temp.hash(ignore_chirality=self.ignore_chirality)
product_reaction_tree[h] = [(None, -1, [])]
original_compound_map[h] = G_temp
current_product_map[h] = G_temp
print >> self.outstream, "Product: " + h
time_per_compound = 0
substrate_map_depth = 0
product_map_depth = 0
while (substrate_map_depth + product_map_depth < max_levels):
print >> self.outstream, "\t*** Level #%d" % (
substrate_map_depth + product_map_depth + 1),
begin_time = time.time()
if (substrate_map_depth <= product_map_depth):
num_current_compounds = len(current_substrate_map)
print >> self.outstream, "- estimated time: %.2f sec" % (
time_per_compound * len(current_substrate_map))
self.expand_tree(current_substrate_map,
set_of_processed_compounds,
reaction_tree=substrate_reaction_tree,
backward=False)
substrate_map_depth += 1
else:
num_current_compounds = len(current_product_map)
print >> self.outstream, "- estimated time: %.2f sec" % (
time_per_compound * len(current_product_map))
self.expand_tree(current_product_map,
set_of_processed_compounds,
reaction_tree=product_reaction_tree,
backward=True)
product_map_depth += 1
if (num_current_compounds == 0):
print >> self.outstream, "Reached a dead end, no new compounds can be created..."
return (original_compound_map, [], -1)
elapsed_time = float(time.time() - begin_time)
time_per_compound = elapsed_time / num_current_compounds
print >> self.outstream, "\t\t- %d substrates + %d products" % (
len(substrate_reaction_tree), len(product_reaction_tree))
bridging_compounds = set(substrate_reaction_tree.keys()) & set(
product_reaction_tree.keys())
if (stop_after_first_solution and len(bridging_compounds) > 0):
break
if (bridging_compounds != set()):
print >> self.outstream, "\t*** found %d bridging compounds" % len(
bridging_compounds)
possible_pathways = []
# for each bridging compound, find the pair of pathways list leading to it
# one from the substrate and one from the product
for h_bridge in bridging_compounds:
# gather all the possible pathways that lead from the substrates
# to the bridging compound, using the substrate reaction-tree
substrate_pathways = self.reaction_DFS(substrate_reaction_tree,
h_bridge,
substrate_map_depth)
# the same but for the products reaction-tree
product_pathways = self.reaction_DFS(product_reaction_tree,
h_bridge,
product_map_depth)
possible_pathways.append(
(substrate_pathways, product_pathways, h_bridge))
return (original_compound_map, possible_pathways,
substrate_map_depth + product_map_depth)
else:
print >> self.outstream, "No path was found, even after %d levels" % max_levels
return (original_compound_map, [], -1)
def find_distance(self, substrates, products, max_levels=4):
"""input is a list of substrates and a list of products
output is the shortest path between any of the substrates to any of the products
"""
# reaction_tree is a dictionary mapping each compound (represented by its hash) to a list,
# the first value is the hash of the same compound with the ignore-attributes flag on
# the second value in the list is the depth of the compound in the tree
# the following members in the list are (predecessor, reaction) pairs, i.e.
# predecessor - the substrate in the reaction to create this product
# reaction - the reaction for creating the product from the substrate
if (max_levels < 1):
raise Exception("max_levels must be at least 1")
set_of_processed_substrates = set()
set_of_processed_products = set()
current_substrate_map = {}
current_product_map = {}
for G in substrates:
G_temp = G.clone()
if (self.ignore_chirality):
G_temp.reset_chiralities()
h = G_temp.hash(ignore_chirality=self.ignore_chirality)
set_of_processed_substrates.add(h)
current_substrate_map[h] = G_temp
print >> self.outstream, "Substrate: " + h
for G in products:
G_temp = G.clone()
if (self.ignore_chirality):
G_temp.reset_chiralities()
h = G_temp.hash(ignore_chirality=self.ignore_chirality)
set_of_processed_products.add(h)
current_product_map[h] = G_temp
print >> self.outstream, "Product: " + h
time_per_compound = 0
for level in range(1, max_levels + 1):
print >> self.outstream, "\t*** Level #%d" % level,
begin_time = time.time()
if (level % 2 == 0):
print >> self.outstream, "- estimated time: %.2f sec" % (
time_per_compound * len(current_substrate_map))
self.expand_tree(current_substrate_map,
set_of_processed_substrates,
backward=False)
num_current_compounds = len(current_substrate_map)
else:
print >> self.outstream, "- estimated time: %.2f sec" % (
time_per_compound * len(current_product_map))
self.expand_tree(current_product_map,
set_of_processed_products,
backward=True)
num_current_compounds = len(current_product_map)
if (num_current_compounds == 0):
print >> self.outstream, "Reached a dead end, no new compounds can be created..."
return -1
elapsed_time = float(time.time() - begin_time)
time_per_compound = elapsed_time / num_current_compounds
print >> self.outstream, "\t\t- %d substrates + %d products" % (
len(set_of_processed_substrates),
len(set_of_processed_products))
bridging_compounds = set_of_processed_substrates & set_of_processed_products
if (len(bridging_compounds) > 0):
print >> self.outstream, "\t*** found %d bridging compounds" % len(
bridging_compounds)
return level
print >> self.outstream, "No path was found, even after %d levels" % max_levels
return -1
def pathway2text(self, G_subs, expanded_reaction_list):
num_reactions = len(expanded_reaction_list)
num_compounds = len(expanded_reaction_list) + 1
i = 0
G = G_subs.clone()
rid = None
s = ""
while True:
if (i == len(expanded_reaction_list)):
break
(rid, mapping, reaction_list) = expanded_reaction_list[i]
s += str(G) + " (" + graph2compound(
G, self.ignore_chirality) + ") - " + str(rid) + " : " + str(
mapping) + "\n"
for reaction in reaction_list:
s += "\t" + str(G) + " (" + graph2compound(
G, self.ignore_chirality) + ") - " + str(
reaction.tostring(mapping)) + "\n"
reaction.apply(G, mapping)
G.update_attributes()
if (self.ignore_chirality):
G.reset_chiralities()
i += 1
s += str(G) + " (" + graph2compound(G, self.ignore_chirality) + ")\n"
return (s, G)
def pathway2svg(self,
G_subs,
expanded_reaction_list,
size_x=300,
size_y=150,
font_size=10):
num_reactions = len(expanded_reaction_list)
num_compounds = len(expanded_reaction_list) + 1
gap_size_x = 100
gap_size_y = 15
scene = Scene()
# first add all the compounds to the graph
i = 0
curr_x = 0
G = G_subs.clone()
rid = None
while True:
if (rid != 'hidden'):
scene.add(G.svg(Scene(size_x, size_y, font_size)),
offset=(curr_x, gap_size_y))
curr_x += size_x
if (i == len(expanded_reaction_list)):
break
(rid, mapping, reaction_list) = expanded_reaction_list[i]
for reaction in reaction_list:
reaction.apply(G, mapping)
G.update_attributes()
if (self.ignore_chirality):
G.reset_chiralities()
if (rid != 'hidden'):
# draw the arrows for the direction of the reactions
scene.add(
ChemicalArrow((curr_x + 30, size_y / 2),
(curr_x + 70, size_y / 2),
stroke_width=2))
scene.add(
Text((curr_x, size_y / 2 - 20),
self.reactor.get_reaction_name(rid),
font_size,
fill_color=red))
scene.add(
Text((curr_x, size_y / 2 + 25),
str(mapping),
font_size,
fill_color=red))
curr_x += gap_size_x
# calculate the cost of this reaction
i += 1
scene.justify()
return (scene, G)
def expand_rid_list(self, rid_list):
""" Attach the list of subreaction corresponding to each Reaction ID in the list
"""
return [(rid, map, self.reactor.get_reaction_list(rid))
for (rid, map) in rid_list]
def apply_rid_list(self, G, rid_list):
for (rid, map) in rid_list:
subreaction_list = self.reactor.get_reaction_list(rid)
for subreaction in subreaction_list:
subreaction.apply(G, map)
G.update_attributes()
return G
def reverse_rid_list(self, rid_list):
return [(self.reactor.reverse_reaction(rid), map)
for (rid, map) in reversed(rid_list)]
def get_all_possible_scenes(self, original_compound_map,
possible_pathways):
def compare_graph_to_hash(G1, h2):
h1 = G1.hash(ignore_chirality=self.ignore_chirality)
return compare_hashes(h1, h2, self.ignore_chirality)
""" returns a list of pairs of (cost, scene) which is a graphical representation of each possible pathway
"""
scene_list = []
# prepare the SVG scenes for all the possible pathways, and calculate their cost
for (substrate_pathways, product_pathways,
h_bridge) in possible_pathways:
# print >> self.outstream, "Bridge: " + h_bridge
for subs_path in substrate_pathways:
G_subs = original_compound_map[subs_path[0]]
subs_reaction_list = self.expand_rid_list(subs_path[1:])
try:
(subs_log,
G_last_subs) = self.pathway2text(G_subs.clone(),
subs_reaction_list)
except ReactionException, msg:
print >> self.outstream, msg
continue
# print >> self.outstream, "*** SUBSTRATE LOG: \n", subs_log
# if (G_last_subs.hash(ignore_chirality=self.ignore_chirality) != h_bridge):
if (compare_graph_to_hash(G_last_subs, h_bridge) != 0):
print "ERROR:"
print "subs: ", G_subs.hash(
ignore_chirality=self.ignore_chirality)
print "last_subs: ", G_last_subs.hash(
ignore_chirality=self.ignore_chirality)
print "bridge: ", h_bridge
sys.exit(-1)
print >> self.outstream, "G_last_subs != G_bridge, check the DFS function..."
raise Exception(
"G_last_subs != G_bridge, check the DFS function...")
for prod_path in product_pathways:
G_prod = original_compound_map[prod_path[0]]
prod_reaction_list = self.expand_rid_list(prod_path[1:])
reverse_prod_reaction_list = self.expand_rid_list(
self.reverse_rid_list(prod_path[1:]))
try:
(prod_log, G_last_prod) = self.pathway2text(
G_prod.clone(), prod_reaction_list)
except ReactionException, msg:
print >> self.outstream, msg
continue
# print >> self.outstream, "*** PRODUCT LOG: \n", prod_log
# if (G_last_prod.hash(ignore_chirality=self.ignore_chirality) != h_bridge):
if (compare_graph_to_hash(G_last_prod, h_bridge) != 0):
print "ERROR:"
print "subs: ", G_subs.hash(
ignore_chirality=self.ignore_chirality)
print "prod: ", G_prod.hash(
ignore_chirality=self.ignore_chirality)
print "last_prod: ", G_last_prod.hash(
ignore_chirality=self.ignore_chirality)
print "bridge: ", h_bridge
sys.exit(-1)
print >> self.outstream, "G_last_prod != G_bridge, check the DFS function..."
raise Exception(
"G_last_prod != G_bridge, check the DFS function..."
)
perm_reaction = self.reactor.get_permutation_reaction(
G_last_subs, G_last_prod)
full_reaction_list = subs_reaction_list + [
perm_reaction
] + reverse_prod_reaction_list
try:
(pathway_scene,
G_last) = self.pathway2svg(G_subs, full_reaction_list)
except ReactionException, msg:
print >> self.outstream, msg
continue
cost = len(subs_reaction_list) + len(
reverse_prod_reaction_list)
scene_list.append((cost, pathway_scene))
#!/usr/bin/python
# -*- coding:utf-8 -*-
__author__ = '*****@*****.**'
from reactor import Reactor
if __name__ == '__main__':
reactor = Reactor()
reactor.server_forever()
def __init__(self, sock):
self._reactor = Reactor()
self._receiver = None
self._output = []
self._socket = sock
self._reactor.register_for_read_events(self)
class PathFinder:
def __init__(self, carbon_only=False, pruning_method=None, ignore_chirality=True, use_antimotifs=True, outstream=sys.stderr, reaction_database_fname="../rec/reaction_templates.dat"):
self.carbon_only = carbon_only
self.ignore_chirality = ignore_chirality
self.use_antimotifs = use_antimotifs
self.pruning_method = pruning_method
self.outstream = outstream
self.reaction_database_fname = reaction_database_fname
self.reactor = Reactor(carbon_only=self.carbon_only, ignore_chirality=self.ignore_chirality, use_antimotifs=self.use_antimotifs, reaction_database_fname=self.reaction_database_fname)
def balance_reaction(self, substrate, product):
""" Balances the reaction (by counting atoms)
"""
atom_gap = compound2graph(substrate).node_bag() - compounds2graph(product).node_bag()
extra_bag = bag.Bag()
extra_bag['CO2'] = atom_gap['C']
extra_bag['H2O'] = atom_gap['O'] + atom_gap['N'] - 2 * atom_gap['C']
extra_bag['PO3'] = atom_gap['PO3']
for (atom, count) in atom_gap.itercounts():
if (not atom in ['C', 'O', 'N', 'PO3'] and count != 0):
raise Exception("cannot balance the number of '%s' atoms, between %s and %s" % (atom, substrate, product))
for (metabolite, count) in extra_bag.itercounts():
if (count > 0):
product += (" + " + metabolite) * count
if (count < 0):
substrate += (" + " + metabolite) * (-count)
return (substrate, product)
def verify_hash(self, hash):
"""Returns True iff the hash passes a basic test (based on general requirements for pathways)
This method is used for pruning the search tree.
"""
if (self.pruning_method == 'PP'): # this method has the same assumptions as Melendez-Hevia's paper about the pentose phosephate cycle
for (nodes, bonds) in parse_hash(hash): # check each of the molecules in the hash
node_bag = bag.Bag()
for atom in nodes:
(base_atom, valence, hydrogens, charge, chirality) = parse_atom(atom)
node_bag[base_atom] += 1
if (node_bag['C'] in [1,2]): # this is a 1 or 2 carbon sugar - invalid!
return False
elif (node_bag['C'] > 0 and node_bag['PO3'] == 0): # this is a unphosphorylated sugar - invalid!
return False
elif (node_bag['C'] == 0): # this is not a sugar (might be PO3 or H2O) - valid!
pass
else: # this is a phosphorylated sugar with at least 3 carbons - valid!
pass
return True
def prune_product_list(self, prod_list):
unique_substrate_product_pairs = set([])
verified_list = []
count_failed_verification = 0
count_hash_duplications = 0
for (h_substrate, G_product, rid, mapping) in prod_list:
h_product = G_product.hash(ignore_chirality=self.ignore_chirality)
if (not self.verify_hash(h_product)):
count_failed_verification += 1
elif ((h_substrate, h_product) in unique_substrate_product_pairs):
count_hash_duplications += 1
else:
verified_list.append((h_substrate, G_product, rid, mapping))
unique_substrate_product_pairs.add((h_substrate, h_product))
return verified_list
def generate_new_compounds(self, compounds, write_progress_bar=True, backward=False):
""" Produce a list of all the new products that can be derived from the given compounds
direction can be: "both", "forward", "backward"
"""
new_product_list = []
total_count = len(compounds)
if (write_progress_bar):
n_dots = 80
n_dots_written = 0
self.outstream.write("\t\t- [")
counter = 0
for (h, G) in compounds.iteritems():
if (write_progress_bar):
dots_to_write = (counter * n_dots / total_count) - n_dots_written
self.outstream.write("." * dots_to_write)
n_dots_written += dots_to_write
counter += 1
for (G_product, rid, mapping) in self.reactor.apply_all_reactions(G, backward):
new_product_list.append((h, G_product, rid, mapping))
if (write_progress_bar):
self.outstream.write("." * (n_dots - n_dots_written) + "]\n")
return self.prune_product_list(new_product_list)
def expand_tree(self, compound_map, set_of_processed_compounds, reaction_tree=None, backward=False):
""" Expands the tree of compounds by one level
* reaction_tree is a multi-map, where the keys are compound hashes, and the values are
lists if 3-tuples, containing (predecessor hash, reaction_id, reaction_mapping)
describing the reaction from the predecessor to the current compound (in the key).
* compound_map is a map from hashes to ChemGraphs, because we need the graph in order
to apply all reactions to it. We discard it in the next round of expand_tree to
save memory.
* set_of_processed_compounds is a set of all the hashes that have been processed,
i.e. entered the compound_map in an earlier stage. We need to know them in order
not to 'expand' the same compound twice. Note the it is common for both
substrate and product compound maps.
"""
new_compound_list = self.generate_new_compounds(compound_map, backward)
compound_map.clear()
for (h_predecessor, G, reaction_id, mapping) in new_compound_list:
h_compound = G.hash(ignore_chirality=self.ignore_chirality)
if (h_compound not in set_of_processed_compounds):
compound_map[h_compound] = G
set_of_processed_compounds.add(h_compound)
if (reaction_tree != None):
# add the reaction to the hash
if (not reaction_tree.has_key(h_compound)):
reaction_tree[h_compound] = []
reaction_tree[h_compound] += [(h_predecessor, reaction_id, mapping)]
def reaction_DFS(self, reaction_tree, h, depth):
""" Returns all the pathways that lead from a seed to the given compound (h)
reaction_tree - is a dictionary mapping compounds to the reactions that create them
h - is a hash of the compound to be created
depth - will be the maximum number of reactions in the returned paths.
"""
if (depth < 0): # this means we exceeded the allowed depth, without reaching a seed, i.e. dead-end
return []
pathways = []
for (h_predecessor, rid, map) in reaction_tree[h]:
if (h_predecessor == None):
pathways += [[h]] # this means 'h' can be creating from nothing, i.e. it is a seed
else:
for pathway in self.reaction_DFS(reaction_tree, h_predecessor, depth-1):
pathways += [pathway + [(rid, map)]]
return pathways
def find_shortest_pathway(self, substrates, products, max_levels=4, stop_after_first_solution=False):
"""input is a list of substrates and a list of products
output is the shortest path between any of the substrates to any of the products
"""
# reaction_tree is a dictionary mapping each compound (represented by its hash) to a list,
# the first value is the hash of the same compound with the ignore-attributes flag on
# the second value in the list is the depth of the compound in the tree
# the following members in the list are (predecessor, reaction) pairs, i.e.
# predecessor - the substrate in the reaction to create this product
# reaction - the reaction for creating the product from the substrate
if (max_levels < 1):
raise Exception("max_levels must be at least 1")
# a map containing only the new compounds (from both trees), mapping hashes to ChemGraphs
# in order to save memory, only hashes of old compounds are saved, and the ChemGraphs discarded
original_compound_map = {}
set_of_processed_compounds = set()
substrate_reaction_tree = {}
product_reaction_tree = {}
current_substrate_map = {}
current_product_map = {}
for G in substrates:
G_temp = G.clone()
if (self.ignore_chirality):
G_temp.reset_chiralities()
h = G_temp.hash(ignore_chirality=self.ignore_chirality)
substrate_reaction_tree[h] = [(None, -1, [])]
original_compound_map[h] = G_temp
current_substrate_map[h] = G_temp
print >> self.outstream, "Substrate: " + h
for G in products:
G_temp = G.clone()
if (self.ignore_chirality):
G_temp.reset_chiralities()
h = G_temp.hash(ignore_chirality=self.ignore_chirality)
product_reaction_tree[h] = [(None, -1, [])]
original_compound_map[h] = G_temp
current_product_map[h] = G_temp
print >> self.outstream, "Product: " + h
time_per_compound = 0
substrate_map_depth = 0
product_map_depth = 0
while (substrate_map_depth + product_map_depth < max_levels):
print >> self.outstream, "\t*** Level #%d" % (substrate_map_depth + product_map_depth + 1),
begin_time = time.time()
if (substrate_map_depth <= product_map_depth):
num_current_compounds = len(current_substrate_map)
print >> self.outstream, "- estimated time: %.2f sec" % (time_per_compound * len(current_substrate_map))
self.expand_tree(current_substrate_map, set_of_processed_compounds, reaction_tree=substrate_reaction_tree, backward=False)
substrate_map_depth += 1
else:
num_current_compounds = len(current_product_map)
print >> self.outstream, "- estimated time: %.2f sec" % (time_per_compound * len(current_product_map))
self.expand_tree(current_product_map, set_of_processed_compounds, reaction_tree=product_reaction_tree, backward=True)
product_map_depth += 1
if (num_current_compounds == 0):
print >> self.outstream, "Reached a dead end, no new compounds can be created..."
return (original_compound_map, [], -1)
elapsed_time = float(time.time() - begin_time)
time_per_compound = elapsed_time / num_current_compounds
print >> self.outstream, "\t\t- %d substrates + %d products" % (len(substrate_reaction_tree), len(product_reaction_tree))
bridging_compounds = set(substrate_reaction_tree.keys()) & set(product_reaction_tree.keys())
if (stop_after_first_solution and len(bridging_compounds) > 0):
break
if (bridging_compounds != set()):
print >> self.outstream, "\t*** found %d bridging compounds" % len(bridging_compounds)
possible_pathways = []
# for each bridging compound, find the pair of pathways list leading to it
# one from the substrate and one from the product
for h_bridge in bridging_compounds:
# gather all the possible pathways that lead from the substrates
# to the bridging compound, using the substrate reaction-tree
substrate_pathways = self.reaction_DFS(substrate_reaction_tree, h_bridge, substrate_map_depth)
# the same but for the products reaction-tree
product_pathways = self.reaction_DFS(product_reaction_tree, h_bridge, product_map_depth)
possible_pathways.append((substrate_pathways, product_pathways, h_bridge))
return (original_compound_map, possible_pathways, substrate_map_depth + product_map_depth)
else:
print >> self.outstream, "No path was found, even after %d levels" % max_levels
return (original_compound_map, [], -1)
def find_distance(self, substrates, products, max_levels=4):
"""input is a list of substrates and a list of products
output is the shortest path between any of the substrates to any of the products
"""
# reaction_tree is a dictionary mapping each compound (represented by its hash) to a list,
# the first value is the hash of the same compound with the ignore-attributes flag on
# the second value in the list is the depth of the compound in the tree
# the following members in the list are (predecessor, reaction) pairs, i.e.
# predecessor - the substrate in the reaction to create this product
# reaction - the reaction for creating the product from the substrate
if (max_levels < 1):
raise Exception("max_levels must be at least 1")
set_of_processed_substrates = set()
set_of_processed_products = set()
current_substrate_map = {}
current_product_map = {}
for G in substrates:
G_temp = G.clone()
if (self.ignore_chirality):
G_temp.reset_chiralities()
h = G_temp.hash(ignore_chirality=self.ignore_chirality)
set_of_processed_substrates.add(h)
current_substrate_map[h] = G_temp
print >> self.outstream, "Substrate: " + h
for G in products:
G_temp = G.clone()
if (self.ignore_chirality):
G_temp.reset_chiralities()
h = G_temp.hash(ignore_chirality=self.ignore_chirality)
set_of_processed_products.add(h)
current_product_map[h] = G_temp
print >> self.outstream, "Product: " + h
time_per_compound = 0
for level in range(1, max_levels+1):
print >> self.outstream, "\t*** Level #%d" % level,
begin_time = time.time()
if (level % 2 == 0):
print >> self.outstream, "- estimated time: %.2f sec" % (time_per_compound * len(current_substrate_map))
self.expand_tree(current_substrate_map, set_of_processed_substrates, backward=False)
num_current_compounds = len(current_substrate_map)
else:
print >> self.outstream, "- estimated time: %.2f sec" % (time_per_compound * len(current_product_map))
self.expand_tree(current_product_map, set_of_processed_products, backward=True)
num_current_compounds = len(current_product_map)
if (num_current_compounds == 0):
print >> self.outstream, "Reached a dead end, no new compounds can be created..."
return -1
elapsed_time = float(time.time() - begin_time)
time_per_compound = elapsed_time / num_current_compounds
print >> self.outstream, "\t\t- %d substrates + %d products" % (len(set_of_processed_substrates), len(set_of_processed_products))
bridging_compounds = set_of_processed_substrates & set_of_processed_products
if (len(bridging_compounds) > 0):
print >> self.outstream, "\t*** found %d bridging compounds" % len(bridging_compounds)
return level
print >> self.outstream, "No path was found, even after %d levels" % max_levels
return -1
def pathway2text(self, G_subs, expanded_reaction_list):
num_reactions = len(expanded_reaction_list)
num_compounds = len(expanded_reaction_list) + 1
i = 0
G = G_subs.clone()
rid = None
s = ""
while True:
if (i == len(expanded_reaction_list)):
break
(rid, mapping, reaction_list) = expanded_reaction_list[i]
s += str(G) + " (" + graph2compound(G, self.ignore_chirality) + ") - " + str(rid) + " : " + str(mapping) + "\n"
for reaction in reaction_list:
s += "\t" + str(G) + " (" + graph2compound(G, self.ignore_chirality) + ") - " + str(reaction.tostring(mapping)) + "\n"
reaction.apply(G, mapping)
G.update_attributes()
if (self.ignore_chirality):
G.reset_chiralities()
i += 1
s += str(G) + " (" + graph2compound(G, self.ignore_chirality) + ")\n"
return (s, G)
def pathway2svg(self, G_subs, expanded_reaction_list, size_x=300, size_y=150, font_size=10):
num_reactions = len(expanded_reaction_list)
num_compounds = len(expanded_reaction_list) + 1
gap_size_x = 100
gap_size_y = 15
scene = Scene()
# first add all the compounds to the graph
i = 0
curr_x = 0
G = G_subs.clone()
rid = None
while True:
if (rid != 'hidden'):
scene.add(G.svg(Scene(size_x, size_y, font_size)), offset=(curr_x, gap_size_y))
curr_x += size_x
if (i == len(expanded_reaction_list)):
break
(rid, mapping, reaction_list) = expanded_reaction_list[i]
for reaction in reaction_list:
reaction.apply(G, mapping)
G.update_attributes()
if (self.ignore_chirality):
G.reset_chiralities()
if (rid != 'hidden'):
# draw the arrows for the direction of the reactions
scene.add(ChemicalArrow((curr_x + 30, size_y / 2), (curr_x + 70, size_y / 2), stroke_width=2))
scene.add(Text((curr_x, size_y / 2 - 20), self.reactor.get_reaction_name(rid), font_size, fill_color=red))
scene.add(Text((curr_x, size_y / 2 + 25), str(mapping), font_size, fill_color=red))
curr_x += gap_size_x
# calculate the cost of this reaction
i += 1
scene.justify()
return (scene, G)
def expand_rid_list(self, rid_list):
""" Attach the list of subreaction corresponding to each Reaction ID in the list
"""
return [(rid, map, self.reactor.get_reaction_list(rid)) for (rid, map) in rid_list]
def apply_rid_list(self, G, rid_list):
for (rid, map) in rid_list:
subreaction_list = self.reactor.get_reaction_list(rid)
for subreaction in subreaction_list:
subreaction.apply(G, map)
G.update_attributes()
return G
def reverse_rid_list(self, rid_list):
return [(self.reactor.reverse_reaction(rid), map) for (rid, map) in reversed(rid_list)]
def get_all_possible_scenes(self, original_compound_map, possible_pathways):
def compare_graph_to_hash(G1, h2):
h1 = G1.hash(ignore_chirality=self.ignore_chirality)
return compare_hashes(h1, h2, self.ignore_chirality)
""" returns a list of pairs of (cost, scene) which is a graphical representation of each possible pathway
"""
scene_list = []
# prepare the SVG scenes for all the possible pathways, and calculate their cost
for (substrate_pathways, product_pathways, h_bridge) in possible_pathways:
# print >> self.outstream, "Bridge: " + h_bridge
for subs_path in substrate_pathways:
G_subs = original_compound_map[subs_path[0]]
subs_reaction_list = self.expand_rid_list(subs_path[1:])
try:
(subs_log, G_last_subs) = self.pathway2text(G_subs.clone(), subs_reaction_list)
except ReactionException, msg:
print >> self.outstream, msg
continue
# print >> self.outstream, "*** SUBSTRATE LOG: \n", subs_log
# if (G_last_subs.hash(ignore_chirality=self.ignore_chirality) != h_bridge):
if (compare_graph_to_hash(G_last_subs, h_bridge) != 0):
print "ERROR:"
print "subs: ", G_subs.hash(ignore_chirality=self.ignore_chirality)
print "last_subs: ", G_last_subs.hash(ignore_chirality=self.ignore_chirality)
print "bridge: ", h_bridge
sys.exit(-1)
print >> self.outstream, "G_last_subs != G_bridge, check the DFS function..."
raise Exception("G_last_subs != G_bridge, check the DFS function...")
for prod_path in product_pathways:
G_prod = original_compound_map[prod_path[0]]
prod_reaction_list = self.expand_rid_list(prod_path[1:])
reverse_prod_reaction_list = self.expand_rid_list(self.reverse_rid_list(prod_path[1:]))
try:
(prod_log, G_last_prod) = self.pathway2text(G_prod.clone(), prod_reaction_list)
except ReactionException, msg:
print >> self.outstream, msg
continue
# print >> self.outstream, "*** PRODUCT LOG: \n", prod_log
# if (G_last_prod.hash(ignore_chirality=self.ignore_chirality) != h_bridge):
if (compare_graph_to_hash(G_last_prod, h_bridge) != 0):
print "ERROR:"
print "subs: ", G_subs.hash(ignore_chirality=self.ignore_chirality)
print "prod: ", G_prod.hash(ignore_chirality=self.ignore_chirality)
print "last_prod: ", G_last_prod.hash(ignore_chirality=self.ignore_chirality)
print "bridge: ", h_bridge
sys.exit(-1)
print >> self.outstream, "G_last_prod != G_bridge, check the DFS function..."
raise Exception("G_last_prod != G_bridge, check the DFS function...")
perm_reaction = self.reactor.get_permutation_reaction(G_last_subs, G_last_prod)
full_reaction_list = subs_reaction_list + [perm_reaction] + reverse_prod_reaction_list
try:
(pathway_scene, G_last) = self.pathway2svg(G_subs, full_reaction_list)
except ReactionException, msg:
print >> self.outstream, msg
continue
cost = len(subs_reaction_list) + len(reverse_prod_reaction_list)
scene_list.append((cost, pathway_scene))
def run_main(listen_fd):
try:
ev_fd = Reactor()
ev_fd.register(listen_fd.fileno(), ev_fd.EV_IN | ev_fd.EV_DISCONNECTED)
except select.error, msg:
logger.error(msg)
def __init__(self, client, filename, port, peer_id):
self._client = client
self._filename = filename
self._port = port
self._peer_id = peer_id
# _peers is a list of peers that the TorrentMgr is trying
# to communicate with
self._peers = []
# _bitfields is a dictionary mapping peers to a bitfield of the pieces
# each has
self._bitfields = {}
try:
self._metainfo = Metainfo(filename)
except (IOError, ValueError) as err:
if isinstance(err, IOError):
message = err.strerror+' ('+filename+')'
else:
message = err.message+' ('+filename+')'
logger.error(message)
raise TorrentMgrError(message)
# _have is the bitfield for this torrent. It is initialized to reflect
# which pieces are already available on disk.
self._filemgr = FileMgr(self._metainfo)
self._have = self._filemgr.have()
try:
self._tracker_proxy = TrackerProxy(self._metainfo, self._port,
self._peer_id)
except TrackerError as err:
logger.critical("Could not connect to tracker at {}"
.format(self._metainfo.announce))
logger.debug(" TrackerError: {}".format(err.message))
raise TorrentMgrError(err.message)
# _needed is a dictionary of pieces which are still needed.
# The value for each piece is a tuple of the number of peers which
# have the piece and a list of those peers.
self._needed = {piece: (0, []) for piece
in list(self._have.findall('0b0'))}
# _interested is a dictionary of peers to whom interest has been
# expressed. The value for each peer is a tuple of the piece that
# has been reserved for the peer, the number of bytes of the piece that
# have already been received, the sha1 hash of the bytes received so
# far and the value of the tick at the time interest was expressed.
self._interested = {}
# _requesting is a dictionary of peers to whom a block request has been
# made. The value for each peer is a tuple of the piece that is being
# requested, the number of bytes that have already been received, the
# shal2 hash of the bytes received so far, the value of the tick at
# the time the request was made and the number of retries that have
# been attempted
self._requesting = {}
# _partial is a list which tracks pieces that were interrupted while
# being downloaded. Each entry is a tuple containing the index of the
# piece, the number of bytes received so far and the sha1 hash of those
# bytes.
self._partial = []
self._reactor = Reactor()
self._reactor.schedule_timer(_TIMER_INTERVAL, self.timer_event)
self._tick = 1
print "Starting to serve torrent {}".format(filename)
self._connect_to_peers(20)
def first():
print(
colored(
"Testcase 1: adding two identical boxes results in only one box",
"red"))
R = Reactor()
R += Box("on x=1..3,y=1..3,z=1..3")
R += Box("on x=1..3,y=1..3,z=1..3")
assert (R.realcubes.__repr__() == "[on x=1..3,y=1..3,z=1..3]")
print(
colored(
"Testcase 2: adding various boxes that overlap but are on the edge with the first, also result in only one box",
"red"))
R = Reactor()
R += Box("on x=1..3,y=1..3,z=1..3")
print(
colored(
" 2a: check that adding sliver on x doesn't cause more boxes",
"yellow"))
R += Box("on x=1..1,y=1..3,z=1..3")
check(R.realcubes.__repr__() == "[on x=1..3,y=1..3,z=1..3]",
[R.size(), R.realcubes, [i.size() for i in R.realcubes]],
f=lambda: R.savefig())
print(
colored(
" 2c: check that adding sliver on z doesn't cause more boxes ",
"yellow"))
R += Box("on x=1..3,y=1..3,z=1..1")
check(R.realcubes.__repr__() == "[on x=1..3,y=1..3,z=1..3]",
[R.size(), R.realcubes, [i.size() for i in R.realcubes]],
f=lambda: R.savefig())
print(
colored(
" 2b: check that adding sliver on y doesn't cause more boxes",
"yellow"))
R += Box("on x=1..3,y=1..1,z=1..3")
check(R.realcubes.__repr__() == "[on x=1..3,y=1..3,z=1..3]",
[R.size(), R.realcubes, [i.size() for i in R.realcubes]],
f=lambda: R.savefig())
print(colored("Testcase 3: merge on X edge"), "red")
R = Reactor()
R += Box("on x=1..1,y=1..1,z=1..1")
R += Box("on x=2..3,y=1..1,z=1..1")
assert (R.realcubes.__repr__() == "[on x=1..3,y=1..1,z=1..1]")
print("Testcase 4: merge on Y edge")
R += Box("on x=1..3,y=2..3,z=1..1")
assert (R.realcubes.__repr__() == "[on x=1..3,y=1..3,z=1..1]")
print("Testcase 5: merge on Z edge")
R += Box("on x=1..3,y=1..3,z=2..3")
assert (R.realcubes.__repr__() == "[on x=1..3,y=1..3,z=1..3]")
print("Testcase 6: merge on (reverse) X edge")
R = Reactor()
R += Box("on x=2..3,y=2..3,z=2..3")
R += Box("on x=1..1,y=2..3,z=2..3")
assert (R.realcubes.__repr__() == "[on x=1..3,y=2..3,z=2..3]")
print("Testcase 7: merge on (reverse) Y edge")
R += Box("on x=1..3,y=1..1,z=2..3")
assert (R.realcubes.__repr__() == "[on x=1..3,y=1..3,z=2..3]")
print("Testcase 8: merge on (reverse) Z edge")
R += Box("on x=1..3,y=1..3,z=1..1")
assert (R.realcubes.__repr__() == "[on x=1..3,y=1..3,z=1..3]")
print(colored("Testcase 9a: Remove a 1x1x1 cube from a 3x3x3 cube", "red"))
R = Reactor()
R += Box("on x=1..3,y=1..3,z=1..3")
assert (R.size() == 27)
R += Box("off x=1..1,y=1..1,z=1..1")
check(R.size() == 26, [R.size(), R.realcubes], f=lambda: R.savefig())
print(colored("Testcase 9b: Remove a 1x1x1 cube from a 3x3x3 cube", "red"))
R = Reactor()
R += Box("on x=1..3,y=1..3,z=1..3")
assert (R.size() == 27)
R += Box("off x=3..3,y=3..3,z=3..3")
check(R.size() == 26, [R.size(), R.realcubes], f=lambda: R.savefig())
print(colored("Testcase 9c: Remove a 1x1x1 cube from a 3x3x3 cube", "red"))
R = Reactor()
R += Box("on x=1..3,y=1..3,z=1..3")
assert (R.size() == 27)
R += Box("off x=2..2,y=2..2,z=2..2")
check(R.size() == 26, [R.size(), R.realcubes], f=lambda: R.savefig())
print(
colored("Testcase 10x: Remove a slab from the upper end of a slab",
"red"))
R = Reactor()
R += Box("on x=1..3,y=1..1,z=1..1")
assert (R.size() == 3)
R += Box("off x=3..3,y=1..1,z=1..1")
check(R.size() == 2, [R.size(), R.realcubes], f=lambda: R.savefig())
print(
colored("Testcase 10y: Remove a slab from the upper end of a slab",
"red"))
R = Reactor()
R += Box("on x=1..1,y=1..3,z=1..1")
assert (R.size() == 3)
R += Box("off x=1..1,y=3..3,z=1..1")
check(R.size() == 2, [R.size(), R.realcubes], f=lambda: R.savefig())
print(
colored("Testcase 10z: Remove a slab from the upper end of a slab",
"red"))
R = Reactor()
R += Box("on x=1..1,y=1..1,z=1..3")
assert (R.size() == 3)
R += Box("off x=1..1,y=1..1,z=3..3")
check(R.size() == 2, [R.size(), R.realcubes], f=lambda: R.savefig())
print(
colored("Testcase 10xy: Remove a corner from the upper end of a slab",
"red"))
R = Reactor()
R += Box("on x=1..3,y=1..3,z=1..1")
assert (R.size() == 9)
R += Box("off x=3..3,y=3..3,z=1..1")
check(R.size() == 8, [R.size(), R.realcubes], f=lambda: R.savefig())
print(
colored("Testcase 10xy: Remove a corner from the upper end of a cube",
"red"))
R = Reactor()
R += Box("on x=1..3,y=1..3,z=1..3")
assert (R.size() == 27)
R += Box("off x=3..3,y=3..3,z=3..3")
check(R.size() == 26, [R.size(), R.realcubes], f=lambda: R.savefig())
print(colored("Testcase 10x-x: Remove a smaller part of the blob", "red"))
R = Reactor()
R += Box("on x=1..4,y=1..1,z=1..1")
R += Box("off x=2..3,y=1..1,z=1..1")
check(R.size() == 2,
[R.size(), R.realcubes, [i.size() for i in R.realcubes]],
f=lambda: R.savefig())
print(colored("Testcase 10y-y: Remove a smaller part of the blob", "red"))
R = Reactor()
R += Box("on x=1..1,y=1..4,z=1..1")
R += Box("off x=1..1,y=2..3,z=1..1")
check(R.size() == 2,
[R.size(), R.realcubes, [i.size() for i in R.realcubes]],
f=lambda: R.savefig())
print(colored("Testcase 10z-z: Remove a smaller part of the blob", "red"))
R = Reactor()
R += Box("on x=1..1,y=1..1,z=1..4")
R += Box("off x=1..1,y=1..1,z=2..3")
check(R.size() == 2,
[R.size(), R.realcubes, [i.size() for i in R.realcubes]],
f=lambda: R.savefig())
# -----------------
print(colored("Testcase 10xyz: Remove a part of a blob", "red"))
# off x=9..11,y=9..11,z=9..11 from on x=10..10,y=10..12,z=10..12
R = Reactor()
R += Box("on x=10..10,y=10..12,z=10..12")
R += Box("off x=9..11,y=9..11,z=9..11")
check(R.size() == 5,
[R.size(), R.realcubes, [i.size() for i in R.realcubes]],
f=lambda: R.savefig())
def fourth():
print("= selftest 4 started")
R = Reactor()
R += Box("on x=1..4,y=1..4,z=1..4")
R += Box("off x=2..3,y=1..4,z=1..4")
R.consistencycheck()
R = Reactor()
# this screws up things... what happens here?
a = Box('on x=-27..23,y=-28..26,z=-21..29')
b = Box("on x=-22..26,y=-27..20,z=-29..19")
a.id = "a"
b.id = "b"
R += a
R += b
# vi tappar bort flaket som ligger mellan y=-28..-27
# nya kuben är mindre än gamla i en av dimensionerna
# check the on set vs the all set
R.consistencycheck()
R = Reactor()
R += Box("on x=-20..26,y=-36..17,z=-47..7")
R += Box("on x=-20..33,y=-21..23,z=-26..28")
R += Box("on x=-22..28,y=-29..23,z=-38..16")
R += Box("on x=-46..7,y=-6..46,z=-50..-1")
R += Box("on x=-49..1,y=-3..46,z=-24..28")
R += Box("on x=2..47,y=-22..22,z=-23..27")
R += Box("on x=-27..23,y=-28..26,z=-21..29")
R += Box("on x=-39..5,y=-6..47,z=-3..44")
R += Box("on x=-30..21,y=-8..43,z=-13..34")
R += Box("on x=-22..26,y=-27..20,z=-29..19")
R += Box("off x=-48..-32,y=26..41,z=-47..-37")
R += Box("on x=-12..35,y=6..50,z=-50..-2")
R += Box("off x=-48..-32,y=-32..-16,z=-15..-5")
R += Box("on x=-18..26,y=-33..15,z=-7..46")
R += Box("off x=-40..-22,y=-38..-28,z=23..41")
R += Box("on x=-16..35,y=-41..10,z=-47..6")
R += Box("off x=-32..-23,y=11..30,z=-14..3")
R += Box("on x=-49..-5,y=-3..45,z=-29..18")
R += Box("off x=18..30,y=-20..-8,z=-3..13")
R += Box("on x=-41..9,y=-7..43,z=-33..15")
# check for the correct size of things
nx = R.drawallblobs()
assert (sum(sum(sum(nx))) == R.size())
assert (R.size() == 590784)
print("= selftest 4 passed")
return R
def example2():
print("= example 2 started")
R = Reactor()
R += Box("on x=-5..47,y=-31..22,z=-19..33")
R += Box("on x=-44..5,y=-27..21,z=-14..35")
R += Box("on x=-49..-1,y=-11..42,z=-10..38")
R += Box("on x=-20..34,y=-40..6,z=-44..1")
R += Box("off x=26..39,y=40..50,z=-2..11")
R += Box("on x=-41..5,y=-41..6,z=-36..8")
R += Box("off x=-43..-33,y=-45..-28,z=7..25")
R += Box("on x=-33..15,y=-32..19,z=-34..11")
R += Box("off x=35..47,y=-46..-34,z=-11..5")
R += Box("on x=-14..36,y=-6..44,z=-16..29")
R += Box("on x=-57795..-6158,y=29564..72030,z=20435..90618")
R += Box("on x=36731..105352,y=-21140..28532,z=16094..90401")
R += Box("on x=30999..107136,y=-53464..15513,z=8553..71215")
R += Box("on x=13528..83982,y=-99403..-27377,z=-24141..23996")
R += Box("on x=-72682..-12347,y=18159..111354,z=7391..80950")
R += Box("on x=-1060..80757,y=-65301..-20884,z=-103788..-16709")
R += Box("on x=-83015..-9461,y=-72160..-8347,z=-81239..-26856")
R += Box("on x=-52752..22273,y=-49450..9096,z=54442..119054")
R += Box("on x=-29982..40483,y=-108474..-28371,z=-24328..38471")
R += Box("on x=-4958..62750,y=40422..118853,z=-7672..65583")
R += Box("on x=55694..108686,y=-43367..46958,z=-26781..48729")
R += Box("on x=-98497..-18186,y=-63569..3412,z=1232..88485")
R += Box("on x=-726..56291,y=-62629..13224,z=18033..85226")
R += Box("on x=-110886..-34664,y=-81338..-8658,z=8914..63723")
R += Box("on x=-55829..24974,y=-16897..54165,z=-121762..-28058")
R += Box("on x=-65152..-11147,y=22489..91432,z=-58782..1780")
R += Box("on x=-120100..-32970,y=-46592..27473,z=-11695..61039")
R += Box("on x=-18631..37533,y=-124565..-50804,z=-35667..28308")
R += Box("on x=-57817..18248,y=49321..117703,z=5745..55881")
R += Box("on x=14781..98692,y=-1341..70827,z=15753..70151")
R += Box("on x=-34419..55919,y=-19626..40991,z=39015..114138")
R += Box("on x=-60785..11593,y=-56135..2999,z=-95368..-26915")
R += Box("on x=-32178..58085,y=17647..101866,z=-91405..-8878")
R += Box("on x=-53655..12091,y=50097..105568,z=-75335..-4862")
R += Box("on x=-111166..-40997,y=-71714..2688,z=5609..50954")
R += Box("on x=-16602..70118,y=-98693..-44401,z=5197..76897")
R += Box("on x=16383..101554,y=4615..83635,z=-44907..18747")
R += Box("off x=-95822..-15171,y=-19987..48940,z=10804..104439")
R += Box("on x=-89813..-14614,y=16069..88491,z=-3297..45228")
R += Box("on x=41075..99376,y=-20427..49978,z=-52012..13762")
R += Box("on x=-21330..50085,y=-17944..62733,z=-112280..-30197")
R += Box("on x=-16478..35915,y=36008..118594,z=-7885..47086")
R += Box("off x=-98156..-27851,y=-49952..43171,z=-99005..-8456")
R += Box("off x=2032..69770,y=-71013..4824,z=7471..94418")
R += Box("on x=43670..120875,y=-42068..12382,z=-24787..38892")
R += Box("off x=37514..111226,y=-45862..25743,z=-16714..54663")
R += Box("off x=25699..97951,y=-30668..59918,z=-15349..69697")
R += Box("off x=-44271..17935,y=-9516..60759,z=49131..112598")
R += Box("on x=-61695..-5813,y=40978..94975,z=8655..80240")
R += Box("off x=-101086..-9439,y=-7088..67543,z=33935..83858")
R += Box("off x=18020..114017,y=-48931..32606,z=21474..89843")
R += Box("off x=-77139..10506,y=-89994..-18797,z=-80..59318")
R += Box("off x=8476..79288,y=-75520..11602,z=-96624..-24783")
R += Box("on x=-47488..-1262,y=24338..100707,z=16292..72967")
R += Box("off x=-84341..13987,y=2429..92914,z=-90671..-1318")
R += Box("off x=-37810..49457,y=-71013..-7894,z=-105357..-13188")
R += Box("off x=-27365..46395,y=31009..98017,z=15428..76570")
R += Box("off x=-70369..-16548,y=22648..78696,z=-1892..86821")
R += Box("on x=-53470..21291,y=-120233..-33476,z=-44150..38147")
R += Box("off x=-93533..-4276,y=-16170..68771,z=-104985..-24507")
s = sum([x.size() for x in R.realcubes])
assert (s == 2758514936282235)
print("= example 2 passed")
def __init__(self):
Reactor.__init__(self)
GPIO.setmode(GPIO.BCM)
GPIO.setup(LEDReactor.GPIO_OUT, GPIO.OUT)
class BasicServent:
"""
This class implements everything that a Servent "must do" as part of Gnutella 0.4 protocol
To add more feature, simply derive this. You can either override entire method or
add more functionality by implement it and then add parent's method.
"""
# Fixed expiration period per ttl in unit of second
FIXED_EXPIRED_INTERVAL = 5
def __init__(self, port=0, files = [], bootstrap_address = None):
self._logger = logging.getLogger("%s(%s)" % (self.__class__.__name__, hex(id(self))[:-1]))
# forwarding table: (message_id, payload_type) -> (connection_handler,
# expiration)
self.forwarding_table = {}
# flood/forward ignore table: message_id -> timestamp
# this table used to prevent loop in flood
# all message send out need to put their message_id and
# timestamp = time.time()+ttl*FIXED_EXPIRED_INTERVAL
self.ignore = {}
# create servent id
self.id = uuid.uuid4().bytes
self.log("id is %s" % self.id.encode('hex_codec'))
# calculate number of file and number of kilobyte shared
self._files = files
self.num_files = len(files)
self.num_kilobytes = 0
for f in files:
self.num_kilobytes += f.file_size
self.num_kilobytes /= 1000 # shrink the unit
# create Reactor class for socket management
self.reactor = Reactor(self, port)
# check if bootstrap_address is given
if bootstrap_address:
self.reactor.bootstrap_connect(bootstrap_address)
return
def on_accept(self):
"""
on event of gnutella connection, accept or refuse depends on
resource management
"""
# Always accept new connection
return True
def on_connect(self, connection_handler):
"""
on event of a servent connects to
"""
return
def on_receive(self, connection_handler, message):
"""
on event of receiving a message from an existing connection
"""
if message.payload_descriptor == GnutellaBodyId.PING:
# check if we saw this ping before. If not, then process
forward_key = (message.message_id, GnutellaBodyId.PONG)
now = time.time()
not_seen_or_expire = forward_key not in self.forwarding_table or (self.forwarding_table[forward_key][1] < now)
not_ignore_or_expire = message.message_id not in self.ignore or (self.ignore[message.message_id] < now)
if not_seen_or_expire and not_ignore_or_expire:
# send Ping to any neighbor that not the one servent recceived the Ping from
self.flood(connection_handler, message)
# add ping to forwarding table to forward PONG
self.put_into_forwarding_table(message, connection_handler)
# reply with Pong (the return trip's ttl should be equals to hops)
pong_message = create_message(GnutellaBodyId.PONG,
message.message_id,
message.hops+1,
ip = self.reactor.ip,
port = self.reactor.port,
num_of_files = self.num_files,
num_of_kb = self.num_kilobytes)
self.log("Sending replied pong %s", pong_message)
self.send_message(pong_message, connection_handler)
elif message.payload_descriptor == GnutellaBodyId.PONG:
# forwarding pong
self.forward(message)
elif message.payload_descriptor == GnutellaBodyId.QUERY:
# check if we saw this query before. If not, then process
forward_key = (message.message_id, GnutellaBodyId.QUERYHIT)
now = time.time()
not_seen_or_expire = forward_key not in self.forwarding_table or (self.forwarding_table[forward_key][1] < now)
not_ignore_or_expire = message.message_id not in self.ignore or (self.ignore[message.message_id] < now)
if not_seen_or_expire and not_ignore_or_expire:
# add to forwarding table to forward QUERYHIT
self.put_into_forwarding_table(message, connection_handler)
# forward query packet to neighbor servent
self.flood(connection_handler, message)
elif message.payload_descriptor == GnutellaBodyId.QUERYHIT:
# don't route query hit if it is meant for this node
if not message.body.servent_id == self.id:
# forwarding query hit
if self.forward(message):
# add to forwarding table to forward PUSH
self.put_into_forwarding_table(message, connection_handler)
elif message.payload_descriptor == GnutellaBodyId.PUSH:
# don't route push if it is meant for this node
if not message.body.servent_id == self.id:
# forward push
self.forward(message)
else:
raise ValueError('message type is not one of PING, PONG, QUERY, QUERYHIT, PUSH')
def on_disconnect(self, connection_handler):
"""
servent behavior when leaving the network
"""
# resource clean up
# clean up forwarding table
remove = [k for k,v in self.forwarding_table.iteritems() if v[0] == connection_handler]
for k in remove:
del self.forwarding_table[k]
return
def on_download(self, event_id, connection_handler):
# DO some logging or resource clean up in here
return
def on_bootstrap(self, peer_address):
# connect to all suggested peer
self.reactor.gnutella_connect(peer_address)
def log(self, msg, *args, **kwargs):
self._logger.debug(msg, *args, **kwargs)
def send_message(self, message, handler):
"""
Servent sends its own message (not as part of flood / forwarding)
By using this method, Servent can keep track which message (by message_id) is its own
by adding it to ignore dictionary.
"""
self.ignore[message.message_id] = time.time()+self.FIXED_EXPIRED_INTERVAL*message.ttl
handler.write(message.serialize())
def put_into_forwarding_table(self, message, handler):
now = time.time()
message_id = copy.deepcopy(message.message_id)
value = (handler, now+self.FIXED_EXPIRED_INTERVAL*message.ttl)
key = None
if message.payload_descriptor == GnutellaBodyId.QUERYHIT:
key = (message_id, message.body.servent_id, GnutellaBodyId.PUSH)
self.forwarding_table[key] = value
elif message.payload_descriptor == GnutellaBodyId.PING:
key = (message_id, GnutellaBodyId.PONG)
self.forwarding_table[key] = value
elif message.payload_descriptor == GnutellaBodyId.QUERY:
key = (message_id, GnutellaBodyId.QUERYHIT)
else:
raise ValueError
# insert into forwarding table
self.forwarding_table[key] = value
def forward(self, message):
"""
Forward message to correct servent in according to forwarding table
"""
if message.ttl < 2:
return False
try:
# create a deep copy
message = copy.deepcopy(message)
message.decrease_ttl()
packet = message.serialize()
key = None
if message.payload_descriptor != GnutellaBodyId.PUSH:
key = (message.message_id, message.payload_descriptor)
else:
key = (message.message_id, message.body.servent_id, message.payload_descriptor)
handler, timestamp = self.forwarding_table[key]
if time.time() < timestamp:
handler.write(packet)
return True
else:
self.log("forwarding timestamp expired -> %s", message)
del self.forwarding_table[key]
return False
except KeyError:
return False
return False
def flood(self, received_handler, message):
"""
Flood other servent's message to every directly connected servent other
than the received handler. This method is used as part of forwarding of ping, query
return number of connection is flooded
"""
if message.ttl < 2:
return 0
# create a deep copy
message = copy.deepcopy(message)
message.decrease_ttl()
return self.reactor.broadcast_except_for(received_handler, message)
def flood_ex(self, message):
"""
Flood your own message to every directly connected servent
return number of connection is flooded
"""
# you can only flood query or ping message by definition of protocol
if message.payload_descriptor == GnutellaBodyId.QUERY or message.payload_descriptor == GnutellaBodyId.PING:
self.ignore[message.message_id] = time.time()+self.FIXED_EXPIRED_INTERVAL*message.ttl
return self.reactor.broadcast_except_for(None, message)
return 0
def set_files(self, files):
# each member of files is a FileInfo
self._files = files
# calculate number of file and number of kilobyte shared
self.num_files = len(files)
self.num_kilobytes = 0
for f in files:
self.num_kilobytes += f.file_size
self.num_kilobytes /= 1000 # shrink the unit
return
def get_files(self):
return self._files
files = property(get_files, set_files)
def check_file(self, file_id):
"""
check if the servent have the file with id = file_id
"""
for fileinfo in self.files:
if fileinfo.file_id == file_id:
return True
return False
def get_file_content(self, file_id, file_name):
if not self.check_file(file_id):
return None
# return dummy content
return "This is (%s, %s)" % (file_name, file_id)
def search(self, criteria):
"""
Return a list of file fit the criteria
Exact match for file name
"""
match = []
for fileinfo in self.files:
if criteria == fileinfo.file_name:
match.append(fileinfo)
return match
