def scenario_offline_beta(self, *classes):
options = [(t[0], float(t[1]))
for t in (s.split(":") for s in classes)]
for class_, offline_beta in options:
if class_ == self.__class:
self.__offline_beta = offline_beta
dprint("choosing offline beta ", self.__offline_beta)
def on_votecast(self, messages):
if self.integrate_with_tribler:
votelist = []
for message in messages:
if __debug__: dprint(message)
dispersy_id = message.packet_id
channel_id = getattr(message, "channel_id", 0)
authentication_member = message.authentication.member
if authentication_member == self._my_member:
peer_id = None
#if channel_id is not found, then this is a manual join
#insert placeholder into database which will be replaced after channelmessage has been received
if not channel_id:
select_channel = "SELECT id FROM _Channels WHERE dispersy_cid = ?"
channel_id = self._channelcast_db._db.fetchone(select_channel, (buffer(message.payload.cid),))
if not channel_id:
insert_channel = "INSERT INTO _Channels (dispersy_cid, peer_id, name) VALUES (?, ?, ?); SELECT last_insert_rowid();"
channel_id = self._channelcast_db._db.fetchone(insert_channel, (buffer(message.payload.cid), -1, ''))
else:
peer_id = self._peer_db.addOrGetPeerID(authentication_member.public_key)
votelist.append((channel_id, peer_id, dispersy_id, message.payload.vote, message.payload.timestamp))
if DEBUG:
print >> sys.stderr, "AllChannelCommunity: got votecast message"
self._votecast_db.on_votes_from_dispersy(votelist)
# this might be a response to a dispersy-missing-message
self._dispersy.handle_missing_messages(messages, MissingMessageCache)
def set(self, origin):
"""
"""
assert isinstance(origin, float)
assert self._origin <= origin
difference = int((origin - self._origin) / CYCLE_SIZE)
if __debug__:
dprint("difference ", difference, " (", origin - self._origin,
"s)")
if difference:
self._origin = origin
# shift
self._bits <<= difference
# remove now obsolete bits
self._bits &= (2**self._size - 1)
# set last bit ACTIVE
self._bits |= 1
return True
else:
if self._bits & 1:
return False
else:
self._bits |= 1
return True
def _observation(self, candidate, member, now, update_record=True):
if not isinstance(candidate, BootstrapCandidate):
# try cache
history = self._observations.get(member.database_id)
if not history:
# fetch from database
try:
timestamp, bytes_ = next(self._database.execute(u"SELECT timestamp, effort FROM observation WHERE member = ?",
(member.database_id,)))
except StopIteration:
# first observation: create new history
history = EffortHistory(now)
else:
history = EffortHistory(str(bytes_), float(timestamp))
# store in cache
self._observations[member.database_id] = history
if len(self._observations) > self._observations_length:
key, value = self._observations.popitem(False)
self._database.execute(u"INSERT OR REPLACE INTO observation (member, timestamp, effort) VALUES (?, ?, ?)",
(key, value.origin, buffer(value.bytes)))
if update_record:
changed = history.set(now)
if changed:
self.try_adding_to_slope(candidate, member, history)
if __debug__: dprint("c", int(now / CYCLE_SIZE), " ", candidate)
return history
def on_votecast(self, messages):
if self.integrate_with_tribler:
votelist = []
for message in messages:
if __debug__: dprint(message)
dispersy_id = message.packet_id
channel_id = getattr(message, "channel_id", 0)
authentication_member = message.authentication.member
if authentication_member == self._my_member:
peer_id = None
#if channel_id is not found, then this is a manual join
#insert placeholder into database which will be replaced after channelmessage has been received
if not channel_id:
select_channel = "SELECT id FROM _Channels WHERE dispersy_cid = ?"
channel_id = self._channelcast_db._db.fetchone(select_channel, (buffer(message.payload.cid),))
if not channel_id:
insert_channel = "INSERT INTO _Channels (dispersy_cid, peer_id, name) VALUES (?, ?, ?); SELECT last_insert_rowid();"
channel_id = self._channelcast_db._db.fetchone(insert_channel, (buffer(message.payload.cid), -1, ''))
else:
peer_id = self._peer_db.addOrGetPeerID(authentication_member.public_key)
votelist.append((channel_id, peer_id, dispersy_id, message.payload.vote, message.payload.timestamp))
if DEBUG:
print >> sys.stderr, "AllChannelCommunity: got votecast message"
self._votecast_db.on_votes_from_dispersy(votelist)
# this might be a response to a dispersy-missing-message
self._dispersy.handle_missing_messages(messages, MissingMessageCache)
def dispersy_cleanup_community(self, message):
if __debug__: dprint()
self._has_been_killed = True
# remove all data from the local database
self._database.cleanup()
# re-classify to prevent loading
return super(EffortCommunity, self).dispersy_cleanup_community(message)
def dispersy_cleanup_community(self, message):
if __debug__: dprint()
self._has_been_killed = True
# remove all data from the local database
self._database.cleanup()
# re-classify to prevent loading
return super(EffortCommunity, self).dispersy_cleanup_community(message)
def set(self, origin):
"""
"""
assert isinstance(origin, float)
assert self._origin <= origin
difference = int((origin - self._origin) / CYCLE_SIZE)
if __debug__: dprint("difference ", difference, " (", origin - self._origin, "s)")
if difference:
self._origin = origin
# shift
self._bits <<= difference
# remove now obsolete bits
self._bits &= (2**self._size - 1)
# set last bit ACTIVE
self._bits |= 1
return True
else:
if self._bits & 1:
return False
else:
self._bits |= 1
return True
def _observation(self, candidate, member, now, update_record=True):
if not isinstance(candidate, BootstrapCandidate):
# try cache
history = self._observations.get(member.database_id)
if not history:
# fetch from database
try:
timestamp, bytes_ = next(self._database.execute(u"SELECT timestamp, effort FROM observation WHERE member = ?",
(member.database_id,)))
except StopIteration:
# first observation: create new history
history = EffortHistory(now)
else:
history = EffortHistory(str(bytes_), float(timestamp))
# store in cache
self._observations[member.database_id] = history
if len(self._observations) > self._observations_length:
key, value = self._observations.popitem(False)
self._database.execute(u"INSERT OR REPLACE INTO observation (member, timestamp, effort) VALUES (?, ?, ?)",
(key, value.origin, buffer(value.bytes)))
if update_record:
changed = history.set(now)
if changed:
self.try_adding_to_slope(candidate, member, history)
if __debug__: dprint("c", int(now / CYCLE_SIZE), " ", candidate)
return history
def scenario_online_beta(self, *classes):
options = [(t[0], float(t[1])) for t in (s.split(":") for s in classes)]
for class_, online_beta in options:
if class_ == self.__class:
self.__online_beta = online_beta
break
dprint("choosing online beta ", self.__online_beta)
def unload_community(self):
if __debug__: dprint()
super(EffortCommunity, self).unload_community()
# unsubscribe from events. 22/03/13 Boudewijn: Dispersy is not allowed to call swift
# because it may cause database locks if people incorrectly keep the session locked.
temp_thread = Callback()
temp_thread.register(self._swift.set_subscribe_channel_close, ("ALL", False, self.i2ithread_channel_close))
temp_thread.register(temp_thread.stop)
temp_thread.start("Temporary-EffortCommunity")
# cancel outstanding pings
for ping_candidate in self._slope.itervalues():
self._dispersy.callback.unregister(ping_candidate.callback_id)
self._slope = {}
# store all cached observations
self._database.executemany(u"INSERT OR REPLACE INTO observation (member, timestamp, effort) VALUES (?, ?, ?)",
[(database_id, history.origin, buffer(history.bytes)) for database_id, history in self._observations.iteritems()])
# update all up and download values
self.download_state_callback([], closing=True)
# store all cached bandwidth guesses
self._database.executemany(u"INSERT OR REPLACE INTO bandwidth_guess (ip, member, timestamp, upload, download) VALUES (?, ?, ?, ?, ?)",
[(unicode(ip), guess.member.database_id if guess.member else 0, guess.timestamp, int(guess.upload), int(guess.download)) for ip, guess in self._bandwidth_guesses.iteritems()])
def on_torrent_collect_request(self, messages):
if __debug__: dprint(len(messages))
candidates = [message.candidate for message in messages]
identifiers = [message.payload.identifier for message in messages]
self._create_pingpong(u"torrent-collect-response", candidates, identifiers)
self.on_torrent_collect_response(messages, verifyRequest = False)
def on_torrent_collect_request(self, messages):
if __debug__: dprint(len(messages))
candidates = [message.candidate for message in messages]
identifiers = [message.payload.identifier for message in messages]
self._create_pingpong(u"torrent-collect-response", candidates, identifiers)
self.on_torrent_collect_response(messages, verifyRequest = False)
def scenario_classes(self, *classes):
options = [(t[0], float(t[1])) for t in (s.split(":") for s in classes)]
value = random() * sum(weight for _, weight in options)
for class_, weight in options:
value -= weight
if value <= 0.0:
self.__class = class_
break
dprint("choosing class ", self.__class)
def _get_channel_community(self, cid):
assert isinstance(cid, str)
assert len(cid) == 20
try:
return self._dispersy.get_community(cid, True)
except KeyError:
if __debug__: dprint("join preview community ", cid.encode("HEX"))
return PreviewChannelCommunity.join_community(DummyMember(cid), self._my_member, self.integrate_with_tribler)
def _get_channel_community(self, cid):
assert isinstance(cid, str)
assert len(cid) == 20
try:
return self._dispersy.get_community(cid, True)
except KeyError:
if __debug__: dprint("join preview community ", cid.encode("HEX"))
return PreviewChannelCommunity.join_community(DummyMember(cid), self._my_member, self.integrate_with_tribler)
def _channel_close(self, roothash_hex, address, raw_bytes_up, raw_bytes_down, cooked_bytes_up, cooked_bytes_down):
assert self._dispersy.callback.is_current_thread, "Must be called on the dispersy.callback thread"
dprint("swift channel close ", address[0], ":", address[1], " with +", cooked_bytes_up, " -", cooked_bytes_down, force=1)
guess = self._get_bandwidth_guess_from_ip(address[0])
guess.upload += cooked_bytes_up / 1024.0
guess.download += cooked_bytes_down / 1024.0
guess.tmp_upload = 0.0
guess.tmp_download = 0.0
def set_online(self):
""" Restore on_incoming_packets and _send functions of
dispersy back to normal.
This simulates a node coming online, since it's able to send
and receive messages.
"""
dprint("Going online")
self._dispersy.on_incoming_packets = self.original_on_incoming_packets
self._dispersy._send = self.original_send
def scenario_classes(self, *classes):
options = [(t[0], float(t[1]))
for t in (s.split(":") for s in classes)]
value = random() * sum(weight for _, weight in options)
for class_, weight in options:
value -= weight
if value <= 0.0:
self.__class = class_
break
dprint("choosing class ", self.__class)
def set_online(self):
""" Restore on_incoming_packets and _send functions of
dispersy back to normal.
This simulates a node coming online, since it's able to send
and receive messages.
"""
dprint("Going online")
self._dispersy.on_incoming_packets = self.original_on_incoming_packets
self._dispersy._send = self.original_send
def create_effort_record(self, second_member, history, forward=True):
"""
Create a dispersy-signature-request that encapsulates an effort-record.
"""
if __debug__: dprint(second_member.mid.encode("HEX"), " = ", bin(history.long))
meta = self.get_meta_message(u"effort-record")
record = meta.impl(authentication=([self._my_member, second_member],),
distribution=(self.claim_global_time(),),
payload=(history.origin, 0.0, history))
return self.create_dispersy_signature_request(record, self.on_signature_response, forward=forward)
def on_introduction_request(self, messages):
try:
return self._original_on_introduction_request(messages)
finally:
now = time()
for message in messages:
if not isinstance(message.candidate, BootstrapCandidate):
for member in message.candidate.get_members(self):
history = self._get_or_create_history(member, now)
changed = history.set(now)
if __debug__: dprint("introduction-request from ", message.candidate, " - ", bin(history.long))
def unload_preview(self):
while True:
yield 60.0
cleanpoint = time() - 300
inactive = [community for community in self.dispersy._communities.itervalues() if isinstance(community, PreviewChannelCommunity) and community.init_timestamp < cleanpoint]
if __debug__:
dprint("cleaning ", len(inactive), "/", len(self.dispersy._communities), " previewchannel communities")
for community in inactive:
community.unload_community()
def unload_preview(self):
while True:
yield 60.0
cleanpoint = time() - 300
inactive = [community for community in self.dispersy._communities.itervalues() if isinstance(community, PreviewChannelCommunity) and community.init_timestamp < cleanpoint]
if __debug__:
dprint("cleaning ", len(inactive), "/", len(self.dispersy._communities), " previewchannel communities")
for community in inactive:
community.unload_community()
def _watchdog(self):
while True:
try:
yield 60.0
# flush changes to disk every 1 minutes
self._database.commit()
except GeneratorExit:
bz2log("effort.log", "unload")
if __debug__: dprint("shutdown")
self._database.commit()
break
def set_offline(self):
""" Replace on_incoming_packets and _sends functions of
dispersy with dummies
This simulates a node going offline, since it's not able to
send or receive any messages
"""
def dummy_function(*params):
return
dprint("Going offline")
self._dispersy.on_incoming_packets = dummy_function
self._dispersy._send = dummy_function
def set_offline(self):
""" Replace on_incoming_packets and _sends functions of
dispersy with dummies
This simulates a node going offline, since it's not able to
send or receive any messages
"""
def dummy_function(*params):
return
dprint("Going offline")
self._dispersy.on_incoming_packets = dummy_function
self._dispersy._send = dummy_function
def on_signature_response(self, old_message, new_message, changed):
"""
A dispersy-signature-response has been received.
Return True or False to either accept or decline the message.
"""
if __debug__: dprint(new_message)
if new_message:
if old_message.meta == new_message.meta:
return True, True, False
return False, False, False
def on_signature_response(self, message, request, retry):
""" Handle a newly created double signed message or a timeout while signing
When request for signing times out I just return. When I receive the signature for the
message that I created then I send it to myself and then store and forward it to others of
the community
"""
if __debug__: dprint(message)
if message:
assert message.name == u"barter-record"
assert message.authentication.is_signed
if __debug__: dprint(message)
# store, update, and forward to the community
self._dispersy.store_update_forward([message], True, True, self.barter_forward_record_on_creation)
log("dispersy.log", "created-barter-record") # TODO: maybe move to barter.log
elif retry < 5:
# signature timeout
# retry
if __debug__:
log("barter.log", "barter-community-signature-request-timeout", retry=retry)
dprint("Signature request timeout. Retry!")
self.create_dispersy_signature_request(request, self.on_signature_response, (request, retry + 1))
else:
# close the transfer
if __debug__:
log("barter.log", "barter-community-signature-request-timeout", retry=retry)
dprint("Signature request timeout")
def allow_signature_request(self, message):
""" Decide whether to reply or not to a signature request
Currently I always sign for testing proposes
"""
assert message.name == u"barter-record"
assert not message.authentication.is_signed
if __debug__: dprint(message)
is_signed, other_member = message.authentication.signed_members[0]
if other_member == self._my_member:
if __debug__: dprint("refuse signature: we should be the first signer", level="warning")
return False
if not is_signed:
if __debug__: dprint("refuse signature: the other member did not sign it", level="warning")
return False
my_member = message.authentication.members[1]
if not my_member == self._my_member:
if __debug__: dprint("refuse signature: we should be the second signer")
return False
# todo: decide if we want to use add our signature to this # record
if True:
# recalculate the limits of what we want to upload to member.authentication.members[0]
return True
# we will not add our signature
return False
def join_community(self, my_member):
self.my_member = my_member
master_key = "3081a7301006072a8648ce3d020106052b81040027038192000403cbbfd2dfb67a7db66c88988df56f93fa6e7f982f9a6a0fa8898492c8b8cae23e10b159ace60b7047012082a5aa4c6e221d7e58107bb550436d57e046c11ab4f51f0ab18fa8f58d0346cc12d1cc2b61fc86fe5ed192309152e11e3f02489e30c7c971dd989e1ce5030ea0fb77d5220a92cceb567cbc94bc39ba246a42e215b55e9315b543ddeff0209e916f77c0d747".decode("HEX")
master = Member.get_instance(master_key)
assert my_member.public_key
assert my_member.private_key
assert master.public_key
assert not master.private_key
dprint("-master- ", master.database_id, " ", id(master), " ", master.mid.encode("HEX"), force=1)
dprint("-my member- ", my_member.database_id, " ", id(my_member), " ", my_member.mid.encode("HEX"), force=1)
return AllChannelCommunity.join_community(master, my_member, my_member, integrate_with_tribler = False)
def on_introduction_response(self, messages):
try:
return self._original_on_introduction_response(messages)
finally:
now = time()
for message in messages:
if not isinstance(message.candidate, BootstrapCandidate):
for member in message.candidate.get_members(self):
history = self._get_or_create_history(member, now)
changed = history.set(now)
if __debug__: dprint("introduction-response from ", message.candidate, " - ", bin(history.long))
if changed:
if __debug__: dprint("changed! (", member.mid.encode("HEX"), ", ", bin(self._histories[member].long), ")")
self.create_effort_record(member, history)
def on_barter_record(self, messages):
""" I handle received barter records
I create or update a row in my database
"""
if __debug__: dprint("storing ", len(messages), " records")
execute = self._database.execute
for message in messages:
if __debug__: log("dispersy.log", "handled-barter-record") # TODO: maybe move to barter.log
# check if there is already a record about this pair
try:
first_member, second_member, global_time = \
execute(u"SELECT first_member, second_member, global_time FROM \
record WHERE (first_member = ? AND second_member = ?) OR \
(first_member = ? AND second_member = ?)",
(message.authentication.members[0].database_id,
message.authentication.members[1].database_id,
message.authentication.members[1].database_id,
message.authentication.members[0].database_id)).next()
except StopIteration:
global_time = -1
first_member = message.authentication.members[0].database_id
second_member = message.authentication.members[1].database_id
if global_time >= message.distribution.global_time:
# ignore the message
if __debug__:
dprint("Ignoring older message")
else:
self._database.execute(u"INSERT OR REPLACE INTO \
record(community, global_time, first_member, second_member, \
upload_first_member, upload_second_member) \
VALUES(?, ?, ?, ?, ?, ?)",
(self._database_id,
message.distribution.global_time,
first_member,
second_member,
message.payload.first_upload,
message.payload.second_upload))
if __debug__:
peer1_id = self._peer_ids.get(message.authentication.members[0].public_key, -1)
peer2_id = self._peer_ids.get(message.authentication.members[1].public_key, -1)
peer1_upload = message.payload.first_upload
peer2_upload = message.payload.second_upload
if peer1_id > peer2_id:
peer1_id, peer2_id = peer2_id, peer1_id
peer1_upload, peer2_upload = peer2_upload, peer1_upload
log("barter.log", "barter-record", first=peer1_id, second=peer2_id, first_upload=peer1_upload, second_upload=peer2_upload)
def _get_or_create_history(self, member, now):
if __debug__:
if member in self._histories:
dprint("member in histories (", member.mid.encode("HEX"), ", ", bin(self._histories[member].long), ")")
else:
dprint("member NOT in histories")
try:
return self._histories[member]
except KeyError:
try:
timestamp, bytes_ = self._database.execute(u"SELECT timestamp, effort FROM observation WHERE community = ? AND member = ?",
(self._database_id, member.database_id)).next()
except StopIteration:
self._histories[member] = history = EffortHistory(8*64, now)
else:
self._histories[member] = history = EffortHistory(bytes_, len(bytes) * 8, float(timestamp))
return history
def unload_community(self):
if __debug__: dprint()
super(EffortCommunity, self).unload_community()
# cancel outstanding pings
for ping_candidate in self._slope.itervalues():
self._dispersy.callback.unregister(ping_candidate.callback_id)
self._slope = {}
# store all cached observations
self._database.executemany(u"INSERT OR REPLACE INTO observation (member, timestamp, effort) VALUES (?, ?, ?)",
[(database_id, history.origin, buffer(history.bytes)) for database_id, history in self._observations.iteritems()])
# update all up and download values
self.download_state_callback([])
# store all cached bandwidth guesses
self._database.executemany(u"INSERT OR REPLACE INTO bandwidth_guess (ip, member, timestamp, upload, download) VALUES (?, ?, ?, ?, ?)",
[(unicode(ip), guess.member.database_id if guess.member else 0, guess.timestamp, int(guess.upload), int(guess.download)) for ip, guess in self._bandwidth_guesses.iteritems()])
def check_votecast(self, messages):
with self._dispersy.database:
communities = {}
for cid in set([message.payload.cid for message in messages]):
channel_id = self._get_channel_id(cid)
if not channel_id:
communities[cid] = self._get_channel_community(message.payload.cid)
for message in messages:
if __debug__: dprint(message)
community = communities.get(message.payload.cid)
if community:
yield DelayMessageReqChannelMessage(message, community, includeSnapshot = message.payload.vote > 0) #request torrents if positive vote
else:
yield message
# ensure that no commits occur
raise IgnoreCommits()
def download_state_callback(self, states):
assert self._dispersy.callback.is_current_thread, "Must be called on the dispersy.callback thread"
assert isinstance(states, list)
timestamp = int(time())
# get all swift downloads that have peers
active = dict((state.get_download().get_def().get_id(), state)
for state
in states
if state.get_download().get_def().get_def_type() == "swift" and state.get_peerlist())
# get global up and download for swift
for state in active.itervalues():
stats = state.stats["stats"]
self._swift_raw_bytes_up = stats.rawUpTotal
self._swift_raw_bytes_down = stats.rawDownTotal
# OLD is used to determine stopped downloads and peers that left. NEW will become the next OLD
old = self._download_states
new = self._download_states = dict()
# find downloads that stopped
for identifier in set(old.iterkeys()).difference(set(active.iterkeys())):
for ip, (up, down) in old[identifier].iteritems():
if __debug__: dprint(identifier.encode("HEX"), "] ", ip, " +", up, " +", down)
guess = self._get_bandwidth_guess_from_ip(ip)
guess.timestamp = timestamp
guess.upload += up
guess.download += down
for identifier, state in active.iteritems():
if identifier in old:
# find peers that left
for ip in set(old[identifier]).difference(set(peer["ip"] for peer in state.get_peerlist())):
up, down = old[identifier][ip]
if __debug__: dprint(identifier.encode("HEX"), "] ", ip, " +", up, " +", down)
guess = self._get_bandwidth_guess_from_ip(ip)
guess.timestamp = timestamp
guess.upload += up
guess.download += down
# set OLD for the next call to DOWNLOAD_STATE_CALLBACK
new[identifier] = dict((peer["ip"], (peer["utotal"], peer["dtotal"])) for peer in state.get_peerlist() if peer["utotal"] > 0.0 or peer["dtotal"] > 0.0)
def on_effort_record(self, messages):
def ordering(message):
# print "RAW_RECORD_IN", message.packet.encode("HEX")
# print \
# "RECORD_IN", \
# bin(message.payload.history.long), \
# message.distribution.global_time, \
# int(message.payload.history.origin / CYCLE_SIZE), \
# message.payload.first_timestamp, \
# message.payload.second_timestamp, \
# message.authentication.members[0].mid.encode("HEX"), \
# message.authentication.members[1].mid.encode("HEX")
if message.authentication.members[0].database_id < message.authentication.members[1].database_id:
return (message.packet_id,
message.distribution.global_time,
message.authentication.members[0].database_id,
message.authentication.members[1].database_id,
int(message.payload.first_timestamp),
int(message.payload.second_timestamp),
buffer(message.payload.history.bytes),
message.payload.first_up,
message.payload.first_down,
message.payload.second_up,
message.payload.second_down)
else:
return (message.packet_id,
message.distribution.global_time,
message.authentication.members[1].database_id,
message.authentication.members[0].database_id,
int(message.payload.second_timestamp),
int(message.payload.first_timestamp),
buffer(message.payload.history.bytes),
message.payload.first_up,
message.payload.first_down,
message.payload.second_up,
message.payload.second_down)
if __debug__: dprint("storing ", len(messages), " effort records")
self._database.executemany(u"INSERT OR REPLACE INTO record (sync, global_time, first_member, second_member, first_timestamp, second_timestamp, effort, first_upload, first_download, second_upload, second_download) VALUES (?, ?, ?, ?, ?, ?, ? ,?, ?, ?, ?)",
(ordering(message) for message in messages))
def on_effort_record(self, messages):
def ordering(message):
# print "RAW_RECORD_IN", message.packet.encode("HEX")
# print \
# "RECORD_IN", \
# bin(message.payload.history.long), \
# message.distribution.global_time, \
# int(message.payload.history.origin / CYCLE_SIZE), \
# message.payload.first_timestamp, \
# message.payload.second_timestamp, \
# message.authentication.members[0].mid.encode("HEX"), \
# message.authentication.members[1].mid.encode("HEX")
if message.authentication.members[0].database_id < message.authentication.members[1].database_id:
return (message.packet_id,
message.distribution.global_time,
message.authentication.members[0].database_id,
message.authentication.members[1].database_id,
int(message.payload.first_timestamp),
int(message.payload.second_timestamp),
buffer(message.payload.history.bytes),
message.payload.first_up,
message.payload.first_down,
message.payload.second_up,
message.payload.second_down)
else:
return (message.packet_id,
message.distribution.global_time,
message.authentication.members[1].database_id,
message.authentication.members[0].database_id,
int(message.payload.second_timestamp),
int(message.payload.first_timestamp),
buffer(message.payload.history.bytes),
message.payload.first_up,
message.payload.first_down,
message.payload.second_up,
message.payload.second_down)
if __debug__: dprint("storing ", len(messages), " effort records")
self._database.executemany(u"INSERT OR REPLACE INTO record (sync, global_time, first_member, second_member, first_timestamp, second_timestamp, effort, first_upload, first_download, second_upload, second_download) VALUES (?, ?, ?, ?, ?, ?, ? ,?, ?, ?, ?)",
(ordering(message) for message in messages))
def create_effort_record(self, second_member, history):
"""
Create a dispersy-signature-request that encapsulates an effort-record.
"""
if __debug__: dprint("asking ", second_member.mid.encode("HEX"), " to sign ", bin(history.long))
guess = self._try_bandwidth_guess_from_member(second_member)
if guess:
first_up = int(guess.upload)
first_down = int(guess.download)
else:
first_up = 0
first_down = 0
self._statistic_outgoing_signature_request += 1
meta = self.get_meta_message(u"effort-record")
record = meta.impl(authentication=([self._my_member, second_member],),
distribution=(self.claim_global_time(),),
payload=(history.origin, history.origin, history, first_up, first_down, 0, 0),
sign=False)
return self.create_dispersy_signature_request(record, self.on_signature_response)
def __init__(self, *args, **kargs):
super(WalktestCommunity, self).__init__(*args, **kargs)
if __debug__:
dprint("cid: ", self.cid.encode("HEX"), force=1)
dprint("mid: ", self.my_member.mid.encode("HEX"), force=1)
try:
hostname = open("/etc/hostname", "r").readline().strip()
except:
hostname = "unknown"
bz2log("walktest.log",
"load",
mid=self.my_member.mid,
hostname=hostname,
**self._default_log())
# redirect introduction-response timeout
self._origional__introduction_response_timeout = self._dispersy.introduction_response_timeout
self._dispersy.introduction_response_timeout = self._replacement__introduction_response_timeout
def create_effort_record(self, second_member, history):
"""
Create a dispersy-signature-request that encapsulates an effort-record.
"""
if __debug__: dprint("asking ", second_member.mid.encode("HEX"), " to sign ", bin(history.long))
guess = self._try_bandwidth_guess_from_member(second_member)
if guess:
first_up = int(guess.upload + guess.tmp_upload)
first_down = int(guess.download + guess.tmp_download)
else:
first_up = 0
first_down = 0
self._statistic_outgoing_signature_request += 1
meta = self.get_meta_message(u"effort-record")
record = meta.impl(authentication=([self._my_member, second_member],),
distribution=(self.claim_global_time(),),
payload=(history.origin, history.origin, history, first_up, first_down, 0, 0),
sign=False)
return self.create_dispersy_signature_request(record, self.on_signature_response)
def __init__(self, *args, **kargs):
super(WalktestCommunity, self).__init__(*args, **kargs)
if __debug__:
dprint("cid: ", self.cid.encode("HEX"), force=1)
dprint("mid: ", self.my_member.mid.encode("HEX"), force=1)
try:
hostname = open("/etc/hostname", "r").readline().strip()
except:
hostname = "unknown"
bz2log("walktest.log",
"load",
mid=self.my_member.mid,
hostname=hostname,
**self._default_log())
# redirect introduction-response timeout
self._origional__introduction_response_timeout = self._dispersy.introduction_response_timeout
self._dispersy.introduction_response_timeout = self._replacement__introduction_response_timeout
def on_signature_response(self, cache, new_message, changed):
"""
A dispersy-signature-response has been received.
Return True or False to either accept or decline the message.
"""
if __debug__: dprint(new_message)
# TODO: we should ensure that new_message is correct (i.e. all checks made above)
if new_message:
self._statistic_outgoing_signature_request_success += 1
self._observation(new_message.candidate, cache.members[0], time())
assert cache.request.payload.message.meta == new_message.meta
return True
else:
self._statistic_outgoing_signature_request_timeout += 1
self.remove_from_slope(cache.members[0])
return False
def on_signature_response(self, cache, new_message, changed):
"""
A dispersy-signature-response has been received.
Return True or False to either accept or decline the message.
"""
if __debug__: dprint(new_message)
# TODO: we should ensure that new_message is correct (i.e. all checks made above)
if new_message:
self._statistic_outgoing_signature_request_success += 1
self._observation(new_message.candidate, cache.members[0], time())
assert cache.request.payload.message.meta == new_message.meta
return True
else:
self._statistic_outgoing_signature_request_timeout += 1
self.remove_from_slope(cache.members[0])
return False
def on_torrent_collect_response(self, messages, verifyRequest = True):
if __debug__: dprint(len(messages))
toInsert = {}
toCollect = {}
toPopularity = {}
for message in messages:
if verifyRequest:
pong_request = self._dispersy.request_cache.pop(message.payload.identifier, SearchCommunity.PingRequestCache)
if __debug__: dprint("pop", pong_request.helper_candidate if pong_request else " (unknown)")
else:
if __debug__: dprint("no-pop")
pong_request = True
if pong_request and message.payload.hashtype == SWIFT_INFOHASHES:
for roothash, infohash, seeders, leechers, ago in message.payload.torrents:
toInsert[infohash] = [infohash, roothash]
toPopularity[infohash] = [seeders, leechers, time() - (ago * 60)]
toCollect.setdefault(infohash, []).append(message.candidate)
if len(toInsert) > 0:
self._torrent_db.on_torrent_collect_response(toInsert.values())
hashes = [hash_ for hash_ in toCollect.keys() if hash_]
if hashes:
hashesToCollect = self._torrent_db.selectSwiftTorrentsToCollect(hashes)
for infohash, roothash in hashesToCollect[:5]:
for candidate in toCollect[infohash]:
if DEBUG:
from Tribler.Core.CacheDB.sqlitecachedb import bin2str
print >> sys.stderr, "SearchCommunity: requesting .torrent after receiving ping/pong ", candidate, bin2str(infohash), bin2str(roothash)
self._rtorrent_handler.download_torrent(candidate, infohash, roothash, prio = LOW_PRIO_COLLECTING, timeout = CANDIDATE_WALK_LIFETIME)
def allow_signature_request(self, message):
"""
A dispersy-signature-request has been received.
Return None or a Message.Implementation.
"""
assert message.name == u"effort-record"
assert not message.authentication.is_signed
if __debug__: dprint(message)
_, first_member = message.authentication.signed_members[0]
_, second_member = message.authentication.signed_members[1]
global_time = message.distribution.global_time if message.distribution.global_time <= self.global_time else self.claim_global_time()
if first_member == self._my_member:
local_history = self._get_or_create_history(second_member, time())
first_timestamp = local_history.origin
second_timestamp = message.payload.second_timestamp
else:
assert second_member == self._my_member
local_history = self._get_or_create_history(first_member, time())
first_timestamp = message.payload.first_timestamp
second_timestamp = local_history.origin
if __debug__: dprint("time diff:", abs(first_timestamp - second_timestamp), "; bits diff:", local_history.long ^ message.payload.history.long)
# TODO shift history and origin for a match
history = EffortHistory(local_history.long & message.payload.history.long, local_history.size, local_history.origin)
bz2log("effort.log", "diff", local=bin(history.long), remote=bin(message.payload.history.long), propose=bin(history.long), time_diff=int(abs(first_timestamp - second_timestamp)))
# return the modified effort-record we propose
meta = self.get_meta_message(u"effort-record")
return meta.impl(authentication=([first_member, second_member],),
distribution=(global_time,),
payload=(first_timestamp, second_timestamp, history))
def check_votecast(self, messages):
with self._dispersy.database:
communities = {}
for cid in set([message.payload.cid for message in messages]):
channel_id = self._get_channel_id(cid)
if not channel_id:
communities[cid] = self._get_channel_community(
message.payload.cid)
for message in messages:
if __debug__: dprint(message)
community = communities.get(message.payload.cid)
if community:
yield DelayMessageReqChannelMessage(
message,
community,
includeSnapshot=message.payload.vote >
0) #request torrents if positive vote
else:
yield message
# ensure that no commits occur
raise IgnoreCommits()