def __init__(self, session, sdef):
self.dllock = NoDispersyRLock()
self.session = session
self.sdef = sdef
self.old_metadir = self.session.get_swift_meta_dir()
# just enough so error saving and get_state() works
self.error = None
# To be able to return the progress of a stopped torrent, how far it got.
self.progressbeforestop = 0.0
# SwiftProcess performing the actual download.
self.sp = None
# spstatus
self.dlstatus = DLSTATUS_WAITING4HASHCHECK
self.dynasize = 0
self.progress = 0.0
self.curspeeds = {DOWNLOAD: 0.0, UPLOAD: 0.0} # bytes/s
self.numleech = 0
self.numseeds = 0
self.contentbytes = {DOWNLOAD: 0, UPLOAD: 0} # bytes
self.done = False # when set it means this download is being removed
self.midict = {}
self.time_seeding = [0, None]
self.total_up = 0
self.total_down = 0
self.lm_network_vod_event_callback = None
self.askmoreinfo = False
def __init__(self, binpath, workdir, zerostatedir, listenport, httpgwport,
cmdgwport, connhandler):
# Called by any thread, assume sessionlock is held
self.splock = NoDispersyRLock()
self.binpath = binpath
self.workdir = workdir
self.zerostatedir = zerostatedir
InstanceConnection.__init__(self, None, connhandler,
self.i2ithread_readlinecallback)
# Main UDP listen socket
if listenport is None:
self.listenport = random.randint(10001, 10999)
else:
self.listenport = listenport
# NSSA control socket
if cmdgwport is None:
self.cmdport = random.randint(11001, 11999)
else:
self.cmdport = cmdgwport
# content web server
if httpgwport is None:
self.httpport = random.randint(12001, 12999)
else:
self.httpport = httpgwport
# Security: only accept commands from localhost, enable HTTP gw,
# no stats/webUI web server
args = []
args.append(str(self.binpath))
# Arno, 2012-05-29: Hack. Win32 getopt code eats first arg when Windows app
# instead of CONSOLE app.
args.append("-j")
args.append("-l") # listen port
args.append("0.0.0.0:" + str(self.listenport))
args.append("-c") # command port
args.append("127.0.0.1:" + str(self.cmdport))
args.append("-g") # HTTP gateway port
args.append("127.0.0.1:" + str(self.httpport))
args.append("-w")
if zerostatedir is not None:
args.append("-e")
args.append(zerostatedir)
#args.append("-B") # DEBUG Hack
if DEBUG:
print >> sys.stderr, "SwiftProcess: __init__: Running", args, "workdir", workdir
if sys.platform == "win32":
creationflags = subprocess.CREATE_NEW_PROCESS_GROUP
else:
creationflags = 0
self.popen = subprocess.Popen(args,
close_fds=True,
cwd=workdir,
creationflags=creationflags)
self.roothash2dl = {}
self.donestate = DONE_STATE_WORKING # shutting down
def __init__(self,session,sdef):
self.dllock = NoDispersyRLock()
self.session = session
self.sdef = sdef
# just enough so error saving and get_state() works
self.error = None
# To be able to return the progress of a stopped torrent, how far it got.
self.progressbeforestop = 0.0
# SwiftProcess performing the actual download.
self.sp = None
# spstatus
self.dlstatus = DLSTATUS_WAITING4HASHCHECK
self.dynasize = 0L
self.progress = 0.0
self.curspeeds = {DOWNLOAD:0.0,UPLOAD:0.0} # bytes/s
self.numleech = 0
self.numseeds = 0
self.contentbytes = {DOWNLOAD:0,UPLOAD:0} # bytes
self.done = False # when set it means this download is being removed
self.midict = {}
self.time_seeding = [0, None]
self.total_up = 0
self.total_down = 0
self.lm_network_vod_event_callback = None
self.askmoreinfo = False
def __init__(self, trsession, ignore_singleton=False):
if not ignore_singleton:
if LibtorrentMgr.__single:
raise RuntimeError("LibtorrentMgr is singleton")
LibtorrentMgr.__single = self
self.trsession = trsession
settings = lt.session_settings()
settings.user_agent = 'Tribler/' + version_id
# Elric: Strip out the -rcX, -beta, -whatever tail on the version string.
fingerprint = ['TL'] + map(int,
version_id.split('-')[0].split('.')) + [0]
# Workaround for libtorrent 0.16.3 segfault (see https://code.google.com/p/libtorrent/issues/detail?id=369)
self.ltsession = lt.session(lt.fingerprint(*fingerprint), flags=1)
self.ltsession.set_settings(settings)
self.ltsession.set_alert_mask(
lt.alert.category_t.stats_notification
| lt.alert.category_t.error_notification
| lt.alert.category_t.status_notification
| lt.alert.category_t.storage_notification
| lt.alert.category_t.performance_warning
| lt.alert.category_t.debug_notification)
self.ltsession.listen_on(self.trsession.get_listen_port(),
self.trsession.get_listen_port() + 10)
self.set_upload_rate_limit(-1)
self.set_download_rate_limit(-1)
self.upnp_mapper = self.ltsession.start_upnp()
print >> sys.stderr, "LibtorrentMgr: listening on %d" % self.ltsession.listen_port(
)
# Start DHT
self.dht_ready = False
try:
dht_state = open(
os.path.join(self.trsession.get_state_dir(),
DHTSTATE_FILENAME)).read()
self.ltsession.start_dht(lt.bdecode(dht_state))
except:
print >> sys.stderr, "LibtorrentMgr: could not restore dht state, starting from scratch"
self.ltsession.start_dht(None)
self.ltsession.add_dht_router('router.bittorrent.com', 6881)
self.ltsession.add_dht_router('router.utorrent.com', 6881)
self.ltsession.add_dht_router('router.bitcomet.com', 6881)
self.torlock = NoDispersyRLock()
self.torrents = {}
self.metainfo_requests = {}
self.metainfo_lock = threading.RLock()
self.metainfo_cache = {}
self.trsession.lm.rawserver.add_task(self.process_alerts, 1)
self.trsession.lm.rawserver.add_task(self.reachability_check, 1)
self.trsession.lm.rawserver.add_task(self.monitor_dht, 5)
self.upnp_mappings = {}
def setUp(self, annotate=True):
TriblerCoreTest.setUp(self, annotate=annotate)
self.lm = TriblerLaunchMany()
self.lm.sesslock = NoDispersyRLock()
self.lm.session = MockObject()
# Ignore notifications
mock_notifier = MockObject()
mock_notifier.notify = lambda *_: None
self.lm.session.notifier = mock_notifier
def __init__(self,binpath,workdir,zerostatedir,listenport,httpgwport,cmdgwport,connhandler):
# Called by any thread, assume sessionlock is held
self.splock = NoDispersyRLock()
self.binpath = binpath
self.workdir = workdir
self.zerostatedir = zerostatedir
InstanceConnection.__init__(self, None, connhandler, self.i2ithread_readlinecallback)
# Main UDP listen socket
if listenport is None:
self.listenport = random.randint(10001,10999)
else:
self.listenport = listenport
# NSSA control socket
if cmdgwport is None:
self.cmdport = random.randint(11001,11999)
else:
self.cmdport = cmdgwport
# content web server
if httpgwport is None:
self.httpport = random.randint(12001,12999)
else:
self.httpport = httpgwport
# Security: only accept commands from localhost, enable HTTP gw,
# no stats/webUI web server
args=[]
args.append(str(self.binpath))
# Arno, 2012-05-29: Hack. Win32 getopt code eats first arg when Windows app
# instead of CONSOLE app.
args.append("-j")
args.append("-l") # listen port
args.append("0.0.0.0:"+str(self.listenport))
args.append("-c") # command port
args.append("127.0.0.1:"+str(self.cmdport))
args.append("-g") # HTTP gateway port
args.append("127.0.0.1:"+str(self.httpport))
args.append("-w")
if zerostatedir is not None:
args.append("-e")
args.append(zerostatedir)
#args.append("-B") # DEBUG Hack
if DEBUG:
print >>sys.stderr,"SwiftProcess: __init__: Running",args,"workdir",workdir
if sys.platform == "win32":
creationflags=subprocess.CREATE_NEW_PROCESS_GROUP
else:
creationflags=0
self.popen = subprocess.Popen(args,close_fds=True,cwd=workdir,creationflags=creationflags)
self.roothash2dl = {}
self.donestate = DONE_STATE_WORKING # shutting down
def __init__(self):
# always use getInstance() to create this object
# ARNOCOMMENT: why isn't the lock used on this read?!
if self.__single != None:
raise RuntimeError, "RePEXScheduler is singleton"
from Tribler.Core.Session import Session # Circular import fix
self.session = Session.get_instance()
self.lock = RLock()
self.active = False
self.current_repex = None # infohash
self.downloads = {} # infohash -> Download; in order to stop Downloads that are done repexing
self.last_attempts = {} # infohash -> ts; in order to prevent starvation when a certain download
def __init__(self,
config=None,
ignore_singleton=False,
autoload_discovery=True):
"""
A Session object is created which is configured with the Tribler configuration object.
Only a single session instance can exist at a time in a process.
:param config: a TriblerConfig object or None, in which case we
look for a saved session in the default location (state dir). If
we can't find it, we create a new TriblerConfig() object to
serve as startup config. Next, the config is saved in the directory
indicated by its 'state_dir' attribute.
:param ignore_singleton: for testing purposes only. Enables the existence of multiple
Session instances.
:param autoload_discovery: only false in the Tunnel community tests
"""
addObserver(self.unhandled_error_observer)
patch_crypto_be_discovery()
if not ignore_singleton:
if Session.__single:
raise RuntimeError("Session is singleton")
Session.__single = self
self._logger = logging.getLogger(self.__class__.__name__)
self.ignore_singleton = ignore_singleton
self.session_lock = NoDispersyRLock()
self.config = config or TriblerConfig()
self.get_ports_in_config()
self.create_state_directory_structure()
if not config.get_megacache_enabled():
config.set_torrent_checking_enabled(False)
self.selected_ports = config.selected_ports
self.init_keypair()
self.lm = TriblerLaunchMany()
self.notifier = Notifier()
self.sqlite_db = None
self.upgrader = None
self.dispersy_member = None
self.autoload_discovery = autoload_discovery
def __init__(self, session, tdef):
super(LibtorrentDownloadImpl, self).__init__()
self._logger = logging.getLogger(self.__class__.__name__)
self.dllock = NoDispersyRLock()
self.session = session
self.tdef = tdef
self.handle = None
self.vod_index = None
self.orig_files = None
self.finished_callback = None
self.finished_callback_already_called = False
# To be able to return the progress of a stopped torrent, how far it got.
self.progressbeforestop = 0.0
self.filepieceranges = []
# Libtorrent session manager, can be None at this point as the core could have
# not been started. Will set in create_engine wrapper
self.ltmgr = None
# Libtorrent status
self.lt_status = None
self.error = None
self.done = False
self.pause_after_next_hashcheck = False
self.checkpoint_after_next_hashcheck = False
self.tracker_status = {} # {url: [num_peers, status_str]}
self.prebuffsize = 5 * 1024 * 1024
self.endbuffsize = 0
self.vod_seekpos = 0
self.max_prebuffsize = 5 * 1024 * 1024
self.pstate_for_restart = None
self.cew_scheduled = False
self.askmoreinfo = False
self.correctedinfoname = u""
self._checkpoint_disabled = False
self.deferreds_resume = []
self.deferreds_handle = []
self.deferred_added = Deferred()
self.deferred_removed = Deferred()
self.handle_check_lc = self.register_task("handle_check", LoopingCall(self.check_handle))
def setUp(self, annotate=True):
TriblerCoreTest.setUp(self, annotate=annotate)
self.lm = TriblerLaunchMany()
self.lm.session_lock = NoDispersyRLock()
self.lm.session = MockObject()
self.lm.session.config = MockObject()
self.lm.session.config.get_max_upload_rate = lambda: 100
self.lm.session.config.get_max_download_rate = lambda: 100
self.lm.session.config.get_default_number_hops = lambda: 0
# Ignore notifications
mock_notifier = MockObject()
mock_notifier.notify = lambda *_: None
self.lm.session.notifier = mock_notifier
def __init__(self, session, sdef):
self.dllock = NoDispersyRLock()
self.session = session
self.sdef = sdef
# just enough so error saving and get_state() works
self.error = None
# To be able to return the progress of a stopped torrent, how far it got.
self.progressbeforestop = 0.0
# SwiftProcess performing the actual download.
self.sp = None
# spstatus
self.dlstatus = DLSTATUS_WAITING4HASHCHECK
self.dynasize = 0L
self.progress = 0.0
self.curspeeds = {DOWNLOAD: 0.0, UPLOAD: 0.0} # bytes/s
self.numleech = 0
self.numseeds = 0
self.done = False
self.midict = {}
self.lm_network_vod_event_callback = None
def __init__(self, trsession, utility):
if LibtorrentMgr.__single:
raise RuntimeError, "LibtorrentMgr is singleton"
LibtorrentMgr.__single = self
self.utility = utility
self.trsession = trsession
settings = lt.session_settings()
settings.user_agent = 'Tribler/' + version_id
fingerprint = ['TL'] + map(int, version_id.split('.')) + [0]
# Workaround for libtorrent 0.16.3 segfault (see https://code.google.com/p/libtorrent/issues/detail?id=369)
self.ltsession = lt.session(lt.fingerprint(*fingerprint), flags=1)
self.ltsession.set_settings(settings)
self.ltsession.set_alert_mask(
lt.alert.category_t.stats_notification
| lt.alert.category_t.error_notification
| lt.alert.category_t.status_notification
| lt.alert.category_t.storage_notification
| lt.alert.category_t.performance_warning)
self.ltsession.listen_on(self.trsession.get_listen_port(),
self.trsession.get_listen_port() + 10)
self.set_upload_rate_limit(-1)
self.set_download_rate_limit(-1)
self.ltsession.start_upnp()
# Start DHT
try:
dht_state = open(
os.path.join(self.trsession.get_state_dir(),
DHTSTATE_FILENAME)).read()
self.ltsession.start_dht(lt.bdecode(dht_state))
except:
print >> sys.stderr, "LibtorrentMgr: could not restore dht state, starting from scratch"
self.ltsession.start_dht(None)
self.ltsession.add_dht_router('router.bittorrent.com', 6881)
self.ltsession.add_dht_router('router.utorrent.com', 6881)
self.ltsession.add_dht_router('router.bitcomet.com', 6881)
self.torlock = NoDispersyRLock()
self.torrents = {}
self.trsession.lm.rawserver.add_task(self.process_alerts, 1)
self.trsession.lm.rawserver.add_task(self.reachability_check, 1)
def __init__(self, config=None, autoload_discovery=True):
"""
A Session object is created which is configured with the Tribler configuration object.
Only a single session instance can exist at a time in a process.
:param config: a TriblerConfig object or None, in which case we
look for a saved session in the default location (state dir). If
we can't find it, we create a new TriblerConfig() object to
serve as startup config. Next, the config is saved in the directory
indicated by its 'state_dir' attribute.
:param autoload_discovery: only false in the Tunnel community tests
"""
addObserver(self.unhandled_error_observer)
patch_crypto_be_discovery()
self._logger = logging.getLogger(self.__class__.__name__)
self.session_lock = NoDispersyRLock()
self.config = config or TriblerConfig()
self.get_ports_in_config()
self.create_state_directory_structure()
if not self.config.get_megacache_enabled():
self.config.set_torrent_checking_enabled(False)
self.selected_ports = self.config.selected_ports
self.init_keypair()
self.lm = TriblerLaunchMany()
self.notifier = Notifier()
self.sqlite_db = None
self.upgrader_enabled = True
self.dispersy_member = None
self.readable_status = '' # Human-readable string to indicate the status during startup/shutdown of Tribler
self.autoload_discovery = autoload_discovery
def __init__(self, session, tdef):
self.dllock = NoDispersyRLock()
self.session = session
self.tdef = tdef
self.handle = None
# Just enough so error saving and get_state() works
self.error = None
# To be able to return the progress of a stopped torrent, how far it got.
self.progressbeforestop = 0.0
self.filepieceranges = []
# Libtorrent session manager
self.ltmgr = LibtorrentMgr.getInstance()
# Libtorrent status
self.dlstates = [DLSTATUS_WAITING4HASHCHECK, DLSTATUS_HASHCHECKING, DLSTATUS_METADATA, DLSTATUS_DOWNLOADING, DLSTATUS_SEEDING, DLSTATUS_SEEDING, DLSTATUS_ALLOCATING_DISKSPACE, DLSTATUS_HASHCHECKING]
self.dlstate = DLSTATUS_WAITING4HASHCHECK
self.length = 0L
self.progress = 0.0
self.bufferprogress = 0.0
self.curspeeds = {DOWNLOAD:0.0,UPLOAD:0.0} # bytes/s
self.all_time_upload = 0.0
self.all_time_download = 0.0
self.finished_time = 0.0
self.done = False
self.pause_after_next_hashcheck = False
self.checkpoint_after_next_hashcheck = False
self.prebuffsize = 5*1024*1024
self.queue_position = -1
self.vod_readpos = 0
self.vod_pausepos = 0
self.vod_status = ""
self.lm_network_vod_event_callback = None
self.pstate_for_restart = None
self.cew_scheduled = False
class SwiftProcess(InstanceConnection):
""" Representation of an operating-system process running the C++ swift engine.
A swift engine can participate in one or more swarms."""
def __init__(self, binpath, workdir, zerostatedir, listenport, httpgwport,
cmdgwport, connhandler):
# Called by any thread, assume sessionlock is held
self.splock = NoDispersyRLock()
self.binpath = binpath
self.workdir = workdir
self.zerostatedir = zerostatedir
InstanceConnection.__init__(self, None, connhandler,
self.i2ithread_readlinecallback)
# Main UDP listen socket
if listenport is None:
self.listenport = random.randint(10001, 10999)
else:
self.listenport = listenport
# NSSA control socket
if cmdgwport is None:
self.cmdport = random.randint(11001, 11999)
else:
self.cmdport = cmdgwport
# content web server
if httpgwport is None:
self.httpport = random.randint(12001, 12999)
else:
self.httpport = httpgwport
# Security: only accept commands from localhost, enable HTTP gw,
# no stats/webUI web server
args = []
args.append(str(self.binpath))
# Arno, 2012-05-29: Hack. Win32 getopt code eats first arg when Windows app
# instead of CONSOLE app.
args.append("-j")
args.append("-l") # listen port
args.append("0.0.0.0:" + str(self.listenport))
args.append("-c") # command port
args.append("127.0.0.1:" + str(self.cmdport))
args.append("-g") # HTTP gateway port
args.append("127.0.0.1:" + str(self.httpport))
args.append("-w")
if zerostatedir is not None:
args.append("-e")
args.append(zerostatedir)
#args.append("-B") # DEBUG Hack
if DEBUG:
print >> sys.stderr, "SwiftProcess: __init__: Running", args, "workdir", workdir
if sys.platform == "win32":
creationflags = subprocess.CREATE_NEW_PROCESS_GROUP
else:
creationflags = 0
self.popen = subprocess.Popen(args,
close_fds=True,
cwd=workdir,
creationflags=creationflags)
self.roothash2dl = {}
self.donestate = DONE_STATE_WORKING # shutting down
#
# Instance2Instance
#
def start_cmd_connection(self):
# Called by any thread, assume sessionlock is held
if self.is_alive():
self.singsock = self.connhandler.start_connection(
("127.0.0.1", self.cmdport), self)
else:
print >> sys.stderr, "sp: start_cmd_connection: Process dead? returncode", self.popen.returncode, "pid", self.popen.pid
def i2ithread_readlinecallback(self, ic, cmd):
#if DEBUG:
# print >>sys.stderr,"sp: Got command #"+cmd+"#"
words = cmd.split()
if words[0] == "TUNNELRECV":
address, session = words[1].split("/")
host, port = address.split(":")
port = int(port)
session = session.decode("HEX")
length = int(words[2])
# require LENGTH bytes
if len(ic.buffer) < length:
return length - len(ic.buffer)
data = ic.buffer[:length]
ic.buffer = ic.buffer[length:]
self.roothash2dl["dispersy"].i2ithread_data_came_in(
session, (host, port), data)
else:
roothash = binascii.unhexlify(words[1])
if words[0] == "ERROR":
print >> sys.stderr, "sp: i2ithread_readlinecallback:", cmd
self.splock.acquire()
try:
if roothash not in self.roothash2dl.keys():
print >> sys.stderr, "sp: i2ithread_readlinecallback: unknown roothash", words[
1]
return
d = self.roothash2dl[roothash]
except:
#print >>sys.stderr,"GOT", words
#print >>sys.stderr,"HAVE", [key.encode("HEX") for key in self.roothash2dl.keys()]
raise
finally:
self.splock.release()
# Hide NSSA interface for SwiftDownloadImpl
if words[0] == "INFO": # INFO HASH status dl/total
dlstatus = int(words[2])
pargs = words[3].split("/")
dynasize = int(pargs[1])
if dynasize == 0:
progress = 0.0
else:
progress = float(pargs[0]) / float(pargs[1])
dlspeed = float(words[4])
ulspeed = float(words[5])
numleech = int(words[6])
numseeds = int(words[7])
d.i2ithread_info_callback(dlstatus, progress, dynasize,
dlspeed, ulspeed, numleech, numseeds)
elif words[0] == "PLAY":
#print >>sys.stderr,"sp: i2ithread_readlinecallback: Got PLAY",cmd
httpurl = words[2]
d.i2ithread_vod_event_callback(VODEVENT_START, httpurl)
elif words[0] == "MOREINFO":
jsondata = cmd[len("MOREINFO ") + 40 + 1:]
midict = json.loads(jsondata)
d.i2ithread_moreinfo_callback(midict)
#
# Swift Mgmt interface
#
def start_download(self, d):
self.splock.acquire()
try:
if self.donestate != DONE_STATE_WORKING or not self.is_alive():
return
roothash = d.get_def().get_roothash()
roothash_hex = d.get_def().get_roothash_as_hex()
# Before send to handle INFO msgs
self.roothash2dl[roothash] = d
url = d.get_def().get_url()
# MULTIFILE
if len(d.get_selected_files()) == 1:
specpath = d.get_selected_files()[0]
qpath = urllib.quote(specpath)
url += "/" + qpath
# Default is unlimited, so don't send MAXSPEED then
maxdlspeed = d.get_max_speed(DOWNLOAD)
if maxdlspeed == 0:
maxdlspeed = None
maxulspeed = d.get_max_speed(UPLOAD)
if maxulspeed == 0:
maxulspeed = None
self.send_start(url,
roothash_hex=roothash_hex,
maxdlspeed=maxdlspeed,
maxulspeed=maxulspeed,
destdir=d.get_dest_dir())
finally:
self.splock.release()
def add_download(self, d):
self.splock.acquire()
try:
roothash = d.get_def().get_roothash()
# Before send to handle INFO msgs
self.roothash2dl[roothash] = d
finally:
self.splock.release()
def remove_download(self, d, removestate, removecontent):
self.splock.acquire()
try:
if self.donestate != DONE_STATE_WORKING or not self.is_alive():
return
roothash_hex = d.get_def().get_roothash_as_hex()
self.send_remove(roothash_hex, removestate, removecontent)
# After send to handle INFO msgs
roothash = d.get_def().get_roothash()
del self.roothash2dl[roothash]
finally:
self.splock.release()
def get_downloads(self):
self.splock.acquire()
try:
return self.roothash2dl.values()
finally:
self.splock.release()
def get_pid(self):
if self.popen is not None:
return self.popen.pid
else:
return -1
def get_listen_port(self):
return self.listenport
def set_max_speed(self, d, direct, speed):
self.splock.acquire()
try:
if self.donestate != DONE_STATE_WORKING or not self.is_alive():
return
roothash_hex = d.get_def().get_roothash_as_hex()
# In Tribler Core API = unlimited. In Swift CMDGW API
# 0 = none.
if speed == 0.0:
speed = 4294967296.0
self.send_max_speed(roothash_hex, direct, speed)
finally:
self.splock.release()
def checkpoint_download(self, d):
self.splock.acquire()
try:
# Arno, 2012-05-15: Allow during shutdown.
if not self.is_alive():
return
roothash_hex = d.get_def().get_roothash_as_hex()
self.send_checkpoint(roothash_hex)
finally:
self.splock.release()
def set_moreinfo_stats(self, d, enable):
self.splock.acquire()
try:
if self.donestate != DONE_STATE_WORKING or not self.is_alive():
return
roothash_hex = d.get_def().get_roothash_as_hex()
self.send_setmoreinfo(roothash_hex, enable)
finally:
self.splock.release()
def add_peer(self, d, addr):
self.splock.acquire()
try:
if self.donestate != DONE_STATE_WORKING or not self.is_alive():
return
addrstr = addr[0] + ':' + str(addr[1])
roothash_hex = d.get_def().get_roothash_as_hex()
self.send_peer_addr(roothash_hex, addrstr)
finally:
self.splock.release()
def early_shutdown(self):
# Called by any thread, assume sessionlock is held
# May get called twice, once by spm.release_sp() and spm.shutdown()
if self.donestate == DONE_STATE_WORKING:
self.donestate = DONE_STATE_EARLY_SHUTDOWN
else:
return
if self.popen is not None:
# Tell engine to shutdown so it can deregister dls from tracker
print >> sys.stderr, "sp: Telling process to shutdown"
self.send_shutdown()
def network_shutdown(self):
# Called by network thread, assume sessionlock is held
if self.donestate == DONE_STATE_EARLY_SHUTDOWN:
self.donestate = DONE_STATE_SHUTDOWN
else:
return
if self.popen is not None:
try:
print >> sys.stderr, "sp: Terminating process"
self.popen.terminate()
self.popen.wait()
self.popen = None
except:
print_exc()
# self.singsock auto closed by killing proc.
#
# Internal methods
#
def send_start(self,
url,
roothash_hex=None,
maxdlspeed=None,
maxulspeed=None,
destdir=None):
# assume splock is held to avoid concurrency on socket
print >> sys.stderr, "sp: send_start:", url, "destdir", destdir
cmd = 'START ' + url
if destdir is not None:
cmd += ' ' + destdir.encode("UTF-8")
cmd += '\r\n'
if maxdlspeed is not None:
cmd += 'MAXSPEED ' + roothash_hex + ' DOWNLOAD ' + str(
float(maxdlspeed)) + '\r\n'
if maxulspeed is not None:
cmd += 'MAXSPEED ' + roothash_hex + ' UPLOAD ' + str(
float(maxulspeed)) + '\r\n'
self.singsock.write(cmd)
def send_remove(self, roothash_hex, removestate, removecontent):
# assume splock is held to avoid concurrency on socket
self.singsock.write('REMOVE ' + roothash_hex + ' ' +
str(int(removestate)) + ' ' +
str(int(removecontent)) + '\r\n')
def send_checkpoint(self, roothash_hex):
# assume splock is held to avoid concurrency on socket
self.singsock.write('CHECKPOINT ' + roothash_hex + '\r\n')
def send_shutdown(self):
# assume splock is held to avoid concurrency on socket
self.singsock.write('SHUTDOWN\r\n')
def send_max_speed(self, roothash_hex, direct, speed):
# assume splock is held to avoid concurrency on socket
cmd = 'MAXSPEED ' + roothash_hex
if direct == DOWNLOAD:
cmd += ' DOWNLOAD '
else:
cmd += ' UPLOAD '
cmd += str(float(speed)) + '\r\n'
self.singsock.write(cmd)
def send_tunnel(self, session, address, data):
# assume splock is held to avoid concurrency on socket
if DEBUG:
print >> sys.stderr, "sp: send_tunnel:", len(
data), "bytes -> %s:%d" % address
self.singsock.write(
"TUNNELSEND %s:%d/%s %d\r\n" %
(address[0], address[1], session.encode("HEX"), len(data)))
self.singsock.write(data)
def send_setmoreinfo(self, roothash_hex, enable):
# assume splock is held to avoid concurrency on socket
onoff = "0"
if enable:
onoff = "1"
self.singsock.write('SETMOREINFO ' + roothash_hex + ' ' + onoff +
'\r\n')
def send_peer_addr(self, roothash_hex, addrstr):
# assume splock is held to avoid concurrency on socket
self.singsock.write('PEERADDR ' + roothash_hex + ' ' + addrstr +
'\r\n')
def is_alive(self):
if self.popen:
self.popen.poll()
return self.popen.returncode is None
return False
def __init__(self, scfg=None, ignore_singleton=False):
"""
A Session object is created which is configured following a copy of the
SessionStartupConfig scfg. (copy constructor used internally)
@param scfg SessionStartupConfig object or None, in which case we
look for a saved session in the default location (state dir). If
we can't find it, we create a new SessionStartupConfig() object to
serve as startup config. Next, the config is saved in the directory
indicated by its 'state_dir' attribute.
In the current implementation only a single session instance can exist
at a time in a process. The ignore_singleton flag is used for testing.
"""
if not ignore_singleton:
if Session.__single:
raise RuntimeError, "Session is singleton"
Session.__single = self
self.sesslock = NoDispersyRLock()
# Determine startup config to use
if scfg is None: # If no override
try:
# Then try to read from default location
state_dir = Session.get_default_state_dir()
cfgfilename = Session.get_default_config_filename(state_dir)
scfg = SessionStartupConfig.load(cfgfilename)
except:
# If that fails, create a fresh config with factory defaults
print_exc()
scfg = SessionStartupConfig()
self.sessconfig = scfg.sessconfig
else: # overrides any saved config
# Work from copy
self.sessconfig = copy.copy(scfg.sessconfig)
# Niels: 11/05/2012, turning off overlay
self.sessconfig["overlay"] = 0
self.sessconfig["crawler"] = 0
# Create dir for session state, if not exist
state_dir = self.sessconfig["state_dir"]
if state_dir is None:
state_dir = Session.get_default_state_dir()
self.sessconfig["state_dir"] = state_dir
if not os.path.isdir(state_dir):
os.makedirs(state_dir)
collected_torrent_dir = self.sessconfig["torrent_collecting_dir"]
if not collected_torrent_dir:
collected_torrent_dir = os.path.join(self.sessconfig["state_dir"], STATEDIR_TORRENTCOLL_DIR)
self.sessconfig["torrent_collecting_dir"] = collected_torrent_dir
collected_subtitles_dir = self.sessconfig.get("subtitles_collecting_dir", None)
if not collected_subtitles_dir:
collected_subtitles_dir = os.path.join(self.sessconfig["state_dir"], STATEDIR_SUBSCOLL_DIR)
self.sessconfig["subtitles_collecting_dir"] = collected_subtitles_dir
if not os.path.exists(collected_torrent_dir):
os.makedirs(collected_torrent_dir)
if not self.sessconfig["peer_icon_path"]:
self.sessconfig["peer_icon_path"] = os.path.join(self.sessconfig["state_dir"], STATEDIR_PEERICON_DIR)
# PERHAPS: load default TorrentDef and DownloadStartupConfig from state dir
# Let user handle that, he's got default_state_dir, etc.
# Core init
# print >>sys.stderr,'Session: __init__ config is', self.sessconfig
if GOTM2CRYPTO:
permidmod.init()
#
# Set params that depend on state_dir
#
# 1. keypair
#
pairfilename = os.path.join(self.sessconfig["state_dir"], "ec.pem")
if self.sessconfig["eckeypairfilename"] is None:
self.sessconfig["eckeypairfilename"] = pairfilename
if os.access(self.sessconfig["eckeypairfilename"], os.F_OK):
# May throw exceptions
self.keypair = permidmod.read_keypair(self.sessconfig["eckeypairfilename"])
else:
self.keypair = permidmod.generate_keypair()
# Save keypair
pubfilename = os.path.join(self.sessconfig["state_dir"], "ecpub.pem")
permidmod.save_keypair(self.keypair, pairfilename)
permidmod.save_pub_key(self.keypair, pubfilename)
# 2. Downloads persistent state dir
dlpstatedir = os.path.join(self.sessconfig["state_dir"], STATEDIR_DLPSTATE_DIR)
if not os.path.isdir(dlpstatedir):
os.mkdir(dlpstatedir)
# 3. tracker
trackerdir = self.get_internal_tracker_dir()
if not os.path.exists(trackerdir):
os.mkdir(trackerdir)
if self.sessconfig["tracker_dfile"] is None:
self.sessconfig["tracker_dfile"] = os.path.join(trackerdir, "tracker.db")
if self.sessconfig["tracker_allowed_dir"] is None:
self.sessconfig["tracker_allowed_dir"] = trackerdir
if self.sessconfig["tracker_logfile"] is None:
if sys.platform == "win32":
# Not "Nul:" but "nul" is /dev/null on Win32
sink = "nul"
else:
sink = "/dev/null"
self.sessconfig["tracker_logfile"] = sink
# 5. peer_icon_path
if self.sessconfig["peer_icon_path"] is None:
self.sessconfig["peer_icon_path"] = os.path.join(self.sessconfig["state_dir"], STATEDIR_PEERICON_DIR)
if not os.path.isdir(self.sessconfig["peer_icon_path"]):
os.mkdir(self.sessconfig["peer_icon_path"])
# 6. Poor man's versioning of SessionConfig, add missing
# default values. Really should use PERSISTENTSTATE_CURRENTVERSION
# and do conversions.
for key, defvalue in sessdefaults.iteritems():
if key not in self.sessconfig:
self.sessconfig[key] = defvalue
if not "live_aux_seeders" in self.sessconfig:
# Poor man's versioning, really should update PERSISTENTSTATE_CURRENTVERSION
self.sessconfig["live_aux_seeders"] = sessdefaults["live_aux_seeders"]
if not "nat_detect" in self.sessconfig:
self.sessconfig["nat_detect"] = sessdefaults["nat_detect"]
if not "puncturing_internal_port" in self.sessconfig:
self.sessconfig["puncturing_internal_port"] = sessdefaults["puncturing_internal_port"]
if not "stun_servers" in self.sessconfig:
self.sessconfig["stun_servers"] = sessdefaults["stun_servers"]
if not "pingback_servers" in self.sessconfig:
self.sessconfig["pingback_servers"] = sessdefaults["pingback_servers"]
if not "mainline_dht" in self.sessconfig:
self.sessconfig["mainline_dht"] = sessdefaults["mainline_dht"]
# SWIFTPROC
if self.sessconfig["swiftpath"] is None:
if sys.platform == "win32":
self.sessconfig["swiftpath"] = os.path.join(self.sessconfig["install_dir"], "swift.exe")
else:
self.sessconfig["swiftpath"] = os.path.join(self.sessconfig["install_dir"], "swift")
# Checkpoint startup config
self.save_pstate_sessconfig()
# Create handler for calling back the user via separate threads
self.uch = UserCallbackHandler(self)
# Create engine with network thread
self.lm = TriblerLaunchMany()
self.lm.register(self, self.sesslock)
self.lm.start()
class Session(SessionRuntimeConfig):
"""
A Session is a running instance of the Tribler Core and the Core's central
class. It implements the SessionConfigInterface which can be used to change
session parameters at runtime (for selected parameters).
cf. libtorrent session
"""
__single = None
def __init__(self, scfg=None, ignore_singleton=False):
"""
A Session object is created which is configured following a copy of the
SessionStartupConfig scfg. (copy constructor used internally)
@param scfg SessionStartupConfig object or None, in which case we
look for a saved session in the default location (state dir). If
we can't find it, we create a new SessionStartupConfig() object to
serve as startup config. Next, the config is saved in the directory
indicated by its 'state_dir' attribute.
In the current implementation only a single session instance can exist
at a time in a process. The ignore_singleton flag is used for testing.
"""
if not ignore_singleton:
if Session.__single:
raise RuntimeError, "Session is singleton"
Session.__single = self
self.sesslock = NoDispersyRLock()
# Determine startup config to use
if scfg is None: # If no override
try:
# Then try to read from default location
state_dir = Session.get_default_state_dir()
cfgfilename = Session.get_default_config_filename(state_dir)
scfg = SessionStartupConfig.load(cfgfilename)
except:
# If that fails, create a fresh config with factory defaults
print_exc()
scfg = SessionStartupConfig()
self.sessconfig = scfg.sessconfig
else: # overrides any saved config
# Work from copy
self.sessconfig = copy.copy(scfg.sessconfig)
# Niels: 11/05/2012, turning off overlay
self.sessconfig["overlay"] = 0
self.sessconfig["crawler"] = 0
# Create dir for session state, if not exist
state_dir = self.sessconfig["state_dir"]
if state_dir is None:
state_dir = Session.get_default_state_dir()
self.sessconfig["state_dir"] = state_dir
if not os.path.isdir(state_dir):
os.makedirs(state_dir)
collected_torrent_dir = self.sessconfig["torrent_collecting_dir"]
if not collected_torrent_dir:
collected_torrent_dir = os.path.join(self.sessconfig["state_dir"], STATEDIR_TORRENTCOLL_DIR)
self.sessconfig["torrent_collecting_dir"] = collected_torrent_dir
collected_subtitles_dir = self.sessconfig.get("subtitles_collecting_dir", None)
if not collected_subtitles_dir:
collected_subtitles_dir = os.path.join(self.sessconfig["state_dir"], STATEDIR_SUBSCOLL_DIR)
self.sessconfig["subtitles_collecting_dir"] = collected_subtitles_dir
if not os.path.exists(collected_torrent_dir):
os.makedirs(collected_torrent_dir)
if not self.sessconfig["peer_icon_path"]:
self.sessconfig["peer_icon_path"] = os.path.join(self.sessconfig["state_dir"], STATEDIR_PEERICON_DIR)
# PERHAPS: load default TorrentDef and DownloadStartupConfig from state dir
# Let user handle that, he's got default_state_dir, etc.
# Core init
# print >>sys.stderr,'Session: __init__ config is', self.sessconfig
if GOTM2CRYPTO:
permidmod.init()
#
# Set params that depend on state_dir
#
# 1. keypair
#
pairfilename = os.path.join(self.sessconfig["state_dir"], "ec.pem")
if self.sessconfig["eckeypairfilename"] is None:
self.sessconfig["eckeypairfilename"] = pairfilename
if os.access(self.sessconfig["eckeypairfilename"], os.F_OK):
# May throw exceptions
self.keypair = permidmod.read_keypair(self.sessconfig["eckeypairfilename"])
else:
self.keypair = permidmod.generate_keypair()
# Save keypair
pubfilename = os.path.join(self.sessconfig["state_dir"], "ecpub.pem")
permidmod.save_keypair(self.keypair, pairfilename)
permidmod.save_pub_key(self.keypair, pubfilename)
# 2. Downloads persistent state dir
dlpstatedir = os.path.join(self.sessconfig["state_dir"], STATEDIR_DLPSTATE_DIR)
if not os.path.isdir(dlpstatedir):
os.mkdir(dlpstatedir)
# 3. tracker
trackerdir = self.get_internal_tracker_dir()
if not os.path.exists(trackerdir):
os.mkdir(trackerdir)
if self.sessconfig["tracker_dfile"] is None:
self.sessconfig["tracker_dfile"] = os.path.join(trackerdir, "tracker.db")
if self.sessconfig["tracker_allowed_dir"] is None:
self.sessconfig["tracker_allowed_dir"] = trackerdir
if self.sessconfig["tracker_logfile"] is None:
if sys.platform == "win32":
# Not "Nul:" but "nul" is /dev/null on Win32
sink = "nul"
else:
sink = "/dev/null"
self.sessconfig["tracker_logfile"] = sink
# 5. peer_icon_path
if self.sessconfig["peer_icon_path"] is None:
self.sessconfig["peer_icon_path"] = os.path.join(self.sessconfig["state_dir"], STATEDIR_PEERICON_DIR)
if not os.path.isdir(self.sessconfig["peer_icon_path"]):
os.mkdir(self.sessconfig["peer_icon_path"])
# 6. Poor man's versioning of SessionConfig, add missing
# default values. Really should use PERSISTENTSTATE_CURRENTVERSION
# and do conversions.
for key, defvalue in sessdefaults.iteritems():
if key not in self.sessconfig:
self.sessconfig[key] = defvalue
if not "live_aux_seeders" in self.sessconfig:
# Poor man's versioning, really should update PERSISTENTSTATE_CURRENTVERSION
self.sessconfig["live_aux_seeders"] = sessdefaults["live_aux_seeders"]
if not "nat_detect" in self.sessconfig:
self.sessconfig["nat_detect"] = sessdefaults["nat_detect"]
if not "puncturing_internal_port" in self.sessconfig:
self.sessconfig["puncturing_internal_port"] = sessdefaults["puncturing_internal_port"]
if not "stun_servers" in self.sessconfig:
self.sessconfig["stun_servers"] = sessdefaults["stun_servers"]
if not "pingback_servers" in self.sessconfig:
self.sessconfig["pingback_servers"] = sessdefaults["pingback_servers"]
if not "mainline_dht" in self.sessconfig:
self.sessconfig["mainline_dht"] = sessdefaults["mainline_dht"]
# SWIFTPROC
if self.sessconfig["swiftpath"] is None:
if sys.platform == "win32":
self.sessconfig["swiftpath"] = os.path.join(self.sessconfig["install_dir"], "swift.exe")
else:
self.sessconfig["swiftpath"] = os.path.join(self.sessconfig["install_dir"], "swift")
# Checkpoint startup config
self.save_pstate_sessconfig()
# Create handler for calling back the user via separate threads
self.uch = UserCallbackHandler(self)
# Create engine with network thread
self.lm = TriblerLaunchMany()
self.lm.register(self, self.sesslock)
self.lm.start()
#
# Class methods
#
def get_instance(*args, **kw):
""" Returns the Session singleton if it exists or otherwise
creates it first, in which case you need to pass the constructor
params.
@return Session."""
if Session.__single is None:
Session(*args, **kw)
return Session.__single
get_instance = staticmethod(get_instance)
def has_instance():
return Session.__single != None
has_instance = staticmethod(has_instance)
def del_instance():
Session.__single = None
del_instance = staticmethod(del_instance)
def get_default_state_dir(homedirpostfix=".Tribler"):
""" Returns the factory default directory for storing session state
on the current platform (Win32,Mac,Unix).
@return An absolute path name. """
# Allow override
statedirvar = "${TSTATEDIR}"
statedir = os.path.expandvars(statedirvar)
if statedir and statedir != statedirvar:
return statedir
if os.path.isdir(homedirpostfix):
return os.path.abspath(homedirpostfix)
appdir = get_appstate_dir()
statedir = os.path.join(appdir, homedirpostfix)
return statedir
get_default_state_dir = staticmethod(get_default_state_dir)
#
# Public methods
#
def start_download(self, cdef, dcfg=None, initialdlstatus=None, hidden=False):
"""
Creates a Download object and adds it to the session. The passed
ContentDef and DownloadStartupConfig are copied into the new Download
object. The Download is then started and checkpointed.
If a checkpointed version of the Download is found, that is restarted
overriding the saved DownloadStartupConfig if "dcfg" is not None.
@param cdef A finalized TorrentDef or a SwiftDef
@param dcfg DownloadStartupConfig or None, in which case
a new DownloadStartupConfig() is created with its default settings
and the result becomes the runtime config of this Download.
@param initialdlstatus The initial download status of this Download
or None. This enables the caller to create a Download in e.g.
DLSTATUS_STOPPED state instead.
@param hidden Whether this torrent should be added to the mypreference table
@return Download
"""
# locking by lm
if cdef.get_def_type() == "torrent":
return self.lm.add(cdef, dcfg, initialdlstatus=initialdlstatus, hidden=hidden)
else:
# SWIFTPROC
return self.lm.swift_add(cdef, dcfg, initialdlstatus=initialdlstatus, hidden=hidden)
def resume_download_from_file(self, filename):
"""
Recreates Download from resume file
@return a Download object.
Note: this cannot be made into a method of Download, as the Download
needs to be bound to a session, it cannot exist independently.
"""
raise NotYetImplementedException()
def get_downloads(self):
"""
Returns a copy of the list of Downloads.
@return A list of Download objects.
"""
# locking by lm
return self.lm.get_downloads()
def get_download(self, hash):
"""
Returns the Download object for this hash.
@return A Donwload Object.
"""
# locking by lm
return self.lm.get_download(hash)
def remove_download(self, d, removecontent=False, removestate=True, hidden=False):
"""
Stops the download and removes it from the session.
@param d The Download to remove
@param removecontent Whether to delete the already downloaded content
from disk.
@param removestate Whether to delete the metadata files of the downloaded
content from disk.
@param hidden Whether this torrent is added to the mypreference table and this entry should be
removed
"""
# locking by lm
if d.get_def().get_def_type() == "torrent":
self.lm.remove(d, removecontent=removecontent, removestate=removestate, hidden=hidden)
else:
# SWIFTPROC
self.lm.swift_remove(d, removecontent=removecontent, removestate=removestate, hidden=hidden)
def remove_download_by_id(self, id, removecontent=False, removestate=True):
"""
@param infohash The Download to remove
@param removecontent Whether to delete the already downloaded content
from disk.
!We can only remove content when the download object is found, otherwise only
the state is removed.
"""
downloadList = self.get_downloads()
for download in downloadList:
if download.get_def().get_id() == id:
self.remove_download(download, removecontent, removestate)
return
self.lm.remove_id(id)
self.uch.perform_removestate_callback(id, [], False)
def set_download_states_callback(self, usercallback, getpeerlist=False):
"""
See Download.set_state_callback. Calls usercallback with a list of
DownloadStates, one for each Download in the Session as first argument.
The usercallback must return a tuple (when,getpeerlist) that indicates
when to reinvoke the callback again (as a number of seconds from now,
or < 0.0 if not at all) and whether to also include the details of
the connected peers in the DownloadStates on that next call.
The callback will be called by a popup thread which can be used
indefinitely (within reason) by the higher level code.
@param usercallback A function adhering to the above spec.
"""
self.lm.set_download_states_callback(usercallback, getpeerlist)
#
# Config parameters that only exist at runtime
#
def get_permid(self):
""" Returns the PermID of the Session, as determined by the
SessionConfig.set_permid() parameter. A PermID is a public key
@return The PermID encoded in a string in DER format. """
self.sesslock.acquire()
try:
return str(self.keypair.pub().get_der())
finally:
self.sesslock.release()
def get_external_ip(self):
""" Returns the external IP address of this Session, i.e., by which
it is reachable from the Internet. This address is determined via
various mechanisms such as the UPnP protocol, our dialback mechanism,
and an inspection of the local network configuration.
@return A string. """
# locking done by lm
return self.lm.get_ext_ip()
def get_externally_reachable(self):
""" Returns whether the Session is externally reachable, i.e., its
listen port is not firewalled. Use add_observer() with NTFY_REACHABLE
to register to the event of detecting reachablility. Note that due to
the use of UPnP a Session may become reachable some time after
startup and due to the Dialback mechanism, this method may return
False while the Session is actually already reachable. Note that True
doesn't mean the Session is reachable from the open Internet, could just
be from the local (otherwise firewalled) LAN.
@return A boolean. """
# Arno, LICHT: make it throw exception when used in LITE versie.
raise NotYetImplementedException()
def get_current_startup_config_copy(self):
""" Returns a SessionStartupConfig that is a copy of the current runtime
SessionConfig.
@return SessionStartupConfig
"""
# Called by any thread
self.sesslock.acquire()
try:
sessconfig = copy.copy(self.sessconfig)
return SessionStartupConfig(sessconfig=sessconfig)
finally:
self.sesslock.release()
#
# Internal tracker
#
def get_internal_tracker_url(self):
""" Returns the announce URL for the internal tracker.
@return URL """
# Called by any thread
self.sesslock.acquire()
try:
url = None
if "tracker_url" in self.sessconfig:
url = self.sessconfig["tracker_url"] # user defined override, e.g. specific hostname
if url is None:
ip = self.lm.get_ext_ip()
port = self.get_listen_port()
url = "http://" + ip + ":" + str(port) + "/announce/"
return url
finally:
self.sesslock.release()
def get_internal_tracker_dir(self):
""" Returns the directory containing the torrents tracked by the internal
tracker (and associated databases).
@return An absolute path. """
# Called by any thread
self.sesslock.acquire()
try:
if self.sessconfig["state_dir"] is None:
return None
else:
return os.path.join(self.sessconfig["state_dir"], STATEDIR_ITRACKER_DIR)
finally:
self.sesslock.release()
def add_to_internal_tracker(self, tdef):
""" Add a torrent def to the list of torrents tracked by the internal
tracker. Use this method to use the Session as a standalone tracker.
@param tdef A finalized TorrentDef.
"""
# Called by any thread
self.sesslock.acquire()
try:
infohash = tdef.get_infohash()
filename = self.get_internal_tracker_torrentfilename(infohash)
tdef.save(filename)
print >>sys.stderr, "Session: add_to_int_tracker: saving to", filename, "url-compat", tdef.get_url_compat()
# Bring to attention of Tracker thread
self.lm.tracker_rescan_dir()
finally:
self.sesslock.release()
def remove_from_internal_tracker(self, tdef):
""" Remove a torrent def from the list of torrents tracked by the
internal tracker. Use this method to use the Session as a standalone
tracker.
@param tdef A finalized TorrentDef.
"""
infohash = tdef.get_infohash()
self.remove_from_internal_tracker_by_infohash(infohash)
def remove_from_internal_tracker_by_infohash(self, infohash):
""" Remove a torrent def from the list of torrents tracked by the
internal tracker. Use this method to use the Session as a standalone
tracker.
@param infohash Identifier of the torrent def to remove.
"""
# Called by any thread
self.sesslock.acquire()
try:
filename = self.get_internal_tracker_torrentfilename(infohash)
if DEBUG:
print >>sys.stderr, "Session: removing itracker entry", filename
if os.access(filename, os.F_OK):
os.remove(filename)
# Bring to attention of Tracker thread
self.lm.tracker_rescan_dir()
finally:
self.sesslock.release()
#
# Notification of events in the Session
#
def add_observer(self, func, subject, changeTypes=[NTFY_UPDATE, NTFY_INSERT, NTFY_DELETE], objectID=None, cache=0):
""" Add an observer function function to the Session. The observer
function will be called when one of the specified events (changeTypes)
occurs on the specified subject.
The function will be called by a popup thread which can be used
indefinitely (within reason) by the higher level code.
@param func The observer function. It should accept as its first argument
the subject, as second argument the changeType, as third argument an
objectID (e.g. the primary key in the observed database) and an
optional list of arguments.
@param subject The subject to observe, one of NTFY_* subjects (see
simpledefs).
@param changeTypes The list of events to be notified of one of NTFY_*
events.
@param objectID The specific object in the subject to monitor (e.g. a
specific primary key in a database to monitor for updates.)
@param cache The time to bundle/cache events matching this function
TODO: Jelle will add per-subject/event description here ;o)
"""
# Called by any thread
self.uch.notifier.add_observer(func, subject, changeTypes, objectID, cache=cache) # already threadsafe
def remove_observer(self, func):
""" Remove observer function. No more callbacks will be made.
@param func The observer function to remove. """
# Called by any thread
self.uch.notifier.remove_observer(func) # already threadsafe
def open_dbhandler(self, subject):
""" Opens a connection to the specified database. Only the thread
calling this method may use this connection. The connection must be
closed with close_dbhandler() when this thread exits.
@param subject The database to open. Must be one of the subjects
specified here.
@return A reference to a DBHandler class for the specified subject or
None when the Session was not started with megacaches enabled.
<pre> NTFY_PEERS -> PeerDBHandler
NTFY_TORRENTS -> TorrentDBHandler
NTFY_MYPREFERENCES -> MyPreferenceDBHandler
NTFY_VOTECAST -> VotecastDBHandler
NTFY_CHANNELCAST -> ChannelCastDBHandler
</pre>
"""
# Called by any thread
self.sesslock.acquire()
try:
if subject == NTFY_PEERS:
return self.lm.peer_db
elif subject == NTFY_TORRENTS:
return self.lm.torrent_db
elif subject == NTFY_MYPREFERENCES:
return self.lm.mypref_db
elif subject == NTFY_SEEDINGSTATS:
return self.lm.seedingstats_db
elif subject == NTFY_SEEDINGSTATSSETTINGS:
return self.lm.seedingstatssettings_db
elif subject == NTFY_VOTECAST:
return self.lm.votecast_db
elif subject == NTFY_CHANNELCAST:
return self.lm.channelcast_db
else:
raise ValueError("Cannot open DB subject: " + subject)
finally:
self.sesslock.release()
def close_dbhandler(self, dbhandler):
""" Closes the given database connection """
dbhandler.close()
#
# Persistence and shutdown
#
def load_checkpoint(self, initialdlstatus=None, initialdlstatus_dict={}):
""" Restart Downloads from checkpoint, if any.
This method allows the API user to manage restoring downloads.
E.g. a video player that wants to start the torrent the user clicked
on first, and only then restart any sleeping torrents (e.g. seeding).
The optional initialdlstatus parameter can be set to DLSTATUS_STOPPED
to restore all the Downloads in DLSTATUS_STOPPED state.
The options initialdlstatus_dict parameter can be used to specify a
state overriding the initaldlstatus parameter per download id.
"""
self.lm.load_checkpoint(initialdlstatus, initialdlstatus_dict)
def checkpoint(self):
""" Saves the internal session state to the Session's state dir. """
# Called by any thread
self.checkpoint_shutdown(stop=False, checkpoint=True, gracetime=None, hacksessconfcheckpoint=False)
def shutdown(self, checkpoint=True, gracetime=2.0, hacksessconfcheckpoint=True):
""" Checkpoints the session and closes it, stopping the download engine.
@param checkpoint Whether to checkpoint the Session state on shutdown.
@param gracetime Time to allow for graceful shutdown + signoff (seconds).
"""
# Called by any thread
self.lm.early_shutdown()
self.checkpoint_shutdown(
stop=True, checkpoint=checkpoint, gracetime=gracetime, hacksessconfcheckpoint=hacksessconfcheckpoint
)
# Arno, 2010-08-09: now shutdown after gracetime
# self.uch.shutdown()
def has_shutdown(self):
""" Whether the Session has completely shutdown, i.e., its internal
threads are finished and it is safe to quit the process the Session
is running in.
@return A Boolean.
"""
return self.lm.sessdoneflag.isSet()
def get_downloads_pstate_dir(self):
""" Returns the directory in which to checkpoint the Downloads in this
Session. """
# Called by network thread
self.sesslock.acquire()
try:
return os.path.join(self.sessconfig["state_dir"], STATEDIR_DLPSTATE_DIR)
finally:
self.sesslock.release()
def download_torrentfile(self, infohash=None, roothash=None, usercallback=None, prio=0):
""" Try to download the torrentfile without a known source.
A possible source could be the DHT.
If the torrent is succesfully
received, the usercallback method is called with the infohash as first
and the contents of the torrentfile (bencoded dict) as second parameter.
If the torrent could not be obtained, the callback is not called.
The torrent will have been added to the TorrentDBHandler (if enabled)
at the time of the call.
@param infohash The infohash of the torrent.
@param usercallback A function adhering to the above spec.
"""
from Tribler.Core.RemoteTorrentHandler import RemoteTorrentHandler
rtorrent_handler = RemoteTorrentHandler.getInstance()
rtorrent_handler.download_torrent(None, infohash, roothash, usercallback, prio)
def download_torrentfile_from_peer(self, candidate, infohash=None, roothash=None, usercallback=None, prio=0):
""" Ask the designated peer to send us the torrentfile for the torrent
identified by the passed infohash. If the torrent is succesfully
received, the usercallback method is called with the infohash as first
and the contents of the torrentfile (bencoded dict) as second parameter.
If the torrent could not be obtained, the callback is not called.
The torrent will have been added to the TorrentDBHandler (if enabled)
at the time of the call.
@param permid The PermID of the peer to query.
@param infohash The infohash of the torrent.
@param usercallback A function adhering to the above spec.
"""
from Tribler.Core.RemoteTorrentHandler import RemoteTorrentHandler
rtorrent_handler = RemoteTorrentHandler.getInstance()
rtorrent_handler.download_torrent(candidate, infohash, roothash, usercallback, prio)
def download_torrentmessages_from_peer(self, candidate, infohashes, usercallback, prio=0):
""" Ask the designated peer to send us the torrentfile for the torrent
identified by the passed infohash. If the torrent is succesfully
received, the usercallback method is called with the infohash as first
and the contents of the torrentfile (bencoded dict) as second parameter.
If the torrent could not be obtained, the callback is not called.
The torrent will have been added to the TorrentDBHandler (if enabled)
at the time of the call.
@param permid The PermID of the peer to query.
@param infohash The infohash of the torrent.
@param usercallback A function adhering to the above spec.
"""
from Tribler.Core.RemoteTorrentHandler import RemoteTorrentHandler
rtorrent_handler = RemoteTorrentHandler.getInstance()
rtorrent_handler.download_torrentmessages(candidate, infohashes, usercallback, prio)
#
# Internal persistence methods
#
def checkpoint_shutdown(self, stop, checkpoint, gracetime, hacksessconfcheckpoint):
""" Checkpoints the Session and optionally shuts down the Session.
@param stop Whether to shutdown the Session as well.
@param checkpoint Whether to checkpoint at all, or just to stop.
@param gracetime Time to allow for graceful shutdown + signoff (seconds).
"""
# Called by any thread
self.sesslock.acquire()
try:
# Arno: Make checkpoint optional on shutdown. At the moment setting
# the config at runtime is not possible (see SessionRuntimeConfig)
# so this has little use, and interferes with our way of
# changing the startup config, which is to write a new
# config to disk that will be read at start up.
if hacksessconfcheckpoint:
try:
self.save_pstate_sessconfig()
except Exception, e:
self.lm.rawserver_nonfatalerrorfunc(e)
# Checkpoint all Downloads and stop NetworkThread
if DEBUG or stop:
print >>sys.stderr, "Session: checkpoint_shutdown"
self.lm.checkpoint(stop=stop, checkpoint=checkpoint, gracetime=gracetime)
finally:
def __init__(self, binpath, workdir, zerostatedir, listenport, httpgwport,
cmdgwport, spmgr):
# Called by any thread, assume sessionlock is held
self.splock = NoDispersyRLock()
self.binpath = binpath
self.workdir = workdir
self.zerostatedir = zerostatedir
self.spmgr = spmgr
# Main UDP listen socket
if listenport is None:
self.listenport = random.randint(10001, 10999)
else:
self.listenport = listenport
# NSSA control socket
if cmdgwport is None:
self.cmdport = random.randint(11001, 11999)
else:
self.cmdport = cmdgwport
# content web server
if httpgwport is None:
self.httpport = random.randint(12001, 12999)
else:
self.httpport = httpgwport
# Security: only accept commands from localhost, enable HTTP gw,
# no stats/webUI web server
args = []
# Arno, 2012-07-09: Unicode problems with popen
args.append(self.binpath.encode(sys.getfilesystemencoding()))
# Arno, 2012-05-29: Hack. Win32 getopt code eats first arg when Windows app
# instead of CONSOLE app.
args.append("-j")
args.append("-l") # listen port
args.append("0.0.0.0:" + str(self.listenport))
args.append("-c") # command port
args.append("127.0.0.1:" + str(self.cmdport))
args.append("-g") # HTTP gateway port
args.append("127.0.0.1:" + str(self.httpport))
args.append("-w")
if zerostatedir is not None:
if sys.platform == "win32":
# Swift on Windows expects command line arguments as UTF-16.
# popen doesn't allow us to pass params in UTF-16, hence workaround.
# Format = hex encoded UTF-8
args.append("-3")
zssafe = binascii.hexlify(zerostatedir.encode("UTF-8"))
args.append(zssafe) # encoding that swift expects
else:
args.append("-e")
args.append(zerostatedir)
args.append("-T") # zero state connection timeout
args.append("180") # seconds
#args.append("-B") # Enable debugging on swift
if True or DEBUG:
print >> sys.stderr, "SwiftProcess: __init__: Running", args, "workdir", workdir
if sys.platform == "win32":
creationflags = subprocess.CREATE_NEW_PROCESS_GROUP
else:
creationflags = 0
# See also SwiftDef::finalize popen
self.popen = subprocess.Popen(args,
close_fds=True,
cwd=workdir,
creationflags=creationflags)
self.roothash2dl = {}
self.donestate = DONE_STATE_WORKING # shutting down
self.fastconn = None
# callbacks for when swift detect a channel close
self._channel_close_callbacks = defaultdict(list)
class SwiftDownloadImpl(SwiftDownloadRuntimeConfig):
""" Download subclass that represents a swift download.
The actual swift download takes places in a SwiftProcess.
"""
def __init__(self, session, sdef):
self.dllock = NoDispersyRLock()
self.session = session
self.sdef = sdef
self.old_metadir = self.session.get_swift_meta_dir()
# just enough so error saving and get_state() works
self.error = None
# To be able to return the progress of a stopped torrent, how far it got.
self.progressbeforestop = 0.0
# SwiftProcess performing the actual download.
self.sp = None
# spstatus
self.dlstatus = DLSTATUS_WAITING4HASHCHECK
self.dynasize = 0
self.progress = 0.0
self.curspeeds = {DOWNLOAD: 0.0, UPLOAD: 0.0} # bytes/s
self.numleech = 0
self.numseeds = 0
self.contentbytes = {DOWNLOAD: 0, UPLOAD: 0} # bytes
self.done = False # when set it means this download is being removed
self.midict = {}
self.time_seeding = [0, None]
self.total_up = 0
self.total_down = 0
self.lm_network_vod_event_callback = None
self.askmoreinfo = False
#
# Download Interface
#
def get_def(self):
return self.sdef
#
# DownloadImpl
#
#
# Creating a Download
#
def setup(self,
dcfg=None,
pstate=None,
initialdlstatus=None,
lm_network_engine_wrapper_created_callback=None,
lm_network_vod_event_callback=None):
"""
Create a Download object. Used internally by Session.
@param dcfg DownloadStartupConfig or None (in which case
a new DownloadConfig() is created and the result
becomes the runtime config of this Download.
"""
# Called by any thread, assume sessionlock is held
try:
self.dllock.acquire(
) # not really needed, no other threads know of this object
# Copy dlconfig, from default if not specified
if dcfg is None:
cdcfg = DownloadStartupConfig()
else:
cdcfg = dcfg
self.dlconfig = copy.copy(cdcfg.dlconfig)
# Things that only exist at runtime
self.dlruntimeconfig = {}
self.dlruntimeconfig['max_desired_upload_rate'] = 0
self.dlruntimeconfig['max_desired_download_rate'] = 0
if pstate and 'dlstate' in pstate:
dlstate = pstate['dlstate']
if 'time_seeding' in dlstate:
self.time_seeding = [dlstate['time_seeding'], None]
if 'total_up' in dlstate:
self.total_up = dlstate['total_up']
if 'total_down' in dlstate:
self.total_down = dlstate['total_down']
if DEBUG:
print >> sys.stderr, "SwiftDownloadImpl: setup: initialdlstatus", repr(
self.sdef.get_roothash_as_hex()), initialdlstatus
# Note: initialdlstatus now only works for STOPPED
if initialdlstatus != DLSTATUS_STOPPED:
self.create_engine_wrapper(
lm_network_engine_wrapper_created_callback, pstate,
lm_network_vod_event_callback)
self.dllock.release()
except Exception as e:
print_exc()
self.set_error(e)
self.dllock.release()
def create_engine_wrapper(self,
lm_network_engine_wrapper_created_callback,
pstate,
lm_network_vod_event_callback,
initialdlstatus=None):
network_create_engine_wrapper_lambda = lambda: self.network_create_engine_wrapper(
lm_network_engine_wrapper_created_callback, pstate,
lm_network_vod_event_callback, initialdlstatus)
self.session.lm.rawserver.add_task(
network_create_engine_wrapper_lambda)
def network_create_engine_wrapper(
self,
lm_network_engine_wrapper_created_callback,
pstate,
lm_network_vod_event_callback,
initialdlstatus=None):
""" Called by any thread, assume dllock already acquired """
if DEBUG:
print >> sys.stderr, "SwiftDownloadImpl: create_engine_wrapper()"
if self.get_mode() == DLMODE_VOD:
self.lm_network_vod_event_callback = lm_network_vod_event_callback
move_files = ('swiftmetadir' not in self.dlconfig
) and not os.path.isdir(self.get_dest_dir())
metadir = self.get_swift_meta_dir()
if not metadir:
metadir = self.session.get_swift_meta_dir()
self.set_swift_meta_dir(metadir)
if not os.path.exists(metadir):
os.makedirs(metadir)
if move_files:
# We must be dealing with a checkpoint from a previous release (<6.1.0). Move the swift metadata to the right directory.
is_multifile = self.get_dest_dir().endswith(
"." + self.get_def().get_roothash_as_hex())
path_old = self.get_dest_dir()
path_new = os.path.join(
metadir,
self.get_def().get_roothash_as_hex()
if is_multifile else os.path.split(self.get_dest_dir())[1])
try:
if is_multifile:
shutil.move(path_old, path_new + '.mfspec')
self.dlconfig['saveas'] = os.path.split(
self.get_dest_dir())[0]
shutil.move(path_old + '.mhash', path_new + '.mhash')
shutil.move(path_old + '.mbinmap', path_new + '.mbinmap')
except:
print_exc()
# Synchronous: starts process if needed
self.sp = self.session.lm.spm.get_or_create_sp(
self.session.get_swift_working_dir(),
self.session.get_torrent_collecting_dir(),
self.get_swift_listen_port(), self.get_swift_httpgw_listen_port(),
self.get_swift_cmdgw_listen_port())
if self.sp:
self.sp.start_download(self)
self.session.lm.rawserver.add_task(self.network_check_swift_alive,
SWIFT_ALIVE_CHECK_INTERVAL)
# Arno: if used, make sure to switch to network thread first!
# if lm_network_engine_wrapper_created_callback is not None:
# sp = self.sp
# exc = self.error
# lm_network_engine_wrapper_created_callback(self,sp,exc,pstate)
#
# SwiftProcess callbacks
#
def i2ithread_info_callback(self, dlstatus, progress, dynasize, dlspeed,
ulspeed, numleech, numseeds, contentdl,
contentul):
self.dllock.acquire()
try:
if dlstatus == DLSTATUS_SEEDING and self.dlstatus != dlstatus:
# started seeding
self.time_seeding[0] = self.get_seeding_time()
self.time_seeding[1] = time.time()
elif dlstatus != DLSTATUS_SEEDING and self.dlstatus != dlstatus:
# stopped seeding
self.time_seeding[0] = self.get_seeding_time()
self.time_seeding[1] = None
self.dlstatus = dlstatus
self.dynasize = dynasize
self.progress = progress
self.curspeeds[DOWNLOAD] = dlspeed
self.curspeeds[UPLOAD] = ulspeed
self.numleech = numleech
self.numseeds = numseeds
self.contentbytes = {DOWNLOAD: contentdl, UPLOAD: contentul}
finally:
self.dllock.release()
def i2ithread_vod_event_callback(self, event, httpurl):
if DEBUG:
print >> sys.stderr, "SwiftDownloadImpl: i2ithread_vod_event_callback: ENTER", event, httpurl, "mode", self.get_mode(
)
self.dllock.acquire()
try:
if event == VODEVENT_START:
if self.get_mode() != DLMODE_VOD:
return
# Fix firefox idiosyncrasies
duration = self.sdef.get_duration()
if duration is not None:
httpurl += '@' + duration
vod_usercallback_wrapper = lambda event, params: self.session.uch.perform_vod_usercallback(
self, self.dlconfig['vod_usercallback'], event, params)
videoinfo = {}
videoinfo['usercallback'] = vod_usercallback_wrapper
# ARNOSMPTODO: if complete, return file directly
# Allow direct connection of video renderer with swift HTTP server
# via new "url" param.
#
if DEBUG:
print >> sys.stderr, "SwiftDownloadImpl: i2ithread_vod_event_callback", event, httpurl
# Arno: No threading violation, lm_network_* is safe at the moment
self.lm_network_vod_event_callback(
videoinfo,
VODEVENT_START,
{
"complete": False,
"filename": None,
"mimetype": 'application/octet-stream', # ARNOSMPTODO
"stream": None,
"length": self.get_dynasize(),
"bitrate": None, # ARNOSMPTODO
"url": httpurl,
})
finally:
self.dllock.release()
def i2ithread_moreinfo_callback(self, midict):
self.dllock.acquire()
try:
# print >>sys.stderr,"SwiftDownloadImpl: Got moreinfo",midict.keys()
self.midict = midict
finally:
self.dllock.release()
#
# Retrieving DownloadState
#
def get_status(self):
""" Returns the status of the download.
@return DLSTATUS_* """
self.dllock.acquire()
try:
return self.dlstatus
finally:
self.dllock.release()
def get_dynasize(self):
""" Returns the size of the swift content. Note this may vary
(generally ~1KiB because of dynamic size determination by the
swift protocol
@return long
"""
self.dllock.acquire()
try:
return self.dynasize
finally:
self.dllock.release()
def get_progress(self):
""" Return fraction of content downloaded.
@return float 0..1
"""
self.dllock.acquire()
try:
return self.progress
finally:
self.dllock.release()
def get_current_speed(self, dir):
""" Return last reported speed in KB/s
@return float
"""
self.dllock.acquire()
try:
return self.curspeeds[dir] / 1024.0
finally:
self.dllock.release()
def get_moreinfo_stats(self, dir):
""" Return last reported more info dict
@return dict
"""
self.dllock.acquire()
try:
return self.midict
finally:
self.dllock.release()
def get_seeding_time(self):
return self.time_seeding[0] + (time.time() - self.time_seeding[1]
if self.time_seeding[1] != None else 0)
def get_total_up(self):
return self.total_up + self.contentbytes[UPLOAD]
def get_total_down(self):
return self.total_down + self.contentbytes[DOWNLOAD]
def get_seeding_statistics(self):
seeding_stats = {}
seeding_stats['total_up'] = self.get_total_up()
seeding_stats['total_down'] = self.get_total_down()
seeding_stats['time_seeding'] = self.get_seeding_time()
return seeding_stats
def network_get_stats(self, getpeerlist):
"""
@return (status,stats,logmsgs,coopdl_helpers,coopdl_coordinator)
"""
# dllock held
# ARNOSMPTODO: Have a status for when swift is hashchecking the file on disk
if self.sp is None:
status = DLSTATUS_STOPPED
else:
status = self.dlstatus
stats = {}
stats['down'] = self.curspeeds[DOWNLOAD]
stats['up'] = self.curspeeds[UPLOAD]
stats['frac'] = self.progress
stats['stats'] = self.network_create_statistics_reponse()
stats['time'] = self.network_calc_eta()
stats['vod_prebuf_frac'] = self.network_calc_prebuf_frac()
stats['vod'] = True
# ARNOSMPTODO: no hard check for suff bandwidth, unlike BT1Download
stats['vod_playable'] = self.progress == 1.0 or (
self.network_calc_prebuf_frac() == 1.0
and self.curspeeds[DOWNLOAD] > 0.0)
stats['vod_playable_after'] = self.network_calc_prebuf_eta()
stats['vod_stats'] = self.network_get_vod_stats()
stats['spew'] = self.network_create_spew_from_peerlist()
seeding_stats = self.get_seeding_statistics()
logmsgs = []
return (status, stats, seeding_stats, logmsgs)
def network_create_statistics_reponse(self):
return SwiftStatisticsResponse(self.numleech, self.numseeds,
self.midict)
def network_calc_eta(self):
bytestogof = (1.0 - self.progress) * float(self.dynasize)
dlspeed = max(0.000001, self.curspeeds[DOWNLOAD])
return bytestogof / dlspeed
def network_calc_prebuf_frac(self):
gotbytesf = self.progress * float(self.dynasize)
prebuff = float(CMDGW_PREBUFFER_BYTES)
return min(1.0, gotbytesf / prebuff)
def network_calc_prebuf_eta(self):
bytestogof = (1.0 - self.network_calc_prebuf_frac()
) * float(CMDGW_PREBUFFER_BYTES)
dlspeed = max(0.000001, self.curspeeds[DOWNLOAD])
return bytestogof / dlspeed
def network_get_vod_stats(self):
# More would have to be sent from swift process to set these correctly
d = {}
d['played'] = None
d['late'] = None
d['dropped'] = None
d['stall'] = None
d['pos'] = None
d['prebuf'] = None
d['firstpiece'] = 0
d['npieces'] = ((self.dynasize + 1023) / 1024)
return d
def network_create_spew_from_peerlist(self):
if not 'channels' in self.midict:
return []
plist = []
channels = self.midict['channels']
for channel in channels:
d = {}
d['ip'] = channel['ip']
d['port'] = channel['port']
d['utotal'] = channel['bytes_up'] / 1024.0
d['dtotal'] = channel['bytes_down'] / 1024.0
plist.append(d)
return plist
#
# Retrieving DownloadState
#
def set_state_callback(self, usercallback, getpeerlist=False, delay=0.0):
""" Called by any thread """
self.dllock.acquire()
try:
network_get_state_lambda = lambda: self.network_get_state(
usercallback, getpeerlist)
# First time on general rawserver
self.session.lm.rawserver.add_task(network_get_state_lambda, delay)
finally:
self.dllock.release()
def network_get_state(self,
usercallback,
getpeerlist,
sessioncalling=False):
""" Called by network thread """
self.dllock.acquire()
try:
if self.sp is None:
if DEBUG:
print >> sys.stderr, "SwiftDownloadImpl: network_get_state: Download not running"
ds = DownloadState(self,
DLSTATUS_STOPPED,
self.error,
self.progressbeforestop,
seeding_stats=self.get_seeding_statistics())
else:
(status, stats, seeding_stats,
logmsgs) = self.network_get_stats(getpeerlist)
ds = DownloadState(self,
status,
self.error,
self.get_progress(),
stats=stats,
seeding_stats=seeding_stats,
logmsgs=logmsgs)
self.progressbeforestop = ds.get_progress()
if sessioncalling:
return ds
# Invoke the usercallback function via a new thread.
# After the callback is invoked, the return values will be passed to
# the returncallback for post-callback processing.
if not self.done:
self.session.uch.perform_getstate_usercallback(
usercallback, ds, self.sesscb_get_state_returncallback)
finally:
self.dllock.release()
def sesscb_get_state_returncallback(self, usercallback, when,
newgetpeerlist):
""" Called by SessionCallbackThread """
self.dllock.acquire()
try:
if when > 0.0 and not self.done:
# Schedule next invocation, either on general or DL specific
# Note this continues when dl is stopped.
network_get_state_lambda = lambda: self.network_get_state(
usercallback, newgetpeerlist)
self.session.lm.rawserver.add_task(network_get_state_lambda,
when)
finally:
self.dllock.release()
#
# Download stop/resume
#
def stop(self):
""" Called by any thread """
self.stop_remove(False, removestate=False, removecontent=False)
def stop_remove(self, removedl, removestate=False, removecontent=False):
""" Called by any thread. Called on Session.remove_download() """
# Arno, 2013-01-29: This download is being removed, not just stopped.
self.done = removedl
self.network_stop(removestate=removestate, removecontent=removecontent)
def network_stop(self, removestate, removecontent):
""" Called by network thread, but safe for any """
self.dllock.acquire()
try:
if DEBUG:
print >> sys.stderr, "SwiftDownloadImpl: network_stop", repr(
self.sdef.get_name())
pstate = self.network_get_persistent_state()
if self.sp is not None:
self.sp.remove_download(self, removestate, removecontent)
self.session.lm.spm.release_sp(self.sp)
self.sp = None
self.time_seeding = [self.get_seeding_time(), None]
# Offload the removal of the dlcheckpoint to another thread
if removestate:
# To remove:
# 1. Core checkpoint (if any)
# 2. .mhash file
# 3. content (if so desired)
# content and .mhash file is removed by swift engine if requested
roothash = self.sdef.get_roothash()
self.session.uch.perform_removestate_callback(
roothash, None, False)
return (self.sdef.get_roothash(), pstate)
finally:
self.dllock.release()
def get_content_dest(self):
""" Returns the file to which the downloaded content is saved. """
return os.path.join(self.get_dest_dir(),
self.sdef.get_roothash_as_hex())
def restart(self, initialdlstatus=None):
""" Restart the Download """
# Called by any thread
if DEBUG:
print >> sys.stderr, "SwiftDownloadImpl: restart:", repr(
self.sdef.get_name())
self.dllock.acquire()
try:
if self.sp is None:
self.error = None # assume fatal error is reproducible
self.create_engine_wrapper(
self.session.lm.network_engine_wrapper_created_callback,
None,
self.session.lm.network_vod_event_callback,
initialdlstatus=initialdlstatus)
# No exception if already started, for convenience
finally:
self.dllock.release()
#
# Config parameters that only exists at runtime
#
def set_max_desired_speed(self, direct, speed):
if DEBUG:
print >> sys.stderr, "Download: set_max_desired_speed", direct, speed
# if speed < 10:
# print_stack()
self.dllock.acquire()
if direct == UPLOAD:
self.dlruntimeconfig['max_desired_upload_rate'] = speed
else:
self.dlruntimeconfig['max_desired_download_rate'] = speed
self.dllock.release()
def get_max_desired_speed(self, direct):
self.dllock.acquire()
try:
if direct == UPLOAD:
return self.dlruntimeconfig['max_desired_upload_rate']
else:
return self.dlruntimeconfig['max_desired_download_rate']
finally:
self.dllock.release()
def get_dest_files(self, exts=None):
"""
Returns (None,destfilename)
"""
if exts is not None:
raise OperationNotEnabledByConfigurationException()
f2dlist = []
diskfn = self.get_content_dest()
f2dtuple = (None, diskfn)
f2dlist.append(f2dtuple)
return f2dlist
#
# Persistence
#
def checkpoint(self):
""" Called by any thread """
# Arno, 2012-05-15. Currently this is safe to call from any thread.
# Need this for torrent collecting via swift.
self.network_checkpoint()
def network_checkpoint(self):
""" Called by network thread """
self.dllock.acquire()
try:
pstate = self.network_get_persistent_state()
if self.sp is not None:
self.sp.checkpoint_download(self)
return (self.sdef.get_roothash(), pstate)
finally:
self.dllock.release()
def network_get_persistent_state(self):
""" Assume dllock already held """
pstate = {}
pstate['version'] = PERSISTENTSTATE_CURRENTVERSION
pstate['metainfo'] = self.sdef.get_url_with_meta() # assumed immutable
dlconfig = copy.copy(self.dlconfig)
dlconfig['name'] = self.sdef.get_name()
# Reset unpicklable params
dlconfig['vod_usercallback'] = None
dlconfig['mode'] = DLMODE_NORMAL # no callback, no VOD
# Reset default metadatadir
if self.get_swift_meta_dir() == self.old_metadir:
dlconfig['swiftmetadir'] = None
pstate['dlconfig'] = dlconfig
pstate['dlstate'] = {}
ds = self.network_get_state(None, False, sessioncalling=True)
pstate['dlstate']['status'] = ds.get_status()
pstate['dlstate']['progress'] = ds.get_progress()
pstate['dlstate']['swarmcache'] = None
pstate['dlstate'].update(ds.get_seeding_statistics())
if DEBUG:
print >> sys.stderr, "SwiftDownloadImpl: netw_get_pers_state: status", dlstatus_strings[
ds.get_status()], "progress", ds.get_progress()
# Swift stores own state in .mhash and .mbinmap file
pstate['engineresumedata'] = None
return pstate
#
# Coop download
#
def get_coopdl_role_object(self, role):
""" Called by network thread """
return None
def recontact_tracker(self):
""" Called by any thread """
pass
#
# MOREINFO
#
def set_moreinfo_stats(self, enable):
""" Called by any thread """
# Arno, 2012-07-31: slight risk if process killed in between
if self.askmoreinfo == enable:
return
self.askmoreinfo = enable
if self.sp is not None:
self.sp.set_moreinfo_stats(self, enable)
#
# External addresses
#
def add_peer(self, addr):
""" Add a peer address from 3rd source (not tracker, not DHT) to this
Download.
@param (hostname_ip,port) tuple
"""
if self.sp is not None:
self.sp.add_peer(self, addr)
#
# Internal methods
#
def set_error(self, e):
self.dllock.acquire()
self.error = e
self.dllock.release()
#
# Auto restart after swift crash
#
def network_check_swift_alive(self):
self.dllock.acquire()
try:
if self.sp is not None and not self.done:
if not self.sp.is_alive():
print >> sys.stderr, "SwiftDownloadImpl: network_check_swift_alive: Restarting", repr(
self.sdef.get_name())
self.sp = None
self.restart()
except:
print_exc()
finally:
self.dllock.release()
if not self.done:
self.session.lm.rawserver.add_task(self.network_check_swift_alive,
SWIFT_ALIVE_CHECK_INTERVAL)
def __init__(self,binpath,workdir,zerostatedir,listenport,httpgwport,cmdgwport,spmgr):
# Called by any thread, assume sessionlock is held
self.splock = NoDispersyRLock()
self.binpath = binpath
self.workdir = workdir
self.zerostatedir = zerostatedir
self.spmgr = spmgr
# Main UDP listen socket
if listenport is None:
self.listenport = random.randint(10001,10999)
else:
self.listenport = listenport
# NSSA control socket
if cmdgwport is None:
self.cmdport = random.randint(11001,11999)
else:
self.cmdport = cmdgwport
# content web server
if httpgwport is None:
self.httpport = random.randint(12001,12999)
else:
self.httpport = httpgwport
# Security: only accept commands from localhost, enable HTTP gw,
# no stats/webUI web server
args=[]
# Arno, 2012-07-09: Unicode problems with popen
args.append(self.binpath.encode(sys.getfilesystemencoding()))
# Arno, 2012-05-29: Hack. Win32 getopt code eats first arg when Windows app
# instead of CONSOLE app.
args.append("-j")
args.append("-l") # listen port
args.append("0.0.0.0:"+str(self.listenport))
args.append("-c") # command port
args.append("127.0.0.1:"+str(self.cmdport))
args.append("-g") # HTTP gateway port
args.append("127.0.0.1:"+str(self.httpport))
args.append("-w")
if zerostatedir is not None:
if sys.platform == "win32":
# Swift on Windows expects command line arguments as UTF-16.
# popen doesn't allow us to pass params in UTF-16, hence workaround.
# Format = hex encoded UTF-8
args.append("-3")
zssafe = binascii.hexlify(zerostatedir.encode("UTF-8"))
args.append(zssafe) # encoding that swift expects
else:
args.append("-e")
args.append(zerostatedir)
args.append("-T") # zero state connection timeout
args.append("180") # seconds
#args.append("-B") # Enable debugging on swift
if True or DEBUG:
print >>sys.stderr,"SwiftProcess: __init__: Running",args,"workdir",workdir
if sys.platform == "win32":
creationflags=subprocess.CREATE_NEW_PROCESS_GROUP
else:
creationflags=0
# See also SwiftDef::finalize popen
self.popen = subprocess.Popen(args,close_fds=True,cwd=workdir,creationflags=creationflags)
self.roothash2dl = {}
self.donestate = DONE_STATE_WORKING # shutting down
self.fastconn = None
def __init__(self, scfg=None, ignore_singleton=False, autoload_discovery=True):
"""
A Session object is created which is configured following a copy of the
SessionStartupConfig scfg. (copy constructor used internally)
@param scfg SessionStartupConfig object or None, in which case we
look for a saved session in the default location (state dir). If
we can't find it, we create a new SessionStartupConfig() object to
serve as startup config. Next, the config is saved in the directory
indicated by its 'state_dir' attribute.
In the current implementation only a single session instance can exist
at a time in a process. The ignore_singleton flag is used for testing.
"""
if not ignore_singleton:
if Session.__single:
raise RuntimeError("Session is singleton")
Session.__single = self
self._logger = logging.getLogger(self.__class__.__name__)
self.ignore_singleton = ignore_singleton
self.sesslock = NoDispersyRLock()
# Determine startup config to use
if scfg is None: # If no override
scfg = SessionStartupConfig.load()
else: # overrides any saved config
# Work from copy
scfg = SessionStartupConfig(copy.copy(scfg.sessconfig))
def create_dir(fullpath):
if not os.path.isdir(fullpath):
os.makedirs(fullpath)
def set_and_create_dir(dirname, setter, default_dir):
if dirname is None:
setter(default_dir)
create_dir(dirname or default_dir)
state_dir = scfg.get_state_dir()
set_and_create_dir(state_dir, scfg.set_state_dir, state_dir)
set_and_create_dir(scfg.get_torrent_store_dir(),
scfg.set_torrent_store_dir,
os.path.join(scfg.get_state_dir(), STATEDIR_TORRENT_STORE_DIR))
# metadata store
set_and_create_dir(scfg.get_metadata_store_dir(),
scfg.set_metadata_store_dir,
os.path.join(scfg.get_state_dir(), STATEDIR_METADATA_STORE_DIR))
set_and_create_dir(scfg.get_peer_icon_path(), scfg.set_peer_icon_path,
os.path.join(scfg.get_state_dir(), STATEDIR_PEERICON_DIR))
create_dir(os.path.join(scfg.get_state_dir(), u"sqlite"))
create_dir(os.path.join(scfg.get_state_dir(), STATEDIR_DLPSTATE_DIR))
# Reset the nickname to something not related to the host name, it was
# really silly to have this default on the first place.
# TODO: Maybe move this to the upgrader?
if socket.gethostname() in scfg.get_nickname():
scfg.set_nickname("Tribler user")
if GOTM2CRYPTO:
permidmod.init()
# Set params that depend on state_dir
#
# 1. keypair
#
pairfilename = scfg.get_permid_keypair_filename()
if os.access(pairfilename, os.F_OK):
# May throw exceptions
self.keypair = permidmod.read_keypair(pairfilename)
else:
self.keypair = permidmod.generate_keypair()
# Save keypair
pubfilename = os.path.join(scfg.get_state_dir(), 'ecpub.pem')
permidmod.save_keypair(self.keypair, pairfilename)
permidmod.save_pub_key(self.keypair, pubfilename)
if not scfg.get_megacache():
scfg.set_torrent_checking(0)
self.sessconfig = scfg.sessconfig
self.sessconfig.lock = self.sesslock
self.selected_ports = scfg.selected_ports
# Claim all random ports
self.get_listen_port()
self.get_dispersy_port()
self.get_mainline_dht_listen_port()
self.get_videoplayer_port()
self.get_anon_listen_port()
self.get_tunnel_community_socks5_listen_ports()
# Create handler for calling back the user via separate threads
self.lm = TriblerLaunchMany()
self.notifier = Notifier(use_pool=True)
# Checkpoint startup config
self.save_pstate_sessconfig()
self.sqlite_db = None
self.autoload_discovery = autoload_discovery
def __init__(self,scfg=None,ignore_singleton=False):
"""
A Session object is created which is configured following a copy of the
SessionStartupConfig scfg. (copy constructor used internally)
@param scfg SessionStartupConfig object or None, in which case we
look for a saved session in the default location (state dir). If
we can't find it, we create a new SessionStartupConfig() object to
serve as startup config. Next, the config is saved in the directory
indicated by its 'state_dir' attribute.
In the current implementation only a single session instance can exist
at a time in a process. The ignore_singleton flag is used for testing.
"""
# ProxyService 90s Test_
# self.start_time = time.time()
# _ProxyService 90s Test
if not ignore_singleton:
if Session.__single:
raise RuntimeError, "Session is singleton"
Session.__single = self
self.sesslock = NoDispersyRLock()
# Determine startup config to use
if scfg is None: # If no override
try:
# Then try to read from default location
state_dir = Session.get_default_state_dir()
cfgfilename = Session.get_default_config_filename(state_dir)
scfg = SessionStartupConfig.load(cfgfilename)
except:
# If that fails, create a fresh config with factory defaults
print_exc()
scfg = SessionStartupConfig()
self.sessconfig = scfg.sessconfig
else: # overrides any saved config
# Work from copy
self.sessconfig = copy.copy(scfg.sessconfig)
#Niels: 11/05/2012, turning off overlay
self.sessconfig['overlay'] = 0
self.sessconfig['crawler'] = 0
# Create dir for session state, if not exist
state_dir = self.sessconfig['state_dir']
if state_dir is None:
state_dir = Session.get_default_state_dir()
self.sessconfig['state_dir'] = state_dir
if not os.path.isdir(state_dir):
os.makedirs(state_dir)
collected_torrent_dir = self.sessconfig['torrent_collecting_dir']
if not collected_torrent_dir:
collected_torrent_dir = os.path.join(self.sessconfig['state_dir'], STATEDIR_TORRENTCOLL_DIR)
self.sessconfig['torrent_collecting_dir'] = collected_torrent_dir
collected_subtitles_dir = self.sessconfig.get('subtitles_collecting_dir',None)
if not collected_subtitles_dir:
collected_subtitles_dir = os.path.join(self.sessconfig['state_dir'], STATEDIR_SUBSCOLL_DIR)
self.sessconfig['subtitles_collecting_dir'] = collected_subtitles_dir
if not os.path.exists(collected_torrent_dir):
os.makedirs(collected_torrent_dir)
if not self.sessconfig['peer_icon_path']:
self.sessconfig['peer_icon_path'] = os.path.join(self.sessconfig['state_dir'], STATEDIR_PEERICON_DIR)
# PERHAPS: load default TorrentDef and DownloadStartupConfig from state dir
# Let user handle that, he's got default_state_dir, etc.
# Core init
#print >>sys.stderr,'Session: __init__ config is', self.sessconfig
if GOTM2CRYPTO:
permidmod.init()
#
# Set params that depend on state_dir
#
# 1. keypair
#
pairfilename = os.path.join(self.sessconfig['state_dir'],'ec.pem')
if self.sessconfig['eckeypairfilename'] is None:
self.sessconfig['eckeypairfilename'] = pairfilename
if os.access(self.sessconfig['eckeypairfilename'],os.F_OK):
# May throw exceptions
self.keypair = permidmod.read_keypair(self.sessconfig['eckeypairfilename'])
else:
self.keypair = permidmod.generate_keypair()
# Save keypair
pubfilename = os.path.join(self.sessconfig['state_dir'],'ecpub.pem')
permidmod.save_keypair(self.keypair,pairfilename)
permidmod.save_pub_key(self.keypair,pubfilename)
# 2. Downloads persistent state dir
dlpstatedir = os.path.join(self.sessconfig['state_dir'],STATEDIR_DLPSTATE_DIR)
if not os.path.isdir(dlpstatedir):
os.mkdir(dlpstatedir)
# 3. tracker
trackerdir = self.get_internal_tracker_dir()
if not os.path.exists(trackerdir):
os.mkdir(trackerdir)
if self.sessconfig['tracker_dfile'] is None:
self.sessconfig['tracker_dfile'] = os.path.join(trackerdir,'tracker.db')
if self.sessconfig['tracker_allowed_dir'] is None:
self.sessconfig['tracker_allowed_dir'] = trackerdir
if self.sessconfig['tracker_logfile'] is None:
if sys.platform == "win32":
# Not "Nul:" but "nul" is /dev/null on Win32
sink = 'nul'
else:
sink = '/dev/null'
self.sessconfig['tracker_logfile'] = sink
# 4. superpeer.txt and crawler.txt
if self.sessconfig['superpeer_file'] is None:
self.sessconfig['superpeer_file'] = os.path.join(self.sessconfig['install_dir'],LIBRARYNAME,'Core','superpeer.txt')
if 'crawler_file' not in self.sessconfig or self.sessconfig['crawler_file'] is None:
self.sessconfig['crawler_file'] = os.path.join(self.sessconfig['install_dir'], LIBRARYNAME,'Core','Statistics','crawler.txt')
# 5. peer_icon_path
if self.sessconfig['peer_icon_path'] is None:
self.sessconfig['peer_icon_path'] = os.path.join(self.sessconfig['state_dir'],STATEDIR_PEERICON_DIR)
if not os.path.isdir(self.sessconfig['peer_icon_path']):
os.mkdir(self.sessconfig['peer_icon_path'])
# 6. Poor man's versioning of SessionConfig, add missing
# default values. Really should use PERSISTENTSTATE_CURRENTVERSION
# and do conversions.
for key,defvalue in sessdefaults.iteritems():
if key not in self.sessconfig:
self.sessconfig[key] = defvalue
# 7. proxyservice_dir
if self.sessconfig['overlay']: #NIELS: proxyservice_on/off is set at runtime, always make sure proxyservice_ and self.sessconfig['proxyservice_status'] == PROXYSERVICE_ON:
if self.sessconfig['proxyservice_dir'] is None:
self.sessconfig['proxyservice_dir'] = os.path.join(get_default_dest_dir(), PROXYSERVICE_DESTDIR)
# Jelle: under linux, default_dest_dir can be /tmp. Then proxyservice_dir can be deleted in between
# sessions.
if not os.path.isdir(self.sessconfig['proxyservice_dir']):
os.makedirs(self.sessconfig['proxyservice_dir'])
if not 'live_aux_seeders' in self.sessconfig:
# Poor man's versioning, really should update PERSISTENTSTATE_CURRENTVERSION
self.sessconfig['live_aux_seeders'] = sessdefaults['live_aux_seeders']
if not 'nat_detect' in self.sessconfig:
self.sessconfig['nat_detect'] = sessdefaults['nat_detect']
if not 'puncturing_internal_port' in self.sessconfig:
self.sessconfig['puncturing_internal_port'] = sessdefaults['puncturing_internal_port']
if not 'stun_servers' in self.sessconfig:
self.sessconfig['stun_servers'] = sessdefaults['stun_servers']
if not 'pingback_servers' in self.sessconfig:
self.sessconfig['pingback_servers'] = sessdefaults['pingback_servers']
if not 'mainline_dht' in self.sessconfig:
self.sessconfig['mainline_dht'] = sessdefaults['mainline_dht']
# SWIFTPROC
if self.sessconfig['swiftpath'] is None:
if sys.platform == "win32":
self.sessconfig['swiftpath'] = os.path.join(self.sessconfig['install_dir'],"swift.exe")
else:
self.sessconfig['swiftpath'] = os.path.join(self.sessconfig['install_dir'],"swift")
# Checkpoint startup config
self.save_pstate_sessconfig()
# Create handler for calling back the user via separate threads
self.uch = UserCallbackHandler(self)
# Create engine with network thread
self.lm = TriblerLaunchMany()
self.lm.register(self,self.sesslock)
self.lm.start()
def __init__(self,
scfg=None,
ignore_singleton=False,
autoload_discovery=True):
"""
A Session object is created which is configured following a copy of the
SessionStartupConfig scfg. (copy constructor used internally)
@param scfg SessionStartupConfig object or None, in which case we
look for a saved session in the default location (state dir). If
we can't find it, we create a new SessionStartupConfig() object to
serve as startup config. Next, the config is saved in the directory
indicated by its 'state_dir' attribute.
In the current implementation only a single session instance can exist
at a time in a process. The ignore_singleton flag is used for testing.
"""
addObserver(self.unhandled_error_observer)
patch_crypto_be_discovery()
if not ignore_singleton:
if Session.__single:
raise RuntimeError("Session is singleton")
Session.__single = self
self._logger = logging.getLogger(self.__class__.__name__)
self.ignore_singleton = ignore_singleton
self.sesslock = NoDispersyRLock()
# Determine startup config to use
if scfg is None: # If no override
scfg = SessionStartupConfig.load()
else: # overrides any saved config
# Work from copy
scfg = SessionStartupConfig(copy.copy(scfg.sessconfig))
def create_dir(fullpath):
if not os.path.isdir(fullpath):
os.makedirs(fullpath)
def set_and_create_dir(dirname, setter, default_dir):
if dirname is None:
setter(default_dir)
create_dir(dirname or default_dir)
state_dir = scfg.get_state_dir()
set_and_create_dir(state_dir, scfg.set_state_dir, state_dir)
set_and_create_dir(
scfg.get_torrent_store_dir(), scfg.set_torrent_store_dir,
os.path.join(scfg.get_state_dir(), STATEDIR_TORRENT_STORE_DIR))
# metadata store
set_and_create_dir(
scfg.get_metadata_store_dir(), scfg.set_metadata_store_dir,
os.path.join(scfg.get_state_dir(), STATEDIR_METADATA_STORE_DIR))
set_and_create_dir(
scfg.get_peer_icon_path(), scfg.set_peer_icon_path,
os.path.join(scfg.get_state_dir(), STATEDIR_PEERICON_DIR))
create_dir(os.path.join(scfg.get_state_dir(), u"sqlite"))
create_dir(os.path.join(scfg.get_state_dir(), STATEDIR_DLPSTATE_DIR))
if GOTM2CRYPTO:
permidmod.init()
# Set params that depend on state_dir
#
# 1. keypair
#
pairfilename = scfg.get_permid_keypair_filename()
if os.path.exists(pairfilename):
self.keypair = permidmod.read_keypair(pairfilename)
else:
self.keypair = permidmod.generate_keypair()
# Save keypair
pubfilename = os.path.join(scfg.get_state_dir(), 'ecpub.pem')
permidmod.save_keypair(self.keypair, pairfilename)
permidmod.save_pub_key(self.keypair, pubfilename)
multichain_pairfilename = scfg.get_multichain_permid_keypair_filename(
)
if os.path.exists(multichain_pairfilename):
self.multichain_keypair = permidmod.read_keypair_multichain(
multichain_pairfilename)
else:
self.multichain_keypair = permidmod.generate_keypair_multichain(
)
# Save keypair
multichain_pubfilename = os.path.join(scfg.get_state_dir(),
'ecpub_multichain.pem')
permidmod.save_keypair_multichain(self.multichain_keypair,
multichain_pairfilename)
permidmod.save_pub_key_multichain(self.multichain_keypair,
multichain_pubfilename)
if not scfg.get_megacache():
scfg.set_torrent_checking(0)
self.sessconfig = scfg.sessconfig
self.sessconfig.lock = self.sesslock
self.selected_ports = scfg.selected_ports
# Claim all random ports
self.get_listen_port()
self.get_dispersy_port()
self.get_mainline_dht_listen_port()
self.get_videoserver_port()
self.get_anon_listen_port()
self.get_tunnel_community_socks5_listen_ports()
# Create handler for calling back the user via separate threads
self.lm = TriblerLaunchMany()
self.notifier = Notifier()
# Checkpoint startup config
self.save_session_config()
self.sqlite_db = None
self.upgrader_enabled = True
self.dispersy_member = None
self.readable_status = '' # Human-readable string to indicate the status during startup/shutdown of Tribler
self.autoload_discovery = autoload_discovery
self.tribler_config = TriblerConfig(self)
self.setup_tribler_gui_config()
class SwiftDownloadImpl(SwiftDownloadRuntimeConfig):
""" Download subclass that represents a swift download.
The actual swift download takes places in a SwiftProcess.
"""
def __init__(self, session, sdef):
self.dllock = NoDispersyRLock()
self.session = session
self.sdef = sdef
self.old_metadir = self.session.get_swift_meta_dir()
# just enough so error saving and get_state() works
self.error = None
# To be able to return the progress of a stopped torrent, how far it got.
self.progressbeforestop = 0.0
# SwiftProcess performing the actual download.
self.sp = None
# spstatus
self.dlstatus = DLSTATUS_WAITING4HASHCHECK
self.dynasize = 0
self.progress = 0.0
self.curspeeds = {DOWNLOAD: 0.0, UPLOAD: 0.0} # bytes/s
self.numleech = 0
self.numseeds = 0
self.contentbytes = {DOWNLOAD: 0, UPLOAD: 0} # bytes
self.done = False # when set it means this download is being removed
self.midict = {}
self.time_seeding = [0, None]
self.total_up = 0
self.total_down = 0
self.lm_network_vod_event_callback = None
self.askmoreinfo = False
#
# Download Interface
#
def get_def(self):
return self.sdef
#
# DownloadImpl
#
#
# Creating a Download
#
def setup(
self,
dcfg=None,
pstate=None,
initialdlstatus=None,
lm_network_engine_wrapper_created_callback=None,
lm_network_vod_event_callback=None,
):
"""
Create a Download object. Used internally by Session.
@param dcfg DownloadStartupConfig or None (in which case
a new DownloadConfig() is created and the result
becomes the runtime config of this Download.
"""
# Called by any thread, assume sessionlock is held
try:
self.dllock.acquire() # not really needed, no other threads know of this object
# Copy dlconfig, from default if not specified
if dcfg is None:
cdcfg = DownloadStartupConfig()
else:
cdcfg = dcfg
self.dlconfig = copy.copy(cdcfg.dlconfig)
# Things that only exist at runtime
self.dlruntimeconfig = {}
self.dlruntimeconfig["max_desired_upload_rate"] = 0
self.dlruntimeconfig["max_desired_download_rate"] = 0
if pstate and "dlstate" in pstate:
dlstate = pstate["dlstate"]
if "time_seeding" in dlstate:
self.time_seeding = [dlstate["time_seeding"], None]
if "total_up" in dlstate:
self.total_up = dlstate["total_up"]
if "total_down" in dlstate:
self.total_down = dlstate["total_down"]
if DEBUG:
print >>sys.stderr, "SwiftDownloadImpl: setup: initialdlstatus", repr(
self.sdef.get_roothash_as_hex()
), initialdlstatus
# Note: initialdlstatus now only works for STOPPED
if initialdlstatus != DLSTATUS_STOPPED:
self.create_engine_wrapper(
lm_network_engine_wrapper_created_callback, pstate, lm_network_vod_event_callback
)
self.dllock.release()
except Exception as e:
print_exc()
self.set_error(e)
self.dllock.release()
def create_engine_wrapper(
self, lm_network_engine_wrapper_created_callback, pstate, lm_network_vod_event_callback, initialdlstatus=None
):
network_create_engine_wrapper_lambda = lambda: self.network_create_engine_wrapper(
lm_network_engine_wrapper_created_callback, pstate, lm_network_vod_event_callback, initialdlstatus
)
self.session.lm.rawserver.add_task(network_create_engine_wrapper_lambda)
def network_create_engine_wrapper(
self, lm_network_engine_wrapper_created_callback, pstate, lm_network_vod_event_callback, initialdlstatus=None
):
""" Called by any thread, assume dllock already acquired """
if DEBUG:
print >>sys.stderr, "SwiftDownloadImpl: create_engine_wrapper()"
if self.get_mode() == DLMODE_VOD:
self.lm_network_vod_event_callback = lm_network_vod_event_callback
move_files = ("swiftmetadir" not in self.dlconfig) and not os.path.isdir(self.get_dest_dir())
metadir = self.get_swift_meta_dir()
if not metadir:
metadir = self.session.get_swift_meta_dir()
self.set_swift_meta_dir(metadir)
if not os.path.exists(metadir):
os.makedirs(metadir)
if move_files:
# We must be dealing with a checkpoint from a previous release (<6.1.0). Move the swift metadata to the right directory.
is_multifile = self.get_dest_dir().endswith("." + self.get_def().get_roothash_as_hex())
path_old = self.get_dest_dir()
path_new = os.path.join(
metadir, self.get_def().get_roothash_as_hex() if is_multifile else os.path.split(self.get_dest_dir())[1]
)
try:
if is_multifile:
shutil.move(path_old, path_new + ".mfspec")
self.dlconfig["saveas"] = os.path.split(self.get_dest_dir())[0]
shutil.move(path_old + ".mhash", path_new + ".mhash")
shutil.move(path_old + ".mbinmap", path_new + ".mbinmap")
except:
print_exc()
# Synchronous: starts process if needed
self.sp = self.session.lm.spm.get_or_create_sp(
self.session.get_swift_working_dir(),
self.session.get_torrent_collecting_dir(),
self.get_swift_listen_port(),
self.get_swift_httpgw_listen_port(),
self.get_swift_cmdgw_listen_port(),
)
if self.sp:
self.sp.start_download(self)
self.session.lm.rawserver.add_task(self.network_check_swift_alive, SWIFT_ALIVE_CHECK_INTERVAL)
# Arno: if used, make sure to switch to network thread first!
# if lm_network_engine_wrapper_created_callback is not None:
# sp = self.sp
# exc = self.error
# lm_network_engine_wrapper_created_callback(self,sp,exc,pstate)
#
# SwiftProcess callbacks
#
def i2ithread_info_callback(
self, dlstatus, progress, dynasize, dlspeed, ulspeed, numleech, numseeds, contentdl, contentul
):
self.dllock.acquire()
try:
if dlstatus == DLSTATUS_SEEDING and self.dlstatus != dlstatus:
# started seeding
self.time_seeding[0] = self.get_seeding_time()
self.time_seeding[1] = time.time()
elif dlstatus != DLSTATUS_SEEDING and self.dlstatus != dlstatus:
# stopped seeding
self.time_seeding[0] = self.get_seeding_time()
self.time_seeding[1] = None
self.dlstatus = dlstatus
self.dynasize = dynasize
self.progress = progress
self.curspeeds[DOWNLOAD] = dlspeed
self.curspeeds[UPLOAD] = ulspeed
self.numleech = numleech
self.numseeds = numseeds
self.contentbytes = {DOWNLOAD: contentdl, UPLOAD: contentul}
finally:
self.dllock.release()
def i2ithread_vod_event_callback(self, event, httpurl):
if DEBUG:
print >>sys.stderr, "SwiftDownloadImpl: i2ithread_vod_event_callback: ENTER", event, httpurl, "mode", self.get_mode()
self.dllock.acquire()
try:
if event == VODEVENT_START:
if self.get_mode() != DLMODE_VOD:
return
# Fix firefox idiosyncrasies
duration = self.sdef.get_duration()
if duration is not None:
httpurl += "@" + duration
vod_usercallback_wrapper = lambda event, params: self.session.uch.perform_vod_usercallback(
self, self.dlconfig["vod_usercallback"], event, params
)
videoinfo = {}
videoinfo["usercallback"] = vod_usercallback_wrapper
# ARNOSMPTODO: if complete, return file directly
# Allow direct connection of video renderer with swift HTTP server
# via new "url" param.
#
if DEBUG:
print >>sys.stderr, "SwiftDownloadImpl: i2ithread_vod_event_callback", event, httpurl
# Arno: No threading violation, lm_network_* is safe at the moment
self.lm_network_vod_event_callback(
videoinfo,
VODEVENT_START,
{
"complete": False,
"filename": None,
"mimetype": "application/octet-stream", # ARNOSMPTODO
"stream": None,
"length": self.get_dynasize(),
"bitrate": None, # ARNOSMPTODO
"url": httpurl,
},
)
finally:
self.dllock.release()
def i2ithread_moreinfo_callback(self, midict):
self.dllock.acquire()
try:
# print >>sys.stderr,"SwiftDownloadImpl: Got moreinfo",midict.keys()
self.midict = midict
finally:
self.dllock.release()
#
# Retrieving DownloadState
#
def get_status(self):
""" Returns the status of the download.
@return DLSTATUS_* """
self.dllock.acquire()
try:
return self.dlstatus
finally:
self.dllock.release()
def get_dynasize(self):
""" Returns the size of the swift content. Note this may vary
(generally ~1KiB because of dynamic size determination by the
swift protocol
@return long
"""
self.dllock.acquire()
try:
return self.dynasize
finally:
self.dllock.release()
def get_progress(self):
""" Return fraction of content downloaded.
@return float 0..1
"""
self.dllock.acquire()
try:
return self.progress
finally:
self.dllock.release()
def get_current_speed(self, dir):
""" Return last reported speed in KB/s
@return float
"""
self.dllock.acquire()
try:
return self.curspeeds[dir] / 1024.0
finally:
self.dllock.release()
def get_moreinfo_stats(self, dir):
""" Return last reported more info dict
@return dict
"""
self.dllock.acquire()
try:
return self.midict
finally:
self.dllock.release()
def get_seeding_time(self):
return self.time_seeding[0] + (time.time() - self.time_seeding[1] if self.time_seeding[1] != None else 0)
def get_total_up(self):
return self.total_up + self.contentbytes[UPLOAD]
def get_total_down(self):
return self.total_down + self.contentbytes[DOWNLOAD]
def get_seeding_statistics(self):
seeding_stats = {}
seeding_stats["total_up"] = self.get_total_up()
seeding_stats["total_down"] = self.get_total_down()
seeding_stats["time_seeding"] = self.get_seeding_time()
return seeding_stats
def network_get_stats(self, getpeerlist):
"""
@return (status,stats,logmsgs,coopdl_helpers,coopdl_coordinator)
"""
# dllock held
# ARNOSMPTODO: Have a status for when swift is hashchecking the file on disk
if self.sp is None:
status = DLSTATUS_STOPPED
else:
status = self.dlstatus
stats = {}
stats["down"] = self.curspeeds[DOWNLOAD]
stats["up"] = self.curspeeds[UPLOAD]
stats["frac"] = self.progress
stats["stats"] = self.network_create_statistics_reponse()
stats["time"] = self.network_calc_eta()
stats["vod_prebuf_frac"] = self.network_calc_prebuf_frac()
stats["vod"] = True
# ARNOSMPTODO: no hard check for suff bandwidth, unlike BT1Download
stats["vod_playable"] = self.progress == 1.0 or (
self.network_calc_prebuf_frac() == 1.0 and self.curspeeds[DOWNLOAD] > 0.0
)
stats["vod_playable_after"] = self.network_calc_prebuf_eta()
stats["vod_stats"] = self.network_get_vod_stats()
stats["spew"] = self.network_create_spew_from_peerlist()
seeding_stats = self.get_seeding_statistics()
logmsgs = []
return (status, stats, seeding_stats, logmsgs)
def network_create_statistics_reponse(self):
return SwiftStatisticsResponse(self.numleech, self.numseeds, self.midict)
def network_calc_eta(self):
bytestogof = (1.0 - self.progress) * float(self.dynasize)
dlspeed = max(0.000001, self.curspeeds[DOWNLOAD])
return bytestogof / dlspeed
def network_calc_prebuf_frac(self):
gotbytesf = self.progress * float(self.dynasize)
prebuff = float(CMDGW_PREBUFFER_BYTES)
return min(1.0, gotbytesf / prebuff)
def network_calc_prebuf_eta(self):
bytestogof = (1.0 - self.network_calc_prebuf_frac()) * float(CMDGW_PREBUFFER_BYTES)
dlspeed = max(0.000001, self.curspeeds[DOWNLOAD])
return bytestogof / dlspeed
def network_get_vod_stats(self):
# More would have to be sent from swift process to set these correctly
d = {}
d["played"] = None
d["late"] = None
d["dropped"] = None
d["stall"] = None
d["pos"] = None
d["prebuf"] = None
d["firstpiece"] = 0
d["npieces"] = (self.dynasize + 1023) / 1024
return d
def network_create_spew_from_peerlist(self):
if not "channels" in self.midict:
return []
plist = []
channels = self.midict["channels"]
for channel in channels:
d = {}
d["ip"] = channel["ip"]
d["port"] = channel["port"]
d["utotal"] = channel["bytes_up"] / 1024.0
d["dtotal"] = channel["bytes_down"] / 1024.0
plist.append(d)
return plist
#
# Retrieving DownloadState
#
def set_state_callback(self, usercallback, getpeerlist=False, delay=0.0):
""" Called by any thread """
self.dllock.acquire()
try:
network_get_state_lambda = lambda: self.network_get_state(usercallback, getpeerlist)
# First time on general rawserver
self.session.lm.rawserver.add_task(network_get_state_lambda, delay)
finally:
self.dllock.release()
def network_get_state(self, usercallback, getpeerlist, sessioncalling=False):
""" Called by network thread """
self.dllock.acquire()
try:
if self.sp is None:
if DEBUG:
print >>sys.stderr, "SwiftDownloadImpl: network_get_state: Download not running"
ds = DownloadState(
self,
DLSTATUS_STOPPED,
self.error,
self.progressbeforestop,
seeding_stats=self.get_seeding_statistics(),
)
else:
(status, stats, seeding_stats, logmsgs) = self.network_get_stats(getpeerlist)
ds = DownloadState(
self,
status,
self.error,
self.get_progress(),
stats=stats,
seeding_stats=seeding_stats,
logmsgs=logmsgs,
)
self.progressbeforestop = ds.get_progress()
if sessioncalling:
return ds
# Invoke the usercallback function via a new thread.
# After the callback is invoked, the return values will be passed to
# the returncallback for post-callback processing.
if not self.done:
self.session.uch.perform_getstate_usercallback(usercallback, ds, self.sesscb_get_state_returncallback)
finally:
self.dllock.release()
def sesscb_get_state_returncallback(self, usercallback, when, newgetpeerlist):
""" Called by SessionCallbackThread """
self.dllock.acquire()
try:
if when > 0.0 and not self.done:
# Schedule next invocation, either on general or DL specific
# Note this continues when dl is stopped.
network_get_state_lambda = lambda: self.network_get_state(usercallback, newgetpeerlist)
self.session.lm.rawserver.add_task(network_get_state_lambda, when)
finally:
self.dllock.release()
#
# Download stop/resume
#
def stop(self):
""" Called by any thread """
self.stop_remove(False, removestate=False, removecontent=False)
def stop_remove(self, removedl, removestate=False, removecontent=False):
""" Called by any thread. Called on Session.remove_download() """
# Arno, 2013-01-29: This download is being removed, not just stopped.
self.done = removedl
self.network_stop(removestate=removestate, removecontent=removecontent)
def network_stop(self, removestate, removecontent):
""" Called by network thread, but safe for any """
self.dllock.acquire()
try:
if DEBUG:
print >>sys.stderr, "SwiftDownloadImpl: network_stop", repr(self.sdef.get_name())
pstate = self.network_get_persistent_state()
if self.sp is not None:
self.sp.remove_download(self, removestate, removecontent)
self.session.lm.spm.release_sp(self.sp)
self.sp = None
self.time_seeding = [self.get_seeding_time(), None]
# Offload the removal of the dlcheckpoint to another thread
if removestate:
# To remove:
# 1. Core checkpoint (if any)
# 2. .mhash file
# 3. content (if so desired)
# content and .mhash file is removed by swift engine if requested
roothash = self.sdef.get_roothash()
self.session.uch.perform_removestate_callback(roothash, None, False)
return (self.sdef.get_roothash(), pstate)
finally:
self.dllock.release()
def get_content_dest(self):
""" Returns the file to which the downloaded content is saved. """
return os.path.join(self.get_dest_dir(), self.sdef.get_roothash_as_hex())
def restart(self, initialdlstatus=None):
""" Restart the Download """
# Called by any thread
if DEBUG:
print >>sys.stderr, "SwiftDownloadImpl: restart:", repr(self.sdef.get_name())
self.dllock.acquire()
try:
if self.sp is None:
self.error = None # assume fatal error is reproducible
self.create_engine_wrapper(
self.session.lm.network_engine_wrapper_created_callback,
None,
self.session.lm.network_vod_event_callback,
initialdlstatus=initialdlstatus,
)
# No exception if already started, for convenience
finally:
self.dllock.release()
#
# Config parameters that only exists at runtime
#
def set_max_desired_speed(self, direct, speed):
if DEBUG:
print >>sys.stderr, "Download: set_max_desired_speed", direct, speed
# if speed < 10:
# print_stack()
self.dllock.acquire()
if direct == UPLOAD:
self.dlruntimeconfig["max_desired_upload_rate"] = speed
else:
self.dlruntimeconfig["max_desired_download_rate"] = speed
self.dllock.release()
def get_max_desired_speed(self, direct):
self.dllock.acquire()
try:
if direct == UPLOAD:
return self.dlruntimeconfig["max_desired_upload_rate"]
else:
return self.dlruntimeconfig["max_desired_download_rate"]
finally:
self.dllock.release()
def get_dest_files(self, exts=None):
"""
Returns (None,destfilename)
"""
if exts is not None:
raise OperationNotEnabledByConfigurationException()
f2dlist = []
diskfn = self.get_content_dest()
f2dtuple = (None, diskfn)
f2dlist.append(f2dtuple)
return f2dlist
#
# Persistence
#
def checkpoint(self):
""" Called by any thread """
# Arno, 2012-05-15. Currently this is safe to call from any thread.
# Need this for torrent collecting via swift.
self.network_checkpoint()
def network_checkpoint(self):
""" Called by network thread """
self.dllock.acquire()
try:
pstate = self.network_get_persistent_state()
if self.sp is not None:
self.sp.checkpoint_download(self)
return (self.sdef.get_roothash(), pstate)
finally:
self.dllock.release()
def network_get_persistent_state(self):
""" Assume dllock already held """
pstate = {}
pstate["version"] = PERSISTENTSTATE_CURRENTVERSION
pstate["metainfo"] = self.sdef.get_url_with_meta() # assumed immutable
dlconfig = copy.copy(self.dlconfig)
dlconfig["name"] = self.sdef.get_name()
# Reset unpicklable params
dlconfig["vod_usercallback"] = None
dlconfig["mode"] = DLMODE_NORMAL # no callback, no VOD
# Reset default metadatadir
if self.get_swift_meta_dir() == self.old_metadir:
dlconfig["swiftmetadir"] = None
pstate["dlconfig"] = dlconfig
pstate["dlstate"] = {}
ds = self.network_get_state(None, False, sessioncalling=True)
pstate["dlstate"]["status"] = ds.get_status()
pstate["dlstate"]["progress"] = ds.get_progress()
pstate["dlstate"]["swarmcache"] = None
pstate["dlstate"].update(ds.get_seeding_statistics())
if DEBUG:
print >>sys.stderr, "SwiftDownloadImpl: netw_get_pers_state: status", dlstatus_strings[
ds.get_status()
], "progress", ds.get_progress()
# Swift stores own state in .mhash and .mbinmap file
pstate["engineresumedata"] = None
return pstate
#
# Coop download
#
def get_coopdl_role_object(self, role):
""" Called by network thread """
return None
def recontact_tracker(self):
""" Called by any thread """
pass
#
# MOREINFO
#
def set_moreinfo_stats(self, enable):
""" Called by any thread """
# Arno, 2012-07-31: slight risk if process killed in between
if self.askmoreinfo == enable:
return
self.askmoreinfo = enable
if self.sp is not None:
self.sp.set_moreinfo_stats(self, enable)
#
# External addresses
#
def add_peer(self, addr):
""" Add a peer address from 3rd source (not tracker, not DHT) to this
Download.
@param (hostname_ip,port) tuple
"""
if self.sp is not None:
self.sp.add_peer(self, addr)
#
# Internal methods
#
def set_error(self, e):
self.dllock.acquire()
self.error = e
self.dllock.release()
#
# Auto restart after swift crash
#
def network_check_swift_alive(self):
self.dllock.acquire()
try:
if self.sp is not None and not self.done:
if not self.sp.is_alive():
print >>sys.stderr, "SwiftDownloadImpl: network_check_swift_alive: Restarting", repr(
self.sdef.get_name()
)
self.sp = None
self.restart()
except:
print_exc()
finally:
self.dllock.release()
if not self.done:
self.session.lm.rawserver.add_task(self.network_check_swift_alive, SWIFT_ALIVE_CHECK_INTERVAL)
class SwiftDownloadImpl(SwiftDownloadRuntimeConfig):
""" Download subclass that represents a swift download.
The actual swift download takes places in a SwiftProcess.
"""
def __init__(self,session,sdef):
self.dllock = NoDispersyRLock()
self.session = session
self.sdef = sdef
# just enough so error saving and get_state() works
self.error = None
# To be able to return the progress of a stopped torrent, how far it got.
self.progressbeforestop = 0.0
# SwiftProcess performing the actual download.
self.sp = None
# spstatus
self.dlstatus = DLSTATUS_WAITING4HASHCHECK
self.dynasize = 0L
self.progress = 0.0
self.curspeeds = {DOWNLOAD:0.0,UPLOAD:0.0} # bytes/s
self.numleech = 0
self.numseeds = 0
self.contentbytes = {DOWNLOAD:0,UPLOAD:0} # bytes
self.done = False # when set it means this download is being removed
self.midict = {}
self.time_seeding = [0, None]
self.total_up = 0
self.total_down = 0
self.lm_network_vod_event_callback = None
self.askmoreinfo = False
#
# Download Interface
#
def get_def(self):
return self.sdef
#
# DownloadImpl
#
#
# Creating a Download
#
def setup(self,dcfg=None,pstate=None,initialdlstatus=None,lm_network_engine_wrapper_created_callback=None,lm_network_vod_event_callback=None):
"""
Create a Download object. Used internally by Session.
@param dcfg DownloadStartupConfig or None (in which case
a new DownloadConfig() is created and the result
becomes the runtime config of this Download.
"""
# Called by any thread, assume sessionlock is held
try:
self.dllock.acquire() # not really needed, no other threads know of this object
# Copy dlconfig, from default if not specified
if dcfg is None:
cdcfg = DownloadStartupConfig()
else:
cdcfg = dcfg
self.dlconfig = copy.copy(cdcfg.dlconfig)
# Things that only exist at runtime
self.dlruntimeconfig= {}
self.dlruntimeconfig['max_desired_upload_rate'] = 0
self.dlruntimeconfig['max_desired_download_rate'] = 0
if pstate and pstate.has_key('dlstate'):
dlstate = pstate['dlstate']
if dlstate.has_key('time_seeding'):
self.time_seeding = [dlstate['time_seeding'], None]
if dlstate.has_key('total_up'):
self.total_up = dlstate['total_up']
if dlstate.has_key('total_down'):
self.total_down = dlstate['total_down']
if DEBUG:
print >>sys.stderr,"SwiftDownloadImpl: setup: initialdlstatus",`self.sdef.get_roothash_as_hex()`,initialdlstatus
# Note: initialdlstatus now only works for STOPPED
if initialdlstatus != DLSTATUS_STOPPED:
self.create_engine_wrapper(lm_network_engine_wrapper_created_callback,pstate,lm_network_vod_event_callback)
self.dllock.release()
except Exception,e:
print_exc()
self.set_error(e)
self.dllock.release()
class SwiftDownloadImpl(SwiftDownloadRuntimeConfig):
""" Download subclass that represents a swift download.
The actual swift download takes places in a SwiftProcess.
"""
def __init__(self, session, sdef):
self.dllock = NoDispersyRLock()
self.session = session
self.sdef = sdef
# just enough so error saving and get_state() works
self.error = None
# To be able to return the progress of a stopped torrent, how far it got.
self.progressbeforestop = 0.0
# SwiftProcess performing the actual download.
self.sp = None
# spstatus
self.dlstatus = DLSTATUS_WAITING4HASHCHECK
self.dynasize = 0L
self.progress = 0.0
self.curspeeds = {DOWNLOAD: 0.0, UPLOAD: 0.0} # bytes/s
self.numleech = 0
self.numseeds = 0
self.done = False
self.midict = {}
self.lm_network_vod_event_callback = None
#
# Download Interface
#
def get_def(self):
return self.sdef
#
# DownloadImpl
#
#
# Creating a Download
#
def setup(self,
dcfg=None,
pstate=None,
initialdlstatus=None,
lm_network_engine_wrapper_created_callback=None,
lm_network_vod_event_callback=None):
"""
Create a Download object. Used internally by Session.
@param dcfg DownloadStartupConfig or None (in which case
a new DownloadConfig() is created and the result
becomes the runtime config of this Download.
"""
# Called by any thread, assume sessionlock is held
try:
self.dllock.acquire(
) # not really needed, no other threads know of this object
# Copy dlconfig, from default if not specified
if dcfg is None:
cdcfg = DownloadStartupConfig()
else:
cdcfg = dcfg
self.dlconfig = copy.copy(cdcfg.dlconfig)
# Things that only exist at runtime
self.dlruntimeconfig = {}
self.dlruntimeconfig['max_desired_upload_rate'] = 0
self.dlruntimeconfig['max_desired_download_rate'] = 0
if DEBUG:
print >> sys.stderr, "SwiftDownloadImpl: setup: initialdlstatus", ` self.sdef.get_roothash_as_hex(
) `, initialdlstatus
# Note: initialdlstatus now only works for STOPPED
if initialdlstatus != DLSTATUS_STOPPED:
self.create_engine_wrapper(
lm_network_engine_wrapper_created_callback, pstate,
lm_network_vod_event_callback)
self.dllock.release()
except Exception, e:
print_exc()
self.set_error(e)
self.dllock.release()
class SwiftProcess:
""" Representation of an operating-system process running the C++ swift engine.
A swift engine can participate in one or more swarms."""
def __init__(self, binpath, workdir, zerostatedir, listenport, httpgwport,
cmdgwport, spmgr):
# Called by any thread, assume sessionlock is held
self.splock = NoDispersyRLock()
self.binpath = binpath
self.workdir = workdir
self.zerostatedir = zerostatedir
self.spmgr = spmgr
# Main UDP listen socket
if listenport is None:
self.listenport = random.randint(10001, 10999)
else:
self.listenport = listenport
# NSSA control socket
if cmdgwport is None:
self.cmdport = random.randint(11001, 11999)
else:
self.cmdport = cmdgwport
# content web server
if httpgwport is None:
self.httpport = random.randint(12001, 12999)
else:
self.httpport = httpgwport
# Security: only accept commands from localhost, enable HTTP gw,
# no stats/webUI web server
args = []
# Arno, 2012-07-09: Unicode problems with popen
args.append(self.binpath.encode(sys.getfilesystemencoding()))
# Arno, 2012-05-29: Hack. Win32 getopt code eats first arg when Windows app
# instead of CONSOLE app.
args.append("-j")
args.append("-l") # listen port
args.append("0.0.0.0:" + str(self.listenport))
args.append("-c") # command port
args.append("127.0.0.1:" + str(self.cmdport))
args.append("-g") # HTTP gateway port
args.append("127.0.0.1:" + str(self.httpport))
args.append("-w")
if zerostatedir is not None:
if sys.platform == "win32":
# Swift on Windows expects command line arguments as UTF-16.
# popen doesn't allow us to pass params in UTF-16, hence workaround.
# Format = hex encoded UTF-8
args.append("-3")
zssafe = binascii.hexlify(zerostatedir.encode("UTF-8"))
args.append(zssafe) # encoding that swift expects
else:
args.append("-e")
args.append(zerostatedir)
args.append("-T") # zero state connection timeout
args.append("180") # seconds
#args.append("-B") # Enable debugging on swift
if True or DEBUG:
print >> sys.stderr, "SwiftProcess: __init__: Running", args, "workdir", workdir
if sys.platform == "win32":
creationflags = subprocess.CREATE_NEW_PROCESS_GROUP
else:
creationflags = 0
# See also SwiftDef::finalize popen
self.popen = subprocess.Popen(args,
close_fds=True,
cwd=workdir,
creationflags=creationflags)
self.roothash2dl = {}
self.donestate = DONE_STATE_WORKING # shutting down
self.fastconn = None
# callbacks for when swift detect a channel close
self._channel_close_callbacks = defaultdict(list)
#
# Instance2Instance
#
def start_cmd_connection(self):
# Called by any thread, assume sessionlock is held
if self.is_alive():
self.fastconn = FastI2IConnection(self.cmdport,
self.i2ithread_readlinecallback,
self.connection_lost)
else:
print >> sys.stderr, "sp: start_cmd_connection: Process dead? returncode", self.popen.returncode, "pid", self.popen.pid
def i2ithread_readlinecallback(self, ic, cmd):
#if DEBUG:
# print >>sys.stderr,"sp: Got command #"+cmd+"#"
if self.donestate != DONE_STATE_WORKING:
return
words = cmd.split()
if words[0] == "TUNNELRECV":
address, session = words[1].split("/")
host, port = address.split(":")
port = int(port)
session = session.decode("HEX")
length = int(words[2])
# require LENGTH bytes
if len(ic.buffer) < length:
return length - len(ic.buffer)
data = ic.buffer[:length]
ic.buffer = ic.buffer[length:]
self.roothash2dl["dispersy"].i2ithread_data_came_in(
session, (host, port), data)
else:
roothash = binascii.unhexlify(words[1])
if words[0] == "ERROR":
print >> sys.stderr, "sp: i2ithread_readlinecallback:", cmd
elif words[0] == "CLOSE_EVENT":
roothash_hex = words[1]
address = words[2].split(":")
raw_bytes_up = int(words[3])
raw_bytes_down = int(words[4])
cooked_bytes_up = int(words[5])
cooked_bytes_down = int(words[6])
if roothash_hex in self._channel_close_callbacks:
for callback in self._channel_close_callbacks[
roothash_hex]:
try:
callback(roothash_hex, address, raw_bytes_up,
raw_bytes_down, cooked_bytes_up,
cooked_bytes_down)
except:
pass
for callback in self._channel_close_callbacks["ALL"]:
try:
callback(roothash_hex, address, raw_bytes_up,
raw_bytes_down, cooked_bytes_up,
cooked_bytes_down)
except:
pass
self.splock.acquire()
try:
if roothash not in self.roothash2dl.keys():
if DEBUG:
print >> sys.stderr, "sp: i2ithread_readlinecallback: unknown roothash", words[
1]
return
d = self.roothash2dl[roothash]
except:
#print >>sys.stderr,"GOT", words
#print >>sys.stderr,"HAVE", [key.encode("HEX") for key in self.roothash2dl.keys()]
raise
finally:
self.splock.release()
# Hide NSSA interface for SwiftDownloadImpl
if words[0] == "INFO": # INFO HASH status dl/total
dlstatus = int(words[2])
pargs = words[3].split("/")
dynasize = int(pargs[1])
if dynasize == 0:
progress = 0.0
else:
progress = float(pargs[0]) / float(pargs[1])
dlspeed = float(words[4])
ulspeed = float(words[5])
numleech = int(words[6])
numseeds = int(words[7])
contentdl = 0 # bytes
contentul = 0 # bytes
if len(words) > 8:
contentdl = int(words[8])
contentul = int(words[9])
d.i2ithread_info_callback(dlstatus, progress, dynasize,
dlspeed, ulspeed, numleech, numseeds,
contentdl, contentul)
elif words[0] == "PLAY":
#print >>sys.stderr,"sp: i2ithread_readlinecallback: Got PLAY",cmd
httpurl = words[2]
d.i2ithread_vod_event_callback(VODEVENT_START, httpurl)
elif words[0] == "MOREINFO":
jsondata = cmd[len("MOREINFO ") + 40 + 1:]
midict = json.loads(jsondata)
d.i2ithread_moreinfo_callback(midict)
elif words[0] == "ERROR":
d.i2ithread_info_callback(DLSTATUS_STOPPED_ON_ERROR, 0.0, 0,
0.0, 0.0, 0, 0, 0, 0)
#
# Swift Mgmt interface
#
def start_download(self, d):
self.splock.acquire()
try:
if self.donestate != DONE_STATE_WORKING or not self.is_alive():
return
roothash = d.get_def().get_roothash()
roothash_hex = d.get_def().get_roothash_as_hex()
# Before send to handle INFO msgs
self.roothash2dl[roothash] = d
url = d.get_def().get_url()
# MULTIFILE
if len(d.get_selected_files()) == 1:
specpath = d.get_selected_files()[0]
qpath = urllib.quote(specpath)
url += "/" + qpath
# Default is unlimited, so don't send MAXSPEED then
maxdlspeed = d.get_max_speed(DOWNLOAD)
if maxdlspeed == 0:
maxdlspeed = None
maxulspeed = d.get_max_speed(UPLOAD)
if maxulspeed == 0:
maxulspeed = None
metadir = d.get_swift_meta_dir()
self.send_start(url,
roothash_hex=roothash_hex,
maxdlspeed=maxdlspeed,
maxulspeed=maxulspeed,
destdir=d.get_dest_dir(),
metadir=metadir)
finally:
self.splock.release()
def add_download(self, d):
self.splock.acquire()
try:
roothash = d.get_def().get_roothash()
# Before send to handle INFO msgs
self.roothash2dl[roothash] = d
finally:
self.splock.release()
def remove_download(self, d, removestate, removecontent):
self.splock.acquire()
try:
if self.donestate != DONE_STATE_WORKING or not self.is_alive():
return
roothash_hex = d.get_def().get_roothash_as_hex()
self.send_remove(roothash_hex, removestate, removecontent)
# After send to handle INFO msgs
roothash = d.get_def().get_roothash()
del self.roothash2dl[roothash]
finally:
self.splock.release()
def get_downloads(self):
self.splock.acquire()
try:
return self.roothash2dl.values()
finally:
self.splock.release()
def get_pid(self):
if self.popen is not None:
return self.popen.pid
else:
return -1
def get_listen_port(self):
return self.listenport
def set_max_speed(self, d, direct, speed):
self.splock.acquire()
try:
if self.donestate != DONE_STATE_WORKING or not self.is_alive():
return
roothash_hex = d.get_def().get_roothash_as_hex()
# In Tribler Core API = unlimited. In Swift CMDGW API
# 0 = none.
if speed == 0.0:
speed = 4294967296.0
self.send_max_speed(roothash_hex, direct, speed)
finally:
self.splock.release()
def checkpoint_download(self, d):
self.splock.acquire()
try:
# Arno, 2012-05-15: Allow during shutdown.
if not self.is_alive():
return
roothash_hex = d.get_def().get_roothash_as_hex()
self.send_checkpoint(roothash_hex)
finally:
self.splock.release()
def set_moreinfo_stats(self, d, enable):
self.splock.acquire()
try:
if self.donestate != DONE_STATE_WORKING or not self.is_alive():
return
roothash_hex = d.get_def().get_roothash_as_hex()
self.send_setmoreinfo(roothash_hex, enable)
finally:
self.splock.release()
def set_subscribe_channel_close(self, download, enable, callback):
# Note that CALLBACK is called on the i2ithread, and hence should not lock
self.splock.acquire()
try:
if self.donestate != DONE_STATE_WORKING or not self.is_alive():
return
roothash_hex = download.get_def().get_roothash_as_hex() if (
download is None or download != "ALL") else "ALL"
if enable:
if not self._channel_close_callbacks[roothash_hex]:
self.send_subscribe(roothash_hex, "CHANNEL_CLOSE", True)
self._channel_close_callbacks[roothash_hex].append(callback)
else:
self._channel_close_callbacks[roothash_hex].remove(callback)
if not self._channel_close_callbacks[roothash_hex]:
self.send_subscribe(roothash_hex, "CHANNEL_CLOSE", False)
finally:
self.splock.release()
def add_peer(self, d, addr):
self.splock.acquire()
try:
if self.donestate != DONE_STATE_WORKING or not self.is_alive():
return
addrstr = addr[0] + ':' + str(addr[1])
roothash_hex = d.get_def().get_roothash_as_hex()
self.send_peer_addr(roothash_hex, addrstr)
finally:
self.splock.release()
def early_shutdown(self):
# Called by any thread, assume sessionlock is held
# May get called twice, once by spm.release_sp() and spm.shutdown()
if self.donestate == DONE_STATE_WORKING:
self.donestate = DONE_STATE_EARLY_SHUTDOWN
else:
return
if self.popen is not None:
# Tell engine to shutdown so it can deregister dls from tracker
print >> sys.stderr, "sp: Telling process to shutdown"
self.send_shutdown()
def network_shutdown(self):
# Called by network thread, assume sessionlock is held
if self.donestate == DONE_STATE_EARLY_SHUTDOWN:
self.donestate = DONE_STATE_SHUTDOWN
else:
return
# could do fastconn.close() here
if self.popen is not None:
try:
print >> sys.stderr, "sp: Terminating process"
self.popen.terminate()
self.popen.wait()
self.popen = None
except WindowsError:
pass
except:
print_exc()
if self.fastconn:
self.fastconn.stop()
#
# Internal methods
#
def send_start(self,
url,
roothash_hex=None,
maxdlspeed=None,
maxulspeed=None,
destdir=None,
metadir=None):
# assume splock is held to avoid concurrency on socket
if DEBUG:
print >> sys.stderr, "sp: send_start:", url, "destdir", destdir
cmd = 'START ' + url
if destdir is not None:
cmd += ' ' + destdir.encode("UTF-8")
if metadir is not None:
cmd += ' ' + metadir.encode("UTF-8")
cmd += '\r\n'
if maxdlspeed is not None:
cmd += 'MAXSPEED ' + roothash_hex + ' DOWNLOAD ' + str(
float(maxdlspeed)) + '\r\n'
if maxulspeed is not None:
cmd += 'MAXSPEED ' + roothash_hex + ' UPLOAD ' + str(
float(maxulspeed)) + '\r\n'
self.write(cmd)
def send_remove(self, roothash_hex, removestate, removecontent):
# assume splock is held to avoid concurrency on socket
self.write('REMOVE ' + roothash_hex + ' ' + str(int(removestate)) +
' ' + str(int(removecontent)) + '\r\n')
def send_checkpoint(self, roothash_hex):
# assume splock is held to avoid concurrency on socket
self.write('CHECKPOINT ' + roothash_hex + '\r\n')
def send_shutdown(self):
# assume splock is held to avoid concurrency on socket
self.write('SHUTDOWN\r\n')
def send_max_speed(self, roothash_hex, direct, speed):
# assume splock is held to avoid concurrency on socket
cmd = 'MAXSPEED ' + roothash_hex
if direct == DOWNLOAD:
cmd += ' DOWNLOAD '
else:
cmd += ' UPLOAD '
cmd += str(float(speed)) + '\r\n'
self.write(cmd)
def send_tunnel(self, session, address, data):
# assume splock is held to avoid concurrency on socket
if DEBUG:
print >> sys.stderr, "sp: send_tunnel:", len(
data), "bytes -> %s:%d" % address
self.write("TUNNELSEND %s:%d/%s %d\r\n" %
(address[0], address[1], session.encode("HEX"), len(data)))
self.write(data)
def send_setmoreinfo(self, roothash_hex, enable):
# assume splock is held to avoid concurrency on socket
onoff = "0"
if enable:
onoff = "1"
self.write('SETMOREINFO ' + roothash_hex + ' ' + onoff + '\r\n')
def send_subscribe(self, roothash_hex, event_type, enable):
"""
Subscribe to a libswift event.
ROOTHASH_HEX can currently only be "ALL"
EVENT_TYPE can currently only be "CHANNEL_CLOSE"
ENABLE can be either True or False
"""
assert roothash_hex == "ALL"
assert event_type == "CHANNEL_CLOSE"
assert isinstance(enable, bool), type(enable)
# assume splock is held to avoid concurrency on socket
if DEBUG:
print >> sys.stderr, "sp: send_subscribe:", roothash_hex, event_type, enable
self.write("SUBSCRIBE %s %s %d\r\n" % (
roothash_hex,
event_type,
int(enable),
))
def send_peer_addr(self, roothash_hex, addrstr):
# assume splock is held to avoid concurrency on socket
self.write('PEERADDR ' + roothash_hex + ' ' + addrstr + '\r\n')
def is_alive(self):
if self.popen:
self.popen.poll()
return self.popen.returncode is None
return False
def write(self, msg):
self.fastconn.write(msg)
def get_cmdport(self):
return self.cmdport
def connection_lost(self, port):
self.spmgr.connection_lost(port)
class Session(SessionRuntimeConfig):
"""
A Session is a running instance of the Tribler Core and the Core's central
class. It implements the SessionConfigInterface which can be used to change
session parameters at runtime (for selected parameters).
cf. libtorrent session
"""
__single = None
def __init__(self, scfg=None, ignore_singleton=False):
"""
A Session object is created which is configured following a copy of the
SessionStartupConfig scfg. (copy constructor used internally)
@param scfg SessionStartupConfig object or None, in which case we
look for a saved session in the default location (state dir). If
we can't find it, we create a new SessionStartupConfig() object to
serve as startup config. Next, the config is saved in the directory
indicated by its 'state_dir' attribute.
In the current implementation only a single session instance can exist
at a time in a process. The ignore_singleton flag is used for testing.
"""
if not ignore_singleton:
if Session.__single:
raise RuntimeError, "Session is singleton"
Session.__single = self
self.sesslock = NoDispersyRLock()
# Determine startup config to use
if scfg is None: # If no override
try:
# Then try to read from default location
state_dir = Session.get_default_state_dir()
cfgfilename = Session.get_default_config_filename(state_dir)
scfg = SessionStartupConfig.load(cfgfilename)
except:
# If that fails, create a fresh config with factory defaults
print_exc()
scfg = SessionStartupConfig()
self.sessconfig = scfg.sessconfig
else: # overrides any saved config
# Work from copy
self.sessconfig = copy.copy(scfg.sessconfig)
# Niels: 11/05/2012, turning off overlay
self.sessconfig['overlay'] = 0
self.sessconfig['crawler'] = 0
# Create dir for session state, if not exist
state_dir = self.sessconfig['state_dir']
if state_dir and not os.path.isdir(state_dir):
try:
os.makedirs(state_dir)
except:
state_dir = None
if state_dir is None:
state_dir = Session.get_default_state_dir()
self.sessconfig['state_dir'] = state_dir
if not os.path.isdir(state_dir):
os.makedirs(state_dir)
collected_torrent_dir = self.sessconfig['torrent_collecting_dir']
if not collected_torrent_dir:
collected_torrent_dir = os.path.join(self.sessconfig['state_dir'], STATEDIR_TORRENTCOLL_DIR)
self.sessconfig['torrent_collecting_dir'] = collected_torrent_dir
collected_subtitles_dir = self.sessconfig.get('subtitles_collecting_dir', None)
if not collected_subtitles_dir:
collected_subtitles_dir = os.path.join(self.sessconfig['state_dir'], STATEDIR_SUBSCOLL_DIR)
self.sessconfig['subtitles_collecting_dir'] = collected_subtitles_dir
if not os.path.exists(collected_torrent_dir):
os.makedirs(collected_torrent_dir)
if not self.sessconfig['peer_icon_path']:
self.sessconfig['peer_icon_path'] = os.path.join(self.sessconfig['state_dir'], STATEDIR_PEERICON_DIR)
# PERHAPS: load default TorrentDef and DownloadStartupConfig from state dir
# Let user handle that, he's got default_state_dir, etc.
# Core init
# print >>sys.stderr,'Session: __init__ config is', self.sessconfig
if GOTM2CRYPTO:
permidmod.init()
#
# Set params that depend on state_dir
#
# 1. keypair
#
pairfilename = os.path.join(self.sessconfig['state_dir'], 'ec.pem')
if self.sessconfig['eckeypairfilename'] is None:
self.sessconfig['eckeypairfilename'] = pairfilename
if os.access(self.sessconfig['eckeypairfilename'], os.F_OK):
# May throw exceptions
self.keypair = permidmod.read_keypair(self.sessconfig['eckeypairfilename'])
else:
self.keypair = permidmod.generate_keypair()
# Save keypair
pubfilename = os.path.join(self.sessconfig['state_dir'], 'ecpub.pem')
permidmod.save_keypair(self.keypair, pairfilename)
permidmod.save_pub_key(self.keypair, pubfilename)
# 2. Downloads persistent state dir
dlpstatedir = os.path.join(self.sessconfig['state_dir'], STATEDIR_DLPSTATE_DIR)
if not os.path.isdir(dlpstatedir):
os.mkdir(dlpstatedir)
# 3. tracker
trackerdir = self.get_internal_tracker_dir()
if not os.path.exists(trackerdir):
os.mkdir(trackerdir)
if self.sessconfig['tracker_dfile'] is None:
self.sessconfig['tracker_dfile'] = os.path.join(trackerdir, 'tracker.db')
if self.sessconfig['tracker_allowed_dir'] is None:
self.sessconfig['tracker_allowed_dir'] = trackerdir
if self.sessconfig['tracker_logfile'] is None:
if sys.platform == "win32":
# Not "Nul:" but "nul" is /dev/null on Win32
sink = 'nul'
else:
sink = '/dev/null'
self.sessconfig['tracker_logfile'] = sink
# 5. peer_icon_path
if self.sessconfig['peer_icon_path'] is None:
self.sessconfig['peer_icon_path'] = os.path.join(self.sessconfig['state_dir'], STATEDIR_PEERICON_DIR)
if not os.path.isdir(self.sessconfig['peer_icon_path']):
os.mkdir(self.sessconfig['peer_icon_path'])
# 6. Poor man's versioning of SessionConfig, add missing
# default values. Really should use PERSISTENTSTATE_CURRENTVERSION
# and do conversions.
for key, defvalue in sessdefaults.iteritems():
if key not in self.sessconfig:
self.sessconfig[key] = defvalue
if not 'live_aux_seeders' in self.sessconfig:
# Poor man's versioning, really should update PERSISTENTSTATE_CURRENTVERSION
self.sessconfig['live_aux_seeders'] = sessdefaults['live_aux_seeders']
if not 'nat_detect' in self.sessconfig:
self.sessconfig['nat_detect'] = sessdefaults['nat_detect']
if not 'puncturing_internal_port' in self.sessconfig:
self.sessconfig['puncturing_internal_port'] = sessdefaults['puncturing_internal_port']
if not 'stun_servers' in self.sessconfig:
self.sessconfig['stun_servers'] = sessdefaults['stun_servers']
if not 'pingback_servers' in self.sessconfig:
self.sessconfig['pingback_servers'] = sessdefaults['pingback_servers']
if not 'mainline_dht' in self.sessconfig:
self.sessconfig['mainline_dht'] = sessdefaults['mainline_dht']
# SWIFTPROC
if self.sessconfig['swiftpath'] is None:
if sys.platform == "win32":
self.sessconfig['swiftpath'] = os.path.join(self.sessconfig['install_dir'], "swift.exe")
else:
self.sessconfig['swiftpath'] = os.path.join(self.sessconfig['install_dir'], "swift")
# Checkpoint startup config
self.save_pstate_sessconfig()
# Create handler for calling back the user via separate threads
self.uch = UserCallbackHandler(self)
# Create engine with network thread
self.lm = TriblerLaunchMany()
self.lm.register(self, self.sesslock)
self.lm.start()
#
# Class methods
#
def get_instance(*args, **kw):
""" Returns the Session singleton if it exists or otherwise
creates it first, in which case you need to pass the constructor
params.
@return Session."""
if Session.__single is None:
Session(*args, **kw)
return Session.__single
get_instance = staticmethod(get_instance)
def has_instance():
return Session.__single != None
has_instance = staticmethod(has_instance)
def del_instance():
Session.__single = None
del_instance = staticmethod(del_instance)
def get_default_state_dir(homedirpostfix='.Tribler'):
""" Returns the factory default directory for storing session state
on the current platform (Win32,Mac,Unix).
@return An absolute path name. """
# Allow override
statedirvar = '${TSTATEDIR}'
statedir = os.path.expandvars(statedirvar)
if statedir and statedir != statedirvar:
return statedir
if os.path.isdir(homedirpostfix):
return os.path.abspath(homedirpostfix)
appdir = get_appstate_dir()
statedir = os.path.join(appdir, homedirpostfix)
return statedir
get_default_state_dir = staticmethod(get_default_state_dir)
#
# Public methods
#
def start_download(self, cdef, dcfg=None, initialdlstatus=None, hidden=False):
"""
Creates a Download object and adds it to the session. The passed
ContentDef and DownloadStartupConfig are copied into the new Download
object. The Download is then started and checkpointed.
If a checkpointed version of the Download is found, that is restarted
overriding the saved DownloadStartupConfig if "dcfg" is not None.
@param cdef A finalized TorrentDef or a SwiftDef
@param dcfg DownloadStartupConfig or None, in which case
a new DownloadStartupConfig() is created with its default settings
and the result becomes the runtime config of this Download.
@param initialdlstatus The initial download status of this Download
or None. This enables the caller to create a Download in e.g.
DLSTATUS_STOPPED state instead.
@param hidden Whether this torrent should be added to the mypreference table
@return Download
"""
# locking by lm
if cdef.get_def_type() == "torrent":
return self.lm.add(cdef, dcfg, initialdlstatus=initialdlstatus, hidden=hidden)
else:
# SWIFTPROC
return self.lm.swift_add(cdef, dcfg, initialdlstatus=initialdlstatus, hidden=hidden)
def resume_download_from_file(self, filename):
"""
Recreates Download from resume file
@return a Download object.
Note: this cannot be made into a method of Download, as the Download
needs to be bound to a session, it cannot exist independently.
"""
raise NotYetImplementedException()
def get_downloads(self):
"""
Returns a copy of the list of Downloads.
@return A list of Download objects.
"""
# locking by lm
return self.lm.get_downloads()
def get_download(self, hash):
"""
Returns the Download object for this hash.
@return A Donwload Object.
"""
# locking by lm
return self.lm.get_download(hash)
def remove_download(self, d, removecontent=False, removestate=True, hidden=False):
"""
Stops the download and removes it from the session.
@param d The Download to remove
@param removecontent Whether to delete the already downloaded content
from disk.
@param removestate Whether to delete the metadata files of the downloaded
content from disk.
@param hidden Whether this torrent is added to the mypreference table and this entry should be
removed
"""
# locking by lm
if d.get_def().get_def_type() == "torrent":
self.lm.remove(d, removecontent=removecontent, removestate=removestate, hidden=hidden)
else:
# SWIFTPROC
self.lm.swift_remove(d, removecontent=removecontent, removestate=removestate, hidden=hidden)
def remove_download_by_id(self, id, removecontent=False, removestate=True):
"""
@param infohash The Download to remove
@param removecontent Whether to delete the already downloaded content
from disk.
!We can only remove content when the download object is found, otherwise only
the state is removed.
"""
downloadList = self.get_downloads()
for download in downloadList:
if download.get_def().get_id() == id:
self.remove_download(download, removecontent, removestate)
return
self.lm.remove_id(id)
self.uch.perform_removestate_callback(id, [], False)
def set_download_states_callback(self, usercallback, getpeerlist=False):
"""
See Download.set_state_callback. Calls usercallback with a list of
DownloadStates, one for each Download in the Session as first argument.
The usercallback must return a tuple (when,getpeerlist) that indicates
when to reinvoke the callback again (as a number of seconds from now,
or < 0.0 if not at all) and whether to also include the details of
the connected peers in the DownloadStates on that next call.
The callback will be called by a popup thread which can be used
indefinitely (within reason) by the higher level code.
@param usercallback A function adhering to the above spec.
"""
self.lm.set_download_states_callback(usercallback, getpeerlist)
#
# Config parameters that only exist at runtime
#
def get_permid(self):
""" Returns the PermID of the Session, as determined by the
SessionConfig.set_permid() parameter. A PermID is a public key
@return The PermID encoded in a string in DER format. """
self.sesslock.acquire()
try:
return str(self.keypair.pub().get_der())
finally:
self.sesslock.release()
def get_external_ip(self):
""" Returns the external IP address of this Session, i.e., by which
it is reachable from the Internet. This address is determined via
various mechanisms such as the UPnP protocol, our dialback mechanism,
and an inspection of the local network configuration.
@return A string. """
# locking done by lm
return self.lm.get_ext_ip()
def get_externally_reachable(self):
""" Returns whether the Session is externally reachable, i.e., its
listen port is not firewalled. Use add_observer() with NTFY_REACHABLE
to register to the event of detecting reachablility. Note that due to
the use of UPnP a Session may become reachable some time after
startup and due to the Dialback mechanism, this method may return
False while the Session is actually already reachable. Note that True
doesn't mean the Session is reachable from the open Internet, could just
be from the local (otherwise firewalled) LAN.
@return A boolean. """
# Arno, LICHT: make it throw exception when used in LITE versie.
raise NotYetImplementedException()
def get_current_startup_config_copy(self):
""" Returns a SessionStartupConfig that is a copy of the current runtime
SessionConfig.
@return SessionStartupConfig
"""
# Called by any thread
self.sesslock.acquire()
try:
sessconfig = copy.copy(self.sessconfig)
return SessionStartupConfig(sessconfig=sessconfig)
finally:
self.sesslock.release()
#
# Internal tracker
#
def get_internal_tracker_url(self):
""" Returns the announce URL for the internal tracker.
@return URL """
# Called by any thread
self.sesslock.acquire()
try:
url = None
if 'tracker_url' in self.sessconfig:
url = self.sessconfig['tracker_url'] # user defined override, e.g. specific hostname
if url is None:
ip = self.lm.get_ext_ip()
port = self.get_listen_port()
url = 'http://' + ip + ':' + str(port) + '/announce/'
return url
finally:
self.sesslock.release()
def get_internal_tracker_dir(self):
""" Returns the directory containing the torrents tracked by the internal
tracker (and associated databases).
@return An absolute path. """
# Called by any thread
self.sesslock.acquire()
try:
if self.sessconfig['state_dir'] is None:
return None
else:
return os.path.join(self.sessconfig['state_dir'], STATEDIR_ITRACKER_DIR)
finally:
self.sesslock.release()
def add_to_internal_tracker(self, tdef):
""" Add a torrent def to the list of torrents tracked by the internal
tracker. Use this method to use the Session as a standalone tracker.
@param tdef A finalized TorrentDef.
"""
# Called by any thread
self.sesslock.acquire()
try:
infohash = tdef.get_infohash()
filename = self.get_internal_tracker_torrentfilename(infohash)
tdef.save(filename)
print >> sys.stderr, "Session: add_to_int_tracker: saving to", filename, "url-compat", tdef.get_url_compat()
# Bring to attention of Tracker thread
self.lm.tracker_rescan_dir()
finally:
self.sesslock.release()
def remove_from_internal_tracker(self, tdef):
""" Remove a torrent def from the list of torrents tracked by the
internal tracker. Use this method to use the Session as a standalone
tracker.
@param tdef A finalized TorrentDef.
"""
infohash = tdef.get_infohash()
self.remove_from_internal_tracker_by_infohash(infohash)
def remove_from_internal_tracker_by_infohash(self, infohash):
""" Remove a torrent def from the list of torrents tracked by the
internal tracker. Use this method to use the Session as a standalone
tracker.
@param infohash Identifier of the torrent def to remove.
"""
# Called by any thread
self.sesslock.acquire()
try:
filename = self.get_internal_tracker_torrentfilename(infohash)
if DEBUG:
print >> sys.stderr, "Session: removing itracker entry", filename
if os.access(filename, os.F_OK):
os.remove(filename)
# Bring to attention of Tracker thread
self.lm.tracker_rescan_dir()
finally:
self.sesslock.release()
#
# Notification of events in the Session
#
def add_observer(self, func, subject, changeTypes=[NTFY_UPDATE, NTFY_INSERT, NTFY_DELETE], objectID=None, cache=0):
""" Add an observer function function to the Session. The observer
function will be called when one of the specified events (changeTypes)
occurs on the specified subject.
The function will be called by a popup thread which can be used
indefinitely (within reason) by the higher level code.
@param func The observer function. It should accept as its first argument
the subject, as second argument the changeType, as third argument an
objectID (e.g. the primary key in the observed database) and an
optional list of arguments.
@param subject The subject to observe, one of NTFY_* subjects (see
simpledefs).
@param changeTypes The list of events to be notified of one of NTFY_*
events.
@param objectID The specific object in the subject to monitor (e.g. a
specific primary key in a database to monitor for updates.)
@param cache The time to bundle/cache events matching this function
TODO: Jelle will add per-subject/event description here ;o)
"""
# Called by any thread
self.uch.notifier.add_observer(func, subject, changeTypes, objectID, cache=cache) # already threadsafe
def remove_observer(self, func):
""" Remove observer function. No more callbacks will be made.
@param func The observer function to remove. """
# Called by any thread
self.uch.notifier.remove_observer(func) # already threadsafe
def open_dbhandler(self, subject):
""" Opens a connection to the specified database. Only the thread
calling this method may use this connection. The connection must be
closed with close_dbhandler() when this thread exits.
@param subject The database to open. Must be one of the subjects
specified here.
@return A reference to a DBHandler class for the specified subject or
None when the Session was not started with megacaches enabled.
<pre> NTFY_PEERS -> PeerDBHandler
NTFY_TORRENTS -> TorrentDBHandler
NTFY_MYPREFERENCES -> MyPreferenceDBHandler
NTFY_VOTECAST -> VotecastDBHandler
NTFY_CHANNELCAST -> ChannelCastDBHandler
</pre>
"""
# Called by any thread
self.sesslock.acquire()
try:
if subject == NTFY_PEERS:
return self.lm.peer_db
elif subject == NTFY_TORRENTS:
return self.lm.torrent_db
elif subject == NTFY_MYPREFERENCES:
return self.lm.mypref_db
elif subject == NTFY_SEEDINGSTATS:
return self.lm.seedingstats_db
elif subject == NTFY_SEEDINGSTATSSETTINGS:
return self.lm.seedingstatssettings_db
elif subject == NTFY_VOTECAST:
return self.lm.votecast_db
elif subject == NTFY_CHANNELCAST:
return self.lm.channelcast_db
else:
raise ValueError('Cannot open DB subject: ' + subject)
finally:
self.sesslock.release()
def close_dbhandler(self, dbhandler):
""" Closes the given database connection """
dbhandler.close()
#
# Persistence and shutdown
#
def load_checkpoint(self, initialdlstatus=None, initialdlstatus_dict={}):
""" Restart Downloads from checkpoint, if any.
This method allows the API user to manage restoring downloads.
E.g. a video player that wants to start the torrent the user clicked
on first, and only then restart any sleeping torrents (e.g. seeding).
The optional initialdlstatus parameter can be set to DLSTATUS_STOPPED
to restore all the Downloads in DLSTATUS_STOPPED state.
The options initialdlstatus_dict parameter can be used to specify a
state overriding the initaldlstatus parameter per download id.
"""
self.lm.load_checkpoint(initialdlstatus, initialdlstatus_dict)
def checkpoint(self):
""" Saves the internal session state to the Session's state dir. """
# Called by any thread
self.checkpoint_shutdown(stop=False, checkpoint=True, gracetime=None, hacksessconfcheckpoint=False)
def shutdown(self, checkpoint=True, gracetime=2.0, hacksessconfcheckpoint=True):
""" Checkpoints the session and closes it, stopping the download engine.
@param checkpoint Whether to checkpoint the Session state on shutdown.
@param gracetime Time to allow for graceful shutdown + signoff (seconds).
"""
# Called by any thread
self.lm.early_shutdown()
self.checkpoint_shutdown(stop=True, checkpoint=checkpoint, gracetime=gracetime, hacksessconfcheckpoint=hacksessconfcheckpoint)
# Arno, 2010-08-09: now shutdown after gracetime
# self.uch.shutdown()
def has_shutdown(self):
""" Whether the Session has completely shutdown, i.e., its internal
threads are finished and it is safe to quit the process the Session
is running in.
@return A Boolean.
"""
return self.lm.sessdoneflag.isSet()
def get_downloads_pstate_dir(self):
""" Returns the directory in which to checkpoint the Downloads in this
Session. """
# Called by network thread
self.sesslock.acquire()
try:
return os.path.join(self.sessconfig['state_dir'], STATEDIR_DLPSTATE_DIR)
finally:
self.sesslock.release()
def download_torrentfile(self, infohash=None, roothash=None, usercallback=None, prio=0):
""" Try to download the torrentfile without a known source.
A possible source could be the DHT.
If the torrent is succesfully
received, the usercallback method is called with the infohash as first
and the contents of the torrentfile (bencoded dict) as second parameter.
If the torrent could not be obtained, the callback is not called.
The torrent will have been added to the TorrentDBHandler (if enabled)
at the time of the call.
@param infohash The infohash of the torrent.
@param usercallback A function adhering to the above spec.
"""
from Tribler.Core.RemoteTorrentHandler import RemoteTorrentHandler
rtorrent_handler = RemoteTorrentHandler.getInstance()
rtorrent_handler.download_torrent(None, infohash, roothash, usercallback, prio)
def download_torrentfile_from_peer(self, candidate, infohash=None, roothash=None, usercallback=None, prio=0):
""" Ask the designated peer to send us the torrentfile for the torrent
identified by the passed infohash. If the torrent is succesfully
received, the usercallback method is called with the infohash as first
and the contents of the torrentfile (bencoded dict) as second parameter.
If the torrent could not be obtained, the callback is not called.
The torrent will have been added to the TorrentDBHandler (if enabled)
at the time of the call.
@param permid The PermID of the peer to query.
@param infohash The infohash of the torrent.
@param usercallback A function adhering to the above spec.
"""
from Tribler.Core.RemoteTorrentHandler import RemoteTorrentHandler
rtorrent_handler = RemoteTorrentHandler.getInstance()
rtorrent_handler.download_torrent(candidate, infohash, roothash, usercallback, prio)
def download_torrentmessages_from_peer(self, candidate, infohashes, usercallback, prio=0):
""" Ask the designated peer to send us the torrentfile for the torrent
identified by the passed infohash. If the torrent is succesfully
received, the usercallback method is called with the infohash as first
and the contents of the torrentfile (bencoded dict) as second parameter.
If the torrent could not be obtained, the callback is not called.
The torrent will have been added to the TorrentDBHandler (if enabled)
at the time of the call.
@param permid The PermID of the peer to query.
@param infohash The infohash of the torrent.
@param usercallback A function adhering to the above spec.
"""
from Tribler.Core.RemoteTorrentHandler import RemoteTorrentHandler
rtorrent_handler = RemoteTorrentHandler.getInstance()
rtorrent_handler.download_torrentmessages(candidate, infohashes, usercallback, prio)
#
# Internal persistence methods
#
def checkpoint_shutdown(self, stop, checkpoint, gracetime, hacksessconfcheckpoint):
""" Checkpoints the Session and optionally shuts down the Session.
@param stop Whether to shutdown the Session as well.
@param checkpoint Whether to checkpoint at all, or just to stop.
@param gracetime Time to allow for graceful shutdown + signoff (seconds).
"""
# Called by any thread
self.sesslock.acquire()
try:
# Arno: Make checkpoint optional on shutdown. At the moment setting
# the config at runtime is not possible (see SessionRuntimeConfig)
# so this has little use, and interferes with our way of
# changing the startup config, which is to write a new
# config to disk that will be read at start up.
if hacksessconfcheckpoint:
try:
self.save_pstate_sessconfig()
except Exception, e:
self.lm.rawserver_nonfatalerrorfunc(e)
# Checkpoint all Downloads and stop NetworkThread
if DEBUG or stop:
print >> sys.stderr, "Session: checkpoint_shutdown"
self.lm.checkpoint(stop=stop, checkpoint=checkpoint, gracetime=gracetime)
finally:
class Session(SessionRuntimeConfig):
"""
A Session is a running instance of the Tribler Core and the Core's central
class. It implements the SessionConfigInterface which can be used to change
session parameters at runtime (for selected parameters).
cf. libtorrent session
"""
__single = None
def __init__(self,scfg=None,ignore_singleton=False):
"""
A Session object is created which is configured following a copy of the
SessionStartupConfig scfg. (copy constructor used internally)
@param scfg SessionStartupConfig object or None, in which case we
look for a saved session in the default location (state dir). If
we can't find it, we create a new SessionStartupConfig() object to
serve as startup config. Next, the config is saved in the directory
indicated by its 'state_dir' attribute.
In the current implementation only a single session instance can exist
at a time in a process. The ignore_singleton flag is used for testing.
"""
# ProxyService 90s Test_
# self.start_time = time.time()
# _ProxyService 90s Test
if not ignore_singleton:
if Session.__single:
raise RuntimeError, "Session is singleton"
Session.__single = self
self.sesslock = NoDispersyRLock()
# Determine startup config to use
if scfg is None: # If no override
try:
# Then try to read from default location
state_dir = Session.get_default_state_dir()
cfgfilename = Session.get_default_config_filename(state_dir)
scfg = SessionStartupConfig.load(cfgfilename)
except:
# If that fails, create a fresh config with factory defaults
print_exc()
scfg = SessionStartupConfig()
self.sessconfig = scfg.sessconfig
else: # overrides any saved config
# Work from copy
self.sessconfig = copy.copy(scfg.sessconfig)
#Niels: 11/05/2012, turning off overlay
self.sessconfig['overlay'] = 0
self.sessconfig['crawler'] = 0
# Create dir for session state, if not exist
state_dir = self.sessconfig['state_dir']
if state_dir is None:
state_dir = Session.get_default_state_dir()
self.sessconfig['state_dir'] = state_dir
if not os.path.isdir(state_dir):
os.makedirs(state_dir)
collected_torrent_dir = self.sessconfig['torrent_collecting_dir']
if not collected_torrent_dir:
collected_torrent_dir = os.path.join(self.sessconfig['state_dir'], STATEDIR_TORRENTCOLL_DIR)
self.sessconfig['torrent_collecting_dir'] = collected_torrent_dir
collected_subtitles_dir = self.sessconfig.get('subtitles_collecting_dir',None)
if not collected_subtitles_dir:
collected_subtitles_dir = os.path.join(self.sessconfig['state_dir'], STATEDIR_SUBSCOLL_DIR)
self.sessconfig['subtitles_collecting_dir'] = collected_subtitles_dir
if not os.path.exists(collected_torrent_dir):
os.makedirs(collected_torrent_dir)
if not self.sessconfig['peer_icon_path']:
self.sessconfig['peer_icon_path'] = os.path.join(self.sessconfig['state_dir'], STATEDIR_PEERICON_DIR)
# PERHAPS: load default TorrentDef and DownloadStartupConfig from state dir
# Let user handle that, he's got default_state_dir, etc.
# Core init
#print >>sys.stderr,'Session: __init__ config is', self.sessconfig
if GOTM2CRYPTO:
permidmod.init()
#
# Set params that depend on state_dir
#
# 1. keypair
#
pairfilename = os.path.join(self.sessconfig['state_dir'],'ec.pem')
if self.sessconfig['eckeypairfilename'] is None:
self.sessconfig['eckeypairfilename'] = pairfilename
if os.access(self.sessconfig['eckeypairfilename'],os.F_OK):
# May throw exceptions
self.keypair = permidmod.read_keypair(self.sessconfig['eckeypairfilename'])
else:
self.keypair = permidmod.generate_keypair()
# Save keypair
pubfilename = os.path.join(self.sessconfig['state_dir'],'ecpub.pem')
permidmod.save_keypair(self.keypair,pairfilename)
permidmod.save_pub_key(self.keypair,pubfilename)
# 2. Downloads persistent state dir
dlpstatedir = os.path.join(self.sessconfig['state_dir'],STATEDIR_DLPSTATE_DIR)
if not os.path.isdir(dlpstatedir):
os.mkdir(dlpstatedir)
# 3. tracker
trackerdir = self.get_internal_tracker_dir()
if not os.path.exists(trackerdir):
os.mkdir(trackerdir)
if self.sessconfig['tracker_dfile'] is None:
self.sessconfig['tracker_dfile'] = os.path.join(trackerdir,'tracker.db')
if self.sessconfig['tracker_allowed_dir'] is None:
self.sessconfig['tracker_allowed_dir'] = trackerdir
if self.sessconfig['tracker_logfile'] is None:
if sys.platform == "win32":
# Not "Nul:" but "nul" is /dev/null on Win32
sink = 'nul'
else:
sink = '/dev/null'
self.sessconfig['tracker_logfile'] = sink
# 4. superpeer.txt and crawler.txt
if self.sessconfig['superpeer_file'] is None:
self.sessconfig['superpeer_file'] = os.path.join(self.sessconfig['install_dir'],LIBRARYNAME,'Core','superpeer.txt')
if 'crawler_file' not in self.sessconfig or self.sessconfig['crawler_file'] is None:
self.sessconfig['crawler_file'] = os.path.join(self.sessconfig['install_dir'], LIBRARYNAME,'Core','Statistics','crawler.txt')
# 5. peer_icon_path
if self.sessconfig['peer_icon_path'] is None:
self.sessconfig['peer_icon_path'] = os.path.join(self.sessconfig['state_dir'],STATEDIR_PEERICON_DIR)
if not os.path.isdir(self.sessconfig['peer_icon_path']):
os.mkdir(self.sessconfig['peer_icon_path'])
# 6. Poor man's versioning of SessionConfig, add missing
# default values. Really should use PERSISTENTSTATE_CURRENTVERSION
# and do conversions.
for key,defvalue in sessdefaults.iteritems():
if key not in self.sessconfig:
self.sessconfig[key] = defvalue
# 7. proxyservice_dir
if self.sessconfig['overlay']: #NIELS: proxyservice_on/off is set at runtime, always make sure proxyservice_ and self.sessconfig['proxyservice_status'] == PROXYSERVICE_ON:
if self.sessconfig['proxyservice_dir'] is None:
self.sessconfig['proxyservice_dir'] = os.path.join(get_default_dest_dir(), PROXYSERVICE_DESTDIR)
# Jelle: under linux, default_dest_dir can be /tmp. Then proxyservice_dir can be deleted in between
# sessions.
if not os.path.isdir(self.sessconfig['proxyservice_dir']):
os.makedirs(self.sessconfig['proxyservice_dir'])
if not 'live_aux_seeders' in self.sessconfig:
# Poor man's versioning, really should update PERSISTENTSTATE_CURRENTVERSION
self.sessconfig['live_aux_seeders'] = sessdefaults['live_aux_seeders']
if not 'nat_detect' in self.sessconfig:
self.sessconfig['nat_detect'] = sessdefaults['nat_detect']
if not 'puncturing_internal_port' in self.sessconfig:
self.sessconfig['puncturing_internal_port'] = sessdefaults['puncturing_internal_port']
if not 'stun_servers' in self.sessconfig:
self.sessconfig['stun_servers'] = sessdefaults['stun_servers']
if not 'pingback_servers' in self.sessconfig:
self.sessconfig['pingback_servers'] = sessdefaults['pingback_servers']
if not 'mainline_dht' in self.sessconfig:
self.sessconfig['mainline_dht'] = sessdefaults['mainline_dht']
# SWIFTPROC
if self.sessconfig['swiftpath'] is None:
if sys.platform == "win32":
self.sessconfig['swiftpath'] = os.path.join(self.sessconfig['install_dir'],"swift.exe")
else:
self.sessconfig['swiftpath'] = os.path.join(self.sessconfig['install_dir'],"swift")
# Checkpoint startup config
self.save_pstate_sessconfig()
# Create handler for calling back the user via separate threads
self.uch = UserCallbackHandler(self)
# Create engine with network thread
self.lm = TriblerLaunchMany()
self.lm.register(self,self.sesslock)
self.lm.start()
#
# Class methods
#
def get_instance(*args, **kw):
""" Returns the Session singleton if it exists or otherwise
creates it first, in which case you need to pass the constructor
params.
@return Session."""
if Session.__single is None:
Session(*args, **kw)
return Session.__single
get_instance = staticmethod(get_instance)
def get_default_state_dir(homedirpostfix='.Tribler'):
""" Returns the factory default directory for storing session state
on the current platform (Win32,Mac,Unix).
@return An absolute path name. """
# Allow override
statedirvar = '${TSTATEDIR}'
statedir = os.path.expandvars(statedirvar)
if statedir and statedir != statedirvar:
return statedir
appdir = get_appstate_dir()
statedir = os.path.join(appdir, homedirpostfix)
return statedir
get_default_state_dir = staticmethod(get_default_state_dir)
#
# Public methods
#
def start_download(self, cdef, dcfg=None, initialdlstatus=None, hidden=False):
"""
Creates a Download object and adds it to the session. The passed
ContentDef and DownloadStartupConfig are copied into the new Download
object. The Download is then started and checkpointed.
If a checkpointed version of the Download is found, that is restarted
overriding the saved DownloadStartupConfig if "dcfg" is not None.
@param cdef A finalized TorrentDef or a SwiftDef
@param dcfg DownloadStartupConfig or None, in which case
a new DownloadStartupConfig() is created with its default settings
and the result becomes the runtime config of this Download.
@param initialdlstatus The initial download status of this Download
or None. This enables the caller to create a Download in e.g.
DLSTATUS_REPEXING state instead.
@return Download
"""
# locking by lm
if cdef.get_def_type() == "torrent":
return self.lm.add(cdef,dcfg,initialdlstatus=initialdlstatus,hidden=hidden)
else:
# SWIFTPROC
return self.lm.swift_add(cdef,dcfg,initialdlstatus=initialdlstatus,hidden=hidden)
def resume_download_from_file(self,filename):
"""
Recreates Download from resume file
@return a Download object.
Note: this cannot be made into a method of Download, as the Download
needs to be bound to a session, it cannot exist independently.
"""
raise NotYetImplementedException()
def get_downloads(self):
"""
Returns a copy of the list of Downloads.
@return A list of Download objects.
"""
# locking by lm
return self.lm.get_downloads()
def get_download(self, hash):
"""
Returns the Download object for this hash.
@return A Donwload Object.
"""
# locking by lm
return self.lm.get_download(hash)
def remove_download(self,d,removecontent=False, removestate=True):
"""
Stops the download and removes it from the session.
@param d The Download to remove
@param removecontent Whether to delete the already downloaded content
from disk.
"""
# locking by lm
if d.get_def().get_def_type() == "torrent":
self.lm.remove(d,removecontent=removecontent,removestate=removestate)
else:
# SWIFTPROC
self.lm.swift_remove(d,removecontent=removecontent,removestate=removestate)
def remove_download_by_id(self, id, removecontent=False, removestate=True):
"""
@param infohash The Download to remove
@param removecontent Whether to delete the already downloaded content
from disk.
!We can only remove content when the download object is found, otherwise only
the state is removed.
"""
downloadList = self.get_downloads()
for download in downloadList:
if download.get_def().get_id() == id:
self.remove_download(download,removecontent,removestate)
return
self.lm.remove_id(id)
self.uch.perform_removestate_callback(id, [], False)
def set_download_states_callback(self,usercallback,getpeerlist=False):
"""
See Download.set_state_callback. Calls usercallback with a list of
DownloadStates, one for each Download in the Session as first argument.
The usercallback must return a tuple (when,getpeerlist) that indicates
when to reinvoke the callback again (as a number of seconds from now,
or < 0.0 if not at all) and whether to also include the details of
the connected peers in the DownloadStates on that next call.
The callback will be called by a popup thread which can be used
indefinitely (within reason) by the higher level code.
@param usercallback A function adhering to the above spec.
"""
self.lm.set_download_states_callback(usercallback,getpeerlist)
#
# Config parameters that only exist at runtime
#
def get_permid(self):
""" Returns the PermID of the Session, as determined by the
SessionConfig.set_permid() parameter. A PermID is a public key
@return The PermID encoded in a string in DER format. """
self.sesslock.acquire()
try:
return str(self.keypair.pub().get_der())
finally:
self.sesslock.release()
def get_external_ip(self):
""" Returns the external IP address of this Session, i.e., by which
it is reachable from the Internet. This address is determined via
various mechanisms such as the UPnP protocol, our dialback mechanism,
and an inspection of the local network configuration.
@return A string. """
# locking done by lm
return self.lm.get_ext_ip()
def get_externally_reachable(self):
""" Returns whether the Session is externally reachable, i.e., its
listen port is not firewalled. Use add_observer() with NTFY_REACHABLE
to register to the event of detecting reachablility. Note that due to
the use of UPnP a Session may become reachable some time after
startup and due to the Dialback mechanism, this method may return
False while the Session is actually already reachable. Note that True
doesn't mean the Session is reachable from the open Internet, could just
be from the local (otherwise firewalled) LAN.
@return A boolean. """
# Arno, LICHT: make it throw exception when used in LITE versie.
from Tribler.Core.NATFirewall.DialbackMsgHandler import DialbackMsgHandler
return DialbackMsgHandler.getInstance().isConnectable()
def get_current_startup_config_copy(self):
""" Returns a SessionStartupConfig that is a copy of the current runtime
SessionConfig.
@return SessionStartupConfig
"""
# Called by any thread
self.sesslock.acquire()
try:
sessconfig = copy.copy(self.sessconfig)
return SessionStartupConfig(sessconfig=sessconfig)
finally:
self.sesslock.release()
#
# Internal tracker
#
def get_internal_tracker_url(self):
""" Returns the announce URL for the internal tracker.
@return URL """
# Called by any thread
self.sesslock.acquire()
try:
url = None
if 'tracker_url' in self.sessconfig:
url = self.sessconfig['tracker_url'] # user defined override, e.g. specific hostname
if url is None:
ip = self.lm.get_ext_ip()
port = self.get_listen_port()
url = 'http://'+ip+':'+str(port)+'/announce/'
return url
finally:
self.sesslock.release()
def get_internal_tracker_dir(self):
""" Returns the directory containing the torrents tracked by the internal
tracker (and associated databases).
@return An absolute path. """
# Called by any thread
self.sesslock.acquire()
try:
if self.sessconfig['state_dir'] is None:
return None
else:
return os.path.join(self.sessconfig['state_dir'],STATEDIR_ITRACKER_DIR)
finally:
self.sesslock.release()
def add_to_internal_tracker(self,tdef):
""" Add a torrent def to the list of torrents tracked by the internal
tracker. Use this method to use the Session as a standalone tracker.
@param tdef A finalized TorrentDef.
"""
# Called by any thread
self.sesslock.acquire()
try:
infohash = tdef.get_infohash()
filename = self.get_internal_tracker_torrentfilename(infohash)
tdef.save(filename)
print >>sys.stderr,"Session: add_to_int_tracker: saving to",filename,"url-compat",tdef.get_url_compat()
# Bring to attention of Tracker thread
self.lm.tracker_rescan_dir()
finally:
self.sesslock.release()
def remove_from_internal_tracker(self,tdef):
""" Remove a torrent def from the list of torrents tracked by the
internal tracker. Use this method to use the Session as a standalone
tracker.
@param tdef A finalized TorrentDef.
"""
infohash = tdef.get_infohash()
self.remove_from_internal_tracker_by_infohash(infohash)
def remove_from_internal_tracker_by_infohash(self,infohash):
""" Remove a torrent def from the list of torrents tracked by the
internal tracker. Use this method to use the Session as a standalone
tracker.
@param infohash Identifier of the torrent def to remove.
"""
# Called by any thread
self.sesslock.acquire()
try:
filename = self.get_internal_tracker_torrentfilename(infohash)
if DEBUG:
print >>sys.stderr,"Session: removing itracker entry",filename
if os.access(filename,os.F_OK):
os.remove(filename)
# Bring to attention of Tracker thread
self.lm.tracker_rescan_dir()
finally:
self.sesslock.release()
#
# Notification of events in the Session
#
def add_observer(self, func, subject, changeTypes = [NTFY_UPDATE, NTFY_INSERT, NTFY_DELETE], objectID = None):
""" Add an observer function function to the Session. The observer
function will be called when one of the specified events (changeTypes)
occurs on the specified subject.
The function will be called by a popup thread which can be used
indefinitely (within reason) by the higher level code.
@param func The observer function. It should accept as its first argument
the subject, as second argument the changeType, as third argument an
objectID (e.g. the primary key in the observed database) and an
optional list of arguments.
@param subject The subject to observe, one of NTFY_* subjects (see
simpledefs).
@param changeTypes The list of events to be notified of one of NTFY_*
events.
@param objectID The specific object in the subject to monitor (e.g. a
specific primary key in a database to monitor for updates.)
TODO: Jelle will add per-subject/event description here ;o)
"""
#Called by any thread
self.uch.notifier.add_observer(func, subject, changeTypes, objectID) # already threadsafe
def remove_observer(self, func):
""" Remove observer function. No more callbacks will be made.
@param func The observer function to remove. """
#Called by any thread
self.uch.notifier.remove_observer(func) # already threadsafe
def open_dbhandler(self,subject):
""" Opens a connection to the specified database. Only the thread
calling this method may use this connection. The connection must be
closed with close_dbhandler() when this thread exits.
@param subject The database to open. Must be one of the subjects
specified here.
@return A reference to a DBHandler class for the specified subject or
None when the Session was not started with megacaches enabled.
<pre> NTFY_PEERS -> PeerDBHandler
NTFY_TORRENTS -> TorrentDBHandler
NTFY_PREFERENCES -> PreferenceDBHandler
NTFY_SUPERPEERS -> SuperpeerDBHandler
NTFY_FRIENDS -> FriendsDBHandler
NTFY_MYPREFERENCES -> MyPreferenceDBHandler
NTFY_BARTERCAST -> BartercastDBHandler
NTFY_SEARCH -> SearchDBHandler
NTFY_TERM -> TermDBHandler
NTFY_VOTECAST -> VotecastDBHandler
NTFY_CHANNELCAST -> ChannelCastDBHandler
NTFY_RICH_METADATA -> MetadataDBHandler
</pre>
"""
# Called by any thread
self.sesslock.acquire()
try:
if subject == NTFY_PEERS:
return self.lm.peer_db
elif subject == NTFY_TORRENTS:
return self.lm.torrent_db
elif subject == NTFY_PREFERENCES:
return self.lm.pref_db
elif subject == NTFY_SUPERPEERS:
return self.lm.superpeer_db
elif subject == NTFY_FRIENDS:
return self.lm.friend_db
elif subject == NTFY_MYPREFERENCES:
return self.lm.mypref_db
elif subject == NTFY_BARTERCAST:
return self.lm.bartercast_db
elif subject == NTFY_SEEDINGSTATS:
return self.lm.seedingstats_db
elif subject == NTFY_SEEDINGSTATSSETTINGS:
return self.lm.seedingstatssettings_db
elif subject == NTFY_VOTECAST:
return self.lm.votecast_db
elif subject == NTFY_SEARCH:
return self.lm.search_db
elif subject == NTFY_TERM:
return self.lm.term_db
elif subject == NTFY_CHANNELCAST:
return self.lm.channelcast_db
elif subject == NTFY_RICH_METADATA:
return self.lm.richmetadataDbHandler
else:
raise ValueError('Cannot open DB subject: '+subject)
finally:
self.sesslock.release()
def close_dbhandler(self,dbhandler):
""" Closes the given database connection """
dbhandler.close()
#
# Access control
#
def set_overlay_request_policy(self, reqpol):
"""
Set a function which defines which overlay requests (e.g. proxy relay request, rquery msg)
will be answered or will be denied.
The function will be called by a network thread and must return
as soon as possible to prevent performance problems.
@param reqpol is a Tribler.Core.RequestPolicy.AbstractRequestPolicy
object.
"""
# Called by any thread
# to protect self.sessconfig
self.sesslock.acquire()
try:
overlay_loaded = self.sessconfig['overlay']
finally:
self.sesslock.release()
if overlay_loaded:
self.lm.overlay_apps.setRequestPolicy(reqpol) # already threadsafe
elif DEBUG:
print >>sys.stderr,"Session: overlay is disabled, so no overlay request policy needed"
#
# Persistence and shutdown
#
def load_checkpoint(self,initialdlstatus=None):
""" Restart Downloads from checkpoint, if any.
This method allows the API user to manage restoring downloads.
E.g. a video player that wants to start the torrent the user clicked
on first, and only then restart any sleeping torrents (e.g. seeding).
The optional initialdlstatus parameter can be set to DLSTATUS_STOPPED
to restore all the Downloads in DLSTATUS_STOPPED state.
"""
self.lm.load_checkpoint(initialdlstatus)
def checkpoint(self):
""" Saves the internal session state to the Session's state dir. """
#Called by any thread
self.checkpoint_shutdown(stop=False,checkpoint=True,gracetime=None,hacksessconfcheckpoint=False)
def shutdown(self,checkpoint=True,gracetime=2.0,hacksessconfcheckpoint=True):
""" Checkpoints the session and closes it, stopping the download engine.
@param checkpoint Whether to checkpoint the Session state on shutdown.
@param gracetime Time to allow for graceful shutdown + signoff (seconds).
"""
# Called by any thread
self.lm.early_shutdown()
self.checkpoint_shutdown(stop=True,checkpoint=checkpoint,gracetime=gracetime,hacksessconfcheckpoint=hacksessconfcheckpoint)
# Arno, 2010-08-09: now shutdown after gracetime
#self.uch.shutdown()
def has_shutdown(self):
""" Whether the Session has completely shutdown, i.e., its internal
threads are finished and it is safe to quit the process the Session
is running in.
@return A Boolean.
"""
return self.lm.sessdoneflag.isSet()
def get_downloads_pstate_dir(self):
""" Returns the directory in which to checkpoint the Downloads in this
Session. """
# Called by network thread
self.sesslock.acquire()
try:
return os.path.join(self.sessconfig['state_dir'],STATEDIR_DLPSTATE_DIR)
finally:
self.sesslock.release()
#
# Tribler Core special features
#
def query_connected_peers(self,query,usercallback,max_peers_to_query=None):
""" Ask all Tribler peers we're currently connected to resolve the
specified query and return the hits. For each peer that returns
hits the usercallback method is called with first parameter the
permid of the peer, as second parameter the query string and
as third parameter a dictionary of hits. The number of times the
usercallback method will be called is undefined.
The callback will be called by a popup thread which can be used
indefinitely (within reason) by the higher level code.
At the moment we support three types of query, which are all queries for
torrent files that match a set of keywords. The format of the
query string is "SIMPLE kw1 kw2 kw3" (type 1) or "SIMPLE+METADATA kw1 kw2
kw3" (type 3). In the future we plan to support full SQL queries.
For SIMPLE queries the dictionary of hits consists of
(infohash,torrentrecord) pairs. The torrentrecord is a
dictionary that contains the following keys:
<pre>
* 'content_name': The 'name' field of the torrent as Unicode string.
* 'length': The total size of the content in the torrent.
* 'leecher': The currently known number of downloaders.
* 'seeder': The currently known number of seeders.
* 'category': A list of category strings the torrent was classified into
by the remote peer.
</pre>
From Session API version 1.0.2 the following keys were added
to the torrentrecord:
<pre>
* 'torrent_size': The size of the .torrent file.
</pre>
From Session API version 1.0.4 the following keys were added
to the torrentrecord:
<pre>
* 'channel_permid': PermID of the channel this torrent belongs to (or '')
* 'channel_name': channel name as Unicode string (or '').
For SIMPLE+METADATA queries there is an extra field
<pre>
* 'torrent_file': Bencoded contents of the .torrent file.
</pre>
The torrents *not* be automatically added to the TorrentDBHandler
(if enabled) at the time of the call.
The third type of query: search for channels. It is used to query for
channels: either a particular channel matching the permid in the query,
or a list of channels whose names match the keywords in the query
by sending the query to connected peers.
The format of the query in the corresponding scenarios should be:
a. keyword-based query: "CHANNEL k bbc"
('k' stands for keyword-based and ' '{space} is a separator followed by
the keywords)
b. permid-based query: "CHANNEL p f34wrf2345wfer2345wefd3r34r54"
('p' stands for permid-based and ' '{space} is a separator followed by
the permid)
In each of the above 2 cases, the format of the hits that is returned
by the queried peer is a dictionary of hits of (signature,channelrecord).
The channelrecord is a dictionary the contains following keys:
<pre>
* 'publisher_id': a PermID
* 'publisher_name': as Unicode string
* 'infohash': 20-byte SHA1 hash
* 'torrenthash': 20-byte SHA1 hash
* 'torrentname': as Unicode string
* 'time_stamp': as integer
</pre>
@param query A Unicode query string adhering to the above spec.
@param usercallback A function adhering to the above spec.
@return currently connected peers with olversion 6 or higher
"""
self.sesslock.acquire()
try:
if self.sessconfig['overlay']:
if not (query.startswith('SIMPLE ') or query.startswith('SIMPLE+METADATA ')) and not query.startswith('CHANNEL '):
raise ValueError("Query does not start with SIMPLE or SIMPLE+METADATA or CHANNEL (%s)"%query)
from Tribler.Core.SocialNetwork.RemoteQueryMsgHandler import RemoteQueryMsgHandler
rqmh = RemoteQueryMsgHandler.getInstance()
rqmh.send_query(query,usercallback,max_peers_to_query=max_peers_to_query)
return len(rqmh.get_connected_peers(OLPROTO_VER_SIXTH))
else:
raise OperationNotEnabledByConfigurationException("Overlay not enabled")
finally:
self.sesslock.release()
return 0
def query_peers(self,query,peers,usercallback):
""" Equal to query_connected_peers only now for a limited subset of peers.
If there is no active connnection to a peer in the list, a connection
will be made.
"""
self.sesslock.acquire()
try:
if self.sessconfig['overlay']:
if not (query.startswith('SIMPLE ') or query.startswith('SIMPLE+METADATA ')) and not query.startswith('CHANNEL '):
raise ValueError('Query does not start with SIMPLE or SIMPLE+METADATA or CHANNEL')
from Tribler.Core.SocialNetwork.RemoteQueryMsgHandler import RemoteQueryMsgHandler
rqmh = RemoteQueryMsgHandler.getInstance()
rqmh.send_query_to_peers(query,peers,usercallback)
else:
raise OperationNotEnabledByConfigurationException("Overlay not enabled")
finally:
self.sesslock.release()
def download_torrentfile(self, infohash = None, roothash = None, usercallback = None, prio = 0):
""" Try to download the torrentfile without a known source.
A possible source could be the DHT.
If the torrent is succesfully
received, the usercallback method is called with the infohash as first
and the contents of the torrentfile (bencoded dict) as second parameter.
If the torrent could not be obtained, the callback is not called.
The torrent will have been added to the TorrentDBHandler (if enabled)
at the time of the call.
@param infohash The infohash of the torrent.
@param usercallback A function adhering to the above spec.
"""
from Tribler.Core.RemoteTorrentHandler import RemoteTorrentHandler
rtorrent_handler = RemoteTorrentHandler.getInstance()
rtorrent_handler.download_torrent(None,infohash,roothash,usercallback,prio)
def download_torrentfile_from_peer(self, candidate, infohash=None, roothash=None, usercallback=None, prio = 0):
""" Ask the designated peer to send us the torrentfile for the torrent
identified by the passed infohash. If the torrent is succesfully
received, the usercallback method is called with the infohash as first
and the contents of the torrentfile (bencoded dict) as second parameter.
If the torrent could not be obtained, the callback is not called.
The torrent will have been added to the TorrentDBHandler (if enabled)
at the time of the call.
@param permid The PermID of the peer to query.
@param infohash The infohash of the torrent.
@param usercallback A function adhering to the above spec.
"""
from Tribler.Core.RemoteTorrentHandler import RemoteTorrentHandler
rtorrent_handler = RemoteTorrentHandler.getInstance()
rtorrent_handler.download_torrent(candidate,infohash,roothash,usercallback,prio)
def download_torrentmessages_from_peer(self, candidate, infohashes, usercallback, prio = 0):
""" Ask the designated peer to send us the torrentfile for the torrent
identified by the passed infohash. If the torrent is succesfully
received, the usercallback method is called with the infohash as first
and the contents of the torrentfile (bencoded dict) as second parameter.
If the torrent could not be obtained, the callback is not called.
The torrent will have been added to the TorrentDBHandler (if enabled)
at the time of the call.
@param permid The PermID of the peer to query.
@param infohash The infohash of the torrent.
@param usercallback A function adhering to the above spec.
"""
from Tribler.Core.RemoteTorrentHandler import RemoteTorrentHandler
rtorrent_handler = RemoteTorrentHandler.getInstance()
rtorrent_handler.download_torrentmessages(candidate,infohashes,usercallback,prio)
#
# Internal persistence methods
#
def checkpoint_shutdown(self,stop,checkpoint,gracetime,hacksessconfcheckpoint):
""" Checkpoints the Session and optionally shuts down the Session.
@param stop Whether to shutdown the Session as well.
@param checkpoint Whether to checkpoint at all, or just to stop.
@param gracetime Time to allow for graceful shutdown + signoff (seconds).
"""
# Called by any thread
self.sesslock.acquire()
try:
# Arno: Make checkpoint optional on shutdown. At the moment setting
# the config at runtime is not possible (see SessionRuntimeConfig)
# so this has little use, and interferes with our way of
# changing the startup config, which is to write a new
# config to disk that will be read at start up.
if hacksessconfcheckpoint:
try:
self.save_pstate_sessconfig()
except Exception,e:
self.lm.rawserver_nonfatalerrorfunc(e)
# Checkpoint all Downloads and stop NetworkThread
if DEBUG or stop:
print >>sys.stderr,"Session: checkpoint_shutdown"
self.lm.checkpoint(stop=stop,checkpoint=checkpoint,gracetime=gracetime)
finally:
def __init__(self, scfg=None, ignore_singleton=False):
"""
A Session object is created which is configured following a copy of the
SessionStartupConfig scfg. (copy constructor used internally)
@param scfg SessionStartupConfig object or None, in which case we
look for a saved session in the default location (state dir). If
we can't find it, we create a new SessionStartupConfig() object to
serve as startup config. Next, the config is saved in the directory
indicated by its 'state_dir' attribute.
In the current implementation only a single session instance can exist
at a time in a process. The ignore_singleton flag is used for testing.
"""
if not ignore_singleton:
if Session.__single:
raise RuntimeError("Session is singleton")
Session.__single = self
self.ignore_singleton = ignore_singleton
self.sesslock = NoDispersyRLock()
# Determine startup config to use
if scfg is None: # If no override
try:
# Then try to read from default location
state_dir = Session.get_default_state_dir()
cfgfilename = Session.get_default_config_filename(state_dir)
scfg = SessionStartupConfig.load(cfgfilename)
except:
# If that fails, create a fresh config with factory defaults
print_exc()
scfg = SessionStartupConfig()
self.sessconfig = scfg.sessconfig
else: # overrides any saved config
# Work from copy
self.sessconfig = copy.copy(scfg.sessconfig)
def create_dir(fullpath):
if not os.path.isdir(fullpath):
os.makedirs(fullpath)
def set_and_create_dir(config, name, default_dir):
dirname = config.get(name, None)
if dirname is None:
config[name] = default_dir
create_dir(config[name])
set_and_create_dir(self.sessconfig, 'state_dir', Session.get_default_state_dir())
set_and_create_dir(self.sessconfig, 'torrent_collecting_dir', os.path.join(self.sessconfig['state_dir'], STATEDIR_TORRENTCOLL_DIR))
set_and_create_dir(self.sessconfig, 'swiftmetadir', os.path.join(self.sessconfig['state_dir'], STATEDIR_SWIFTRESEED_DIR))
set_and_create_dir(self.sessconfig, 'peer_icon_path', os.path.join(self.sessconfig['state_dir'], STATEDIR_PEERICON_DIR))
create_dir(os.path.join(self.sessconfig['state_dir'], STATEDIR_DLPSTATE_DIR))
# Poor man's versioning of SessionConfig, add missing
# default values. Really should use PERSISTENTSTATE_CURRENTVERSION
# and do conversions.
for key, defvalue in sessdefaults.iteritems():
if key not in self.sessconfig:
self.sessconfig[key] = defvalue
if self.sessconfig['nickname'] == '__default_name__':
self.sessconfig['nickname'] = socket.gethostname()
# SWIFTPROC
if self.sessconfig['swiftpath'] is None:
if sys.platform == "win32":
self.sessconfig['swiftpath'] = os.path.join(self.sessconfig['install_dir'], "swift.exe")
else:
self.sessconfig['swiftpath'] = os.path.join(self.sessconfig['install_dir'], "swift")
if GOTM2CRYPTO:
permidmod.init()
# Set params that depend on state_dir
#
# 1. keypair
#
pairfilename = os.path.join(self.sessconfig['state_dir'], 'ec.pem')
if self.sessconfig['eckeypairfilename'] is None:
self.sessconfig['eckeypairfilename'] = pairfilename
if os.access(self.sessconfig['eckeypairfilename'], os.F_OK):
# May throw exceptions
self.keypair = permidmod.read_keypair(self.sessconfig['eckeypairfilename'])
else:
self.keypair = permidmod.generate_keypair()
# Save keypair
pubfilename = os.path.join(self.sessconfig['state_dir'], 'ecpub.pem')
permidmod.save_keypair(self.keypair, pairfilename)
permidmod.save_pub_key(self.keypair, pubfilename)
# Checkpoint startup config
self.save_pstate_sessconfig()
class SwiftProcess:
""" Representation of an operating-system process running the C++ swift engine.
A swift engine can participate in one or more swarms."""
def __init__(self,binpath,workdir,zerostatedir,listenport,httpgwport,cmdgwport,spmgr):
# Called by any thread, assume sessionlock is held
self.splock = NoDispersyRLock()
self.binpath = binpath
self.workdir = workdir
self.zerostatedir = zerostatedir
self.spmgr = spmgr
# Main UDP listen socket
if listenport is None:
self.listenport = random.randint(10001,10999)
else:
self.listenport = listenport
# NSSA control socket
if cmdgwport is None:
self.cmdport = random.randint(11001,11999)
else:
self.cmdport = cmdgwport
# content web server
if httpgwport is None:
self.httpport = random.randint(12001,12999)
else:
self.httpport = httpgwport
# Security: only accept commands from localhost, enable HTTP gw,
# no stats/webUI web server
args=[]
# Arno, 2012-07-09: Unicode problems with popen
args.append(self.binpath.encode(sys.getfilesystemencoding()))
# Arno, 2012-05-29: Hack. Win32 getopt code eats first arg when Windows app
# instead of CONSOLE app.
args.append("-j")
args.append("-l") # listen port
args.append("0.0.0.0:"+str(self.listenport))
args.append("-c") # command port
args.append("127.0.0.1:"+str(self.cmdport))
args.append("-g") # HTTP gateway port
args.append("127.0.0.1:"+str(self.httpport))
args.append("-w")
if zerostatedir is not None:
if sys.platform == "win32":
# Swift on Windows expects command line arguments as UTF-16.
# popen doesn't allow us to pass params in UTF-16, hence workaround.
# Format = hex encoded UTF-8
args.append("-3")
zssafe = binascii.hexlify(zerostatedir.encode("UTF-8"))
args.append(zssafe) # encoding that swift expects
else:
args.append("-e")
args.append(zerostatedir)
args.append("-T") # zero state connection timeout
args.append("180") # seconds
#args.append("-B") # Enable debugging on swift
if True or DEBUG:
print >>sys.stderr,"SwiftProcess: __init__: Running",args,"workdir",workdir
if sys.platform == "win32":
creationflags=subprocess.CREATE_NEW_PROCESS_GROUP
else:
creationflags=0
# See also SwiftDef::finalize popen
self.popen = subprocess.Popen(args,close_fds=True,cwd=workdir,creationflags=creationflags)
self.roothash2dl = {}
self.donestate = DONE_STATE_WORKING # shutting down
self.fastconn = None
#
# Instance2Instance
#
def start_cmd_connection(self):
# Called by any thread, assume sessionlock is held
if self.is_alive():
self.fastconn = FastI2IConnection(self.cmdport,self.i2ithread_readlinecallback,self.connection_lost)
else:
print >>sys.stderr,"sp: start_cmd_connection: Process dead? returncode",self.popen.returncode,"pid",self.popen.pid
def i2ithread_readlinecallback(self,ic,cmd):
#if DEBUG:
# print >>sys.stderr,"sp: Got command #"+cmd+"#"
if self.donestate != DONE_STATE_WORKING:
return
words = cmd.split()
if words[0] == "TUNNELRECV":
address, session = words[1].split("/")
host, port = address.split(":")
port = int(port)
session = session.decode("HEX")
length = int(words[2])
# require LENGTH bytes
if len(ic.buffer) < length:
return length - len(ic.buffer)
data = ic.buffer[:length]
ic.buffer = ic.buffer[length:]
self.roothash2dl["dispersy"].i2ithread_data_came_in(session, (host, port), data)
else:
roothash = binascii.unhexlify(words[1])
if words[0] == "ERROR":
print >>sys.stderr,"sp: i2ithread_readlinecallback:",cmd
self.splock.acquire()
try:
if roothash not in self.roothash2dl.keys():
if DEBUG:
print >>sys.stderr,"sp: i2ithread_readlinecallback: unknown roothash",words[1]
return
d = self.roothash2dl[roothash]
except:
#print >>sys.stderr,"GOT", words
#print >>sys.stderr,"HAVE", [key.encode("HEX") for key in self.roothash2dl.keys()]
raise
finally:
self.splock.release()
# Hide NSSA interface for SwiftDownloadImpl
if words[0] == "INFO": # INFO HASH status dl/total
dlstatus = int(words[2])
pargs = words[3].split("/")
dynasize = int(pargs[1])
if dynasize == 0:
progress = 0.0
else:
progress = float(pargs[0])/float(pargs[1])
dlspeed = float(words[4])
ulspeed = float(words[5])
numleech = int(words[6])
numseeds = int(words[7])
contentdl = 0 # bytes
contentul = 0 # bytes
if len(words) > 8:
contentdl = int(words[8])
contentul = int(words[9])
d.i2ithread_info_callback(dlstatus,progress,dynasize,dlspeed,ulspeed,numleech,numseeds,contentdl,contentul)
elif words[0] == "PLAY":
#print >>sys.stderr,"sp: i2ithread_readlinecallback: Got PLAY",cmd
httpurl = words[2]
d.i2ithread_vod_event_callback(VODEVENT_START,httpurl)
elif words[0] == "MOREINFO":
jsondata = cmd[len("MOREINFO ")+40+1:]
midict = json.loads(jsondata)
d.i2ithread_moreinfo_callback(midict)
elif words[0] == "ERROR":
d.i2ithread_info_callback(DLSTATUS_STOPPED_ON_ERROR,0.0,0,0.0,0.0,0,0,0,0)
#
# Swift Mgmt interface
#
def start_download(self,d):
self.splock.acquire()
try:
if self.donestate != DONE_STATE_WORKING or not self.is_alive():
return
roothash = d.get_def().get_roothash()
roothash_hex = d.get_def().get_roothash_as_hex()
# Before send to handle INFO msgs
self.roothash2dl[roothash] = d
url = d.get_def().get_url()
# MULTIFILE
if len(d.get_selected_files()) == 1:
specpath = d.get_selected_files()[0]
qpath = urllib.quote(specpath)
url += "/" + qpath
# Default is unlimited, so don't send MAXSPEED then
maxdlspeed=d.get_max_speed(DOWNLOAD)
if maxdlspeed == 0:
maxdlspeed = None
maxulspeed=d.get_max_speed(UPLOAD)
if maxulspeed == 0:
maxulspeed = None
self.send_start(url,roothash_hex=roothash_hex,maxdlspeed=maxdlspeed,maxulspeed=maxulspeed,destdir=d.get_dest_dir())
finally:
self.splock.release()
def add_download(self,d):
self.splock.acquire()
try:
roothash = d.get_def().get_roothash()
# Before send to handle INFO msgs
self.roothash2dl[roothash] = d
finally:
self.splock.release()
def remove_download(self,d,removestate,removecontent):
self.splock.acquire()
try:
if self.donestate != DONE_STATE_WORKING or not self.is_alive():
return
roothash_hex = d.get_def().get_roothash_as_hex()
self.send_remove(roothash_hex,removestate,removecontent)
# After send to handle INFO msgs
roothash = d.get_def().get_roothash()
del self.roothash2dl[roothash]
finally:
self.splock.release()
def get_downloads(self):
self.splock.acquire()
try:
return self.roothash2dl.values()
finally:
self.splock.release()
def get_pid(self):
if self.popen is not None:
return self.popen.pid
else:
return -1
def get_listen_port(self):
return self.listenport
def set_max_speed(self,d,direct,speed):
self.splock.acquire()
try:
if self.donestate != DONE_STATE_WORKING or not self.is_alive():
return
roothash_hex = d.get_def().get_roothash_as_hex()
# In Tribler Core API = unlimited. In Swift CMDGW API
# 0 = none.
if speed == 0.0:
speed = 4294967296.0
self.send_max_speed(roothash_hex,direct,speed)
finally:
self.splock.release()
def checkpoint_download(self,d):
self.splock.acquire()
try:
# Arno, 2012-05-15: Allow during shutdown.
if not self.is_alive():
return
roothash_hex = d.get_def().get_roothash_as_hex()
self.send_checkpoint(roothash_hex)
finally:
self.splock.release()
def set_moreinfo_stats(self,d,enable):
self.splock.acquire()
try:
if self.donestate != DONE_STATE_WORKING or not self.is_alive():
return
roothash_hex = d.get_def().get_roothash_as_hex()
self.send_setmoreinfo(roothash_hex,enable)
finally:
self.splock.release()
def add_peer(self,d,addr):
self.splock.acquire()
try:
if self.donestate != DONE_STATE_WORKING or not self.is_alive():
return
addrstr = addr[0]+':'+str(addr[1])
roothash_hex = d.get_def().get_roothash_as_hex()
self.send_peer_addr(roothash_hex,addrstr)
finally:
self.splock.release()
def early_shutdown(self):
# Called by any thread, assume sessionlock is held
# May get called twice, once by spm.release_sp() and spm.shutdown()
if self.donestate == DONE_STATE_WORKING:
self.donestate = DONE_STATE_EARLY_SHUTDOWN
else:
return
if self.popen is not None:
# Tell engine to shutdown so it can deregister dls from tracker
print >>sys.stderr,"sp: Telling process to shutdown"
self.send_shutdown()
def network_shutdown(self):
# Called by network thread, assume sessionlock is held
if self.donestate == DONE_STATE_EARLY_SHUTDOWN:
self.donestate = DONE_STATE_SHUTDOWN
else:
return
# could do fastconn.close() here
if self.popen is not None:
try:
print >>sys.stderr,"sp: Terminating process"
self.popen.terminate()
self.popen.wait()
self.popen = None
except WindowsError:
pass
except:
print_exc()
if self.fastconn:
self.fastconn.stop()
#
# Internal methods
#
def send_start(self,url,roothash_hex=None,maxdlspeed=None,maxulspeed=None,destdir=None):
# assume splock is held to avoid concurrency on socket
if DEBUG: print >>sys.stderr,"sp: send_start:",url,"destdir",destdir
cmd = 'START '+url
if destdir is not None:
cmd += ' '+destdir.encode("UTF-8")
cmd += '\r\n'
if maxdlspeed is not None:
cmd += 'MAXSPEED '+roothash_hex+' DOWNLOAD '+str(float(maxdlspeed))+'\r\n'
if maxulspeed is not None:
cmd += 'MAXSPEED '+roothash_hex+' UPLOAD '+str(float(maxulspeed))+'\r\n'
self.write(cmd)
def send_remove(self,roothash_hex,removestate,removecontent):
# assume splock is held to avoid concurrency on socket
self.write('REMOVE '+roothash_hex+' '+str(int(removestate))+' '+str(int(removecontent))+'\r\n')
def send_checkpoint(self,roothash_hex):
# assume splock is held to avoid concurrency on socket
self.write('CHECKPOINT '+roothash_hex+'\r\n')
def send_shutdown(self):
# assume splock is held to avoid concurrency on socket
self.write('SHUTDOWN\r\n')
def send_max_speed(self,roothash_hex,direct,speed):
# assume splock is held to avoid concurrency on socket
cmd = 'MAXSPEED '+roothash_hex
if direct == DOWNLOAD:
cmd += ' DOWNLOAD '
else:
cmd += ' UPLOAD '
cmd += str(float(speed))+'\r\n'
self.write(cmd)
def send_tunnel(self,session,address,data):
# assume splock is held to avoid concurrency on socket
if DEBUG:
print >>sys.stderr,"sp: send_tunnel:",len(data),"bytes -> %s:%d" % address
self.write("TUNNELSEND %s:%d/%s %d\r\n" % (address[0], address[1], session.encode("HEX"), len(data)))
self.write(data)
def send_setmoreinfo(self,roothash_hex,enable):
# assume splock is held to avoid concurrency on socket
onoff = "0"
if enable:
onoff = "1"
self.write('SETMOREINFO '+roothash_hex+' '+onoff+'\r\n')
def send_peer_addr(self,roothash_hex,addrstr):
# assume splock is held to avoid concurrency on socket
self.write('PEERADDR '+roothash_hex+' '+addrstr+'\r\n')
def is_alive(self):
if self.popen:
self.popen.poll()
return self.popen.returncode is None
return False
def write(self,msg):
self.fastconn.write(msg)
def get_cmdport(self):
return self.cmdport
def connection_lost(self,port):
self.spmgr.connection_lost(port)
class RePEXScheduler(RePEXerStatusCallback):
"""
The RePEXScheduler periodically requests a list of DownloadStates from
the Session and repexes the stopped downloads in a round robin fashion.
"""
__single = None # used for multithreaded singletons pattern
lock = RLock()
@classmethod
def getInstance(cls, *args, **kw):
# Singleton pattern with double-checking to ensure that it can only create one object
if cls.__single is None:
cls.lock.acquire()
try:
if cls.__single is None:
cls.__single = cls(*args, **kw)
finally:
cls.lock.release()
return cls.__single
def __init__(self):
# always use getInstance() to create this object
# ARNOCOMMENT: why isn't the lock used on this read?!
if self.__single != None:
raise RuntimeError, "RePEXScheduler is singleton"
from Tribler.Core.Session import Session # Circular import fix
self.session = Session.get_instance()
self.lock = RLock()
self.active = False
self.current_repex = None # infohash
self.downloads = {} # infohash -> Download; in order to stop Downloads that are done repexing
self.last_attempts = {} # infohash -> ts; in order to prevent starvation when a certain download
# keeps producing empty SwarmCaches
def start(self):
""" Starts the RePEX scheduler. """
if DEBUG:
print >>sys.stderr, "RePEXScheduler: start"
self.lock.acquire()
try:
if self.active:
return
self.active = True
self.session.set_download_states_callback(self.network_scan)
RePEXer.attach_observer(self)
finally:
self.lock.release()
def stop(self):
""" Stops the RePEX scheduler. """
if DEBUG:
print >>sys.stderr, "RePEXScheduler: stop"
self.lock.acquire()
try:
if not self.active:
return
RePEXer.detach_observer(self)
self.active = False
self.session.set_download_states_callback(self.network_stop_repex)
finally:
self.lock.release()
def network_scan(self, dslist):
"""
Called by session thread. Scans for stopped downloads and stores
them in a queue.
@param dslist List of DownloadStates"""
# TODO: only repex last X Downloads instead of all.
if DEBUG:
print >>sys.stderr, "RePEXScheduler: network_scan: %s DownloadStates" % len(dslist)
self.lock.acquire()
exception = None
try:
try:
if not self.active or self.current_repex is not None:
return -1, False
now = ts_now()
found_infohash = None
found_download = None
found_age = -1
for ds in dslist:
download = ds.get_download()
infohash = download.get_def().get_id()
debug_msg = None
if DEBUG:
print >>sys.stderr, "RePEXScheduler: network_scan: checking", `download.get_def().get_name()`
if ds.get_status() == DLSTATUS_STOPPED and ds.get_progress()==1.0:
# TODO: only repex finished downloads or also prematurely stopped ones?
age = now - (swarmcache_ts(ds.get_swarmcache()) or 0)
last_attempt_ago = now - self.last_attempts.get(infohash, 0)
if last_attempt_ago < REPEX_MIN_INTERVAL:
debug_msg = "...too soon to try again, last attempt was %ss ago" % last_attempt_ago
elif age < REPEX_INTERVAL:
debug_msg = "...SwarmCache too fresh: %s seconds" % age
else:
if age >= REPEX_INTERVAL:
debug_msg = "...suitable for RePEX!"
if age > found_age:
found_download = download
found_infohash = infohash
found_age = age
else:
debug_msg = "...not repexable: %s %s%%" % (dlstatus_strings[ds.get_status()], ds.get_progress()*100)
if DEBUG:
print >>sys.stderr, "RePEXScheduler: network_scan:", debug_msg
if found_download is None:
if DEBUG:
print >>sys.stderr, "RePEXScheduler: network_scan: nothing found yet"
return REPEX_SCAN_INTERVAL, False
else:
if DEBUG:
print >>sys.stderr, "RePEXScheduler: network_scan: found %s, starting RePEX phase." % `found_download.get_def().get_name()`
self.current_repex = found_infohash
self.downloads[found_infohash] = found_download
found_download.set_mode(DLMODE_NORMAL)
found_download.restart(initialdlstatus=DLSTATUS_REPEXING)
return -1, False
except Exception, e:
exception = e
finally:
self.lock.release()
if exception is not None:
# [E0702, RePEXScheduler.network_scan] Raising NoneType
# while only classes, instances or string are allowed
# pylint: disable-msg=E0702
raise exception
def network_stop_repex(self, dslist):
"""Called by network thread.
@param dslist List of DownloadStates"""
if DEBUG:
print >>sys.stderr, "RePEXScheduler: network_stop_repex:"
for d in [ds.get_download() for ds in dslist if ds.get_status() == DLSTATUS_REPEXING]:
if DEBUG:
print >>sys.stderr, "\t...",`d.get_def().get_name()`
d.stop()
return -1, False
#
# RePEXerStatusCallback interface (called by network thread)
#
def repex_aborted(self, repexer, dlstatus=None):
if DEBUG:
if dlstatus is None:
status_string = str(None)
else:
status_string = dlstatus_strings[dlstatus]
print >>sys.stderr, "RePEXScheduler: repex_aborted:", b2a_hex(repexer.infohash), status_string
self.current_repex = None
self.last_attempts[repexer.infohash] = ts_now()
self.session.set_download_states_callback(self.network_scan)
def repex_done(self, repexer, swarmcache, shufflecount, shufflepeers, bootstrapcount, datacost):
if DEBUG:
print >>sys.stderr, 'RePEXScheduler: repex_done: %s\n\ttable size/shuffle/bootstrap %s/%s/%s' % (
b2a_hex(repexer.infohash), len(swarmcache), shufflecount, bootstrapcount)
self.current_repex = None
self.last_attempts[repexer.infohash] = ts_now()
self.downloads[repexer.infohash].stop()
self.session.set_download_states_callback(self.network_scan)