def init_libtorrent(self, max_download_speed=0, max_upload_speed=0):
"""Perform the initialization of things that should be initialized once"""
if self.session:
return
settings = libtorrent.session_settings()
settings.user_agent = encode_utf8(
'Torrent Launcher (libtorrent/{})'.format(
decode_utf8(libtorrent.version)))
"""When running on a network where the bandwidth is in such an abundance
that it's virtually infinite, this algorithm is no longer necessary, and
might even be harmful to throughput. It is adviced to experiment with the
session_setting::mixed_mode_algorithm, setting it to session_settings::prefer_tcp.
This setting entirely disables the balancing and unthrottles all connections."""
settings.mixed_mode_algorithm = 0
# Fingerprint = 'LT1080' == LibTorrent 1.0.8.0
fingerprint = libtorrent.fingerprint(
b'LT', *(int(i) for i in libtorrent.version.split('.')))
self.session = libtorrent.session(fingerprint=fingerprint)
self.session.listen_on(
6881, 6891
) # This is just a port suggestion. On failure, the port is automatically selected.
# Prevent conversion to C int error
settings.download_rate_limit = min(max_download_speed, 999999) * 1024
settings.upload_rate_limit = min(max_upload_speed, 999999) * 1024
self.session.set_settings(settings)
def create_torrent(directory, announces=None, output=None, comment=None, web_seeds=None):
if not output:
output = directory + ".torrent"
# "If a piece size of 0 is specified, a piece_size will be calculated such that the torrent file is roughly 40 kB."
piece_size_multiplier = 0
piece_size = (16 * 1024) * piece_size_multiplier # Must be multiple of 16KB
# http://www.libtorrent.org/make_torrent.html#create-torrent
flags = libtorrent.create_torrent_flags_t.calculate_file_hashes
if not os.path.isdir(directory):
raise Exception("The path {} is not a directory".format(directory))
fs = libtorrent.file_storage()
is_not_whitelisted = lambda node: not is_whitelisted(unicode_helpers.decode_utf8(node))
libtorrent.add_files(fs, unicode_helpers.encode_utf8(directory), is_not_whitelisted, flags=flags)
t = libtorrent.create_torrent(fs, piece_size=piece_size, flags=flags)
for announce in announces:
t.add_tracker(unicode_helpers.encode_utf8(announce))
if comment:
t.set_comment(unicode_helpers.encode_utf8(comment))
for web_seed in web_seeds:
t.add_url_seed(unicode_helpers.encode_utf8(web_seed))
# t.add_http_seed("http://...")
libtorrent.set_piece_hashes(t, unicode_helpers.encode_utf8(os.path.dirname(directory)))
with open(output, "wb") as file_handle:
file_handle.write(libtorrent.bencode(t.generate()))
return output
def log_torrent_progress(self, s, mod_name):
"""Just log the download progress for now.
Do not log anything if the torrent is 100% completed to prevent spamming
while seeding."""
# download_fraction = s.progress
download_kBps = s.download_rate / 1024
upload_kBps = s.upload_rate / 1024
state = decode_utf8(s.state.name)
if s.progress == 1:
return
Logger.info(
'Progress: [{}] {:.2f}% complete (down: {:.1f} kB/s up: {:.1f} kB/s connections: {}) {}'
.format(mod_name, s.progress * 100, download_kBps, upload_kBps,
s.num_peers, state))
def get_session_logs(self):
"""Get alerts from torrent engine and forward them to the manager process"""
torrent_log = []
alerts = self.session.pop_alerts(
) # Important: these are messages for the whole session, not only one torrent!
# Use alert.handle in the future to get the torrent handle
for alert in alerts:
# Filter with: alert.category() & libtorrent.alert.category_t.error_notification
message = decode_utf8(alert.message(), errors='ignore')
Logger.info("Alerts: Category: {}, Message: {}".format(
alert.category(), message))
torrent_log.append({
'message': message,
'category': alert.category()
})
return torrent_log
def sync(self, force_sync=False, just_seed=False):
"""
Synchronize the mod directory contents to contain exactly the files that
are described in the torrent file.
force_sync - Assume no resume data is available. Manually recheck all the
checksums for all the files in the torrent description.
Individual torrent states:
1) Downloading -> Wait until it starts seeding
2) Seeding -> Pause the torrent to sync it to disk
3) Paused -> Data has been synced, We can start seeding while
waiting for the other torrents to download.
4) Waiting seed -> When all torrents are waiting seeds, pause to stop
5) Paused to stop -> When all torrents are paused to stop, stop syncing
"""
sync_success = True
self.result_queue.progress(
{
'msg': 'Downloading metadata...',
'log': [],
}, 0)
for mod in self.mods:
try:
self.prepare_libtorrent_params(mod, force_sync, just_seed)
except (PrepareParametersException,
torrent_utils.AdminRequiredError) as ex:
self.result_queue.reject({'msg': ex.args[0]})
sync_success = False
return sync_success
if self.force_termination:
Logger.info(
'Sync: Downloading process was requested to stop before starting the download.'
)
self.result_queue.reject({
'details':
'Downloading process was requested to stop before starting the download.'
})
return
for mod in self.mods:
# Launch the download of the torrent
Logger.info('Sync: Downloading {} to {}'.format(
mod.torrent_url, mod.parent_location))
torrent_handle = self.session.add_torrent(mod.libtorrent_params)
mod.torrent_handle = torrent_handle
self.get_torrents_status()
self.eta = Eta()
# Loop until state (5). All torrents finished and paused
while not self.is_syncing_finished():
self.handle_messages()
self.log_session_progress()
for mod in self.mods:
if not mod.torrent_handle.is_valid():
Logger.info(
'Sync: Torrent {} - torrent handle is invalid. Terminating'
.format(mod.foldername))
self.force_termination = True # reject will be made once all torrents are done
continue
# It is assumed that all torrents below have a valid torrent_handle
self.log_torrent_progress(mod.status, mod.foldername)
if mod.status.error:
Logger.info(
'Sync: Torrent {} in error state. Terminating. Error string: {}'
.format(mod.foldername, decode_utf8(mod.status.error)))
self.force_termination = True # reject will be made once all torrents are done
continue # Torrent is now paused
# Allow saving fast-resume data only after finishing checking the files of the torrent
# If we save the data from a torrent while being checked, this will result
# in marking the torrent as having only a fraction of data it really has.
if mod.status.state in (libtorrent.torrent_status.downloading,
libtorrent.torrent_status.finished,
libtorrent.torrent_status.seeding):
mod.can_save_resume_data = True
# Shut the torrent if we are terminating
if self.force_termination:
if not mod.torrent_handle.is_paused(): # Don't spam logs
Logger.info(
'Sync: Pausing torrent {} for termination'.format(
mod.foldername))
self.pause_torrent(mod)
# If state (2). Request pausing the torrent to synchronize data to disk
if not mod.finished_hook_ran and mod.torrent_handle.is_seed():
if not mod.torrent_handle.is_paused():
Logger.info(
'Sync: Pausing torrent {} for disk syncing'.format(
mod.foldername))
self.pause_torrent(mod)
# If state (3). Run the hooks and maybe start waiting-seed
if not mod.finished_hook_ran and mod.torrent_handle.is_seed(
) and mod.torrent_handle.is_paused():
Logger.info(
'Sync: Torrent {} paused. Running finished_hook'.
format(mod.foldername))
hook_successful = self.torrent_finished_hook(mod)
if not hook_successful:
self.result_queue.reject({
'msg':
'Could not perform mod {} cleanup. Make sure the files are not in use by another program.'
.format(mod.foldername)
})
Logger.info(
'Sync: Could not perform mod {} cleanup. Make sure the files are not in use by another program.'
.format(mod.foldername))
sync_success = False
self.force_termination = True
mod.finished_hook_ran = True
# Do not go into state (4) if we are terminating
if not self.force_termination:
Logger.info(
'Sync: Seeding {} again until all downloads are done.'
.format(mod.foldername))
self.resume_torrent(mod)
# If all are in state (4)
if self.all_torrents_ran_finished_hooks() and not just_seed:
Logger.info('Sync: Pausing all torrents for syncing end.')
self.pause_all_torrents()
sleep(self._update_interval)
self.get_torrents_status()
Logger.info('Sync: Main loop exited')
for mod in self.mods:
if not mod.torrent_handle.is_valid():
self.result_queue.reject({
'details':
'Mod {} torrent handle is invalid'.format(mod.foldername)
})
sync_success = False
continue
self.save_resume_data(mod)
self.log_torrent_progress(mod.status, mod.foldername)
if mod.status.error:
self.result_queue.reject({
'details':
'An error occured: Libtorrent error: {}'.format(
decode_utf8(mod.status.error))
})
sync_success = False
return sync_success
def get_mod_torrent_metadata(self, mod, metadata_file):
"""Retrieve torrent metadata either from the metadata_file or from associated the file, if not present.
return torrent_info, torrent_contents
torrent_contents may be None. If that's the case, don't cache it.
"""
torrent_info = None
torrent_content = metadata_file.get_torrent_content()
if torrent_content:
try:
torrent_info = torrent_utils.get_torrent_info_from_bytestring(
torrent_content)
except RuntimeError as ex: # Raised by libtorrent.torrent_info()
error_message = decode_utf8(ex.args[0])
Logger.error(
'TorrentSyncer: could not parse torrent cached metadata: {}'
.format(error_message))
# If no cached torrent metadata content, download it now and cache it
if not torrent_info:
if mod.torrent_url.startswith(
'file://'): # Local torrent from file
try:
torrent_info = self.get_torrent_info_from_file(
mod.torrent_url[len('file://'):])
except RuntimeError as ex: # Raised by libtorrent.torrent_info()
error_message = 'Could not parse local torrent metadata: {}'.format(
decode_utf8(ex.args[0]))
Logger.error('TorrentSyncer: {}'.format(error_message))
raise PrepareParametersException(error_message)
return torrent_info, None # Don't cache torrent_content
else: # Torrent from url
try:
Logger.info('TorrentSyncer: Fetching torrent: {}'.format(
mod.torrent_url))
res = requests_wrapper.download_url(None,
mod.torrent_url,
timeout=5)
except requests_wrapper.DownloadException as ex:
error_message = 'Downloading metadata: {}'.format(
ex.args[0])
raise PrepareParametersException(error_message)
if res.status_code == 404:
message = textwrap.dedent('''\
Torrent file could not be downloaded from the master server.
Reason: file not found on the server (HTTP 404).
This may be because the mods are updated on the server right now.
Please try again in a few minutes.
''')
raise PrepareParametersException(message)
elif res.status_code != 200:
message = textwrap.dedent('''\
Torrent file could not be downloaded from the master server.
HTTP error code: {}
Contact the master server owner to fix this issue.
'''.format(unicode(res.status_code)))
raise PrepareParametersException(message)
try:
torrent_content = res.content
torrent_info = torrent_utils.get_torrent_info_from_bytestring(
res.content)
except RuntimeError as ex: # Raised by libtorrent.torrent_info()
error_message = 'Could not parse torrent metadata: {}\nContact the master server owner to fix this issue.'.format(
decode_utf8(ex.args[0]))
Logger.error('TorrentSyncer: {}'.format(error_message))
raise PrepareParametersException(error_message)
return torrent_info, torrent_content