def _start(self, items):
""" Start some items if conditions are met.
"""
# TODO: Filter by a custom date field, for scheduled downloads starting at a certain time, or after a given delay
# TODO: Don't start anything more if download BW is used >= config threshold in %
# Check if anything more is ready to start downloading
startable = [i for i in items if self.config.startable.match(i)]
if not startable:
self.LOG.debug("Checked %d item(s), none startable" % (len(items),))
return
# TODO: sort by priority, then loaded time
# Stick to "start_at_once" parameter, unless "downloading_min" is violated
downloading = [i for i in items if self.config.downloading.match(i)]
start_now = max(self.config.start_at_once, self.config.downloading_min - len(downloading))
start_now = min(start_now, len(startable))
#down_traffic = sum(i.down for i in downloading)
##self.LOG.info("%d downloading, down %d" % (len(downloading), down_traffic))
# Start eligible items
for idx, item in enumerate(startable):
# Check if we reached 'start_now' in this run
if idx >= start_now:
self.LOG.debug("Only starting %d item(s) in this run, %d more could be downloading" % (
start_now, len(startable)-idx,))
break
# TODO: Prevent start of more torrents that can fit on the drive (taking "off" files into account)
# (restarts items that were stopped due to the "low_diskspace" schedule, and also avoids triggering it at all)
# Only check the other conditions when we have `downloading_min` covered
if len(downloading) < self.config.downloading_min:
self.LOG.debug("Catching up from %d to a minimum of %d downloading item(s)" % (
len(downloading), self.config.downloading_min))
else:
# Limit to the given maximum of downloading items
if len(downloading) >= self.config.downloading_max:
self.LOG.debug("Already downloading %d item(s) out of %d max, %d more could be downloading" % (
len(downloading), self.config.downloading_max, len(startable)-idx,))
break
# If we made it here, start it!
downloading.append(item)
self.LOG.info("%s '%s' [%s, #%s]" % (
"WOULD start" if self.config.dry_run else "Starting",
fmt.to_utf8(item.name), item.alias, item.hash))
if not self.config.dry_run:
item.start()
if not self.config.quiet:
self.proxy.log('', "%s: Started '%s' {%s}" % (
self.__class__.__name__, fmt.to_utf8(item.name), item.alias,
))
def mainloop(self):
""" The main loop.
"""
# Print usage if not enough args
if len(self.args) < 2:
self.parser.print_help()
self.parser.exit()
# TODO: Add mode to move tied metafiles, without losing the tie
# Target handling
target = self.args[-1]
if "//" in target.rstrip('/'):
# Create parts of target path
existing, _ = target.split("//", 1)
if not os.path.isdir(existing):
self.fatal("Path before '//' MUST exists in %s" % (pretty_path(target),))
# Possibly create the rest
target = target.replace("//", "/")
if not os.path.exists(target):
self.guarded(os.makedirs, target)
# Preparation
# TODO: Handle cases where target is the original download path correctly!
# i.e. rtmv foo/ foo AND rtmv foo/ . (in the download dir)
proxy = config.engine.open()
download_path = os.path.realpath(os.path.expanduser(proxy.get_directory().rstrip(os.sep)))
target = self.resolve_slashed(target)
source_paths = [self.resolve_slashed(i) for i in self.args[:-1]]
source_realpaths = [os.path.realpath(i) for i in source_paths]
source_items = defaultdict(list) # map of source path to item
items = list(config.engine.items(prefetch=self.PREFETCH_FIELDS))
# Validate source paths and find matching items
for item in items:
if not item.path:
continue
realpath = None
try:
realpath = os.path.realpath(item.path)
except (EnvironmentError, UnicodeError), exc:
self.LOG.warning("Cannot realpath %r (%s)" % (item.path, exc))
# Look if item matches a source path
# TODO: Handle download items nested into each other!
try:
path_idx = source_realpaths.index(realpath or fmt.to_utf8(item.path))
except ValueError:
continue
if realpath:
self.LOG.debug('Item path %s resolved to %s' % (pretty_path(item.path), pretty_path(realpath)))
self.LOG.debug('Found "%s" for %s' % (fmt.to_utf8(item.name), pretty_path(source_paths[path_idx])))
source_items[source_paths[path_idx]].append(item)
def load(self):
""" Load metafile into client.
"""
if not self.ns.info_hash and not self.parse():
return
self.addinfo()
# TODO: dry_run
try:
# TODO: Scrub metafile if requested
# Determine target state
start_it = self.job.config.load_mode.lower() in ("start", "started")
queue_it = self.job.config.queued
if "start" in self.ns.flags:
start_it = True
elif "load" in self.ns.flags:
start_it = False
if "queue" in self.ns.flags:
queue_it = True
# Load metafile into client
load_cmd = self.job.proxy.load.verbose
if queue_it:
if not start_it:
self.ns.commands.append("d.priority.set=0")
elif start_it:
load_cmd = self.job.proxy.load.start_verbose
self.job.LOG.debug("Templating values are:\n %s" % "\n ".join("%s=%s" % (key, repr(val))
for key, val in sorted(self.ns.items())
))
load_cmd(xmlrpc.NOHASH, self.ns.pathname, *tuple(self.ns.commands))
time.sleep(.05) # let things settle
# Announce new item
if not self.job.config.quiet:
msg = "%s: Loaded '%s' from '%s/'%s%s" % (
self.job.__class__.__name__,
fmt.to_utf8(self.job.proxy.d.name(self.ns.info_hash, fail_silently=True)),
os.path.dirname(self.ns.pathname).rstrip(os.sep),
" [queued]" if queue_it else "",
(" [startable]" if queue_it else " [started]") if start_it else " [normal]",
)
self.job.proxy.log(xmlrpc.NOHASH, msg)
# TODO: Evaluate fields and set client values
# TODO: Add metadata to tied file if requested
# TODO: Execute commands AFTER adding the item, with full templating
# Example: Labeling - add items to a persistent view, i.e. "postcmd = view.set_visible={{label}}"
# could also be done automatically from the path, see above under "flags" (autolabel = True)
# and add traits to the flags, too, in that case
except xmlrpc.ERRORS as exc:
self.job.LOG.error("While loading #%s: %s" % (self.ns.info_hash, exc))
def load(self):
""" Load metafile into client.
"""
if not self.ns.info_hash and not self.parse():
return
self.addinfo()
# TODO: dry_run
try:
# TODO: Scrub metafile if requested
# Determine target state
start_it = self.job.config.load_mode.lower() in ("start", "started")
queue_it = self.job.config.queued
if "start" in self.ns.flags:
start_it = True
elif "load" in self.ns.flags:
start_it = False
if "queue" in self.ns.flags:
queue_it = True
# Load metafile into client
load_cmd = self.job.proxy.load_verbose
if queue_it:
if not start_it:
self.ns.commands.append("d.set_priority=0")
elif start_it:
load_cmd = self.job.proxy.load_start_verbose
self.job.LOG.debug("Templating values are:\n %s" % "\n ".join("%s=%s" % (key, repr(val))
for key, val in sorted(self.ns.items())
))
load_cmd(self.ns.pathname, *tuple(self.ns.commands))
time.sleep(.05) # let things settle
# Announce new item
if not self.job.config.quiet:
msg = "%s: Loaded '%s' from '%s/'%s%s" % (
self.job.__class__.__name__,
fmt.to_utf8(self.job.proxy.d.get_name(self.ns.info_hash, fail_silently=True)),
os.path.dirname(self.ns.pathname).rstrip(os.sep),
" [queued]" if queue_it else "",
(" [startable]" if queue_it else " [started]") if start_it else " [normal]",
)
self.job.proxy.log('', msg)
# TODO: Evaluate fields and set client values
# TODO: Add metadata to tied file if requested
# TODO: Execute commands AFTER adding the item, with full templating
# Example: Labeling - add items to a persistent view, i.e. "postcmd = view.set_visible={{label}}"
# could also be done automatically from the path, see above under "flags" (autolabel = True)
# and add traits to the flags, too, in that case
except xmlrpc.ERRORS, exc:
self.job.LOG.error("While loading #%s: %s" % (self.ns.info_hash, exc))
def pretty_path(path):
""" Prettify path for logging.
"""
path = fmt.to_utf8(path)
home_dir = os.path.expanduser("~")
if path.startswith(home_dir):
path = "~" + path[len(home_dir):]
return '"%s"' % (path,)
def assign_fields(meta, assignments, options_debug=False):
""" Takes a list of C{key=value} strings and assigns them to the
given metafile. If you want to set nested keys (e.g. "info.source"),
you have to use a dot as a separator. For exotic keys *containing*
a dot, double that dot ("dotted..key").
Numeric values starting with "+" or "-" are converted to integers.
If just a key name is given (no '='), the field is removed.
"""
for assignment in assignments:
assignment = fmt.to_unicode(assignment)
try:
if '=' in assignment:
field, val = assignment.split('=', 1)
else:
field, val = assignment, None
if val and val[0] in "+-" and val[1:].isdigit():
val = int(val, 10)
# TODO: Allow numerical indices, and "+" for append
namespace = meta
keypath = [
i.replace('\0', '.')
for i in field.replace('..', '\0').split('.')
]
for key in keypath[:-1]:
# Create missing dicts as we go...
namespace = namespace.setdefault(fmt.to_utf8(key), {})
except (KeyError, IndexError, TypeError, ValueError) as exc:
if options_debug:
raise
raise error.UserError("Bad assignment %r (%s)!" %
(assignment, exc))
else:
if val is None:
del namespace[fmt.to_utf8(keypath[-1])]
else:
namespace[fmt.to_utf8(keypath[-1])] = fmt.to_utf8(val)
return meta
def add_fast_resume(meta, datapath):
""" Add fast resume data to a metafile dict.
"""
# Get list of files
files = meta["info"].get("files", None)
single = files is None
if single:
if os.path.isdir(datapath):
datapath = os.path.join(datapath, meta["info"]["name"])
files = [
Bunch(
path=[os.path.abspath(datapath)],
length=meta["info"]["length"],
)
]
# Prepare resume data
resume = meta.setdefault("libtorrent_resume", {})
resume["bitfield"] = len(meta["info"]["pieces"]) // 20
resume["files"] = []
piece_length = meta["info"]["piece length"]
offset = 0
for fileinfo in files:
# Get the path into the filesystem
filepath = os.sep.join(fileinfo["path"])
if not single:
filepath = os.path.join(datapath,
fmt.to_utf8(filepath.strip(os.sep)))
# Check file size
if os.path.getsize(filepath) != fileinfo["length"]:
raise OSError(
errno.EINVAL,
"File size mismatch for %r [is %d, expected %d]" % (
filepath,
os.path.getsize(filepath),
fileinfo["length"],
))
# Add resume data for this file
resume["files"].append(
dict(
priority=1,
mtime=int(os.path.getmtime(filepath)),
completed=(offset + fileinfo["length"] + piece_length - 1) //
piece_length - offset // piece_length,
))
offset += fileinfo["length"]
return meta
def parse(self, conditions):
""" Parse filter conditions.
@param conditions: multiple conditions.
@type conditions: list or str
"""
conditions_text = conditions
try:
conditions = shlex.split(fmt.to_utf8(conditions))
except AttributeError:
# Not a string, assume parsed tree
conditions_text = self._tree2str(conditions)
# Empty list?
if not conditions:
raise FilterError("No conditions given at all!")
# NOT *must* appear at the start of a group
negate = conditions[:1] == ["NOT"]
if negate:
conditions = conditions[1:]
if not conditions:
raise FilterError("NOT must be followed by some conditions!")
# Handle grouping
if '[' in conditions:
tree = [[]]
for term in conditions:
if term == '[':
tree.append([]) # new grouping
elif term == ']':
subtree = tree.pop()
if not tree:
raise FilterError(
"Unbalanced brackets, too many closing ']' in condition %r"
% (conditions_text, ))
tree[-1].append(
subtree) # append finished group to containing level
else:
tree[-1].append(term) # append to current level
if len(tree) > 1:
raise FilterError(
"Unbalanced brackets, too many open '[' in condition %r" %
(conditions_text, ))
conditions = tree[0]
# Prepare root matcher
conditions = list(conditions)
matcher = CompoundFilterAll()
if "OR" in conditions:
root = CompoundFilterAny()
root.append(matcher)
else:
root = matcher
# Go through conditions and parse them
for condition in conditions:
if condition == "OR":
# Leading OR, or OR OR in sequence?
if not matcher:
raise FilterError("Left-hand side of OR missing in %r!" %
(conditions_text, ))
# Start next run of AND conditions
matcher = CompoundFilterAll()
root.append(matcher)
elif isinstance(condition, list):
matcher.append(self.parse(condition))
else:
matcher.append(self._create_filter(condition))
# Trailing OR?
if not matcher:
raise FilterError("Right-hand side of OR missing in %r!" %
(conditions_text, ))
return NegateFilter(root) if negate else root
def __str__(self):
return fmt.to_utf8("%s=%s" % (self._name, self._condition))
def mainloop(self):
""" The main loop.
"""
if not self.args:
self.parser.print_help()
self.parser.exit()
for idx, filename in enumerate(self.args):
torrent = metafile.Metafile(filename)
if idx and not self.options.output:
print('')
print("~" * 79)
try:
# Read and check metafile
try:
data = metafile.checked_open(
filename,
log=self.LOG if self.options.skip_validation else None,
quiet=(self.options.quiet
and (self.options.output or self.options.raw)))
except EnvironmentError as exc:
self.fatal("Can't read '%s' (%s)" % (
filename,
str(exc).replace(": '%s'" % filename, ""),
))
raise
listing = None
if self.options.raw or self.options.json:
if not self.options.reveal and "info" in data:
# Shorten useless binary piece hashes
data["info"]["pieces"] = "<%d piece hashes>" % (
len(data["info"]["pieces"]) /
len(hashlib.sha1().digest()) # bogus pylint: disable=E1101
)
if self.options.json:
listing = json.dumps(data,
default=repr,
indent=4,
sort_keys=True)
else:
pprinter = (pprint.PrettyPrinter if self.options.reveal
else metafile.MaskingPrettyPrinter)()
listing = pprinter.pformat(data)
elif self.options.output:
def splitter(fields):
"Yield single names for a list of comma-separated strings."
for flist in fields:
for field in flist.split(','):
yield field.strip()
data["__file__"] = filename
if 'info' in data:
data["__hash__"] = metafile.info_hash(data)
data["__size__"] = metafile.data_size(data)
values = []
for field in splitter(self.options.output):
try:
val = data
for key in field.split('.'):
val = val[key]
except KeyError as exc:
self.LOG.error("%s: Field %r not found (%s)" %
(filename, field, exc))
break
else:
values.append(str(val))
else:
listing = '\t'.join(values)
else:
listing = '\n'.join(
torrent.listing(masked=not self.options.reveal))
except (ValueError, KeyError, bencode.BencodeError) as exc:
if self.options.debug:
raise
self.LOG.warning("Bad metafile %r (%s: %s)" %
(filename, type(exc).__name__, exc))
else:
if listing is not None:
print(fmt.to_utf8(listing))
def _start(self, items):
""" Start some items if conditions are met.
"""
# TODO: Filter by a custom date field, for scheduled downloads starting at a certain time, or after a given delay
# TODO: Don't start anything more if download BW is used >= config threshold in %
# Check if anything more is ready to start downloading
startable = [i for i in items if self.config.startable.match(i)]
if not startable:
self.LOG.debug(
"Checked %d item(s), none startable according to [ %s ]",
len(items), self.config.startable)
return
# Check intermission delay
now = time.time()
if now < self.last_start:
# compensate for summer time and other oddities
self.last_start = now
delayed = int(self.last_start + self.config.intermission - now)
if delayed > 0:
self.LOG.debug("Delaying start of {:d} item(s),"
" due to {:d}s intermission with {:d}s left".format(
len(startable), self.config.intermission,
delayed))
return
# TODO: sort by priority, then loaded time
# Stick to "start_at_once" parameter, unless "downloading_min" is violated
downloading = [i for i in items if self.config.downloading.match(i)]
start_now = max(self.config.start_at_once,
self.config.downloading_min - len(downloading))
start_now = min(start_now, len(startable))
#down_traffic = sum(i.down for i in downloading)
##self.LOG.info("%d downloading, down %d" % (len(downloading), down_traffic))
# Start eligible items
for idx, item in enumerate(startable):
# Check if we reached 'start_now' in this run
if idx >= start_now:
self.LOG.debug(
"Only starting %d item(s) in this run, %d more could be downloading"
% (
start_now,
len(startable) - idx,
))
break
# TODO: Prevent start of more torrents that can fit on the drive (taking "off" files into account)
# (restarts items that were stopped due to the "low_diskspace" schedule, and also avoids triggering it at all)
# Only check the other conditions when we have `downloading_min` covered
if len(downloading) < self.config.downloading_min:
self.LOG.debug(
"Catching up from %d to a minimum of %d downloading item(s)"
% (len(downloading), self.config.downloading_min))
else:
# Limit to the given maximum of downloading items
if len(downloading) >= self.config.downloading_max:
self.LOG.debug(
"Already downloading %d item(s) out of %d max, %d more could be downloading"
% (
len(downloading),
self.config.downloading_max,
len(startable) - idx,
))
break
# If we made it here, start it!
self.last_start = now
downloading.append(item)
self.LOG.info(
"%s '%s' [%s, #%s]" %
("WOULD start" if self.config.dry_run else "Starting",
fmt.to_utf8(item.name), item.alias, item.hash))
if not self.config.dry_run:
item.start()
if not self.config.quiet:
self.proxy.log(
xmlrpc.NOHASH, "%s: Started '%s' {%s}" % (
self.__class__.__name__,
fmt.to_utf8(item.name),
item.alias,
))
def parse(self, conditions):
""" Parse filter conditions.
@param conditions: multiple conditions.
@type conditions: list or str
"""
conditions_text = conditions
try:
conditions = shlex.split(fmt.to_utf8(conditions))
except AttributeError:
# Not a string, assume parsed tree
conditions_text = self._tree2str(conditions)
# Empty list?
if not conditions:
raise FilterError("No conditions given at all!")
# NOT *must* appear at the start of a group
negate = conditions[:1] == ["NOT"]
if negate:
conditions = conditions[1:]
if not conditions:
raise FilterError("NOT must be followed by some conditions!")
# Handle grouping
if '[' in conditions:
tree = [[]]
for term in conditions:
if term == '[':
tree.append([]) # new grouping
elif term == ']':
subtree = tree.pop()
if not tree:
raise FilterError("Unbalanced brackets, too many closing ']' in condition %r" % (conditions_text,))
tree[-1].append(subtree) # append finished group to containing level
else:
tree[-1].append(term) # append to current level
if len(tree) > 1:
raise FilterError("Unbalanced brackets, too many open '[' in condition %r" % (conditions_text,))
conditions = tree[0]
# Prepare root matcher
conditions = list(conditions)
matcher = CompoundFilterAll()
if "OR" in conditions:
root = CompoundFilterAny()
root.append(matcher)
else:
root = matcher
# Go through conditions and parse them
for condition in conditions:
if condition == "OR":
# Leading OR, or OR OR in sequence?
if not matcher:
raise FilterError("Left-hand side of OR missing in %r!" % (conditions_text,))
# Start next run of AND conditions
matcher = CompoundFilterAll()
root.append(matcher)
elif isinstance(condition, list):
matcher.append(self.parse(condition))
else:
matcher.append(self._create_filter(condition))
# Trailing OR?
if not matcher:
raise FilterError("Right-hand side of OR missing in %r!" % (conditions_text,))
return NegateFilter(root) if negate else root
except KeyError, exc:
self.LOG.error("%s: Field %r not found (%s)" %
(filename, field, exc))
break
else:
values.append(str(val))
else:
listing = '\t'.join(values)
else:
listing = '\n'.join(
torrent.listing(masked=not self.options.reveal))
except (ValueError, KeyError, bencode.BencodeError), exc:
if self.options.debug:
raise
self.LOG.warning("Bad metafile %r (%s: %s)" %
(filename, type(exc).__name__, exc))
else:
if listing is not None:
print(fmt.to_utf8(listing))
def run(): #pragma: no cover
""" The entry point.
"""
ScriptBase.setup()
MetafileLister().run()
if __name__ == "__main__":
run()
def __init__(self, text):
shlex.shlex.__init__(self, fmt.to_utf8(text), None, True)
self.whitespace += ','
self.whitespace_split = True
self.commenters = ''
def mainloop(self):
""" The main loop.
"""
# Print usage if not enough args
if len(self.args) < 2:
self.parser.print_help()
self.parser.exit()
# TODO: Add mode to move tied metafiles, without losing the tie
# Target handling
target = self.args[-1]
if "//" in target.rstrip('/'):
# Create parts of target path
existing, _ = target.split("//", 1)
if not os.path.isdir(existing):
self.fatal("Path before '//' MUST exists in %s" %
(pretty_path(target), ))
# Possibly create the rest
target = target.replace("//", "/")
if not os.path.exists(target):
self.guarded(os.makedirs, target)
# Preparation
# TODO: Handle cases where target is the original download path correctly!
# i.e. rtmv foo/ foo AND rtmv foo/ . (in the download dir)
proxy = config.engine.open()
download_path = os.path.realpath(
os.path.expanduser(
proxy.directory.default(xmlrpc.NOHASH).rstrip(os.sep)))
target = self.resolve_slashed(target)
source_paths = [self.resolve_slashed(i) for i in self.args[:-1]]
source_realpaths = [os.path.realpath(i) for i in source_paths]
source_items = defaultdict(list) # map of source path to item
items = list(config.engine.items(prefetch=self.PREFETCH_FIELDS))
# Validate source paths and find matching items
for item in items:
if not item.path:
continue
realpath = None
try:
realpath = os.path.realpath(item.path)
except (EnvironmentError, UnicodeError) as exc:
self.LOG.warning("Cannot realpath %r (%s)" % (item.path, exc))
# Look if item matches a source path
# TODO: Handle download items nested into each other!
try:
path_idx = source_realpaths.index(realpath
or fmt.to_utf8(item.path))
except ValueError:
continue
if realpath:
self.LOG.debug('Item path %s resolved to %s' %
(pretty_path(item.path), pretty_path(realpath)))
self.LOG.debug(
'Found "%s" for %s' %
(fmt.to_utf8(item.name), pretty_path(source_paths[path_idx])))
source_items[source_paths[path_idx]].append(item)
##for path in source_paths: print path, "==>"; print " " + "\n ".join(i.path for i in source_items[path])
if not os.path.isdir(target) and len(source_paths) > 1:
self.fatal(
"Can't move multiple files to %s which is no directory!" %
(pretty_path(target), ))
# Actually move the data
moved_count = 0
for path in source_paths:
item = None # Make sure there's no accidental stale reference
if not source_items[path]:
self.LOG.warn("No download item found for %s, skipping!" %
(pretty_path(path), ))
continue
if len(source_items[path]) > 1:
self.LOG.warn(
"Can't handle multi-item moving yet, skipping %s!" %
(pretty_path(path), ))
continue
if os.path.islink(path):
self.LOG.warn("Won't move symlinks, skipping %s!" %
(pretty_path(path), ))
continue
for item in source_items[path]:
if os.path.islink(item.path) and os.path.realpath(
item.path) != os.readlink(item.path):
self.LOG.warn(
"Can't handle multi-hop symlinks yet, skipping %s!" %
(pretty_path(path), ))
continue
if not item.is_complete:
if self.options.force_incomplete:
self.LOG.warn("Moving incomplete item '%s'!" %
(item.name, ))
else:
self.LOG.warn("Won't move incomplete item '%s'!" %
(item.name, ))
continue
moved_count += 1
dst = target
if os.path.isdir(dst):
dst = os.path.join(dst, os.path.basename(path))
self.LOG.info("Moving to %s..." % (pretty_path(dst), ))
# Pause torrent?
# was_active = item.is_active and not self.options.dry_run
# if was_active: item.pause()
# TODO: move across devices
# TODO: move using "d.directory.set" instead of symlinks
if os.path.islink(item.path):
if os.path.abspath(dst) == os.path.abspath(
item.path.rstrip(os.sep)):
# Moving back to original place
self.LOG.debug("Unlinking %s" %
(pretty_path(item.path), ))
self.guarded(os.remove, item.path)
self.guarded(os.rename, path, dst)
else:
# Moving to another place
self.LOG.debug("Re-linking %s" %
(pretty_path(item.path), ))
self.guarded(os.rename, path, dst)
self.guarded(os.remove, item.path)
self.guarded(os.symlink, os.path.abspath(dst),
item.path)
else:
# Moving download initially
self.LOG.debug("Symlinking %s" %
(pretty_path(item.path), ))
src1, src2 = os.path.join(download_path,
os.path.basename(
item.path)), fmt.to_unicode(
os.path.realpath(path))
assert src1 == src2, 'Item path %r should match %r!' % (
src1, src2)
self.guarded(os.rename, item.path, dst)
self.guarded(os.symlink, os.path.abspath(dst), item.path)
# Resume torrent?
# if was_active: sitem.resume()
# Print stats
self.LOG.debug("XMLRPC stats: %s" % proxy)
self.LOG.log(
logging.DEBUG if self.options.cron else logging.INFO,
"Moved %d path%s (skipped %d)" %
(moved_count, "" if moved_count == 1 else "s",
len(source_paths) - moved_count))
def _make_info(self, piece_size, progress, walker, piece_callback=None):
""" Create info dict.
"""
# These collect the file descriptions and piece hashes
file_list = []
pieces = []
# Initialize progress state
hashing_secs = time.time()
totalsize = -1 if self._fifo else self._calc_size()
totalhashed = 0
# Start a new piece
sha1sum = hashlib.sha1()
done = 0
filename = None
# Hash all files
for filename in walker:
# Assemble file info
filesize = os.path.getsize(filename)
filepath = filename[len(
os.path.dirname(self.datapath) if self._fifo else self.datapath
):].lstrip(os.sep)
file_list.append({
"length":
filesize,
"path": [
fmt.to_utf8(x) for x in fmt.to_unicode(filepath).replace(
os.sep, '/').split('/')
],
})
self.LOG.debug("Hashing %r, size %d..." % (filename, filesize))
# Open file and hash it
fileoffset = 0
handle = open(filename, "rb")
try:
while fileoffset < filesize:
# Read rest of piece or file, whatever is smaller
chunk = handle.read(
min(filesize - fileoffset, piece_size - done))
sha1sum.update(chunk) # bogus pylint: disable=E1101
done += len(chunk)
fileoffset += len(chunk)
totalhashed += len(chunk)
# Piece is done
if done == piece_size:
pieces.append(sha1sum.digest()) # bogus pylint: disable=E1101
if piece_callback:
piece_callback(filename, pieces[-1])
# Start a new piece
sha1sum = hashlib.sha1()
done = 0
# Report progress
if progress:
progress(totalhashed, totalsize)
finally:
handle.close()
# Add hash of partial last piece
if done > 0:
pieces.append(sha1sum.digest()) # bogus pylint: disable=E1103
if piece_callback:
piece_callback(filename, pieces[-1])
# Build the meta dict
metainfo = {
"pieces": b"".join(pieces),
"piece length": piece_size,
"name": os.path.basename(self.datapath),
}
# Handle directory/FIFO vs. single file
if self._fifo or os.path.isdir(self.datapath):
metainfo["files"] = file_list
else:
metainfo["length"] = totalhashed
hashing_secs = time.time() - hashing_secs
self.LOG.info("Hashing of %s took %.1f secs (%s/s)" % (
fmt.human_size(totalhashed).strip(),
hashing_secs,
fmt.human_size(totalhashed / hashing_secs).strip(),
))
# Return validated info dict
return check_info(metainfo), totalhashed
def __str__(self):
return fmt.to_utf8("%s=%s" % (self._name, self._condition))
