def __init__(self, chain):
self._chain = chain
self.d = defer.DeferredList([p.d for p in chain])
def getRFC(n=10, m=3):
"""An example of spawnDeferredWorkers for getting a list a RFC.
n specifies how many RFC (starting from 1) to download.
m specifies how many downloads to run concurrently.
"""
from zope.interface import implements
from twisted.web import client
from twisted.internet import reactor
import time
class HTTPClientWorker(client.HTTPDownloader):
"""
Adaptation of client.HTTPDownloader to IWorker interface
"""
implements(IWorker)
def __init__(self, *args, **kwargs):
client.HTTPDownloader.__init__(self, *args, **kwargs)
def startWork(self):
print("starting downloading %s..." % self.url)
reactor.connectTCP(self.host, self.port, self)
# callback functions
def gotPageList(result):
pages = len(result)
errors = len([item for item in result if not item[0]])
print("\n\ngot %d pages with %d errors" % (pages, errors))
end = time.clock()
print("%d secs elapsed" % (end - start))
reactor.stop()
def savePage(page, worker):
print("got %s" % worker.url)
def error(reason, worker):
print("failed to download %s" % worker.url)
print(reason.value.__class__, reason.value)
return reason
workerList = [
HTTPClientWorker("http://www.ietf.org/rfc/rfc%d.txt" % rfc,
"rfc%d.txt" % rfc) for rfc in range(1, n + 1)
]
deferredList = spawnDeferredWorkers(workerList, m)
for deferred, worker in zip(deferredList, workerList):
deferred.addCallback(savePage, worker).addErrback(error, worker)
deferred = defer.DeferredList(deferredList, consumeErrors=True)
deferred.addCallback(gotPageList)
start = time.clock()
reactor.run()
def generateMeasurements(self):
"""
This is a generator that yields measurements and registers the
callbacks for when a measurement is successful or has failed.
FIXME: If this generator throws exception TaskManager scheduler is
irreversibly damaged.
"""
for test_class, test_methods in self.testCases:
# load a singular input processor for all instances
all_inputs = test_class.inputs
for test_input in all_inputs:
measurements = []
test_instance = test_class()
# Set each instances inputs to a singular input processor
test_instance.inputs = all_inputs
test_instance._setUp()
test_instance.summary = self.summary
for method in test_methods:
try:
measurement = self.makeMeasurement(
test_instance, method, test_input)
except Exception:
log.exception(failure.Failure())
log.err('Failed to run %s %s %s' %
(test_instance, method, test_input))
continue # it's better to skip single measurement...
log.debug("Running %s %s" % (test_instance, method))
measurements.append(measurement.done)
self.state.taskCreated()
yield measurement
# This is to skip setting callbacks on measurements that
# cannot be run.
if len(measurements) == 0:
continue
# When the measurement.done callbacks have all fired
# call the postProcessor before writing the report
if self.report:
post = defer.DeferredList(measurements)
@post.addBoth
def set_runtime(results):
runtime = time.time() - test_instance._start_time
for _, m in results:
m.testInstance.report['test_runtime'] = runtime
test_instance.report['test_runtime'] = runtime
return results
# Call the postProcessor, which must return a single report
# or a deferred
post.addCallback(test_instance.postProcessor)
def noPostProcessor(failure, report):
failure.trap(e.NoPostProcessor)
return report
post.addErrback(noPostProcessor, test_instance.report)
post.addCallback(self.report.write)
if self.report and self.director:
# ghetto hax to keep NetTestState counts are accurate
[post.addBoth(self.doneReport) for _ in measurements]
self.state.allTasksScheduled()
config = Options()
try:
config.parseOptions()
except usage.UsageError, errorText:
print "%s: %s" % (sys.argv[0], errorText)
print "%s: Try --help for usage details." % sys.argv[0]
sys.exit(1)
fmt = config["format"]
if fmt not in ("csv", "xml", "json"):
fmt = "json"
report.requests = config["requests"]
report.show_errmsg = config["errmsg"]
tasks = []
coop = task.Cooperator()
work = randrequests(fmt, report.requests)
for n in xrange(config["concurrent"]):
d = coop.coiterate(work)
tasks.append(d)
dl = defer.DeferredList(tasks)
dl.addCallback(lambda ign: report.summary())
dl.addCallback(lambda ign: reactor.stop())
if __name__ == "__main__":
main()
reactor.run()
def _process_changes(self, newRev, branch):
"""
Read changes since last change.
- Read list of commit hashes.
- Extract details from each commit.
- Add changes to database.
"""
# initial run, don't parse all history
if not self.lastRev:
return
# get the change list
revListArgs = (['--format=%H', '{}'.format(newRev)] +
['^' + rev
for rev in sorted(self.lastRev.values())] + ['--'])
self.changeCount = 0
results = yield self._dovccmd('log', revListArgs, path=self.workdir)
# process oldest change first
revList = results.split()
revList.reverse()
if self.buildPushesWithNoCommits and not revList:
existingRev = self.lastRev.get(branch)
if existingRev != newRev:
revList = [newRev]
if existingRev is None:
# This branch was completely unknown, rebuild
log.msg(
'gitpoller: rebuilding {} for new branch "{}"'.format(
newRev, branch))
else:
# This branch is known, but it now points to a different
# commit than last time we saw it, rebuild.
log.msg('gitpoller: rebuilding {} for updated branch "{}"'.
format(newRev, branch))
self.changeCount = len(revList)
self.lastRev[branch] = newRev
if self.changeCount:
log.msg(
'gitpoller: processing {} changes: {} from "{}" branch "{}"'.
format(self.changeCount, revList, self.repourl, branch))
for rev in revList:
dl = defer.DeferredList([
self._get_commit_timestamp(rev),
self._get_commit_author(rev),
self._get_commit_files(rev),
self._get_commit_comments(rev),
],
consumeErrors=True)
results = yield dl
# check for failures
failures = [r[1] for r in results if not r[0]]
if failures:
for failure in failures:
log.err(
failure, "while processing changes for {} {}".format(
newRev, branch))
# just fail on the first error; they're probably all related!
failures[0].raiseException()
timestamp, author, files, comments = [r[1] for r in results]
yield self.master.data.updates.addChange(
author=author,
revision=bytes2unicode(rev, encoding=self.encoding),
files=files,
comments=comments,
when_timestamp=timestamp,
branch=bytes2unicode(self._removeHeads(branch)),
project=self.project,
repository=bytes2unicode(self.repourl, encoding=self.encoding),
category=self.category,
src='git')
def buildMessage(self, builder_name, build_status, results, step_name):
"""Send an email about the tree closing.
Don't attach the patch as MailNotifier.buildMessage do.
@type builder_name: string
@type build_status: L{buildbot.status.builder.BuildStatus}
@type step_name: name of this step
"""
# TODO(maruel): Update function signature to match
# mail.MailNotifier.buildMessage().
log.msg('About to email')
if (self._last_time_mail_sent and self._last_time_mail_sent >
time.time() - self.minimum_delay_between_alert):
# Rate limit tree alerts.
return
self._last_time_mail_sent = time.time()
# TODO(maruel): Use self.createEmail().
blame_interested_users = self.shouldBlameCommitters(step_name)
project_name = self.master_status.getTitle()
revisions_list = build_utils.getAllRevisions(build_status)
build_url = self.master_status.getURLForThing(build_status)
waterfall_url = self.master_status.getBuildbotURL()
status_text = self.status_header % {
'builder': builder_name,
'steps': step_name,
}
# Use the first line as a title.
status_title = status_text.split('\n', 1)[0]
blame_list = ','.join(build_status.getResponsibleUsers())
revisions_string = ''
latest_revision = 0
if revisions_list:
revisions_string = ', '.join([str(rev) for rev in revisions_list])
latest_revision = max([rev for rev in revisions_list])
if results[0] == FAILURE:
result = 'failure'
else:
result = 'warning'
# Generate a HTML table looking like the waterfall.
# WARNING: Gmail ignores embedded CSS style. I don't know how to fix that so
# meanwhile, I just won't embedded the CSS style.
html_content = (
"""<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<title>%s</title>
</head>
<body>
<a href="%s">%s</a><p>
%s<p>
<a href="%s">%s</a><p>
Revision: %s<br>
""" % (status_title, waterfall_url, waterfall_url,
status_text.replace('\n',
"<br>\n"), build_url, build_url, revisions_string))
# Only include the blame list if relevant.
if blame_interested_users:
html_content += " Blame list: %s<p>\n" % blame_list
html_content += build_utils.EmailableBuildTable(
build_status, waterfall_url)
html_content += "<p>"
# Add the change list descriptions. getChanges() returns a tuple of
# buildbot.changes.changes.Change
for change in build_status.getChanges():
html_content += change.asHTML()
html_content += "</body>\n</html>"
# Simpler text content for non-html aware clients.
text_content = ("""%s
%s
%swaterfall?builder=%s
--=> %s <=--
Revision: %s
Blame list: %s
Buildbot waterfall: http://build.chromium.org/
""" % (status_title, build_url, urllib.quote(waterfall_url, '/:'),
urllib.quote(builder_name), status_text, revisions_string, blame_list))
m = MIMEMultipart('alternative')
# The HTML message, is best and preferred.
m.attach(MIMEText(text_content, 'plain', 'iso-8859-1'))
m.attach(MIMEText(html_content, 'html', 'iso-8859-1'))
m['Date'] = formatdate(localtime=True)
m['Subject'] = self.subject % {
'result': result,
'projectName': project_name,
'builder': builder_name,
'reason': build_status.getReason(),
'revision': str(latest_revision),
}
m['From'] = self.fromaddr
if self.reply_to:
m['Reply-To'] = self.reply_to
recipients = list(self.extraRecipients[:])
if self.sheriffs:
recipients.extend(
BuildSheriffs.GetSheriffs(classes=self.sheriffs,
data_dir=self.public_html))
dl = []
if self.sendToInterestedUsers and self.lookup and blame_interested_users:
for u in build_status.getInterestedUsers():
d = defer.maybeDeferred(self.lookup.getAddress, u)
d.addCallback(recipients.append)
dl.append(d)
defered_object = defer.DeferredList(dl)
defered_object.addCallback(self._gotRecipients, recipients, m)
defered_object.addCallback(self.getFinishedMessage, builder_name,
build_status, step_name)
return defered_object
return
@defer.inlineCallbacks
def crawl(self, spider):
start_request = iter(spider.start_request())
while True:
try:
request = next(start_request)
Q.put(request)
except StopIteration as e:
break
self._next_request()
self._close = defer.Deferred()
yield self._close
_visited = set()
spider = ChoutiSpider()
engine = Engine()
d = engine.crawl(spider)
_visited.add(d)
dd = defer.DeferredList(_visited)
dd.addBoth(lambda _: reactor.stop())
reactor.run()
def politeShutdown(self):
"""Stop inventory on all connected readers."""
protoDeferreds = []
for proto in self.protocols:
protoDeferreds.append(proto.stopPolitely(disconnect=True))
return defer.DeferredList(protoDeferreds)
def tearDown(self):
d = defer.DeferredList([s.stopService() for s in self.services])
d.addCallback(flushEventualQueue)
return d
def get_trac_tickets(start_day, end_day):
"""
Get the last week's worth of tickets, where week ends through yesterday.
"""
def format_trac_tickets(what, body):
tickets = csv.reader(body.splitlines(), delimiter='\t')
# Trac returns a tab-delimited file with the header. Skip it.
next(tickets)
# Returned format is id, summary, type.
summary = [{
'id': t[0],
'summary': t[1],
'type': t[2],
'url': TRAC_BUILDBOT_TICKET_URL % {
'ticket': t[0]
}
} for t in tickets]
return (what, summary)
def summarize_trac_tickets(results):
opened = {}
closed = {}
for success, value in results:
if not success:
continue
what, tickets = value
for t in tickets:
if what == 'Opened':
opened[len(opened)] = t
elif what == 'Closed':
closed[len(closed)] = t
# Convert ticket summary to a table to start the weekly summary.
# Also include a list of every new/reopened and closed tickets.
col_order = ['id', 'type', 'summary']
# Left-justify every cell except the first column. Return None for the
# first column to have it skipped.
opened_table = tablify_dict(opened,
show_header=False,
col_order=col_order,
link_field='id',
link_url_field='url')
opened_overview = '\n'.join(
['<h2>New/Reopened Tickets</h2>', opened_table])
closed_table = tablify_dict(closed,
show_header=False,
col_order=col_order,
link_field='id',
link_url_field='url')
closed_overview = '\n'.join(['<h2>Closed Tickets</h2>', closed_table])
trac_summary = [opened_overview, closed_overview]
return ('trac', '\n\n'.join(trac_summary))
trac_query_url = ('%(trac_url)s/query?%(status)s&format=tab'
'&%(time_arg)s=%(start)s..%(end)s'
'&col=id&col=summary&col=type&col=status&order=id')
url_options = {
'trac_url': TRAC_BUILDBOT_URL,
'start': start_day,
'end': end_day,
}
agent = Agent(reactor)
fetches = []
# Need to make two queries: one to get the new/reopened tickets and a
# second to get the closed tickets.
url_options['status'] = 'status=new&status=reopened'
url_options['time_arg'] = 'time'
new_url = trac_query_url % (url_options)
d = agent.request('GET', new_url, HTTP_HEADERS)
d.addCallback(get_body('Opened', format_trac_tickets))
fetches.append(d)
url_options['status'] = 'status=closed'
url_options['time_arg'] = 'changetime'
closed_url = trac_query_url % (url_options)
d = agent.request('GET', closed_url, HTTP_HEADERS)
d.addCallback(get_body('Closed', format_trac_tickets))
fetches.append(d)
dl = defer.DeferredList(fetches, fireOnOneErrback=True, consumeErrors=True)
dl.addCallback(summarize_trac_tickets)
return dl
def _stop_lbry_files(self):
log.info("Stopping %i lbry files", len(self.lbry_files))
yield defer.DeferredList([
self._stop_lbry_file(lbry_file)
for lbry_file in list(self.lbry_files)
])
def evaluate(self,
tick,
task,
inputs,
nosend_ports=None,
fail_on_unexpected_nosend=False):
"""
Evaluate the given task with the given inputs. The inputs are a dict
with port -> (value-id, worker) mapping.
:param nosend_ports: Set of output ports which must not be pickled.
:param fail_on_unexpected_nosend: Action if output ports that are not in `nosend_ports`
fail to pickle. If `True`, the evaluation is aborted with a :class:`NoPickleError`,
if `False` the operation proceeds as if the ports where in `nosend_ports`.
"""
ports = []
transfers = []
transfer_results = {}
for port, (valueid, worker) in inputs.iteritems():
d = self.fetch_from(worker, valueid)
def transfer_completed(transfer_result, valueid, port):
if transfer_result: # `None` if the value was already present
transfer_results[port] = transfer_result
return self.get_value(valueid)
d.addCallback(transfer_completed, valueid, port)
ports.append(port)
transfers.append(d)
d = defer.DeferredList(transfers)
def run(inputs):
"""
Runs in separate thread.
"""
start = time.clock()
try:
result = task.evaluate(inputs)
except:
result = failure.Failure()
finally:
end = time.clock()
return traverser.EvalResult(result, end - start)
@twistit.yieldefer
def got_all(results):
values = []
for success, result in results:
if not success:
if result.check(pickle.PickleError):
raise pickle.PickleError(
"Failed to unpickle input of %r.%r: %s" %
(tick, port, result))
else:
result.raiseException()
else:
values.append(result)
inputs = dict(zip(ports, values))
evalresult = yield threads.deferToThread(run, inputs)
if not isinstance(evalresult.result, dict) and not isinstance(
evalresult.result, failure.Failure):
raise ValueError(
"Evaluation of task %r did not produce a dict or a failure. Got %r."
% (task, evalresult.result))
defer.returnValue(evalresult)
def task_completed(evalresult):
if isinstance(evalresult.result, dict):
# Injest values into our store and replace the eval results with ValueIds.
outputs = evalresult.result
outs = {}
datasizes = {}
for port, value in outputs.iteritems():
valueid = ValueId(graph.Endpoint(tick, port))
pickle_supported = True
if nosend_ports and port in nosend_ports:
pickle_supported = False
try:
size = self.set_value(
valueid, value, pickle_supported, pickle_supported
and fail_on_unexpected_nosend)
except NoPickleError as e:
e = NoPickleError(
"Value of output port %r cannot be pickled." %
port,
cause=e.cause)
# TODO: memory leak. We should remove the values we've set in
# previous loop iterations.
raise e
outs[port] = valueid
if size is not None:
datasizes[port] = size
evalresult.result = outs
evalresult.datasizes = datasizes
evalresult.transfer_results = transfer_results
return evalresult
d.addCallback(got_all)
d.addCallback(task_completed)
return d
def _create_docs(results):
# create many small files
deferreds = []
for i in range(0, number_of_docs):
deferreds.append(sol.create_doc(json.loads(simple_doc)))
return defer.DeferredList(deferreds)
import random
from twisted.internet import reactor, task, defer
from twisted.python.util import println
delay = lambda: 1e-4 * random.random()
d = defer.DeferredList([
task.deferLater(reactor, delay(), println, line)
for line in 'Enjoy Rosetta Code'.split()
])
d.addBoth(lambda _: reactor.stop())
reactor.run()
def stopService(self):
res = defer.maybeDeferred(AbstractBuildSlave.stopService, self)
if self.conn is not None:
d = self._soft_disconnect()
res = defer.DeferredList([res, d])
return res
def TWISTEDDoIt():
def StartServices(*args, **kwargs):
HydrusData.Print('Starting services\u2026')
for service in services:
service_key = service.GetServiceKey()
service_type = service.GetServiceType()
try:
port = service.GetPort()
if service_type == HC.SERVER_ADMIN:
http_factory = ServerServer.HydrusServiceAdmin(
service)
elif service_type == HC.FILE_REPOSITORY:
http_factory = ServerServer.HydrusServiceRepositoryFile(
service)
elif service_type == HC.TAG_REPOSITORY:
http_factory = ServerServer.HydrusServiceRepositoryTag(
service)
else:
return
(ssl_cert_path, ssl_key_path) = self.db.GetSSLPaths()
sslmethod = twisted.internet.ssl.SSL.TLSv1_2_METHOD
context_factory = twisted.internet.ssl.DefaultOpenSSLContextFactory(
ssl_key_path, ssl_cert_path, sslmethod)
ipv6_port = None
try:
ipv6_port = reactor.listenSSL(port,
http_factory,
context_factory,
interface='::')
except Exception as e:
HydrusData.Print('Could not bind to IPv6:')
HydrusData.Print(str(e))
ipv4_port = None
try:
ipv4_port = reactor.listenSSL(
port, http_factory, context_factory)
except:
if ipv6_port is None:
raise
self._service_keys_to_connected_ports[service_key] = (
ipv4_port, ipv6_port)
if not HydrusNetworking.LocalPortInUse(port):
raise Exception(
'Tried to bind port {} for "{}" but it failed.'
.format(port, service.GetName()))
except Exception as e:
HydrusData.Print(traceback.format_exc())
HydrusData.Print('Services started')
if len(self._service_keys_to_connected_ports) > 0:
HydrusData.Print('Stopping services\u2026')
deferreds = []
for (ipv4_port, ipv6_port
) in self._service_keys_to_connected_ports.values():
if ipv4_port is not None:
deferred = defer.maybeDeferred(ipv4_port.stopListening)
deferreds.append(deferred)
if ipv6_port is not None:
deferred = defer.maybeDeferred(ipv6_port.stopListening)
deferreds.append(deferred)
self._service_keys_to_connected_ports = {}
deferred = defer.DeferredList(deferreds)
if len(services) > 0:
deferred.addCallback(StartServices)
elif len(services) > 0:
StartServices()
def __init__(self, server, **kwargs):
if server.coherence.config.get('use_dbus', 'no') != 'yes':
raise Exception(
'this backend needs use_dbus enabled in the configuration')
BackendStore.__init__(self, server, **kwargs)
self.config = kwargs
self.name = kwargs.get('name', 'Tracker')
self.update_id = 0
self.token = None
self.songs = 0
self.albums = 0
self.artists = 0
self.playlists = 0
self.genres = 0
self.videos = 0
self.images = 0
self.bus = dbus.SessionBus()
tracker_object = self.bus.get_object(BUS_NAME, OBJECT_PATH)
self.tracker_interface = dbus.Interface(tracker_object,
'org.freedesktop.Tracker')
self.search_interface = dbus.Interface(
tracker_object, 'org.freedesktop.Tracker.Search')
self.keywords_interface = dbus.Interface(
tracker_object, 'org.freedesktop.Tracker.Keywords')
self.metadata_interface = dbus.Interface(
tracker_object, 'org.freedesktop.Tracker.Metadata')
self.query_id = -1
self.containers = {}
self.tracks = {}
self.containers[ROOT_CONTAINER_ID] = \
Container(ROOT_CONTAINER_ID, -1, self.name, store=self)
def queries_finished(r):
louie.send('Coherence.UPnP.Backend.init_completed',
None,
backend=self)
def queries_failed(r):
error = ''
louie.send('Coherence.UPnP.Backend.init_failed',
None,
backend=self,
msg=error)
services = kwargs.get('service', 'Music,Videos,Images')
services = [x.strip().lower() for x in services.split(',')]
l = []
mapping = {
'music': self.get_tracks,
'videos': self.get_videos,
'images': self.get_images
}
for service in services:
try:
l.append(mapping[service]())
except KeyError:
self.warning('Wrong Tracker service definition - %r', service)
if len(l) > 0:
dl = defer.DeferredList(l)
dl.addCallback(queries_finished)
dl.addErrback(lambda x: louie.send(
'Coherence.UPnP.Backend.init_failed',
None,
backend=self,
msg='Connection to Tracker service(s) failed!'))
else:
louie.send('Coherence.UPnP.Backend.init_failed',
None,
backend=self,
msg='No Tracker service defined!')
def on_connect_success(self, result, task, config):
"""Gets called when successfully connected to a daemon."""
from deluge.ui.client import client
from twisted.internet import reactor, defer
if not result:
log.debug('on_connect_success returned a failed result. BUG?')
if task.options.test:
log.debug('Test connection to deluge daemon successful.')
client.disconnect()
return
def format_label(label):
"""Makes a string compliant with deluge label naming rules"""
return re.sub('[^\w-]+', '_', label.lower())
def set_torrent_options(torrent_id, entry, opts):
"""Gets called when a torrent was added to the daemon."""
dlist = []
if not torrent_id:
log.error('There was an error adding %s to deluge.' %
entry['title'])
# TODO: Fail entry? How can this happen still now?
return
log.info('%s successfully added to deluge.' % entry['title'])
entry['deluge_id'] = torrent_id
def create_path(result, path):
"""Creates the specified path if deluge is older than 1.3"""
from deluge.common import VersionSplit
# Before 1.3, deluge would not create a non-existent move directory, so we need to.
if VersionSplit('1.3.0') > VersionSplit(self.deluge_version):
if client.is_localhost():
if not os.path.isdir(path):
log.debug('path %s doesn\'t exist, creating' %
path)
os.makedirs(path)
else:
log.warning(
'If path does not exist on the machine running the daemon, move will fail.'
)
if opts.get('movedone'):
dlist.append(
version_deferred.addCallback(create_path,
opts['movedone']))
dlist.append(
client.core.set_torrent_move_completed(torrent_id, True))
dlist.append(
client.core.set_torrent_move_completed_path(
torrent_id, opts['movedone']))
log.debug('%s move on complete set to %s' %
(entry['title'], opts['movedone']))
if opts.get('label'):
def apply_label(result, torrent_id, label):
"""Gets called after labels and torrent were added to deluge."""
return client.label.set_torrent(torrent_id, label)
dlist.append(
label_deferred.addCallback(apply_label, torrent_id,
opts['label']))
if opts.get('queuetotop') is not None:
if opts['queuetotop']:
dlist.append(client.core.queue_top([torrent_id]))
log.debug('%s moved to top of queue' % entry['title'])
else:
dlist.append(client.core.queue_bottom([torrent_id]))
log.debug('%s moved to bottom of queue' % entry['title'])
def on_get_torrent_status(status):
"""Gets called with torrent status, including file info.
Sets the torrent options which require knowledge of the current status of the torrent."""
main_file_dlist = []
# Determine where the file should be
move_now_path = None
if opts.get('movedone'):
if status['progress'] == 100:
move_now_path = opts['movedone']
else:
# Deluge will unset the move completed option if we move the storage, forgo setting proper
# path, in favor of leaving proper final location.
log.debug(
'Not moving storage for %s, as this will prevent movedone.'
% entry['title'])
elif opts.get('path'):
move_now_path = opts['path']
if move_now_path and os.path.normpath(
move_now_path) != os.path.normpath(
status['save_path']):
main_file_dlist.append(
version_deferred.addCallback(create_path,
move_now_path))
log.debug('Moving storage for %s to %s' %
(entry['title'], move_now_path))
main_file_dlist.append(
client.core.move_storage([torrent_id], move_now_path))
if opts.get('content_filename') or opts.get('main_file_only'):
def file_exists(filename):
# Checks the download path as well as the move completed path for existence of the file
if os.path.exists(
os.path.join(status['save_path'], filename)):
return True
elif status.get('move_on_completed') and status.get(
'move_on_completed_path'):
if os.path.exists(
os.path.join(
status['move_on_completed_path'],
filename)):
return True
else:
return False
def unused_name(name):
# If on local computer, tries appending a (#) suffix until a unique filename is found
if client.is_localhost():
counter = 2
while file_exists(name):
name = ''.join([
os.path.splitext(name)[0], " (",
str(counter), ')',
os.path.splitext(name)[1]
])
counter += 1
else:
log.debug(
'Cannot ensure content_filename is unique '
'when adding to a remote deluge daemon.')
return name
def rename(file, new_name):
# Renames a file in torrent
main_file_dlist.append(
client.core.rename_files(
torrent_id, [(file['index'], new_name)]))
log.debug('File %s in %s renamed to %s' %
(file['path'], entry['title'], new_name))
# find a file that makes up more than main_file_ratio (default: 90%) of the total size
main_file = None
for file in status['files']:
if file['size'] > (status['total_size'] *
opts.get('main_file_ratio')):
main_file = file
break
if main_file is not None:
# proceed with renaming only if such a big file is found
# find the subtitle file
keep_subs = opts.get('keep_subs')
sub_file = None
if keep_subs:
sub_exts = [".srt", ".sub"]
for file in status['files']:
ext = os.path.splitext(file['path'])[1]
if ext in sub_exts:
sub_file = file
break
# check for single file torrents so we dont add unnecessary folders
if (os.path.dirname(main_file['path'])
is not ("" or "/")):
# check for top folder in user config
if (opts.get('content_filename')
and os.path.dirname(
opts['content_filename']) is not ""):
top_files_dir = os.path.dirname(
opts['content_filename']) + "/"
else:
top_files_dir = os.path.dirname(
main_file['path']) + "/"
else:
top_files_dir = "/"
if opts.get('content_filename'):
# rename the main file
big_file_name = (
top_files_dir +
os.path.basename(opts['content_filename']) +
os.path.splitext(main_file['path'])[1])
big_file_name = unused_name(big_file_name)
rename(main_file, big_file_name)
# rename subs along with the main file
if sub_file is not None and keep_subs:
sub_file_name = (
os.path.splitext(big_file_name)[0] +
os.path.splitext(sub_file['path'])[1])
rename(sub_file, sub_file_name)
if opts.get('main_file_only'):
# download only the main file (and subs)
file_priorities = [
1 if f == main_file or
(f == sub_file and keep_subs) else 0
for f in status['files']
]
main_file_dlist.append(
client.core.set_torrent_file_priorities(
torrent_id, file_priorities))
if opts.get('hide_sparse_files'):
# hide the other sparse files that are not supposed to download but are created anyway
# http://dev.deluge-torrent.org/ticket/1827
# Made sparse files behave better with deluge http://flexget.com/ticket/2881
sparse_files = [
f for f in status['files']
if f != main_file and (
f != sub_file or (not keep_subs))
]
rename_pairs = [
(f['index'],
top_files_dir + ".sparse_files/" +
os.path.basename(f['path']))
for f in sparse_files
]
main_file_dlist.append(
client.core.rename_files(
torrent_id, rename_pairs))
else:
log.warning(
'No files in "%s" are > %d%% of content size, no files renamed.'
% (entry['title'],
opts.get('main_file_ratio') * 100))
return defer.DeferredList(main_file_dlist)
status_keys = [
'files', 'total_size', 'save_path', 'move_on_completed_path',
'move_on_completed', 'progress'
]
dlist.append(
client.core.get_torrent_status(
torrent_id,
status_keys).addCallback(on_get_torrent_status))
return defer.DeferredList(dlist)
def on_fail(result, task, entry):
"""Gets called when daemon reports a failure adding the torrent."""
log.info('%s was not added to deluge! %s' %
(entry['title'], result))
entry.fail('Could not be added to deluge')
# dlist is a list of deferreds that must complete before we exit
dlist = []
# loop through entries to get a list of labels to add
labels = set([
format_label(entry['label']) for entry in task.accepted
if entry.get('label')
])
if config.get('label'):
labels.add(format_label(config['label']))
label_deferred = defer.succeed(True)
if labels:
# Make sure the label plugin is available and enabled, then add appropriate labels
def on_get_enabled_plugins(plugins):
"""Gets called with the list of enabled deluge plugins."""
def on_label_enabled(result):
""" This runs when we verify the label plugin is enabled. """
def on_get_labels(d_labels):
"""Gets available labels from deluge, and adds any new labels we need."""
dlist = []
for label in labels:
if label not in d_labels:
log.debug('Adding the label %s to deluge' %
label)
dlist.append(client.label.add(label))
return defer.DeferredList(dlist)
return client.label.get_labels().addCallback(on_get_labels)
if 'Label' in plugins:
return on_label_enabled(True)
else:
# Label plugin isn't enabled, so we check if it's available and enable it.
def on_get_available_plugins(plugins):
"""Gets plugins available to deluge, enables Label plugin if available."""
if 'Label' in plugins:
log.debug('Enabling label plugin in deluge')
return client.core.enable_plugin(
'Label').addCallback(on_label_enabled)
else:
log.error(
'Label plugin is not installed in deluge')
return client.core.get_available_plugins().addCallback(
on_get_available_plugins)
label_deferred = client.core.get_enabled_plugins().addCallback(
on_get_enabled_plugins)
dlist.append(label_deferred)
def on_get_daemon_info(ver):
"""Gets called with the daemon version info, stores it in self."""
log.debug('deluge version %s' % ver)
self.deluge_version = ver
version_deferred = client.daemon.info().addCallback(on_get_daemon_info)
dlist.append(version_deferred)
def on_get_session_state(torrent_ids):
"""Gets called with a list of torrent_ids loaded in the deluge session.
Adds new torrents and modifies the settings for ones already in the session."""
dlist = []
# add the torrents
for entry in task.accepted:
@defer.inlineCallbacks
def _wait_for_metadata(torrent_id, timeout):
log.verbose('Waiting %d seconds for "%s" to magnetize' %
(timeout, entry['title']))
for i in range(timeout):
time.sleep(1)
try:
status = yield client.core.get_torrent_status(
torrent_id, ['files'])
except Exception as err:
log.error('wait_for_metadata Error: %s' % err)
break
if len(status['files']) > 0:
log.info('"%s" magnetization successful' %
(entry['title']))
break
else:
log.warning(
'"%s" did not magnetize before the timeout elapsed, '
'file list unavailable for processing.' %
entry['title'])
defer.returnValue(torrent_id)
def add_entry(entry, opts):
"""Adds an entry to the deluge session"""
magnet, filedump = None, None
if entry.get('url', '').startswith('magnet:'):
magnet = entry['url']
else:
if not os.path.exists(entry['file']):
entry.fail(
'Downloaded temp file \'%s\' doesn\'t exist!' %
entry['file'])
del (entry['file'])
return
with open(entry['file'], 'rb') as f:
filedump = base64.encodestring(f.read())
log.verbose('Adding %s to deluge.' % entry['title'])
if magnet:
d = client.core.add_torrent_magnet(magnet, opts)
if config.get('magnetization_timeout'):
d.addCallback(_wait_for_metadata,
config['magnetization_timeout'])
return d
else:
return client.core.add_torrent_file(
entry['title'], filedump, opts)
# Generate deluge options dict for torrent add
add_opts = {}
try:
path = entry.render(entry.get('path', config['path']))
if path:
add_opts['download_location'] = pathscrub(
os.path.expanduser(path))
except RenderError as e:
log.error('Could not set path for %s: %s' %
(entry['title'], e))
for fopt, dopt in self.options.items():
value = entry.get(fopt, config.get(fopt))
if value is not None:
add_opts[dopt] = value
if fopt == 'ratio':
add_opts['stop_at_ratio'] = True
# Make another set of options, that get set after the torrent has been added
modify_opts = {
'label':
format_label(entry.get('label', config['label'])),
'queuetotop':
entry.get('queuetotop', config.get('queuetotop')),
'main_file_only':
entry.get('main_file_only',
config.get('main_file_only', False)),
'main_file_ratio':
entry.get('main_file_ratio',
config.get('main_file_ratio')),
'hide_sparse_files':
entry.get('hide_sparse_files',
config.get('hide_sparse_files', True)),
'keep_subs':
entry.get('keep_subs', config.get('keep_subs', True))
}
try:
movedone = entry.render(
entry.get('movedone', config['movedone']))
modify_opts['movedone'] = pathscrub(
os.path.expanduser(movedone))
except RenderError as e:
log.error('Error setting movedone for %s: %s' %
(entry['title'], e))
try:
content_filename = entry.get(
'content_filename', config.get('content_filename', ''))
modify_opts['content_filename'] = pathscrub(
entry.render(content_filename))
except RenderError as e:
log.error('Error setting content_filename for %s: %s' %
(entry['title'], e))
torrent_id = entry.get('deluge_id') or entry.get(
'torrent_info_hash')
torrent_id = torrent_id and torrent_id.lower()
if torrent_id in torrent_ids:
log.info(
'%s is already loaded in deluge, setting options' %
entry['title'])
# Entry has a deluge id, verify the torrent is still in the deluge session and apply options
# Since this is already loaded in deluge, we may also need to change the path
modify_opts['path'] = add_opts.pop('download_location',
None)
dlist.extend([
set_torrent_options(torrent_id, entry, modify_opts),
client.core.set_torrent_options([torrent_id], add_opts)
])
else:
dlist.append(
add_entry(entry, add_opts).addCallbacks(
set_torrent_options,
on_fail,
callbackArgs=(entry, modify_opts),
errbackArgs=(task, entry)))
return defer.DeferredList(dlist)
dlist.append(
client.core.get_session_state().addCallback(on_get_session_state))
def on_complete(result):
"""Gets called when all of our tasks for deluge daemon are complete."""
client.disconnect()
tasks = defer.DeferredList(dlist).addBoth(on_complete)
def on_timeout(result):
"""Gets called if tasks have not completed in 30 seconds.
Should only happen when something goes wrong."""
log.error('Timed out while adding torrents to deluge.')
log.debug('dlist: %s' % result.resultList)
client.disconnect()
# Schedule a disconnect to happen if FlexGet hangs while connected to Deluge
# Leave the timeout long, to give time for possible lookups to occur
reactor.callLater(600, lambda: tasks.called or on_timeout(tasks))
def on_torrents_status(all_torrents):
global filtertime
tlist = []
torrents_by_tracker = {}
# order torrents by tracker
for torrent_id, status in all_torrents.items():
if status['tracker']:
tracker = urlparse(status['tracker']).hostname
else:
tracker = "No tracker"
torrents_by_tracker.setdefault(tracker, []).append(
(torrent_id, status))
if log.getEffectiveLevel() == logging.getLevelName("DEBUG"):
log.debug("Tracker torrent counts:")
for tracker, torrents in torrents_by_tracker.items():
log.debug("%-25s count: %d", tracker, len(torrents))
total_delete_count = 0
# Delete errored torrents (mostly not registered)
for tracker, torrents in torrents_by_tracker.items():
for torrent_id, status in torrents:
# messages are of format
# <tracker name>: <message>
tracker_status = status["tracker_status"]
if ":" in tracker_status:
tracker, message = tracker_status.split(': ', 1)
if message not in non_fatal_errors:
if message == "Error: torrent not registered with this tracker":
tlist.append(
client.core.remove_torrent(torrent_id, True))
log_removal(status,
"Torrent not registered with tracker")
elif tracker_status == '':
log.info("Empty tracker status, magnet torrent?")
else:
log.error("Unable to split message: '%s'", tracker_status)
# Delete oldest torrents from sites with max count reached
if args['delete_maxcount']:
# go through sites with set maximum linits
for tracker, limit in maxlimits.items():
limit_count = 0
# count number of torrents on tracker
for torrent_id, status in all_torrents.items():
if tracker in status['tracker']:
limit_count += 1
log.debug("%d torrents found for tracker %s (limit %d)",
limit_count, tracker, limit)
# over or at limit, start deleting
if limit_count >= limit:
# delete up to limit + 1 to make room
delete_count = limit_count - limit
delete_count += 1
if is_interactive:
print("Tracker %s is %d over limit" %
(tracker, delete_count))
print("Deleting %d oldest torrents to make room" %
delete_count)
# start deleting from the oldest onwards
for torrent_id, status in \
sorted(all_torrents.items(), key=lambda item: item[1]["time_added"]):
if tracker in status['tracker']:
tlist.append(
client.core.remove_torrent(torrent_id, True))
delete_count = delete_count - 1
total_delete_count += 1
if is_interactive:
log_removal(status,
"tracker %s over limit" % tracker)
if delete_count <= 0:
break
# Remove torrents with no tracker
if args['delete_orphans']:
counter = 0
for torrent_id, status in torrents_by_tracker.get("No tracker", []):
# Don't count timed out trackers as no tracker torrents
# this happens f.ex. when deluge is started with a big torrent load,
# not all torrents can connect in time
if not any(reason in status["tracker_status"]
for reason in non_fatal_errors):
added = datetime.datetime.fromtimestamp(status["time_added"])
delta = datetime.datetime.now() - added
# hours, minutes, seconds = td.seconds // 3600, td.seconds // 60 % 60, td.seconds % 60
# don't delete torrents under 1d old as orphans, it might be a connection issue
if delta.days < 1:
if is_interactive:
print("Skipping %s" % status["name"])
print("Reason: under 1d old")
print_info(status)
continue
tlist.append(client.core.remove_torrent(torrent_id, True))
total_delete_count += 1
log_removal(status, "Orphan torrent")
counter += 1
# Max 5 at a time
if counter > orphan_limit: break
# delete torrents over maximum ratio
if args['maximum_ratio']:
for torrent_id, status in sorted(all_torrents.items(),
key=lambda item: item[1]["ratio"]):
if status['ratio'] > max_ratio:
total_delete_count += 1
tlist.append(client.core.remove_torrent(torrent_id, True))
log_removal(status,
"Torrent over maximum ratio (%d)" % max_ratio)
# Only delete oldest to free space if nothing else has been deleted during this run
# The free space is determined before the run, so this is necessary
if total_delete_count == 0 and args['free_space']:
counter = 0
for torrent_id, status in sorted(
all_torrents.items(), key=lambda item: item[1]["time_added"]):
tlist.append(client.core.remove_torrent(torrent_id, True))
if is_interactive:
log_removal(status, "Free disk space needed")
counter += 1
if counter >= free_space_limit:
break
# for torrent_id, status in all_torrents.items():
# # Filter out torrents with no issues
# if status["tracker_status"].endswith("Announce OKXX"):
# log.debug("Current torrent id is: %s" % (torrent_id))
# log.debug("--Torrent name is: %s" % (status["name"]))
# log.debug("--Torrent state is: %s" % (status["state"]))
# log.debug("--Torrent ratio is: %s" % (status["ratio"]))
# log.debug("--Torrent DL rate is: %s" % (status["download_payload_rate"]))
# log.debug("--Torrent UL rate is: %s" % (status["upload_payload_rate"]))
# log.debug("--Torrent tracker is: %s" % (status["tracker_status"]))
global count
count = len(all_torrents)
defer.DeferredList(tlist).addCallback(printReport)
def set_torrent_options(torrent_id, entry, opts):
"""Gets called when a torrent was added to the daemon."""
dlist = []
if not torrent_id:
log.error('There was an error adding %s to deluge.' %
entry['title'])
# TODO: Fail entry? How can this happen still now?
return
log.info('%s successfully added to deluge.' % entry['title'])
entry['deluge_id'] = torrent_id
def create_path(result, path):
"""Creates the specified path if deluge is older than 1.3"""
from deluge.common import VersionSplit
# Before 1.3, deluge would not create a non-existent move directory, so we need to.
if VersionSplit('1.3.0') > VersionSplit(self.deluge_version):
if client.is_localhost():
if not os.path.isdir(path):
log.debug('path %s doesn\'t exist, creating' %
path)
os.makedirs(path)
else:
log.warning(
'If path does not exist on the machine running the daemon, move will fail.'
)
if opts.get('movedone'):
dlist.append(
version_deferred.addCallback(create_path,
opts['movedone']))
dlist.append(
client.core.set_torrent_move_completed(torrent_id, True))
dlist.append(
client.core.set_torrent_move_completed_path(
torrent_id, opts['movedone']))
log.debug('%s move on complete set to %s' %
(entry['title'], opts['movedone']))
if opts.get('label'):
def apply_label(result, torrent_id, label):
"""Gets called after labels and torrent were added to deluge."""
return client.label.set_torrent(torrent_id, label)
dlist.append(
label_deferred.addCallback(apply_label, torrent_id,
opts['label']))
if opts.get('queuetotop') is not None:
if opts['queuetotop']:
dlist.append(client.core.queue_top([torrent_id]))
log.debug('%s moved to top of queue' % entry['title'])
else:
dlist.append(client.core.queue_bottom([torrent_id]))
log.debug('%s moved to bottom of queue' % entry['title'])
def on_get_torrent_status(status):
"""Gets called with torrent status, including file info.
Sets the torrent options which require knowledge of the current status of the torrent."""
main_file_dlist = []
# Determine where the file should be
move_now_path = None
if opts.get('movedone'):
if status['progress'] == 100:
move_now_path = opts['movedone']
else:
# Deluge will unset the move completed option if we move the storage, forgo setting proper
# path, in favor of leaving proper final location.
log.debug(
'Not moving storage for %s, as this will prevent movedone.'
% entry['title'])
elif opts.get('path'):
move_now_path = opts['path']
if move_now_path and os.path.normpath(
move_now_path) != os.path.normpath(
status['save_path']):
main_file_dlist.append(
version_deferred.addCallback(create_path,
move_now_path))
log.debug('Moving storage for %s to %s' %
(entry['title'], move_now_path))
main_file_dlist.append(
client.core.move_storage([torrent_id], move_now_path))
if opts.get('content_filename') or opts.get('main_file_only'):
def file_exists(filename):
# Checks the download path as well as the move completed path for existence of the file
if os.path.exists(
os.path.join(status['save_path'], filename)):
return True
elif status.get('move_on_completed') and status.get(
'move_on_completed_path'):
if os.path.exists(
os.path.join(
status['move_on_completed_path'],
filename)):
return True
else:
return False
def unused_name(name):
# If on local computer, tries appending a (#) suffix until a unique filename is found
if client.is_localhost():
counter = 2
while file_exists(name):
name = ''.join([
os.path.splitext(name)[0], " (",
str(counter), ')',
os.path.splitext(name)[1]
])
counter += 1
else:
log.debug(
'Cannot ensure content_filename is unique '
'when adding to a remote deluge daemon.')
return name
def rename(file, new_name):
# Renames a file in torrent
main_file_dlist.append(
client.core.rename_files(
torrent_id, [(file['index'], new_name)]))
log.debug('File %s in %s renamed to %s' %
(file['path'], entry['title'], new_name))
# find a file that makes up more than main_file_ratio (default: 90%) of the total size
main_file = None
for file in status['files']:
if file['size'] > (status['total_size'] *
opts.get('main_file_ratio')):
main_file = file
break
if main_file is not None:
# proceed with renaming only if such a big file is found
# find the subtitle file
keep_subs = opts.get('keep_subs')
sub_file = None
if keep_subs:
sub_exts = [".srt", ".sub"]
for file in status['files']:
ext = os.path.splitext(file['path'])[1]
if ext in sub_exts:
sub_file = file
break
# check for single file torrents so we dont add unnecessary folders
if (os.path.dirname(main_file['path'])
is not ("" or "/")):
# check for top folder in user config
if (opts.get('content_filename')
and os.path.dirname(
opts['content_filename']) is not ""):
top_files_dir = os.path.dirname(
opts['content_filename']) + "/"
else:
top_files_dir = os.path.dirname(
main_file['path']) + "/"
else:
top_files_dir = "/"
if opts.get('content_filename'):
# rename the main file
big_file_name = (
top_files_dir +
os.path.basename(opts['content_filename']) +
os.path.splitext(main_file['path'])[1])
big_file_name = unused_name(big_file_name)
rename(main_file, big_file_name)
# rename subs along with the main file
if sub_file is not None and keep_subs:
sub_file_name = (
os.path.splitext(big_file_name)[0] +
os.path.splitext(sub_file['path'])[1])
rename(sub_file, sub_file_name)
if opts.get('main_file_only'):
# download only the main file (and subs)
file_priorities = [
1 if f == main_file or
(f == sub_file and keep_subs) else 0
for f in status['files']
]
main_file_dlist.append(
client.core.set_torrent_file_priorities(
torrent_id, file_priorities))
if opts.get('hide_sparse_files'):
# hide the other sparse files that are not supposed to download but are created anyway
# http://dev.deluge-torrent.org/ticket/1827
# Made sparse files behave better with deluge http://flexget.com/ticket/2881
sparse_files = [
f for f in status['files']
if f != main_file and (
f != sub_file or (not keep_subs))
]
rename_pairs = [
(f['index'],
top_files_dir + ".sparse_files/" +
os.path.basename(f['path']))
for f in sparse_files
]
main_file_dlist.append(
client.core.rename_files(
torrent_id, rename_pairs))
else:
log.warning(
'No files in "%s" are > %d%% of content size, no files renamed.'
% (entry['title'],
opts.get('main_file_ratio') * 100))
return defer.DeferredList(main_file_dlist)
status_keys = [
'files', 'total_size', 'save_path', 'move_on_completed_path',
'move_on_completed', 'progress'
]
dlist.append(
client.core.get_torrent_status(
torrent_id,
status_keys).addCallback(on_get_torrent_status))
return defer.DeferredList(dlist)
def sendmail(tid,
username,
password,
smtp_host,
smtp_port,
security,
from_name,
from_address,
to_address,
subject,
body,
anonymize=True,
socks_host='127.0.0.1',
socks_port=9050):
"""
Send an email using SMTPS/SMTP+TLS and maybe torify the connection.
@param to_address: the 'To:' field of the email
@param subject: the mail subject
@param body: the mail body
@return: a {Deferred} that returns a success {bool} if the message was passed
to the server.
"""
try:
timeout = 30
message = MIME_mail_build(from_name, from_address, to_address,
to_address, subject, body)
log.debug('Sending email to %s using SMTP server [%s:%d] [%s]',
to_address,
smtp_host,
smtp_port,
security,
tid=tid)
context_factory = TLSClientContextFactory()
smtp_deferred = defer.Deferred()
factory = ESMTPSenderFactory(
username.encode('utf-8'),
password.encode('utf-8'),
from_address,
to_address,
message,
smtp_deferred,
contextFactory=context_factory,
requireAuthentication=True,
requireTransportSecurity=(security != 'SSL'),
retries=0,
timeout=timeout)
if security == "SSL":
factory = tls.TLSMemoryBIOFactory(context_factory, True, factory)
if anonymize:
socksProxy = TCP4ClientEndpoint(reactor,
socks_host,
socks_port,
timeout=timeout)
endpoint = SOCKS5ClientEndpoint(smtp_host.encode('utf-8'),
smtp_port, socksProxy)
else:
endpoint = TCP4ClientEndpoint(reactor,
smtp_host.encode('utf-8'),
smtp_port,
timeout=timeout)
conn_deferred = endpoint.connect(factory)
final = defer.DeferredList([conn_deferred, smtp_deferred],
fireOnOneErrback=True,
consumeErrors=True)
def failure_cb(failure):
"""
@param failure {Failure {twisted.internet.FirstError {Failure}}}
"""
log.err("SMTP connection failed (Exception: %s)",
failure.value.subFailure.value,
tid=tid)
log.debug(failure)
return False
def success_cb(results):
"""
@param results {list of (success, return_val) tuples}
"""
return True
final.addCallback(success_cb)
final.addErrback(failure_cb)
return final
except Exception as excep:
# avoids raising an exception inside email logic to avoid chained errors
log.err("Unexpected exception in sendmail: %s", str(excep), tid=tid)
return defer.succeed(False)
def on_get_torrent_status(status):
"""Gets called with torrent status, including file info.
Sets the torrent options which require knowledge of the current status of the torrent."""
main_file_dlist = []
# Determine where the file should be
move_now_path = None
if opts.get('movedone'):
if status['progress'] == 100:
move_now_path = opts['movedone']
else:
# Deluge will unset the move completed option if we move the storage, forgo setting proper
# path, in favor of leaving proper final location.
log.debug(
'Not moving storage for %s, as this will prevent movedone.'
% entry['title'])
elif opts.get('path'):
move_now_path = opts['path']
if move_now_path and os.path.normpath(
move_now_path) != os.path.normpath(
status['save_path']):
main_file_dlist.append(
version_deferred.addCallback(create_path,
move_now_path))
log.debug('Moving storage for %s to %s' %
(entry['title'], move_now_path))
main_file_dlist.append(
client.core.move_storage([torrent_id], move_now_path))
if opts.get('content_filename') or opts.get('main_file_only'):
def file_exists(filename):
# Checks the download path as well as the move completed path for existence of the file
if os.path.exists(
os.path.join(status['save_path'], filename)):
return True
elif status.get('move_on_completed') and status.get(
'move_on_completed_path'):
if os.path.exists(
os.path.join(
status['move_on_completed_path'],
filename)):
return True
else:
return False
def unused_name(name):
# If on local computer, tries appending a (#) suffix until a unique filename is found
if client.is_localhost():
counter = 2
while file_exists(name):
name = ''.join([
os.path.splitext(name)[0], " (",
str(counter), ')',
os.path.splitext(name)[1]
])
counter += 1
else:
log.debug(
'Cannot ensure content_filename is unique '
'when adding to a remote deluge daemon.')
return name
def rename(file, new_name):
# Renames a file in torrent
main_file_dlist.append(
client.core.rename_files(
torrent_id, [(file['index'], new_name)]))
log.debug('File %s in %s renamed to %s' %
(file['path'], entry['title'], new_name))
# find a file that makes up more than main_file_ratio (default: 90%) of the total size
main_file = None
for file in status['files']:
if file['size'] > (status['total_size'] *
opts.get('main_file_ratio')):
main_file = file
break
if main_file is not None:
# proceed with renaming only if such a big file is found
# find the subtitle file
keep_subs = opts.get('keep_subs')
sub_file = None
if keep_subs:
sub_exts = [".srt", ".sub"]
for file in status['files']:
ext = os.path.splitext(file['path'])[1]
if ext in sub_exts:
sub_file = file
break
# check for single file torrents so we dont add unnecessary folders
if (os.path.dirname(main_file['path'])
is not ("" or "/")):
# check for top folder in user config
if (opts.get('content_filename')
and os.path.dirname(
opts['content_filename']) is not ""):
top_files_dir = os.path.dirname(
opts['content_filename']) + "/"
else:
top_files_dir = os.path.dirname(
main_file['path']) + "/"
else:
top_files_dir = "/"
if opts.get('content_filename'):
# rename the main file
big_file_name = (
top_files_dir +
os.path.basename(opts['content_filename']) +
os.path.splitext(main_file['path'])[1])
big_file_name = unused_name(big_file_name)
rename(main_file, big_file_name)
# rename subs along with the main file
if sub_file is not None and keep_subs:
sub_file_name = (
os.path.splitext(big_file_name)[0] +
os.path.splitext(sub_file['path'])[1])
rename(sub_file, sub_file_name)
if opts.get('main_file_only'):
# download only the main file (and subs)
file_priorities = [
1 if f == main_file or
(f == sub_file and keep_subs) else 0
for f in status['files']
]
main_file_dlist.append(
client.core.set_torrent_file_priorities(
torrent_id, file_priorities))
if opts.get('hide_sparse_files'):
# hide the other sparse files that are not supposed to download but are created anyway
# http://dev.deluge-torrent.org/ticket/1827
# Made sparse files behave better with deluge http://flexget.com/ticket/2881
sparse_files = [
f for f in status['files']
if f != main_file and (
f != sub_file or (not keep_subs))
]
rename_pairs = [
(f['index'],
top_files_dir + ".sparse_files/" +
os.path.basename(f['path']))
for f in sparse_files
]
main_file_dlist.append(
client.core.rename_files(
torrent_id, rename_pairs))
else:
log.warning(
'No files in "%s" are > %d%% of content size, no files renamed.'
% (entry['title'],
opts.get('main_file_ratio') * 100))
return defer.DeferredList(main_file_dlist)
Q = queue.Queue()
class Engine(object):
def __init__(self):
self._close = None
def crawl(self, spider):
start_requests = iter(spider.start_requests())
while True:
try:
request = next(start_requests)
Q.put(request)
except StopIteration as e:
break
self._close = defer.Deferred()
yield self._close
spider = ChoutiSpider()
_active = set()
engine = Engine()
d = engine.crawl(spider)
_active.add(d)
dd = defer.DeferredList(_active)
dd.addBoth(lambda a: reactor.stop())
reactor.run()
def on_get_session_state(torrent_ids):
"""Gets called with a list of torrent_ids loaded in the deluge session.
Adds new torrents and modifies the settings for ones already in the session."""
dlist = []
# add the torrents
for entry in task.accepted:
@defer.inlineCallbacks
def _wait_for_metadata(torrent_id, timeout):
log.verbose('Waiting %d seconds for "%s" to magnetize' %
(timeout, entry['title']))
for i in range(timeout):
time.sleep(1)
try:
status = yield client.core.get_torrent_status(
torrent_id, ['files'])
except Exception as err:
log.error('wait_for_metadata Error: %s' % err)
break
if len(status['files']) > 0:
log.info('"%s" magnetization successful' %
(entry['title']))
break
else:
log.warning(
'"%s" did not magnetize before the timeout elapsed, '
'file list unavailable for processing.' %
entry['title'])
defer.returnValue(torrent_id)
def add_entry(entry, opts):
"""Adds an entry to the deluge session"""
magnet, filedump = None, None
if entry.get('url', '').startswith('magnet:'):
magnet = entry['url']
else:
if not os.path.exists(entry['file']):
entry.fail(
'Downloaded temp file \'%s\' doesn\'t exist!' %
entry['file'])
del (entry['file'])
return
with open(entry['file'], 'rb') as f:
filedump = base64.encodestring(f.read())
log.verbose('Adding %s to deluge.' % entry['title'])
if magnet:
d = client.core.add_torrent_magnet(magnet, opts)
if config.get('magnetization_timeout'):
d.addCallback(_wait_for_metadata,
config['magnetization_timeout'])
return d
else:
return client.core.add_torrent_file(
entry['title'], filedump, opts)
# Generate deluge options dict for torrent add
add_opts = {}
try:
path = entry.render(entry.get('path', config['path']))
if path:
add_opts['download_location'] = pathscrub(
os.path.expanduser(path))
except RenderError as e:
log.error('Could not set path for %s: %s' %
(entry['title'], e))
for fopt, dopt in self.options.items():
value = entry.get(fopt, config.get(fopt))
if value is not None:
add_opts[dopt] = value
if fopt == 'ratio':
add_opts['stop_at_ratio'] = True
# Make another set of options, that get set after the torrent has been added
modify_opts = {
'label':
format_label(entry.get('label', config['label'])),
'queuetotop':
entry.get('queuetotop', config.get('queuetotop')),
'main_file_only':
entry.get('main_file_only',
config.get('main_file_only', False)),
'main_file_ratio':
entry.get('main_file_ratio',
config.get('main_file_ratio')),
'hide_sparse_files':
entry.get('hide_sparse_files',
config.get('hide_sparse_files', True)),
'keep_subs':
entry.get('keep_subs', config.get('keep_subs', True))
}
try:
movedone = entry.render(
entry.get('movedone', config['movedone']))
modify_opts['movedone'] = pathscrub(
os.path.expanduser(movedone))
except RenderError as e:
log.error('Error setting movedone for %s: %s' %
(entry['title'], e))
try:
content_filename = entry.get(
'content_filename', config.get('content_filename', ''))
modify_opts['content_filename'] = pathscrub(
entry.render(content_filename))
except RenderError as e:
log.error('Error setting content_filename for %s: %s' %
(entry['title'], e))
torrent_id = entry.get('deluge_id') or entry.get(
'torrent_info_hash')
torrent_id = torrent_id and torrent_id.lower()
if torrent_id in torrent_ids:
log.info(
'%s is already loaded in deluge, setting options' %
entry['title'])
# Entry has a deluge id, verify the torrent is still in the deluge session and apply options
# Since this is already loaded in deluge, we may also need to change the path
modify_opts['path'] = add_opts.pop('download_location',
None)
dlist.extend([
set_torrent_options(torrent_id, entry, modify_opts),
client.core.set_torrent_options([torrent_id], add_opts)
])
else:
dlist.append(
add_entry(entry, add_opts).addCallbacks(
set_torrent_options,
on_fail,
callbackArgs=(entry, modify_opts),
errbackArgs=(task, entry)))
return defer.DeferredList(dlist)
def run(self):
self._start_deferred = defer.Deferred()
unhandled = self._start_unhandled_deferreds = []
self._sync_addlog_deferreds = []
try:
# here's where we set things up for backward compatibility for
# old-style steps, using monkey patches so that new-style steps
# aren't bothered by any of this equipment
# monkey-patch self.step_status.{setText,setText2} back into
# existence for old steps, signalling an update to the summary
self.step_status = BuildStepStatus()
self.step_status.setText = lambda text: self.realUpdateSummary()
self.step_status.setText2 = lambda text: self.realUpdateSummary()
# monkey-patch in support for old statistics functions
self.step_status.setStatistic = self.setStatistic
self.step_status.getStatistic = self.getStatistic
self.step_status.hasStatistic = self.hasStatistic
# monkey-patch an addLog that returns an write-only, sync log
self.addLog = self.addLog_oldStyle
self._logFileWrappers = {}
# and a getLog that returns a read-only, sync log, captured by
# LogObservers installed by addLog_oldStyle
self.getLog = self.getLog_oldStyle
# old-style steps shouldn't be calling updateSummary
def updateSummary():
assert 0, 'updateSummary is only valid on new-style steps'
self.updateSummary = updateSummary
results = yield self.start()
if results is not None:
self._start_deferred.callback(results)
results = yield self._start_deferred
finally:
# hook for tests
# assert so that it is only run in non optimized mode
assert self._run_finished_hook() is None
# wait until all the sync logs have been actually created before
# finishing
yield defer.DeferredList(self._sync_addlog_deferreds,
consumeErrors=True)
self._start_deferred = None
unhandled = self._start_unhandled_deferreds
self.realUpdateSummary()
# Wait for any possibly-unhandled deferreds. If any fail, change the
# result to EXCEPTION and log.
while unhandled:
self._start_unhandled_deferreds = []
unhandled_results = yield defer.DeferredList(
unhandled, consumeErrors=True)
for success, res in unhandled_results:
if not success:
log.err(
res,
"from an asynchronous method executed in an old-style step"
)
results = EXCEPTION
unhandled = self._start_unhandled_deferreds
defer.returnValue(results)
def _getAllDocsFromBothDbs(results):
d1 = sol1.get_all_docs()
d2 = sol2.get_all_docs()
return defer.DeferredList([d1, d2])
def state_variable_change(self, variable):
print "%s %r" % (variable.name, variable.value)
if variable.name == "PhysicalLinkStatus":
if variable.value.lower() == 'up':
self.link_state_image.set_from_pixbuf(self.link_up_icon)
else:
self.link_state_image.set_from_pixbuf(self.link_down_icon)
def request_cb(r):
#print r
self.link_type.set_markup(
"<b>Type:</b> %s (%s/%s)" %
(r['NewWANAccessType'], r['NewLayer1DownstreamMaxBitRate'],
r['NewLayer1UpstreamMaxBitRate']))
action = variable.service.get_action('GetCommonLinkProperties')
d = action.call()
d.addCallback(request_cb)
d.addErrback(self.handle_error)
elif variable.name == "PortMappingNumberOfEntries":
self.nat_store.clear()
if type(variable.value) == int and variable.value > 0:
l = []
for i in range(variable.value):
action = variable.service.get_action(
'GetGenericPortMappingEntry')
d = action.call(NewPortMappingIndex=i)
def add_index(r, index):
r['NewPortMappingIndex'] = index
return r
d.addCallback(add_index, i + 1)
d.addErrback(self.handle_error)
l.append(d)
def request_cb(r, last_updated_timestamp, v):
#print r
#print last_updated_timestamp == v.last_time_touched,last_updated_timestamp,v.last_time_touched
if last_updated_timestamp == v.last_time_touched:
mappings = [m[1] for m in r if m[0] == True]
mappings.sort(
cmp=lambda x, y: cmp(x['NewPortMappingIndex'], y[
'NewPortMappingIndex']))
for mapping in mappings:
#print mapping
self.nat_store.append([
mapping['NewPortMappingIndex'],
mapping['NewEnabled'], mapping['NewProtocol'],
mapping['NewRemoteHost'],
mapping['NewExternalPort'],
mapping['NewInternalClient'],
mapping['NewInternalPort'],
mapping['NewLeaseDuration'],
mapping['NewPortMappingDescription']
])
dl = defer.DeferredList(l)
dl.addCallback(request_cb, variable.last_time_touched,
variable)
dl.addErrback(self.handle_error)
elif variable.name == "ExternalIPAddress":
self.external_ip.set_markup(variable.value)
def _destroyVMs(self):
d1 = VirtualBox.destroyVM('image1')
d2 = VirtualBox.destroyVM('image2')
dl = defer.DeferredList([d1, d2])
return dl
def stop(self):
# 迭代加入defer
return defer.DeferredList([c.stop() for c in list(self.crawlers)])
class Trigger(BuildStep):
name = "trigger"
renderables = [
'alwaysUseLatest',
'parent_relationship',
'schedulerNames',
'set_properties',
'sourceStamps',
'updateSourceStamp'
]
flunkOnFailure = True
def __init__(self, schedulerNames=None, sourceStamp=None, sourceStamps=None,
updateSourceStamp=None, alwaysUseLatest=False,
waitForFinish=False, set_properties=None,
copy_properties=None, parent_relationship="Triggered from", **kwargs):
if schedulerNames is None:
schedulerNames = []
if not schedulerNames:
config.error(
"You must specify a scheduler to trigger")
if (sourceStamp or sourceStamps) and (updateSourceStamp is not None):
config.error(
"You can't specify both sourceStamps and updateSourceStamp")
if (sourceStamp or sourceStamps) and alwaysUseLatest:
config.error(
"You can't specify both sourceStamps and alwaysUseLatest")
if alwaysUseLatest and (updateSourceStamp is not None):
config.error(
"You can't specify both alwaysUseLatest and updateSourceStamp"
)
self.schedulerNames = schedulerNames
self.sourceStamps = sourceStamps or []
if sourceStamp:
self.sourceStamps.append(sourceStamp)
if updateSourceStamp is not None:
self.updateSourceStamp = updateSourceStamp
else:
self.updateSourceStamp = not (alwaysUseLatest or self.sourceStamps)
self.alwaysUseLatest = alwaysUseLatest
self.waitForFinish = waitForFinish
if set_properties is None:
set_properties = {}
if copy_properties is None:
copy_properties = []
properties = {}
properties.update(set_properties)
for i in copy_properties:
properties[i] = Property(i)
self.set_properties = properties
self.parent_relationship = parent_relationship
self.running = False
self.ended = False
self.brids = []
self.triggeredNames = None
BuildStep.__init__(self, **kwargs)
def interrupt(self, reason):
# FIXME: new style step cannot be interrupted anymore by just calling finished()
# (which was a bad idea in the first place)
# we should claim the self.brids, and CANCELLED them
# if they were already claimed, stop the associated builds via data api
# then the big deferredlist will automatically be called
if self.running and not self.ended:
self.ended = True
# Create the properties that are used for the trigger
def createTriggerProperties(self, properties):
# make a new properties object from a dict rendered by the old
# properties object
trigger_properties = Properties()
trigger_properties.update(properties, "Trigger")
return trigger_properties
def getSchedulerByName(self, name):
# we use the fact that scheduler_manager is a multiservice, with schedulers as childs
# this allow to quickly find schedulers instance by name
schedulers = self.master.scheduler_manager.namedServices
if name not in schedulers:
raise ValueError("unknown triggered scheduler: %r" % (name,))
sch = schedulers[name]
if not ITriggerableScheduler.providedBy(sch):
raise ValueError("triggered scheduler is not ITriggerableScheduler: %r" % (name,))
return sch
# This customization enpoint allows users to dynamically select which scheduler and properties to trigger
def getSchedulersAndProperties(self):
return [(sched, self.set_properties) for sched in self.schedulerNames]
def prepareSourcestampListForTrigger(self):
if self.sourceStamps:
ss_for_trigger = {}
for ss in self.sourceStamps:
codebase = ss.get('codebase', '')
assert codebase not in ss_for_trigger, "codebase specified multiple times"
ss_for_trigger[codebase] = ss
return ss_for_trigger.values()
if self.alwaysUseLatest:
return []
# start with the sourcestamps from current build
ss_for_trigger = {}
objs_from_build = self.build.getAllSourceStamps()
for ss in objs_from_build:
ss_for_trigger[ss.codebase] = ss.asDict()
# overrule revision in sourcestamps with got revision
if self.updateSourceStamp:
got = self.build.build_status.getAllGotRevisions()
for codebase in ss_for_trigger:
if codebase in got:
ss_for_trigger[codebase]['revision'] = got[codebase]
return ss_for_trigger.values()
@defer.inlineCallbacks
def worstStatus(self, overall_results, rclist):
for was_cb, results in rclist:
if isinstance(results, tuple):
results, _ = results
if not was_cb:
yield self.addLogWithFailure(results)
results = EXCEPTION
overall_results = worst_status(overall_results, results)
defer.returnValue(overall_results)
@defer.inlineCallbacks
def addBuildUrls(self, rclist):
brids = {}
for was_cb, results in rclist:
if isinstance(results, tuple):
results, brids = results
if was_cb: # errors were already logged in worstStatus
for builderid, br in brids.iteritems():
builderDict = yield self.master.data.get(("builders", builderid))
builds = yield self.master.db.builds.getBuilds(buildrequestid=br)
for build in builds:
num = build['number']
url = self.master.status.getURLForBuild(builderid, num)
yield self.addURL("%s: %s #%d" % (statusToString(results),
builderDict["name"], num), url)
@defer.inlineCallbacks
def run(self):
schedulers_and_props = yield self.getSchedulersAndProperties()
# post process the schedulernames, and raw properties
# we do this out of the loop, as this can result in errors
schedulers_and_props = [(
self.getSchedulerByName(sch),
self.createTriggerProperties(props_to_set))
for sch, props_to_set in schedulers_and_props]
ss_for_trigger = self.prepareSourcestampListForTrigger()
dl = []
triggeredNames = []
results = SUCCESS
self.running = True
for sch, props_to_set in schedulers_and_props:
idsDeferred, resultsDeferred = sch.trigger(
waited_for=self.waitForFinish, sourcestamps=ss_for_trigger,
set_props=props_to_set,
parent_buildid=self.build.buildid,
parent_relationship=self.parent_relationship
)
# we are not in a hurry of starting all in parallel and managing
# the deferred lists, just let the db writes be serial.
try:
bsid, brids = yield idsDeferred
except Exception, e:
yield self.addLogWithException(e)
results = EXCEPTION
self.brids.extend(brids.values())
for brid in brids.values():
# put the url to the brids, so that we can have the status from the beginning
url = self.master.status.getURLForBuildrequest(brid)
yield self.addURL("%s #%d" % (sch.name, brid), url)
dl.append(resultsDeferred)
triggeredNames.append(sch.name)
if self.ended:
defer.returnValue(CANCELLED)
self.triggeredNames = triggeredNames
if self.waitForFinish:
rclist = yield defer.DeferredList(dl, consumeErrors=1)
# we were interrupted, don't bother update status
if self.ended:
defer.returnValue(CANCELLED)
yield self.addBuildUrls(rclist)
results = yield self.worstStatus(results, rclist)
else:
# do something to handle errors
for d in dl:
d.addErrback(log.err,
'(ignored) while invoking Triggerable schedulers:')
defer.returnValue(results)
