def __init__(self,
repourl,
branch=None,
branches=None,
bookmarks=None,
workdir=None,
pollInterval=10 * 60,
hgbin="hg",
usetimestamps=True,
category=None,
project="",
pollinterval=-2,
encoding="utf-8",
name=None,
pollAtLaunch=False,
revlink=lambda branch, revision: (""),
pollRandomDelayMin=0,
pollRandomDelayMax=0):
# for backward compatibility; the parameter used to be spelled with 'i'
if pollinterval != -2:
pollInterval = pollinterval
self.repourl = repourl
if branch and branches:
config.error("HgPoller: can't specify both branch and branches")
elif branch:
self.branches = [branch]
else:
self.branches = branches or []
self.bookmarks = bookmarks or []
if name is None:
name = repourl
if self.bookmarks:
name += "_" + "_".join(self.bookmarks)
if self.branches:
name += "_" + "_".join(self.branches)
if not self.branches and not self.bookmarks:
self.branches = ['default']
if not callable(revlink):
config.error("You need to provide a valid callable for revlink")
super().__init__(name=name,
pollInterval=pollInterval,
pollAtLaunch=pollAtLaunch,
pollRandomDelayMin=pollRandomDelayMin,
pollRandomDelayMax=pollRandomDelayMax)
self.encoding = encoding
self.lastChange = time.time()
self.lastPoll = time.time()
self.hgbin = hgbin
self.workdir = workdir
self.usetimestamps = usetimestamps
self.category = category if callable(category) else bytes2unicode(
category)
self.project = project
self.initLock = defer.DeferredLock()
self.lastRev = {}
self.revlink_callable = revlink
if self.workdir is None:
config.error("workdir is mandatory for now in HgPoller")
class DroneServer(Entity):
"""This model controls the routing of commands from the various services."""
keyRing = RSAKeyRing('%s' % (config.DRONED_KEY_DIR, ))
lock = defer.DeferredLock()
def __init__(self):
self._primes = set()
self.builtins = {
'help': self.help_action,
'ping': self.ping_action,
'list': self.list_action,
'shell': self.shell_action,
'reload': self.reload_action,
'tasks': self.tasks_action,
'cancel': self.cancel_action,
}
#self register this server hostname
if not Server.exists(config.HOSTNAME):
server = Server(config.HOSTNAME)
server.listed = True
else:
server = Server(config.HOSTNAME)
self.server = server
def _license(self, argstr):
return "\n" + copyright.copyright_notice
_license.__doc__ = copyright.copyright
def _version(self, argstr):
"""Shows the server version"""
return "DroneD/%s" % (copyright.version, )
@synchronizedDeferred(lock)
@deferredAsThread #will catch exceptions
def getprime(self):
pfh = open(config.DRONED_PRIMES)
psize = os.stat(config.DRONED_PRIMES)[6]
if (psize % 4) != 0 or psize < 4000:
pfh.close()
raise AssertionError("primes file is corrupt/too small")
try:
fcntl.fcntl(pfh.fileno(), fcntl.F_SETFD, fcntl.FD_CLOEXEC)
except:
pass
result = 0
while True:
try:
r = random.randint(0, (psize - 4) / 4) * 4
pfh.seek(r)
p = pfh.read(4)
prime = struct.unpack("!L", p)[0]
#makes a blocking call from this thread to the reactor
#this is done for thread safety
sync = synchronizedInThread()
trp = sync(self._trackPrime)
result = trp(prime)
break
except AssertionError:
continue
pfh.close()
return result
def _trackPrime(self, prime):
"""Tracks the prime numbers"""
assert prime not in self._primes
self._primes.add(prime)
return prime
@synchronizedDeferred(lock)
def validateMessage(self, magicNumber):
"""Is the message meant for me"""
for prime in self._primes:
if (magicNumber % prime) == 0:
#release the prime to prevent replay attacks
self._primes.discard(prime)
return True
return False
@synchronizedDeferred(lock)
def releasePrime(self, prime):
"""release the prime"""
self._primes.discard(prime)
def get_action(self, action):
"""route commands to perform actions"""
if action == 'license':
return self._license
elif action == 'version':
return self._version
foo = self.builtins.get(action, False)
if foo and hasattr(foo, '__call__'):
return foo
#look for AdminActions
for admin in AdminAction.objects:
if action != admin.action: continue
return admin
raise AssertionError("No such action ... try 'help'")
def tasks_action(self, argstr):
"""Usage: tasks - displays tasks and status"""
results = ['', "completed/succeeded\t'task'\n"]
for action in Action.objects:
r = """\t%s/%s\t'%s'""" % \
(action.completed, action.succeeded, action.description)
results.append(r)
results.append('')
return (0, '\n'.join(results))
#TODO test
def cancel_action(self, argstr):
"""Usage: cancel 'task' - cancels all tasks matching the description"""
cancelled = 0
plural = 's'
for action in Action.objects:
if action.completed: continue
if action.description == argstr:
try:
action.deferred.cancel()
cancelled += 1
except:
pass
if cancelled == 1: plural = ''
return (0, 'cancelled %d task%s' % (cancelled, plural))
def reload_action(self, argstr):
"""Usage: reload - reload droned rsa keys"""
self.keyRing = RSAKeyRing('%s' % (config.DRONED_KEY_DIR, ))
def ping_action(self, argstr):
"""Usage: ping"""
return (42, 'PONG')
def help_action(self, argstr):
"""Usage: help <action>"""
if not argstr:
l = self.builtins.keys() + [ str(i.action) for i in \
AdminAction.objects ] + ['license', 'version']
return '\n'.join(sorted(l))
try:
return self.get_action(argstr).__doc__
except:
return 'Unknown action'
def list_action(self, argstr):
"""Lists all known model instances and their classes"""
r = ''
for obj in gc.get_objects():
if not isinstance(obj, Entity): continue
if not obj.__class__.isValid(obj): continue
#avoid hidden objects
if obj.__class__.__name__.startswith('_'): continue
#avoid romeo key value objects
r += '%s\t%s\n' % (obj.__class__.__name__, str(obj))
return r
#this was tried natively once before w/o success
@deferredAsThread
def shell_action(server, *cmd):
"""Usage: shell <cmd>\nReturns: <exitcode> <stdout>"""
readfd, writefd = os.pipe()
pid = os.fork()
if pid == 0:
devnull = open('/dev/null', 'r')
os.dup2(devnull.fileno(), 0)
os.dup2(writefd, 1)
os.dup2(writefd, 2)
os.execvp('/bin/sh', ['/bin/sh', '-c'] + list(cmd))
os._exit(255)
else:
os.close(writefd)
fh = os.fdopen(readfd)
output = fh.read()
status = os.waitpid(pid, 0)
return (status[1] >> 8, output)
def formatResults(self, response):
"""format results of actions"""
#handle the NoneType case
if not response:
return {'code': 0, 'description': 'None'}
Msg = {
'code': -4,
'description': 'could not format result ' + str(response)
}
#fix errback results
if isinstance(response, Failure):
#see if this is a known droned command failure
if response.check(DroneCommandFailed):
response = response.value.resultContext
else:
response = {
'description': response.getErrorMessage(),
'code': 1,
'stacktrace': response.getTraceback(),
'error': True,
}
#should not have to check like this, but some old code does this wrong
elif isinstance(response, DroneCommandFailed):
response = response.resultContext
if isinstance(response, dict):
if 'description' not in response and 'error' in response and \
isinstance(response['error'], Failure):
response['description'] = response['error'].getErrorMessage()
response['stacktrace'] = response['error'].getTraceback()
server_log(response['stacktrace']) #log to the console log
Msg.update(response)
elif isinstance(response, basestring):
Msg = {'code': 0, 'description': response}
elif isinstance(response, tuple):
code, message = response[0:2]
if not isinstance(code, int): code = -2
Msg = {'code': code, 'description': message}
return Msg
from zope.interface import implementer from zope.interface import Interface, Attribute from .torconfig import TorConfig from .onion import IAuthenticatedOnionClients from .onion import FilesystemOnionService from .onion import IFilesystemOnionService from .onion import EphemeralOnionService from .onion import FilesystemAuthenticatedOnionService from .onion import EphemeralAuthenticatedOnionService from .onion import AuthStealth # , AuthBasic from .torconfig import _endpoint_from_socksport_line from .util import SingleObserver, _Version _global_tor = None _global_tor_lock = defer.DeferredLock() # we need the lock because we (potentially) yield several times while # "creating" the TorConfig instance # in an ideal world, we'd have "get_global_tor()" and it would return # a Tor instance, and all would be well. HOWEVER, "get_global_tor" was # previously released to return a TorConfig instance. So it still # needs to do that, *and* it should be the very same TorConfig # instance attached to the "global Tor instance". # Anyway: get_global_tor_instance() is the newst API, and the one new # code should use (if at all -- ideally just use a .global_tor() # factory-function call instead) @defer.inlineCallbacks def get_global_tor_instance(reactor,
def __setstate__(self, d):
styles.Versioned.__setstate__(self, d)
self._getSourceStampSetId_lock = defer.DeferredLock()
def initServer(self):
self.devices = util.MultiDict() # aliases -> device
self.device_guids = {} # name -> guid
self._next_guid = 0
self._refreshLock = defer.DeferredLock()
return self.refreshDeviceList()
def __init__(self, doc_id=None, future_doc_id=None, **kwargs):
self._doc_id = doc_id
self._future_doc_id = future_doc_id
self._lock = defer.DeferredLock()
super(SoledadDocumentWrapper, self).__init__(**kwargs)
def __setstate__(self, d):
styles.Versioned.__setstate__(self, d)
self._addSourceStampToDatabase_lock = defer.DeferredLock()
async def Acquire(switchid):
if switchid not in lock_tbl:
lock_tbl[switchid] = defer.DeferredLock()
await lock_tbl[switchid].acquire()
def tearDown(self):
from txtorcon import endpoints
endpoints._global_tor_config = None
del endpoints._global_tor_lock
endpoints._global_tor_lock = defer.DeferredLock()
self.patcher.stop()
def __init__(self, device):
super(SIMProtocol, self).__init__(device)
self.queue = defer.DeferredQueue()
self.mutex = defer.DeferredLock()
self._check_queue()
def __init__(self):
MongoServerProtocol.__init__(self)
self.__connection_ready = []
self.__deferreds = {}
self.__auth_lock = defer.DeferredLock()
def __init__(self):
super(GitilesRevisionComparator, self).__init__()
self.sha1_lookup = {}
self.initialized = False
self.initLock = defer.DeferredLock()
def __init__(self,
name,
builderNames,
properties=None,
codebases=DEFAULT_CODEBASES):
super(BaseScheduler, self).__init__(name=name)
ok = True
if interfaces.IRenderable.providedBy(builderNames):
pass
elif isinstance(builderNames, (list, tuple)):
for b in builderNames:
if not isinstance(b, string_types) and \
not interfaces.IRenderable.providedBy(b):
ok = False
else:
ok = False
if not ok:
config.error(
"The builderNames argument to a scheduler must be a list "
"of Builder names or an IRenderable object that will render"
"to a list of builder names.")
self.builderNames = builderNames
if properties is None:
properties = {}
self.properties = Properties()
self.properties.update(properties, "Scheduler")
self.properties.setProperty("scheduler", name, "Scheduler")
self.objectid = None
# Set the codebases that are necessary to process the changes
# These codebases will always result in a sourcestamp with or without
# changes
known_keys = set(['branch', 'repository', 'revision'])
if codebases is None:
config.error("Codebases cannot be None")
elif isinstance(codebases, list):
codebases = dict((codebase, {}) for codebase in codebases)
elif not isinstance(codebases, dict):
config.error(
"Codebases must be a dict of dicts, or list of strings")
else:
for codebase, attrs in iteritems(codebases):
if not isinstance(attrs, dict):
config.error("Codebases must be a dict of dicts")
else:
unk = set(attrs) - known_keys
if unk:
config.error(
"Unknown codebase keys %s for codebase %s" %
(', '.join(unk), codebase))
self.codebases = codebases
# internal variables
self._change_consumer = None
self._enable_consumer = None
self._change_consumption_lock = defer.DeferredLock()
self.enabled = True
def __init__(self, path) -> None:
if path == ':memory:':
self.io = BytesIO()
self.path = path
self._size: Optional[int] = None
self._header_connect_lock = defer.DeferredLock()
def __init__(self, *args, **kwargs):
super(NamedVolume, self).__init__(*args, **kwargs)
self._lock = defer.DeferredLock()
def __init__(self):
self.argd = MAIN_ARGD
# Main deferred, fired on fatal error or final disconnect.
self.deferred = defer.Deferred()
# Class to handle admin stuff. Needs to be accessed here and in
# CommandHandler.
self.admin = AdminHandler()
# Admin should have the EasySettings config options.
self.admin.config = CONFIG
self.admin.argd = self.argd
self.admin.monitor = self.get_config('monitor', False)
self.admin.monitordata = self.get_config('data', False)
self.admin.monitorips = self.get_config('ips', False)
self.admin.nickname = self.get_config('nick', 'pyval')
self.admin.cmdchar = self.get_config('commandchar', '!')
self.admin.noheartbeatlog = self.get_config('noheartbeat', False)
# Give admin access to certain functions.
self.admin.quit = self.quit
self.admin.sendLine = self.sendLine
self.admin.ctcpMakeQuery = self.ctcpMakeQuery
self.admin.do_action = self.me
self.admin.handlinglock = defer.DeferredLock()
# For setting the topic for our own channel if possible.
self.admin.topicfmt = ''.join([
'Python Evaluation Bot (pyval) | ',
'Type {cc}py <code> or {cc}help [cmd] if {nick} is around. | ',
'Use \\n for actual newlines (Enter), or \\\\n for '
'escaped newlines.'
])
self.admin.topicmsg = self.admin.topicfmt.format(
cc=self.admin.cmdchar, nick=self.admin.nickname)
# parse username/password config where 'user:password' is used.
pw = self.get_config('loginpw', None)
if pw:
if ':' in pw:
username, pw = pw.split(':')
else:
username = self.get_config('username', NAME)
else:
username = self.get_config('username', None)
if username:
if ':' in username:
username, pw = username.split(':')
else:
username = NAME
self.admin.username = username
self.password = pw
# IRCClient must hold the nickname/username attribute.
self.nickname = self.admin.nickname
self.username = self.admin.username
# The password attributes are deleted as soon as they are used.
self.nickservpw = self.get_config('password', default=None)
self.erroneousNickFallback = '{}_'.format(self.nickname)
# Settings for client/version replies.
self.versionName = NAME
self.versionNum = VERSION
# parse cmdline args to set attributes.
# self.channels depends on self.nickname for the default channel.
self.channels = self.parse_join_channels(self.get_config('channels'))
# Class to handle messages and commands.
self.commandhandler = CommandHandler(defer_=defer,
reactor_=reactor,
task_=task,
adminhandler=self.admin)
# Save cmdline args to config.
if self.get_config('autosave'):
save_config()
class SoledadMailAdaptor(SoledadIndexMixin):
implements(IMailAdaptor)
store = None
indexes = indexes.MAIL_INDEXES
wait_for_indexes = ['get_or_create_mbox', 'update_mbox', 'get_all_mboxes']
mboxwrapper_klass = MailboxWrapper
atomic = defer.DeferredLock()
log = Logger()
def __init__(self):
SoledadIndexMixin.__init__(self)
# Message handling
def get_msg_from_string(self, MessageClass, raw_msg):
"""
Get an instance of a MessageClass initialized with a MessageWrapper
that contains all the parts obtained from parsing the raw string for
the message.
:param MessageClass: any Message class that can be initialized passing
an instance of an IMessageWrapper implementor.
:type MessageClass: type
:param raw_msg: a string containing the raw email message.
:type raw_msg: str
:rtype: MessageClass instance.
"""
assert (MessageClass is not None)
mdoc, fdoc, hdoc, cdocs = _split_into_parts(raw_msg)
return self.get_msg_from_docs(MessageClass, mdoc, fdoc, hdoc, cdocs)
def get_msg_from_docs(self,
MessageClass,
mdoc,
fdoc,
hdoc,
cdocs=None,
uid=None):
"""
Get an instance of a MessageClass initialized with a MessageWrapper
that contains the passed part documents.
This is not the recommended way of obtaining a message, unless you know
how to take care of ensuring the internal consistency between the part
documents, or unless you are glueing together the part documents that
have been previously generated by `get_msg_from_string`.
:param MessageClass: any Message class that can be initialized passing
an instance of an IMessageWrapper implementor.
:type MessageClass: type
:param fdoc: a dictionary containing values from which a
FlagsDocWrapper can be initialized
:type fdoc: dict
:param hdoc: a dictionary containing values from which a
HeaderDocWrapper can be initialized
:type hdoc: dict
:param cdocs: None, or a dictionary mapping integers (1-indexed) to
dicts from where a ContentDocWrapper can be initialized.
:type cdocs: dict, or None
:rtype: MessageClass instance.
"""
assert (MessageClass is not None)
return MessageClass(MessageWrapper(mdoc, fdoc, hdoc, cdocs), uid=uid)
def get_msg_from_mdoc_id(self,
MessageClass,
store,
mdoc_id,
uid=None,
get_cdocs=False):
def wrap_meta_doc(doc):
cls = MetaMsgDocWrapper
return cls(doc_id=doc.doc_id, **doc.content)
def get_part_docs_from_mdoc_wrapper(wrapper):
d_docs = []
d_docs.append(store.get_doc(wrapper.fdoc))
d_docs.append(store.get_doc(wrapper.hdoc))
for cdoc in wrapper.cdocs:
d_docs.append(store.get_doc(cdoc))
def add_mdoc(doc_list):
return [wrapper.serialize()] + doc_list
d = defer.gatherResults(d_docs)
d.addCallback(add_mdoc)
return d
def get_parts_doc_from_mdoc_id():
mbox = re.findall(constants.METAMSGID_MBOX_RE, mdoc_id)[0]
chash = re.findall(constants.METAMSGID_CHASH_RE, mdoc_id)[0]
def _get_fdoc_id_from_mdoc_id():
return constants.FDOCID.format(mbox_uuid=mbox, chash=chash)
def _get_hdoc_id_from_mdoc_id():
return constants.HDOCID.format(mbox_uuid=mbox, chash=chash)
d_docs = []
fdoc_id = _get_fdoc_id_from_mdoc_id()
hdoc_id = _get_hdoc_id_from_mdoc_id()
d_docs.append(store.get_doc(mdoc_id))
d_docs.append(store.get_doc(fdoc_id))
d_docs.append(store.get_doc(hdoc_id))
d = defer.gatherResults(d_docs)
return d
def _err_log_failure_part_docs(failure):
# See https://leap.se/code/issues/7495.
# This avoids blocks, but the real cause still needs to be
# isolated (0.9.0rc3) -- kali
self.log.debug("BUG ------------------------------------------")
self.log.debug(
"BUG: Error while retrieving part docs for mdoc id %s" %
mdoc_id)
self.log.debug("BUG (please report above info) ---------------")
return []
def _err_log_cannot_find_msg(failure):
self.log.error('BUG: Error while getting msg (uid=%s)' % uid)
return None
if get_cdocs:
d = store.get_doc(mdoc_id)
d.addCallback(wrap_meta_doc)
d.addCallback(get_part_docs_from_mdoc_wrapper)
d.addErrback(_err_log_failure_part_docs)
else:
d = get_parts_doc_from_mdoc_id()
d.addCallback(self._get_msg_from_variable_doc_list,
msg_class=MessageClass,
uid=uid)
d.addErrback(_err_log_cannot_find_msg)
return d
def _get_msg_from_variable_doc_list(self, doc_list, msg_class, uid=None):
if len(doc_list) == 3:
mdoc, fdoc, hdoc = doc_list
cdocs = None
elif len(doc_list) > 3:
# XXX is this case used?
mdoc, fdoc, hdoc = doc_list[:3]
cdocs = dict(enumerate(doc_list[3:], 1))
return self.get_msg_from_docs(msg_class,
mdoc,
fdoc,
hdoc,
cdocs,
uid=uid)
def get_flags_from_mdoc_id(self, store, mdoc_id):
"""
# XXX stuff here...
"""
mbox = re.findall(constants.METAMSGID_MBOX_RE, mdoc_id)[0]
chash = re.findall(constants.METAMSGID_CHASH_RE, mdoc_id)[0]
def _get_fdoc_id_from_mdoc_id():
return constants.FDOCID.format(mbox_uuid=mbox, chash=chash)
fdoc_id = _get_fdoc_id_from_mdoc_id()
def wrap_fdoc(doc):
if not doc:
return
cls = FlagsDocWrapper
return cls(doc_id=doc.doc_id, **doc.content)
def get_flags(fdoc_wrapper):
if not fdoc_wrapper:
return []
return fdoc_wrapper.get_flags()
d = store.get_doc(fdoc_id)
d.addCallback(wrap_fdoc)
d.addCallback(get_flags)
return d
def create_msg(self, store, msg):
"""
:param store: an instance of soledad, or anything that behaves alike
:param msg: a Message object.
:return: a Deferred that is fired when all the underlying documents
have been created.
:rtype: defer.Deferred
"""
wrapper = msg.get_wrapper()
return wrapper.create(store)
def update_msg(self, store, msg):
"""
:param msg: a Message object.
:param store: an instance of soledad, or anything that behaves alike
:return: a Deferred that is fired when all the underlying documents
have been updated (actually, it's only the fdoc that's allowed
to update).
:rtype: defer.Deferred
"""
wrapper = msg.get_wrapper()
return wrapper.update(store)
# batch deletion
def del_all_flagged_messages(self, store, mbox_uuid):
"""
Delete all messages flagged as deleted.
"""
def delete_fdoc_and_mdoc_flagged(fdocs):
# low level here, not using the wrappers...
# get meta doc ids from the flag doc ids
fdoc_ids = [doc.doc_id for doc in fdocs]
mdoc_ids = map(lambda s: "M" + s[1:], fdoc_ids)
def delete_all_docs(mdocs, fdocs):
mdocs = list(mdocs)
doc_ids = [m.doc_id for m in mdocs]
_d = []
docs = mdocs + fdocs
for doc in docs:
_d.append(store.delete_doc(doc))
d = defer.gatherResults(_d)
# return the mdocs ids only
d.addCallback(lambda _: doc_ids)
return d
d = store.get_docs(mdoc_ids)
d.addCallback(delete_all_docs, fdocs)
return d
type_ = FlagsDocWrapper.model.type_
uuid = mbox_uuid.replace('-', '_')
deleted_index = indexes.TYPE_MBOX_DEL_IDX
d = store.get_from_index(deleted_index, type_, uuid, "1")
d.addCallback(delete_fdoc_and_mdoc_flagged)
return d
# count messages
def get_count_unseen(self, store, mbox_uuid):
"""
Get the number of unseen messages for a given mailbox.
:param store: instance of Soledad.
:param mbox_uuid: the uuid for this mailbox.
:rtype: int
"""
type_ = FlagsDocWrapper.model.type_
uuid = mbox_uuid.replace('-', '_')
unseen_index = indexes.TYPE_MBOX_SEEN_IDX
d = store.get_count_from_index(unseen_index, type_, uuid, "0")
d.addErrback(lambda f: self.log.error('Error on count_unseen'))
return d
def get_count_recent(self, store, mbox_uuid):
"""
Get the number of recent messages for a given mailbox.
:param store: instance of Soledad.
:param mbox_uuid: the uuid for this mailbox.
:rtype: int
"""
type_ = FlagsDocWrapper.model.type_
uuid = mbox_uuid.replace('-', '_')
recent_index = indexes.TYPE_MBOX_RECENT_IDX
d = store.get_count_from_index(recent_index, type_, uuid, "1")
d.addErrback(lambda f: self.log.error('Error on count_recent'))
return d
# search api
def get_mdoc_id_from_msgid(self, store, mbox_uuid, msgid):
"""
Get the UID for a message with the passed msgid (the one in the headers
msg-id).
This is used by the MUA to retrieve the recently saved draft.
"""
type_ = HeaderDocWrapper.model.type_
uuid = mbox_uuid.replace('-', '_')
msgid_index = indexes.TYPE_MSGID_IDX
def get_mdoc_id(hdoc):
if not hdoc:
self.log.warn("Could not find a HDOC with MSGID %s" % msgid)
return None
hdoc = hdoc[0]
mdoc_id = hdoc.doc_id.replace("H-", "M-%s-" % uuid)
return mdoc_id
d = store.get_from_index(msgid_index, type_, msgid)
d.addCallback(get_mdoc_id)
return d
# Mailbox handling
def get_or_create_mbox(self, store, name):
"""
Get the mailbox with the given name, or create one if it does not
exist.
:param store: instance of Soledad
:param name: the name of the mailbox
:type name: str
"""
index = indexes.TYPE_MBOX_IDX
mbox = normalize_mailbox(name)
return MailboxWrapper.get_or_create(store, index, mbox)
def update_mbox(self, store, mbox_wrapper):
"""
Update the documents for a given mailbox.
:param mbox_wrapper: MailboxWrapper instance
:type mbox_wrapper: MailboxWrapper
:return: a Deferred that will be fired when the mailbox documents
have been updated.
:rtype: defer.Deferred
"""
leap_assert_type(mbox_wrapper, SoledadDocumentWrapper)
return mbox_wrapper.update(store)
def delete_mbox(self, store, mbox_wrapper):
leap_assert_type(mbox_wrapper, SoledadDocumentWrapper)
return mbox_wrapper.delete(store)
def get_all_mboxes(self, store):
"""
Retrieve a list with wrappers for all the mailboxes.
:return: a deferred that will be fired with a list of all the
MailboxWrappers found.
:rtype: defer.Deferred
"""
return MailboxWrapper.get_all(store)
import logging
from collections import defaultdict
from twisted.internet import defer
from adpay.db import utils as db_utils
from adpay.stats import cache as stats_cache, consts as stats_consts, utils as stats_utils
#: Deferred lock for updating events
ADD_EVENT_LOCK = defer.DeferredLock()
@defer.inlineCallbacks
def calculate_events_payments(campaign_doc,
timestamp,
payment_percentage_cutoff=0.5):
"""
For new users:
1. Assign them max_human_score from the database and CPM value (per campaign)
2. Assign payment score based on average of other users.
3.
:param campaign_doc:
:param timestamp:
:param payment_percentage_cutoff:
:return:
"""
# Check if campaign exists
if campaign_doc is None:
return
def __init__(self):
self.req = None
self.setRawMode()
self.lock = defer.DeferredLock()
self.data = ''
from app import communication_keys
from amazon import suicide_check
from amazon.sqs import WorkerQueue
from amazon.suicide_check import get_cpu_balance
from twisted.web.client import getPage
from twisted.web import xmlrpc, server
from twisted.internet import defer, reactor
import StringIO
import subprocess
import json
import boto.ec2
lock_apk = defer.DeferredLock()
from twisted.internet import protocol
processes = []
class SpawnProcessProtocol(protocol.ProcessProtocol):
def outReceived(self, data):
print 'data:', data
def errReceived(self, data):
print "errReceived! with %d bytes!" % len(data)
print 'data:', data
def errConnectionLost(self):
processes.remove(self)
def __init__(self, name, builderNames, properties,
codebases=DefaultCodebases):
"""
Initialize a Scheduler.
@param name: name of this scheduler (used as a key for state)
@type name: unicode
@param builderNames: list of builders this scheduler may start
@type builderNames: list of unicode
@param properties: properties to add to builds triggered by this
scheduler
@type properties: dictionary
@param codebases: codebases that are necessary to process the changes
@type codebases: dict with following struct:
key: '<codebase>'
value: {'repository':'<repo>', 'branch':'<br>', 'revision:'<rev>'}
@param consumeChanges: true if this scheduler wishes to be informed
about the addition of new changes. Defaults to False. This should
be passed explicitly from subclasses to indicate their interest in
consuming changes.
@type consumeChanges: boolean
"""
service.MultiService.__init__(self)
self.name = name
"name of this scheduler; used to identify replacements on reconfig"
ok = True
if not isinstance(builderNames, (list, tuple)):
ok = False
else:
for b in builderNames:
if not isinstance(b, basestring):
ok = False
if not ok:
config.error(
"The builderNames argument to a scheduler must be a list "
"of Builder names.")
self.builderNames = builderNames
"list of builder names to start in each buildset"
self.properties = Properties()
"properties that are contributed to each buildset"
self.properties.update(properties, "Scheduler")
self.properties.setProperty("scheduler", name, "Scheduler")
self.objectid = None
self.master = None
# Set the codebases that are necessary to process the changes
# These codebases will always result in a sourcestamp with or without changes
if codebases is not None:
if not isinstance(codebases, dict):
config.error("Codebases must be a dict of dicts")
for codebase, codebase_attrs in codebases.iteritems():
if not isinstance(codebase_attrs, dict):
config.error("Codebases must be a dict of dicts")
if (codebases != BaseScheduler.DefaultCodebases and
'repository' not in codebase_attrs):
config.error("The key 'repository' is mandatory in codebases")
else:
config.error("Codebases cannot be None")
self.codebases = codebases
# internal variables
self._change_subscription = None
self._change_consumption_lock = defer.DeferredLock()
def __init__(self, master):
service.MultiService.__init__(self)
self.master = master
self.upstream_subscribers = bbcollections.defaultdict(list)
self._updateLock = defer.DeferredLock()
def __init__(self,
repourl=_repourl,
branch=_branch,
workdir=None,
pollInterval=10 * 60,
gitbin='git',
usetimestamps=True,
category=None,
project=None,
pollinterval=-2,
fetch_refspec=None,
encoding='utf-8',
projects=None):
self.cleanRe = re.compile(r"Require(?:s?)\s*.*\s*clean build",
re.IGNORECASE + re.MULTILINE)
self.cleanCfg = re.compile(r"(CMakeLists\.txt$|\.cmake$|\.cmake\.in$)")
# projects is a list of projects to watch or None to watch all.
if projects:
if isinstance(projects, str) or isinstance(projects, tuple):
projects = [projects]
assert isinstance(projects, list)
assert len(projects) > 0
# Each project to watch is a string (project name) or a tuple
# (project name, branch) like ('llvm', 'branches/release_30').
# But we want it always to be a tuple, so we convert a project
# name string to a tuple (project, 'master').
self.projects = set()
for project in projects:
if isinstance(project, str):
project = (project, branch)
assert isinstance(project, tuple)
self.projects.add(project)
# for backward compatibility; the parameter used to be spelled with 'i'
if pollinterval != -2:
pollInterval = pollinterval
if project is None: project = ''
self.repourl = repourl
self.branch = branch
self.pollInterval = pollInterval
self.fetch_refspec = fetch_refspec
self.encoding = encoding
self.lastChange = time.time()
self.lastPoll = time.time()
self.gitbin = gitbin
self.workdir = workdir
self.usetimestamps = usetimestamps
self.category = category
self.project = project
self.changeCount = 0
self.commitInfo = {}
self.initLock = defer.DeferredLock()
if self.workdir == None:
self.workdir = tempfile.gettempdir() + '/gitpoller_work'
log.msg(
"WARNING: LLVMGitPoller using deprecated temporary workdir " +
"'%s'; consider setting workdir=" % self.workdir)
def __init__(self, owner, slug, **kwargs):
kwargs['name'] = self.build_name(owner, slug)
self.initLock = defer.DeferredLock()
super().__init__(owner, slug, **kwargs)
def __init__(self,
basedir,
configFileName=None,
umask=None,
reactor=None,
config_loader=None):
service.AsyncMultiService.__init__(self)
if reactor is None:
from twisted.internet import reactor
self.reactor = reactor
self.setName("buildmaster")
self.umask = umask
self.basedir = basedir
if basedir is not None: # None is used in tests
assert os.path.isdir(self.basedir)
if config_loader is not None and configFileName is not None:
raise config.ConfigErrors([
"Can't specify both `config_loader` and `configFilename`.",
])
elif config_loader is None:
if configFileName is None:
configFileName = 'master.cfg'
config_loader = config.FileLoader(self.basedir, configFileName)
self.config_loader = config_loader
self.configFileName = configFileName
# flag so we don't try to do fancy things before the master is ready
self._master_initialized = False
self.initLock = defer.DeferredLock()
# set up child services
self.create_child_services()
# db configured values
self.configured_db_url = None
# configuration / reconfiguration handling
self.config = config.MasterConfig()
self.reconfig_active = False
self.reconfig_requested = False
self.reconfig_notifier = None
# this stores parameters used in the tac file, and is accessed by the
# WebStatus to duplicate those values.
self.log_rotation = LogRotation()
# local cache for this master's object ID
self._object_id = None
# Check environment is sensible
check_functional_environment(self.config)
# figure out local hostname
try:
self.hostname = os.uname()[1] # only on unix
except AttributeError:
self.hostname = socket.getfqdn()
# public attributes
self.name = ("%s:%s" %
(self.hostname, os.path.abspath(self.basedir or '.')))
if isinstance(self.name, bytes):
self.name = self.name.decode('ascii', 'replace')
self.masterid = None
def setUp(self):
super(TestMultiLock, self).setUp()
self.lock_one = defer.DeferredLock()
self.lock_two = defer.DeferredLock()
self.multi_lock = MultiLock(self.lock_one, self.lock_two)
self.count = 0
class Janitizer(Service):
minute = property(lambda foo: 60)
hour = property(lambda foo: 3600)
day = property(lambda foo: 86400)
week = property(lambda f: 604800)
oldfiles = {}
#get the watch dictionary from romeo
watchDict = property(lambda s: SERVICECONFIG.wrapped.get('JANITIZE', {}))
#lock aquired before starting a thread that modifies class state
busy = defer.DeferredLock()
def update(self, watchDict):
"""Inspects occurrence for a watchDict parameter and updates
the internal state of Janitizer
@param watchDict (dict)
return None
"""
#break references
tmp = copy.deepcopy(self.watchDict)
tmp.update(watchDict) #apply updates
SERVICECONFIG.JANITIZE = tmp
#this would have blocked the reactor w/o the thread
@synchronizedDeferred(busy)
@deferredAsThread
def garbageCheck(self):
"""Check for file patterns that are removeable"""
watchDict = copy.deepcopy(self.watchDict) #use locals for safety
for directory, garbageList in watchDict.iteritems():
if not os.path.exists(directory): continue
for pattern, limit in garbageList:
#blocking method in a thread
self.cleanupLinks(directory)
files = [os.path.join(directory,f) for f in os.listdir(directory) \
if re.search(pattern,f)]
files = sorted(files)
if len(files) > int(limit):
log('These files matched:\n\t%s' % '\n\t'.join(files))
while len(files) > int(limit):
oldfile = files.pop(0)
log('Deleting %s' % oldfile)
if os.path.islink(oldfile): continue
if os.path.isdir(oldfile):
for base, dirs, myfiles in os.walk(oldfile,
topdown=False):
for name in myfiles:
os.remove(os.path.join(base, name))
for name in dirs:
os.rmdir(os.path.join(base, name))
os.rmdir(oldfile)
else:
os.unlink(oldfile)
#blocking method in a thread
self.cleanupLinks(directory)
#this will block the reactor
def cleanupLinks(self, directory):
"""cleans broken symlinks
@param directory: (string)
return list
"""
files = [os.path.join(directory, f) for f in os.listdir(directory)]
for f in files[:]:
if not os.path.exists(f):
log('Removing broken symlink %s' % f)
os.unlink(f)
files.remove(f)
return files
def clean_old_files(self, directory, age, recurse=True):
"""mark this directory for cleaning at a certain age
@param directory: (string)
@param age: (float)
@param recurse: (bool)
return None
"""
self.oldfiles[directory] = (age, recurse)
#this would have blocked the reactor w/o the thread
@synchronizedDeferred(busy)
@deferredAsThread
def clean_elderly(self):
"""clean old files in a thread"""
for directory in self.oldfiles:
self.recursive_clean(directory, *self.oldfiles[directory])
#this will block the reactor
def recursive_clean(self, directory, age, recurse):
"""recusively clean a directory
@param directory: (string)
@param age: (float)
@param recurse: (bool)
return bool
"""
try:
data = map(lambda n: os.path.join(directory, n),
os.listdir(directory))
except:
log('could not find directory %s' % directory)
return
for node in data:
if os.path.isdir(node) and recurse:
#blocking method in a thread
empty = self.recursive_clean(node, age, recurse)
if empty:
try:
os.rmdir(node)
except:
log('could not remove directory: %s' % node)
continue
if os.path.isdir(node): continue #in case recurse is False
if (time.time() - os.stat(node).st_mtime) > age:
try:
os.remove(node)
except:
log('could not remove file: %s' % node)
return bool(os.listdir(directory))
def startService(self):
"""Start Janitizer Service"""
self.GARBAGE_CHECK = task.LoopingCall(self.garbageCheck)
self.ELDERLY_CHECK = task.LoopingCall(self.clean_elderly)
#start the service
Service.startService(self)
self.GARBAGE_CHECK.start(self.minute * 20)
self.ELDERLY_CHECK.start(self.minute)
def stopService(self):
"""Stop All Janitizer Service"""
try:
if self.GARBAGE_CHECK.running:
self.GARBAGE_CHECK.stop()
if self.ELDERLY_CHECK.running:
self.ELDERLY_CHECK.stop()
except:
pass
Service.stopService(self)
def getMergingLocks(self, build_request_ids):
return [
self.build_merging_locks.setdefault(brid, defer.DeferredLock())
for brid in build_request_ids
]
def doRequest(self, *args):
if self.__lock is None:
self.__lock = defer.DeferredLock()
return self.__lock.run(threads.deferToThread, self._doRequest, *args)
def __init__(self, _ignored):
self.lock = defer.DeferredLock()
self.value = None
