def __init__(self, repourl, branch=None, branches=None, bookmarks=None, workdir=None, pollInterval=10 * 60, hgbin="hg", usetimestamps=True, category=None, project="", pollinterval=-2, encoding="utf-8", name=None, pollAtLaunch=False, revlink=lambda branch, revision: (""), pollRandomDelayMin=0, pollRandomDelayMax=0): # for backward compatibility; the parameter used to be spelled with 'i' if pollinterval != -2: pollInterval = pollinterval self.repourl = repourl if branch and branches: config.error("HgPoller: can't specify both branch and branches") elif branch: self.branches = [branch] else: self.branches = branches or [] self.bookmarks = bookmarks or [] if name is None: name = repourl if self.bookmarks: name += "_" + "_".join(self.bookmarks) if self.branches: name += "_" + "_".join(self.branches) if not self.branches and not self.bookmarks: self.branches = ['default'] if not callable(revlink): config.error("You need to provide a valid callable for revlink") super().__init__(name=name, pollInterval=pollInterval, pollAtLaunch=pollAtLaunch, pollRandomDelayMin=pollRandomDelayMin, pollRandomDelayMax=pollRandomDelayMax) self.encoding = encoding self.lastChange = time.time() self.lastPoll = time.time() self.hgbin = hgbin self.workdir = workdir self.usetimestamps = usetimestamps self.category = category if callable(category) else bytes2unicode( category) self.project = project self.initLock = defer.DeferredLock() self.lastRev = {} self.revlink_callable = revlink if self.workdir is None: config.error("workdir is mandatory for now in HgPoller")
class DroneServer(Entity): """This model controls the routing of commands from the various services.""" keyRing = RSAKeyRing('%s' % (config.DRONED_KEY_DIR, )) lock = defer.DeferredLock() def __init__(self): self._primes = set() self.builtins = { 'help': self.help_action, 'ping': self.ping_action, 'list': self.list_action, 'shell': self.shell_action, 'reload': self.reload_action, 'tasks': self.tasks_action, 'cancel': self.cancel_action, } #self register this server hostname if not Server.exists(config.HOSTNAME): server = Server(config.HOSTNAME) server.listed = True else: server = Server(config.HOSTNAME) self.server = server def _license(self, argstr): return "\n" + copyright.copyright_notice _license.__doc__ = copyright.copyright def _version(self, argstr): """Shows the server version""" return "DroneD/%s" % (copyright.version, ) @synchronizedDeferred(lock) @deferredAsThread #will catch exceptions def getprime(self): pfh = open(config.DRONED_PRIMES) psize = os.stat(config.DRONED_PRIMES)[6] if (psize % 4) != 0 or psize < 4000: pfh.close() raise AssertionError("primes file is corrupt/too small") try: fcntl.fcntl(pfh.fileno(), fcntl.F_SETFD, fcntl.FD_CLOEXEC) except: pass result = 0 while True: try: r = random.randint(0, (psize - 4) / 4) * 4 pfh.seek(r) p = pfh.read(4) prime = struct.unpack("!L", p)[0] #makes a blocking call from this thread to the reactor #this is done for thread safety sync = synchronizedInThread() trp = sync(self._trackPrime) result = trp(prime) break except AssertionError: continue pfh.close() return result def _trackPrime(self, prime): """Tracks the prime numbers""" assert prime not in self._primes self._primes.add(prime) return prime @synchronizedDeferred(lock) def validateMessage(self, magicNumber): """Is the message meant for me""" for prime in self._primes: if (magicNumber % prime) == 0: #release the prime to prevent replay attacks self._primes.discard(prime) return True return False @synchronizedDeferred(lock) def releasePrime(self, prime): """release the prime""" self._primes.discard(prime) def get_action(self, action): """route commands to perform actions""" if action == 'license': return self._license elif action == 'version': return self._version foo = self.builtins.get(action, False) if foo and hasattr(foo, '__call__'): return foo #look for AdminActions for admin in AdminAction.objects: if action != admin.action: continue return admin raise AssertionError("No such action ... try 'help'") def tasks_action(self, argstr): """Usage: tasks - displays tasks and status""" results = ['', "completed/succeeded\t'task'\n"] for action in Action.objects: r = """\t%s/%s\t'%s'""" % \ (action.completed, action.succeeded, action.description) results.append(r) results.append('') return (0, '\n'.join(results)) #TODO test def cancel_action(self, argstr): """Usage: cancel 'task' - cancels all tasks matching the description""" cancelled = 0 plural = 's' for action in Action.objects: if action.completed: continue if action.description == argstr: try: action.deferred.cancel() cancelled += 1 except: pass if cancelled == 1: plural = '' return (0, 'cancelled %d task%s' % (cancelled, plural)) def reload_action(self, argstr): """Usage: reload - reload droned rsa keys""" self.keyRing = RSAKeyRing('%s' % (config.DRONED_KEY_DIR, )) def ping_action(self, argstr): """Usage: ping""" return (42, 'PONG') def help_action(self, argstr): """Usage: help <action>""" if not argstr: l = self.builtins.keys() + [ str(i.action) for i in \ AdminAction.objects ] + ['license', 'version'] return '\n'.join(sorted(l)) try: return self.get_action(argstr).__doc__ except: return 'Unknown action' def list_action(self, argstr): """Lists all known model instances and their classes""" r = '' for obj in gc.get_objects(): if not isinstance(obj, Entity): continue if not obj.__class__.isValid(obj): continue #avoid hidden objects if obj.__class__.__name__.startswith('_'): continue #avoid romeo key value objects r += '%s\t%s\n' % (obj.__class__.__name__, str(obj)) return r #this was tried natively once before w/o success @deferredAsThread def shell_action(server, *cmd): """Usage: shell <cmd>\nReturns: <exitcode> <stdout>""" readfd, writefd = os.pipe() pid = os.fork() if pid == 0: devnull = open('/dev/null', 'r') os.dup2(devnull.fileno(), 0) os.dup2(writefd, 1) os.dup2(writefd, 2) os.execvp('/bin/sh', ['/bin/sh', '-c'] + list(cmd)) os._exit(255) else: os.close(writefd) fh = os.fdopen(readfd) output = fh.read() status = os.waitpid(pid, 0) return (status[1] >> 8, output) def formatResults(self, response): """format results of actions""" #handle the NoneType case if not response: return {'code': 0, 'description': 'None'} Msg = { 'code': -4, 'description': 'could not format result ' + str(response) } #fix errback results if isinstance(response, Failure): #see if this is a known droned command failure if response.check(DroneCommandFailed): response = response.value.resultContext else: response = { 'description': response.getErrorMessage(), 'code': 1, 'stacktrace': response.getTraceback(), 'error': True, } #should not have to check like this, but some old code does this wrong elif isinstance(response, DroneCommandFailed): response = response.resultContext if isinstance(response, dict): if 'description' not in response and 'error' in response and \ isinstance(response['error'], Failure): response['description'] = response['error'].getErrorMessage() response['stacktrace'] = response['error'].getTraceback() server_log(response['stacktrace']) #log to the console log Msg.update(response) elif isinstance(response, basestring): Msg = {'code': 0, 'description': response} elif isinstance(response, tuple): code, message = response[0:2] if not isinstance(code, int): code = -2 Msg = {'code': code, 'description': message} return Msg
from zope.interface import implementer from zope.interface import Interface, Attribute from .torconfig import TorConfig from .onion import IAuthenticatedOnionClients from .onion import FilesystemOnionService from .onion import IFilesystemOnionService from .onion import EphemeralOnionService from .onion import FilesystemAuthenticatedOnionService from .onion import EphemeralAuthenticatedOnionService from .onion import AuthStealth # , AuthBasic from .torconfig import _endpoint_from_socksport_line from .util import SingleObserver, _Version _global_tor = None _global_tor_lock = defer.DeferredLock() # we need the lock because we (potentially) yield several times while # "creating" the TorConfig instance # in an ideal world, we'd have "get_global_tor()" and it would return # a Tor instance, and all would be well. HOWEVER, "get_global_tor" was # previously released to return a TorConfig instance. So it still # needs to do that, *and* it should be the very same TorConfig # instance attached to the "global Tor instance". # Anyway: get_global_tor_instance() is the newst API, and the one new # code should use (if at all -- ideally just use a .global_tor() # factory-function call instead) @defer.inlineCallbacks def get_global_tor_instance(reactor,
def __setstate__(self, d): styles.Versioned.__setstate__(self, d) self._getSourceStampSetId_lock = defer.DeferredLock()
def initServer(self): self.devices = util.MultiDict() # aliases -> device self.device_guids = {} # name -> guid self._next_guid = 0 self._refreshLock = defer.DeferredLock() return self.refreshDeviceList()
def __init__(self, doc_id=None, future_doc_id=None, **kwargs): self._doc_id = doc_id self._future_doc_id = future_doc_id self._lock = defer.DeferredLock() super(SoledadDocumentWrapper, self).__init__(**kwargs)
def __setstate__(self, d): styles.Versioned.__setstate__(self, d) self._addSourceStampToDatabase_lock = defer.DeferredLock()
async def Acquire(switchid): if switchid not in lock_tbl: lock_tbl[switchid] = defer.DeferredLock() await lock_tbl[switchid].acquire()
def tearDown(self): from txtorcon import endpoints endpoints._global_tor_config = None del endpoints._global_tor_lock endpoints._global_tor_lock = defer.DeferredLock() self.patcher.stop()
def __init__(self, device): super(SIMProtocol, self).__init__(device) self.queue = defer.DeferredQueue() self.mutex = defer.DeferredLock() self._check_queue()
def __init__(self): MongoServerProtocol.__init__(self) self.__connection_ready = [] self.__deferreds = {} self.__auth_lock = defer.DeferredLock()
def __init__(self): super(GitilesRevisionComparator, self).__init__() self.sha1_lookup = {} self.initialized = False self.initLock = defer.DeferredLock()
def __init__(self, name, builderNames, properties=None, codebases=DEFAULT_CODEBASES): super(BaseScheduler, self).__init__(name=name) ok = True if interfaces.IRenderable.providedBy(builderNames): pass elif isinstance(builderNames, (list, tuple)): for b in builderNames: if not isinstance(b, string_types) and \ not interfaces.IRenderable.providedBy(b): ok = False else: ok = False if not ok: config.error( "The builderNames argument to a scheduler must be a list " "of Builder names or an IRenderable object that will render" "to a list of builder names.") self.builderNames = builderNames if properties is None: properties = {} self.properties = Properties() self.properties.update(properties, "Scheduler") self.properties.setProperty("scheduler", name, "Scheduler") self.objectid = None # Set the codebases that are necessary to process the changes # These codebases will always result in a sourcestamp with or without # changes known_keys = set(['branch', 'repository', 'revision']) if codebases is None: config.error("Codebases cannot be None") elif isinstance(codebases, list): codebases = dict((codebase, {}) for codebase in codebases) elif not isinstance(codebases, dict): config.error( "Codebases must be a dict of dicts, or list of strings") else: for codebase, attrs in iteritems(codebases): if not isinstance(attrs, dict): config.error("Codebases must be a dict of dicts") else: unk = set(attrs) - known_keys if unk: config.error( "Unknown codebase keys %s for codebase %s" % (', '.join(unk), codebase)) self.codebases = codebases # internal variables self._change_consumer = None self._enable_consumer = None self._change_consumption_lock = defer.DeferredLock() self.enabled = True
def __init__(self, path) -> None: if path == ':memory:': self.io = BytesIO() self.path = path self._size: Optional[int] = None self._header_connect_lock = defer.DeferredLock()
def __init__(self, *args, **kwargs): super(NamedVolume, self).__init__(*args, **kwargs) self._lock = defer.DeferredLock()
def __init__(self): self.argd = MAIN_ARGD # Main deferred, fired on fatal error or final disconnect. self.deferred = defer.Deferred() # Class to handle admin stuff. Needs to be accessed here and in # CommandHandler. self.admin = AdminHandler() # Admin should have the EasySettings config options. self.admin.config = CONFIG self.admin.argd = self.argd self.admin.monitor = self.get_config('monitor', False) self.admin.monitordata = self.get_config('data', False) self.admin.monitorips = self.get_config('ips', False) self.admin.nickname = self.get_config('nick', 'pyval') self.admin.cmdchar = self.get_config('commandchar', '!') self.admin.noheartbeatlog = self.get_config('noheartbeat', False) # Give admin access to certain functions. self.admin.quit = self.quit self.admin.sendLine = self.sendLine self.admin.ctcpMakeQuery = self.ctcpMakeQuery self.admin.do_action = self.me self.admin.handlinglock = defer.DeferredLock() # For setting the topic for our own channel if possible. self.admin.topicfmt = ''.join([ 'Python Evaluation Bot (pyval) | ', 'Type {cc}py <code> or {cc}help [cmd] if {nick} is around. | ', 'Use \\n for actual newlines (Enter), or \\\\n for ' 'escaped newlines.' ]) self.admin.topicmsg = self.admin.topicfmt.format( cc=self.admin.cmdchar, nick=self.admin.nickname) # parse username/password config where 'user:password' is used. pw = self.get_config('loginpw', None) if pw: if ':' in pw: username, pw = pw.split(':') else: username = self.get_config('username', NAME) else: username = self.get_config('username', None) if username: if ':' in username: username, pw = username.split(':') else: username = NAME self.admin.username = username self.password = pw # IRCClient must hold the nickname/username attribute. self.nickname = self.admin.nickname self.username = self.admin.username # The password attributes are deleted as soon as they are used. self.nickservpw = self.get_config('password', default=None) self.erroneousNickFallback = '{}_'.format(self.nickname) # Settings for client/version replies. self.versionName = NAME self.versionNum = VERSION # parse cmdline args to set attributes. # self.channels depends on self.nickname for the default channel. self.channels = self.parse_join_channels(self.get_config('channels')) # Class to handle messages and commands. self.commandhandler = CommandHandler(defer_=defer, reactor_=reactor, task_=task, adminhandler=self.admin) # Save cmdline args to config. if self.get_config('autosave'): save_config()
class SoledadMailAdaptor(SoledadIndexMixin): implements(IMailAdaptor) store = None indexes = indexes.MAIL_INDEXES wait_for_indexes = ['get_or_create_mbox', 'update_mbox', 'get_all_mboxes'] mboxwrapper_klass = MailboxWrapper atomic = defer.DeferredLock() log = Logger() def __init__(self): SoledadIndexMixin.__init__(self) # Message handling def get_msg_from_string(self, MessageClass, raw_msg): """ Get an instance of a MessageClass initialized with a MessageWrapper that contains all the parts obtained from parsing the raw string for the message. :param MessageClass: any Message class that can be initialized passing an instance of an IMessageWrapper implementor. :type MessageClass: type :param raw_msg: a string containing the raw email message. :type raw_msg: str :rtype: MessageClass instance. """ assert (MessageClass is not None) mdoc, fdoc, hdoc, cdocs = _split_into_parts(raw_msg) return self.get_msg_from_docs(MessageClass, mdoc, fdoc, hdoc, cdocs) def get_msg_from_docs(self, MessageClass, mdoc, fdoc, hdoc, cdocs=None, uid=None): """ Get an instance of a MessageClass initialized with a MessageWrapper that contains the passed part documents. This is not the recommended way of obtaining a message, unless you know how to take care of ensuring the internal consistency between the part documents, or unless you are glueing together the part documents that have been previously generated by `get_msg_from_string`. :param MessageClass: any Message class that can be initialized passing an instance of an IMessageWrapper implementor. :type MessageClass: type :param fdoc: a dictionary containing values from which a FlagsDocWrapper can be initialized :type fdoc: dict :param hdoc: a dictionary containing values from which a HeaderDocWrapper can be initialized :type hdoc: dict :param cdocs: None, or a dictionary mapping integers (1-indexed) to dicts from where a ContentDocWrapper can be initialized. :type cdocs: dict, or None :rtype: MessageClass instance. """ assert (MessageClass is not None) return MessageClass(MessageWrapper(mdoc, fdoc, hdoc, cdocs), uid=uid) def get_msg_from_mdoc_id(self, MessageClass, store, mdoc_id, uid=None, get_cdocs=False): def wrap_meta_doc(doc): cls = MetaMsgDocWrapper return cls(doc_id=doc.doc_id, **doc.content) def get_part_docs_from_mdoc_wrapper(wrapper): d_docs = [] d_docs.append(store.get_doc(wrapper.fdoc)) d_docs.append(store.get_doc(wrapper.hdoc)) for cdoc in wrapper.cdocs: d_docs.append(store.get_doc(cdoc)) def add_mdoc(doc_list): return [wrapper.serialize()] + doc_list d = defer.gatherResults(d_docs) d.addCallback(add_mdoc) return d def get_parts_doc_from_mdoc_id(): mbox = re.findall(constants.METAMSGID_MBOX_RE, mdoc_id)[0] chash = re.findall(constants.METAMSGID_CHASH_RE, mdoc_id)[0] def _get_fdoc_id_from_mdoc_id(): return constants.FDOCID.format(mbox_uuid=mbox, chash=chash) def _get_hdoc_id_from_mdoc_id(): return constants.HDOCID.format(mbox_uuid=mbox, chash=chash) d_docs = [] fdoc_id = _get_fdoc_id_from_mdoc_id() hdoc_id = _get_hdoc_id_from_mdoc_id() d_docs.append(store.get_doc(mdoc_id)) d_docs.append(store.get_doc(fdoc_id)) d_docs.append(store.get_doc(hdoc_id)) d = defer.gatherResults(d_docs) return d def _err_log_failure_part_docs(failure): # See https://leap.se/code/issues/7495. # This avoids blocks, but the real cause still needs to be # isolated (0.9.0rc3) -- kali self.log.debug("BUG ------------------------------------------") self.log.debug( "BUG: Error while retrieving part docs for mdoc id %s" % mdoc_id) self.log.debug("BUG (please report above info) ---------------") return [] def _err_log_cannot_find_msg(failure): self.log.error('BUG: Error while getting msg (uid=%s)' % uid) return None if get_cdocs: d = store.get_doc(mdoc_id) d.addCallback(wrap_meta_doc) d.addCallback(get_part_docs_from_mdoc_wrapper) d.addErrback(_err_log_failure_part_docs) else: d = get_parts_doc_from_mdoc_id() d.addCallback(self._get_msg_from_variable_doc_list, msg_class=MessageClass, uid=uid) d.addErrback(_err_log_cannot_find_msg) return d def _get_msg_from_variable_doc_list(self, doc_list, msg_class, uid=None): if len(doc_list) == 3: mdoc, fdoc, hdoc = doc_list cdocs = None elif len(doc_list) > 3: # XXX is this case used? mdoc, fdoc, hdoc = doc_list[:3] cdocs = dict(enumerate(doc_list[3:], 1)) return self.get_msg_from_docs(msg_class, mdoc, fdoc, hdoc, cdocs, uid=uid) def get_flags_from_mdoc_id(self, store, mdoc_id): """ # XXX stuff here... """ mbox = re.findall(constants.METAMSGID_MBOX_RE, mdoc_id)[0] chash = re.findall(constants.METAMSGID_CHASH_RE, mdoc_id)[0] def _get_fdoc_id_from_mdoc_id(): return constants.FDOCID.format(mbox_uuid=mbox, chash=chash) fdoc_id = _get_fdoc_id_from_mdoc_id() def wrap_fdoc(doc): if not doc: return cls = FlagsDocWrapper return cls(doc_id=doc.doc_id, **doc.content) def get_flags(fdoc_wrapper): if not fdoc_wrapper: return [] return fdoc_wrapper.get_flags() d = store.get_doc(fdoc_id) d.addCallback(wrap_fdoc) d.addCallback(get_flags) return d def create_msg(self, store, msg): """ :param store: an instance of soledad, or anything that behaves alike :param msg: a Message object. :return: a Deferred that is fired when all the underlying documents have been created. :rtype: defer.Deferred """ wrapper = msg.get_wrapper() return wrapper.create(store) def update_msg(self, store, msg): """ :param msg: a Message object. :param store: an instance of soledad, or anything that behaves alike :return: a Deferred that is fired when all the underlying documents have been updated (actually, it's only the fdoc that's allowed to update). :rtype: defer.Deferred """ wrapper = msg.get_wrapper() return wrapper.update(store) # batch deletion def del_all_flagged_messages(self, store, mbox_uuid): """ Delete all messages flagged as deleted. """ def delete_fdoc_and_mdoc_flagged(fdocs): # low level here, not using the wrappers... # get meta doc ids from the flag doc ids fdoc_ids = [doc.doc_id for doc in fdocs] mdoc_ids = map(lambda s: "M" + s[1:], fdoc_ids) def delete_all_docs(mdocs, fdocs): mdocs = list(mdocs) doc_ids = [m.doc_id for m in mdocs] _d = [] docs = mdocs + fdocs for doc in docs: _d.append(store.delete_doc(doc)) d = defer.gatherResults(_d) # return the mdocs ids only d.addCallback(lambda _: doc_ids) return d d = store.get_docs(mdoc_ids) d.addCallback(delete_all_docs, fdocs) return d type_ = FlagsDocWrapper.model.type_ uuid = mbox_uuid.replace('-', '_') deleted_index = indexes.TYPE_MBOX_DEL_IDX d = store.get_from_index(deleted_index, type_, uuid, "1") d.addCallback(delete_fdoc_and_mdoc_flagged) return d # count messages def get_count_unseen(self, store, mbox_uuid): """ Get the number of unseen messages for a given mailbox. :param store: instance of Soledad. :param mbox_uuid: the uuid for this mailbox. :rtype: int """ type_ = FlagsDocWrapper.model.type_ uuid = mbox_uuid.replace('-', '_') unseen_index = indexes.TYPE_MBOX_SEEN_IDX d = store.get_count_from_index(unseen_index, type_, uuid, "0") d.addErrback(lambda f: self.log.error('Error on count_unseen')) return d def get_count_recent(self, store, mbox_uuid): """ Get the number of recent messages for a given mailbox. :param store: instance of Soledad. :param mbox_uuid: the uuid for this mailbox. :rtype: int """ type_ = FlagsDocWrapper.model.type_ uuid = mbox_uuid.replace('-', '_') recent_index = indexes.TYPE_MBOX_RECENT_IDX d = store.get_count_from_index(recent_index, type_, uuid, "1") d.addErrback(lambda f: self.log.error('Error on count_recent')) return d # search api def get_mdoc_id_from_msgid(self, store, mbox_uuid, msgid): """ Get the UID for a message with the passed msgid (the one in the headers msg-id). This is used by the MUA to retrieve the recently saved draft. """ type_ = HeaderDocWrapper.model.type_ uuid = mbox_uuid.replace('-', '_') msgid_index = indexes.TYPE_MSGID_IDX def get_mdoc_id(hdoc): if not hdoc: self.log.warn("Could not find a HDOC with MSGID %s" % msgid) return None hdoc = hdoc[0] mdoc_id = hdoc.doc_id.replace("H-", "M-%s-" % uuid) return mdoc_id d = store.get_from_index(msgid_index, type_, msgid) d.addCallback(get_mdoc_id) return d # Mailbox handling def get_or_create_mbox(self, store, name): """ Get the mailbox with the given name, or create one if it does not exist. :param store: instance of Soledad :param name: the name of the mailbox :type name: str """ index = indexes.TYPE_MBOX_IDX mbox = normalize_mailbox(name) return MailboxWrapper.get_or_create(store, index, mbox) def update_mbox(self, store, mbox_wrapper): """ Update the documents for a given mailbox. :param mbox_wrapper: MailboxWrapper instance :type mbox_wrapper: MailboxWrapper :return: a Deferred that will be fired when the mailbox documents have been updated. :rtype: defer.Deferred """ leap_assert_type(mbox_wrapper, SoledadDocumentWrapper) return mbox_wrapper.update(store) def delete_mbox(self, store, mbox_wrapper): leap_assert_type(mbox_wrapper, SoledadDocumentWrapper) return mbox_wrapper.delete(store) def get_all_mboxes(self, store): """ Retrieve a list with wrappers for all the mailboxes. :return: a deferred that will be fired with a list of all the MailboxWrappers found. :rtype: defer.Deferred """ return MailboxWrapper.get_all(store)
import logging from collections import defaultdict from twisted.internet import defer from adpay.db import utils as db_utils from adpay.stats import cache as stats_cache, consts as stats_consts, utils as stats_utils #: Deferred lock for updating events ADD_EVENT_LOCK = defer.DeferredLock() @defer.inlineCallbacks def calculate_events_payments(campaign_doc, timestamp, payment_percentage_cutoff=0.5): """ For new users: 1. Assign them max_human_score from the database and CPM value (per campaign) 2. Assign payment score based on average of other users. 3. :param campaign_doc: :param timestamp: :param payment_percentage_cutoff: :return: """ # Check if campaign exists if campaign_doc is None: return
def __init__(self): self.req = None self.setRawMode() self.lock = defer.DeferredLock() self.data = ''
from app import communication_keys from amazon import suicide_check from amazon.sqs import WorkerQueue from amazon.suicide_check import get_cpu_balance from twisted.web.client import getPage from twisted.web import xmlrpc, server from twisted.internet import defer, reactor import StringIO import subprocess import json import boto.ec2 lock_apk = defer.DeferredLock() from twisted.internet import protocol processes = [] class SpawnProcessProtocol(protocol.ProcessProtocol): def outReceived(self, data): print 'data:', data def errReceived(self, data): print "errReceived! with %d bytes!" % len(data) print 'data:', data def errConnectionLost(self): processes.remove(self)
def __init__(self, name, builderNames, properties, codebases=DefaultCodebases): """ Initialize a Scheduler. @param name: name of this scheduler (used as a key for state) @type name: unicode @param builderNames: list of builders this scheduler may start @type builderNames: list of unicode @param properties: properties to add to builds triggered by this scheduler @type properties: dictionary @param codebases: codebases that are necessary to process the changes @type codebases: dict with following struct: key: '<codebase>' value: {'repository':'<repo>', 'branch':'<br>', 'revision:'<rev>'} @param consumeChanges: true if this scheduler wishes to be informed about the addition of new changes. Defaults to False. This should be passed explicitly from subclasses to indicate their interest in consuming changes. @type consumeChanges: boolean """ service.MultiService.__init__(self) self.name = name "name of this scheduler; used to identify replacements on reconfig" ok = True if not isinstance(builderNames, (list, tuple)): ok = False else: for b in builderNames: if not isinstance(b, basestring): ok = False if not ok: config.error( "The builderNames argument to a scheduler must be a list " "of Builder names.") self.builderNames = builderNames "list of builder names to start in each buildset" self.properties = Properties() "properties that are contributed to each buildset" self.properties.update(properties, "Scheduler") self.properties.setProperty("scheduler", name, "Scheduler") self.objectid = None self.master = None # Set the codebases that are necessary to process the changes # These codebases will always result in a sourcestamp with or without changes if codebases is not None: if not isinstance(codebases, dict): config.error("Codebases must be a dict of dicts") for codebase, codebase_attrs in codebases.iteritems(): if not isinstance(codebase_attrs, dict): config.error("Codebases must be a dict of dicts") if (codebases != BaseScheduler.DefaultCodebases and 'repository' not in codebase_attrs): config.error("The key 'repository' is mandatory in codebases") else: config.error("Codebases cannot be None") self.codebases = codebases # internal variables self._change_subscription = None self._change_consumption_lock = defer.DeferredLock()
def __init__(self, master): service.MultiService.__init__(self) self.master = master self.upstream_subscribers = bbcollections.defaultdict(list) self._updateLock = defer.DeferredLock()
def __init__(self, repourl=_repourl, branch=_branch, workdir=None, pollInterval=10 * 60, gitbin='git', usetimestamps=True, category=None, project=None, pollinterval=-2, fetch_refspec=None, encoding='utf-8', projects=None): self.cleanRe = re.compile(r"Require(?:s?)\s*.*\s*clean build", re.IGNORECASE + re.MULTILINE) self.cleanCfg = re.compile(r"(CMakeLists\.txt$|\.cmake$|\.cmake\.in$)") # projects is a list of projects to watch or None to watch all. if projects: if isinstance(projects, str) or isinstance(projects, tuple): projects = [projects] assert isinstance(projects, list) assert len(projects) > 0 # Each project to watch is a string (project name) or a tuple # (project name, branch) like ('llvm', 'branches/release_30'). # But we want it always to be a tuple, so we convert a project # name string to a tuple (project, 'master'). self.projects = set() for project in projects: if isinstance(project, str): project = (project, branch) assert isinstance(project, tuple) self.projects.add(project) # for backward compatibility; the parameter used to be spelled with 'i' if pollinterval != -2: pollInterval = pollinterval if project is None: project = '' self.repourl = repourl self.branch = branch self.pollInterval = pollInterval self.fetch_refspec = fetch_refspec self.encoding = encoding self.lastChange = time.time() self.lastPoll = time.time() self.gitbin = gitbin self.workdir = workdir self.usetimestamps = usetimestamps self.category = category self.project = project self.changeCount = 0 self.commitInfo = {} self.initLock = defer.DeferredLock() if self.workdir == None: self.workdir = tempfile.gettempdir() + '/gitpoller_work' log.msg( "WARNING: LLVMGitPoller using deprecated temporary workdir " + "'%s'; consider setting workdir=" % self.workdir)
def __init__(self, owner, slug, **kwargs): kwargs['name'] = self.build_name(owner, slug) self.initLock = defer.DeferredLock() super().__init__(owner, slug, **kwargs)
def __init__(self, basedir, configFileName=None, umask=None, reactor=None, config_loader=None): service.AsyncMultiService.__init__(self) if reactor is None: from twisted.internet import reactor self.reactor = reactor self.setName("buildmaster") self.umask = umask self.basedir = basedir if basedir is not None: # None is used in tests assert os.path.isdir(self.basedir) if config_loader is not None and configFileName is not None: raise config.ConfigErrors([ "Can't specify both `config_loader` and `configFilename`.", ]) elif config_loader is None: if configFileName is None: configFileName = 'master.cfg' config_loader = config.FileLoader(self.basedir, configFileName) self.config_loader = config_loader self.configFileName = configFileName # flag so we don't try to do fancy things before the master is ready self._master_initialized = False self.initLock = defer.DeferredLock() # set up child services self.create_child_services() # db configured values self.configured_db_url = None # configuration / reconfiguration handling self.config = config.MasterConfig() self.reconfig_active = False self.reconfig_requested = False self.reconfig_notifier = None # this stores parameters used in the tac file, and is accessed by the # WebStatus to duplicate those values. self.log_rotation = LogRotation() # local cache for this master's object ID self._object_id = None # Check environment is sensible check_functional_environment(self.config) # figure out local hostname try: self.hostname = os.uname()[1] # only on unix except AttributeError: self.hostname = socket.getfqdn() # public attributes self.name = ("%s:%s" % (self.hostname, os.path.abspath(self.basedir or '.'))) if isinstance(self.name, bytes): self.name = self.name.decode('ascii', 'replace') self.masterid = None
def setUp(self): super(TestMultiLock, self).setUp() self.lock_one = defer.DeferredLock() self.lock_two = defer.DeferredLock() self.multi_lock = MultiLock(self.lock_one, self.lock_two) self.count = 0
class Janitizer(Service): minute = property(lambda foo: 60) hour = property(lambda foo: 3600) day = property(lambda foo: 86400) week = property(lambda f: 604800) oldfiles = {} #get the watch dictionary from romeo watchDict = property(lambda s: SERVICECONFIG.wrapped.get('JANITIZE', {})) #lock aquired before starting a thread that modifies class state busy = defer.DeferredLock() def update(self, watchDict): """Inspects occurrence for a watchDict parameter and updates the internal state of Janitizer @param watchDict (dict) return None """ #break references tmp = copy.deepcopy(self.watchDict) tmp.update(watchDict) #apply updates SERVICECONFIG.JANITIZE = tmp #this would have blocked the reactor w/o the thread @synchronizedDeferred(busy) @deferredAsThread def garbageCheck(self): """Check for file patterns that are removeable""" watchDict = copy.deepcopy(self.watchDict) #use locals for safety for directory, garbageList in watchDict.iteritems(): if not os.path.exists(directory): continue for pattern, limit in garbageList: #blocking method in a thread self.cleanupLinks(directory) files = [os.path.join(directory,f) for f in os.listdir(directory) \ if re.search(pattern,f)] files = sorted(files) if len(files) > int(limit): log('These files matched:\n\t%s' % '\n\t'.join(files)) while len(files) > int(limit): oldfile = files.pop(0) log('Deleting %s' % oldfile) if os.path.islink(oldfile): continue if os.path.isdir(oldfile): for base, dirs, myfiles in os.walk(oldfile, topdown=False): for name in myfiles: os.remove(os.path.join(base, name)) for name in dirs: os.rmdir(os.path.join(base, name)) os.rmdir(oldfile) else: os.unlink(oldfile) #blocking method in a thread self.cleanupLinks(directory) #this will block the reactor def cleanupLinks(self, directory): """cleans broken symlinks @param directory: (string) return list """ files = [os.path.join(directory, f) for f in os.listdir(directory)] for f in files[:]: if not os.path.exists(f): log('Removing broken symlink %s' % f) os.unlink(f) files.remove(f) return files def clean_old_files(self, directory, age, recurse=True): """mark this directory for cleaning at a certain age @param directory: (string) @param age: (float) @param recurse: (bool) return None """ self.oldfiles[directory] = (age, recurse) #this would have blocked the reactor w/o the thread @synchronizedDeferred(busy) @deferredAsThread def clean_elderly(self): """clean old files in a thread""" for directory in self.oldfiles: self.recursive_clean(directory, *self.oldfiles[directory]) #this will block the reactor def recursive_clean(self, directory, age, recurse): """recusively clean a directory @param directory: (string) @param age: (float) @param recurse: (bool) return bool """ try: data = map(lambda n: os.path.join(directory, n), os.listdir(directory)) except: log('could not find directory %s' % directory) return for node in data: if os.path.isdir(node) and recurse: #blocking method in a thread empty = self.recursive_clean(node, age, recurse) if empty: try: os.rmdir(node) except: log('could not remove directory: %s' % node) continue if os.path.isdir(node): continue #in case recurse is False if (time.time() - os.stat(node).st_mtime) > age: try: os.remove(node) except: log('could not remove file: %s' % node) return bool(os.listdir(directory)) def startService(self): """Start Janitizer Service""" self.GARBAGE_CHECK = task.LoopingCall(self.garbageCheck) self.ELDERLY_CHECK = task.LoopingCall(self.clean_elderly) #start the service Service.startService(self) self.GARBAGE_CHECK.start(self.minute * 20) self.ELDERLY_CHECK.start(self.minute) def stopService(self): """Stop All Janitizer Service""" try: if self.GARBAGE_CHECK.running: self.GARBAGE_CHECK.stop() if self.ELDERLY_CHECK.running: self.ELDERLY_CHECK.stop() except: pass Service.stopService(self)
def getMergingLocks(self, build_request_ids): return [ self.build_merging_locks.setdefault(brid, defer.DeferredLock()) for brid in build_request_ids ]
def doRequest(self, *args): if self.__lock is None: self.__lock = defer.DeferredLock() return self.__lock.run(threads.deferToThread, self._doRequest, *args)
def __init__(self, _ignored): self.lock = defer.DeferredLock() self.value = None