def __init__(self, delegate, vhost, spec, heartbeat=0, clock=None, insist=False): FrameReceiver.__init__(self, spec) self.delegate = delegate # XXX Cyclic dependency self.delegate.client = self self.vhost = vhost self.channelFactory = type("Channel%s" % self.spec.klass.__name__, (self.channelClass, self.spec.klass), {}) self.channels = {} self.channelLock = defer.DeferredLock() self.outgoing = defer.DeferredQueue() self.work = defer.DeferredQueue() self.started = TwistedEvent() self.disconnected = TwistedEvent() # Fired upon connection shutdown self.closed = False self.queueLock = defer.DeferredLock() self.basic_return_queue = TimeoutDeferredQueue() self.queues = {} self.outgoing.get().addCallback(self.writer) self.work.get().addCallback(self.worker) self.heartbeatInterval = heartbeat self.insist = insist if clock is None: clock = reactor self.clock = clock if self.heartbeatInterval > 0: self.checkHB = self.clock.callLater(self.heartbeatInterval * self.MAX_UNSEEN_HEARTBEAT, self.check_heartbeat) self.sendHB = LoopingCall(self.send_heartbeat) self.sendHB.clock = self.clock d = self.started.wait() d.addCallback(lambda _: self.reschedule_send_heartbeat()) d.addCallback(lambda _: self.reschedule_check_heartbeat()) # If self.started fails, don't start the heartbeat. d.addErrback(lambda _: None)
def connectionMade(self): self.srv_queue = defer.DeferredQueue() self.srv_queue.get().addCallback(self.clientDataReceived) self.master_cli_queue = defer.DeferredQueue() self.slave_cli_queues = [] factory = ProxyClientFactory(self.srv_queue, self.master_cli_queue) master_settings = self.factory.master_settings reactor.connectTCP(master_settings["host"], master_settings["port"], factory) for slave_settings in self.factory.slave_settings: slave_cli_queue = defer.DeferredQueue() self.slave_cli_queues.append(slave_cli_queue) factory = ProxyClientFactory(self.srv_queue, slave_cli_queue) reactor.connectTCP(slave_settings["host"], slave_settings["port"], factory) self.iter_slave_cli_queues = itertools.cycle(self.slave_cli_queues)
def __init__(self, device): super(BufferingStateMachine, self).__init__() self.device = device self.custom = device.custom self.cmd = None self.state = 'idle' # since Python 2.5+ using a string as buffer is efficient m'kay? self.idlebuf = "" self.waitbuf = "" self.notifications = defer.DeferredQueue()
def __init__(self, oper, del_obj=True): self.oper = oper self.queue = defer.DeferredQueue() self.presence = defaultdict(lambda: []) self.qcounter = defaultdict(lambda: 0) self.queue.get().addCallback(self._new_data) task.LoopingCall(self._auto_dispatch).start( 1) # secs between checkings task.LoopingCall(self._counters_cleanup).start( 30) # presence maintenance self.delete_objects = del_obj
def __init__(self, factory, poolsize=10, isolated=False): self.factory = factory self.poolsize = poolsize self.isolated = isolated self.size = 0 self.pool = [] self.connectionQueue = defer.DeferredQueue() self._waitingForConnection = set() self._waitingForEmptyPool = set()
def connectionMade(self): self.srv_queue = defer.DeferredQueue() self.cli_queue = defer.DeferredQueue() self.srv_queue.get().addCallback(self.clientDataReceived) src_addr = self.transport.getPeer().host src_port = self.transport.getPeer().port tuples = self.factory.hosts.get((src_addr, src_port), False) # check if we've got a new tuple to connect to and # if we're still waiting for connections or not while not tuples and self.factory.proxy.running: sleep(0.1) tuples = self.factory.hosts.get((src_addr, src_port), False) if tuples: self.dst_addr, self.dst_port, dst_fid = tuples self.dst_flow = self.factory.get_flow(dst_fid) print "Connecting to %s:%s" % (self.dst_addr, self.dst_port)
class Globals(): """Global variables to keep track of experiment flow, image files, etc. The following are options, to be edited to suit: FULLSCREEN (bool): Toggles fullscreen or windowed mode for pyglet. IMAGES_DIR (str): Path to folder with images. Include trailing slash! IMAGES_SCALE (float): Scaling factor to resize images. LEGAL_FILE_EXTENSIONS (list of str): Types of image files to read from the directory. Case sensitive! Include leading dot! LOG_FILE (str): Path to a text file where the log will be written. Include extension! QTM_PROJECT_DATA_DIR (str): Path to the Data directory in the QTM project. Include trailing slash! QTM_RT_PASSWORD (str): Password for QTM real-time client control. Default is blank (""). QTM_RT_PORT (int): Port number (little endian) for QTM real-time connection. Default is 22223. SCREEN_IDX (int): Index of screen to create pyglet window on. See pyglet documentation for more info. STATUS_* variables can be changed or added (along with new directories for images) to modify experiment flow; this is only a rudimentary example. """ FULLSCREEN = False IMAGES_DIR = "PATH_TO_IMAGES_DIR/" IMAGES_SCALE = 0.65 LEGAL_FILE_EXTENSIONS = ('.JPG', '.jpg') LOG_FILE = "log.txt" QTM_PROJECT_DATA_DIR = "PATH_TO_DATA_DIR/" QTM_RT_PASSWORD = "" QTM_RT_PORT = 22223 SCREEN_IDX = 0 STATUS_BEGIN = 0 STATUS_EXPERIMENT = 1 STATUS_DONE = 2 img_files = [] for f in os.listdir(IMAGES_DIR): if f.endswith(LEGAL_FILE_EXTENSIONS): img_files.append(os.path.join(IMAGES_DIR, f)) print(img_files) random.shuffle(img_files) print(img_files) status = None sprite = None connection = None event_queue = defer.DeferredQueue() startTime = datetime.now().strftime('%m%d%H%M')
def __init__(self, func, width=0, size=None, backlog=None): self._queue = defer.DeferredQueue(size, backlog) self._func = func self._pool = DeferredPool() self._coop = task.Cooperator() self._currentWidth = 0 self._pendingStops = 0 self._underway = set() self.stopped = self.paused = False self.width = int(width) assert self.width >= 0
def start_sniffer_thread(self): self.packet_queue = defer.DeferredQueue() cpu_port_intf = str( self.topo.get_cpu_port_intf(self.sw_name).replace("eth0", "eth1")) self.sniffer_thread = SnifferThread(reactor, self.packet_queue, cpu_port_intf) self.sniffer_thread.daemon = True # die when the main thread dies self.sniffer_thread.start() workers = 4 for i in range(workers): self._consume_from_packet_queue()
def get_bluetooth_discv_queue(self): bus = dbus.SystemBus() items = [(self._rem_dev_name_cb, 'RemoteNameUpdated'), (self._disc_completed_cb, 'DiscoveryCompleted')] for method, signal in items: bus.add_signal_receiver(method, signal, INTERFACE, SERVICE, DEV) self.queue = defer.DeferredQueue() self.adapter.DiscoverDevices() return self.queue
def test_cancelQueueAfterSynchronousGet(self): """ When canceling a L{Deferred} from a L{DeferredQueue} that already has a result, the cancel should have no effect. """ def _failOnErrback(_): self.fail("Unexpected errback call!") queue = defer.DeferredQueue() d = queue.get() d.addErrback(_failOnErrback) queue.put(None) d.cancel()
def __init__(self, nick, connections, *args, **kwargs): """ acts as a connection manager, middle man for incoming events, and processor of outgoing actions. """ #TODO: maybe support a custom internal key to keep track of bots, not just by 'name'... #TODO: if we do it by name, make sure we don't have name dupes, even between networks :-/ # bot id self.nick = nick self.id = str(uuid.uuid1()) self.log = logging.getLogger('{0}.{1}.{2}'.format( self.__class__.__module__, self.__class__.__name__, self.nick)) self.log.info('starting bot') #bot manager instance self.bot_manager = None # setup this bots event queue / consumer, action queue/consumer self.event_queue = defer.DeferredQueue() self._consume_events(self.event_queue) self.action_queue = defer.DeferredQueue() self._consume_actions(self.action_queue) # setup plugins self.enabled_plugins = kwargs.get('enabled_plugins') self.plugin_manager = PluginManager(bot=self) # self.manager.config.PLUGINS: # build connections # TODO: create connection manager self.connection_manager = ConnectionManager(config=connections, bot=self) self.log.debug('connections on {0!r}: {1!r}'.format( self.nick, self.connection_manager)) # should have a 'ready' state that we should check before starting? self.state = OFF self.party_line = None
def __init__(self): self.log = None self.data_buf = '' self.response_data_buf = '' self.stats_client = StatsdClient.get() # Local connection state self.tree_connect_requests = {} self.connected_trees = {} self.file_open_requests = {} self.file_close_requests = {} self.open_files = {} self.session_latest_create_request_filename = None self.session_latest_tree_connect_path = None self.srv_queue = defer.DeferredQueue() self.cli_queue = defer.DeferredQueue() # Manually bind a reaction to the data that comes from the server self.srv_queue.get().addCallback(self.clientDataReceived) # The list of NMB packets from the client waiting to be processed self.client_pending_packets_queue = defer.DeferredQueue() self.client_pending_packets_queue_len = 0 self.client_pending_packets_queue.get().addCallback(self.process_client_pending_packet) # The list of NMB packets from the server waiting to be processed self.server_pending_packets_queue = defer.DeferredQueue() self.server_pending_packets_queue_len = 0 self.server_pending_packets_queue.get().addCallback(self.process_server_pending_packet) # Whether a shutdown has been requested self.shutdown_requested = False self.shutdown_deferred = None # Misc counters self.total_processed_client_packets = 0 self.total_processed_server_packets = 0
def __init__(self, circuit_manager, request, socks): ''' :param oppy.util.exitrequest.ExitRequest request: connection request for this stream :param oppy.socks.socks.OppySOCKSProtocol socks: socks protocol instance this stream should relay data to and from ''' self.stream_id = None self._read_queue = defer.DeferredQueue() self._write_queue = defer.DeferredQueue() self._read_deferred = None self._write_deferred = None self.request = request self.socks = socks self._deliver_window = STREAM_WINDOW_INIT self._package_window = STREAM_WINDOW_INIT self.circuit = None # set this flag if SOCKS closes our connection before the circuit # is done building self._closed = False self._circuit_request = circuit_manager.getOpenCircuit(self) self._circuit_request.addCallback(self._registerNewStream)
def __init__(self, delegate, vhost, spec, heartbeat=0, clock=None, insist=False): FrameReceiver.__init__(self, spec) self.delegate = delegate # XXX Cyclic dependency self.delegate.client = self self.vhost = vhost self.channelFactory = type("Channel%s" % self.spec.klass.__name__, (self.channelClass, self.spec.klass), {}) self.channels = {} self.channelLock = defer.DeferredLock() self.outgoing = defer.DeferredQueue() self.work = defer.DeferredQueue() self.started = TwistedEvent() self.queueLock = defer.DeferredLock() self.basic_return_queue = TimeoutDeferredQueue() self.queues = {} self.outgoing.get().addCallback(self.writer) self.work.get().addCallback(self.worker) self.heartbeatInterval = heartbeat self.insist = insist if self.heartbeatInterval > 0: if clock is None: from twisted.internet import reactor as clock self.clock = clock self.checkHB = self.clock.callLater(self.heartbeatInterval * self.MAX_UNSEEN_HEARTBEAT, self.checkHeartbeat) self.sendHB = LoopingCall(self.sendHeartbeat) d = self.started.wait() d.addCallback(lambda _: self.reschedule_sendHB()) d.addCallback(lambda _: self.reschedule_checkHB())
def __init__(self, keyspace=None, retries=0, credentials=None, decoder=None): self.deferred = defer.Deferred() self.queue = defer.DeferredQueue() self.continueTrying = True self._protos = [] self._pending = [] self.request_retries = retries self.keyspace = keyspace self.credentials = credentials self.decoder = decoder
def main(): logging.basicConfig(level=logging.DEBUG, format="%(asctime)-15s %(message)s") logging.info('Starting up') #sensor_pollers, buffer_unloaders, sqlite_unloaders are worker lists sensor_pollers, buffer_unloaders, sqlite_unloaders = [], [], [] upload_tasks = defer.DeferredQueue() #Setup monitoring of fake sensors #Create sensor instance in sensor list fake_sensors = [fake_sensor()] #Iterate over sensors for source in fake_sensors: name = 'temp%d' % (1, ) #Create a datapointBuffer which will call source_method (with optional source_method_args) to retrieve a datapoint) dp_buffer = sensorup.datapointBuffer( source_method=source.get_temp_channel, source_method_args=0, datastream_id=name) #Add this new datasource (datapointBuffer instance) to worker lists #sensorPollAndBuffer: poll for new data and store in an in-memory list sensor_pollers.append( task.LoopingCall(sensorup.sensorPollAndBuffer, dp_buffer)) #bufferUnloadAndUpload: Flushes in-memory list of datapoints, and attempts uploading to Cosm. Failures go into sqlite. buffer_unloaders.append( task.LoopingCall(sensorup.bufferUnloadAndUpload, dp_buffer, upload_tasks)) #Collect datapoint every 1s for poller in sensor_pollers: poller.start(1.0) #Upload every 30s for poller in buffer_unloaders: poller.start(30.0, False) #interval for sqlite_unloaders should be sufficiently large as to prevent double uploading # sqlite_unloader interval > buffer_unloaders interval # sqlite_unloader interval > client.request_timeout * sensor count #Flush old datapoints from the database every 86400s sqlite_purger = task.LoopingCall(sensorup.sqlite_purge) sqlite_purger.start(86400) #Only one concurrent upload task task.cooperate(sensorup.worker(upload_tasks)) reactor.run()
def load(self): # Chargement d'un plan de test self.plan = plan # Chargement dans une liste des phases a traiter self.queue = defer.DeferredQueue() for phase, servers in self.plan.iteritems(): if 'bridage' in servers: maxparallel = servers['bridage'] del servers['bridage'] else: maxparallel = 0 self.queue.put(Phase(phase, servers, maxparallel, self)) print "Chargement de la premiere phase"
def __init__(self, endpoint, sid, userid): resource.Resource.__init__(self) self.endpoint = endpoint self.broker = self.endpoint.broker self.sessionid = sid self.userid = userid self.queue = defer.DeferredQueue() self.zombie = False self.pollingDefer = None self.putChild('logout', BaseRequest(self, self.logout)) self.putChild('userinfo', BaseRequest(self, self.userinfo)) self.putChild('pending', BaseRequest(self, self.pending)) self.putChild('polling', BaseRequest(self, self.polling)) self.putChild('received', BaseRequest(self, self.received)) self.putChild('message', BaseRequest(self, self.message))
def __init__(self, dispatcher, interval, protocol, address, incoming_account, outgoing_account): self.dispatcher = dispatcher self.interval = interval self.address = address self.incoming = self.getIncoming(protocol, incoming_account) self.outgoing = ApplicationserverSMTPSender(address=self.address, **outgoing_account) self.mailQueue = defer.DeferredQueue(size=100) self.mailProcessor = MailProcessor( self.mailQueue, taskMethod=self.dispatcher.callServiceMethod, send=self.sendMail) self.mailCache = list() self.run()
def __init__(self, conf, passive=False): node.Node.__init__(self, name='file-transfer') self.passive = passive self.conf = conf self.working_queue = {} self.queue = defer.DeferredQueue() self.working = 0 self.QUEUESIZE = int(self.conf.get('transport', 'threads')) self.PENDINGLIMIT = 100 self.timestamp = time() self.stopped = False if not self.passive: self._sched_resume_ft() self._sched_stop_ft() self._requeue()
def testQueue(self): N, M = 2, 2 queue = defer.DeferredQueue(N, M) gotten = [] for i in range(M): queue.get().addCallback(gotten.append) self.assertRaises(defer.QueueUnderflow, queue.get) for i in range(M): queue.put(i) self.assertEquals(gotten, range(i + 1)) for i in range(N): queue.put(N + i) self.assertEquals(gotten, range(M)) self.assertRaises(defer.QueueOverflow, queue.put, None) gotten = [] for i in range(N): queue.get().addCallback(gotten.append) self.assertEquals(gotten, range(N, N + i + 1)) queue = defer.DeferredQueue() gotten = [] for i in range(N): queue.get().addCallback(gotten.append) for i in range(N): queue.put(i) self.assertEquals(gotten, range(N)) queue = defer.DeferredQueue(size=0) self.assertRaises(defer.QueueOverflow, queue.put, None) queue = defer.DeferredQueue(backlog=0) self.assertRaises(defer.QueueUnderflow, queue.get)
def __init__(self, maxParrallel=None, startnow=True): """ @param maxParrallel: the maximum number of tasks that can run at a time. @type maxParallel: int @param startnow: Set True to start the executor immediately. @type startnow: boolean """ queue = defer.DeferredQueue() limit = maxParrallel if maxParrallel is not None else 0 tokens = DynamicDeferredSemaphore(limit) super(TwistedExecutor, self).__init__(queue, tokens, log) if startnow: self.start(reactor)
def startup(self, remoteReference): """ Starts things up with the remote reference in hand. Useful to have this as a separate method when you're subclassing and doing difference constructor stuff. """ # Setup resignation-upon-disconnect self.resignators = [] self.disconnectErrors = (pb.DeadReferenceError, pb.PBConnectionLost) remoteReference.notifyOnDisconnect(self.resign) # Prepare the run request queue self.jobs = [] self.runRequestQueue = defer.DeferredQueue() for k in xrange(self.N): self.runRequestQueue.put(None)
def __init__(self, slots, network_manager_factory, splash_proxy_factory_cls, js_profiles_path, verbosity=1): self.network_manager_factory = network_manager_factory self.splash_proxy_factory_cls = splash_proxy_factory_cls or ( lambda profile_name: None) self.js_profiles_path = js_profiles_path self.active = set() self.queue = defer.DeferredQueue() self.verbosity = verbosity for n in range(slots): self._wait_for_render(None, n, log=False)
def __init__(self, keyspace=None, retries=0, credentials={}, require_api_version=None): self.deferred = defer.Deferred() self.queue = defer.DeferredQueue() self.continueTrying = True self._protos = [] self._pending = [] self.request_retries = retries self.keyspace = keyspace self.credentials = credentials if credentials: self.protocol = AuthenticatedThriftClientProtocol self.require_api_version = require_api_version
def __init__(self, connection_manager, micro_status_entry, link_protocol=3): msg = ("Creating connection task to {}".format( micro_status_entry.address)) logging.debug(msg) # MAp all circuits using this connections self._connection_manager = connection_manager self.micro_status_entry = micro_status_entry self._link_protocol = link_protocol self._read_queue = defer.DeferredQueue() self._buffer = '' self._connection_cert = None self._tasks = None self._current_task = None self._connection = None self._failed = False
def __init__(self, keyspace=None, retries=0, credentials=None, sasl_kwargs=None): self.deferred = defer.Deferred() self.queue = defer.DeferredQueue() self.continueTrying = True self._protos = [] self._pending = [] self.request_retries = retries self.keyspace = keyspace self.credentials = credentials self.sasl_kwargs = sasl_kwargs if credentials: self.protocol = AuthenticatedThriftClientProtocol elif sasl_kwargs: self.protocol = SASLThriftClientProtocol
class InterpreterProtocol(basic.NetstringReceiver): log = Logger() queue = defer.DeferredQueue() def connectionMade(self): self.log.debug("Interpreter connections eastablished") def stringReceived(self, string): response = json.loads(string.decode("utf8", "ignore")) self.responseReceived(response) def sendRequest(self, request): request = json.dumps(request).encode("utf-8") self.sendString(request) return self.queue.get() def responseReceived(self, response): self.queue.put(response)
def test_cancelQueueAfterGet(self): """ When canceling a L{Deferred} from a L{DeferredQueue} that does not have a result (i.e., the L{Deferred} has not fired), the cancel causes a L{defer.CancelledError} failure. If the queue has a result later on, it doesn't try to fire the deferred. """ queue = defer.DeferredQueue() d = queue.get() self.assertFailure(d, defer.CancelledError) d.cancel() def cb(ignore): # If the deferred is still linked with the deferred queue, it will # fail with an AlreadyCalledError queue.put(None) return queue.get().addCallback(self.assertIdentical, None) return d.addCallback(cb)