def __init__(self, buffer, start_callback=None):
self.buffer = buffer
self._done = False
self._flush = DeferredSemaphore(1)
self._waiter = DeferredSemaphore(1)
self._flush.acquire()
self._started = start_callback
self._keepalive = LoopingCall(self._send_keepalive)
def __init__(self, mapsPath, fetchURL, deleteIfNotPresent, tfLevelSounds):
assert isinstance(mapsPath, str) and len(mapsPath)
assert isinstance(fetchURL, str) and len(fetchURL)
self.mapsPath = FilePath(mapsPath)
self.downloadTempPath = self.mapsPath.child('mapupdater')
self.fetchURL = URLPath.fromString(fetchURL)
self.deleteIfNotPresent = deleteIfNotPresent
self.tfLevelSounds = tfLevelSounds
self.semaphore = DeferredSemaphore(1)
self.downloadSemaphore = DeferredSemaphore(4)
for fp in self.downloadTempPath.globChildren('*.bsp.bz2'):
fp.remove()
def collect(self, config):
log.debug('Starting Delivery health collect')
# Runs once at Application level and once more at components level
ip_address = config.manageIp
if not ip_address:
log.error("%s: IP Address cannot be empty", device.id)
returnValue(None)
# Gather the info about applications
applicationList = []
deferreds = []
sem = DeferredSemaphore(1)
for datasource in config.datasources:
applicationComponentID = datasource.params['applicationComponentID']
if applicationComponentID in applicationList:
continue
applicationList.append(applicationComponentID)
applicationNameID = datasource.params['applicationNameID']
serviceURL = datasource.params['serviceURL']
url = self.urls[datasource.datasource].format(serviceURL)
d = sem.run(getPage, url,
headers={
"Accept": "application/json",
"User-Agent": "Mozilla/3.0Gold",
"iv-groups": datasource.zIVGroups,
"iv-user": datasource.zIVUser,
},
)
tag = '{}_{}'.format(datasource.datasource, applicationNameID)
d.addCallback(self.add_tag, tag)
deferreds.append(d)
return DeferredList(deferreds)
def runU1DBQuery(self, meth, *args, **kw):
"""
Execute a U1DB query in a thread, using a pooled connection.
Concurrent threads trying to update the same database may timeout
because of other threads holding the database lock. Because of this,
we will retry SQLCIPHER_MAX_RETRIES times and fail after that.
:param meth: The U1DB wrapper method name.
:type meth: str
:return: a Deferred which will fire the return value of
'self._runU1DBQuery(Transaction(...), *args, **kw)', or a Failure.
:rtype: twisted.internet.defer.Deferred
"""
meth = "u1db_%s" % meth
semaphore = DeferredSemaphore(SQLCIPHER_MAX_RETRIES)
def _run_interaction():
return self.runInteraction(self._runU1DBQuery, meth, *args, **kw)
def _errback(failure):
failure.trap(dbapi2.OperationalError)
if failure.getErrorMessage() == "database is locked":
logger.warn("database operation timed out")
should_retry = semaphore.acquire()
if should_retry:
logger.warn("trying again...")
return _run_interaction()
logger.warn("giving up!")
return failure
d = _run_interaction()
d.addErrback(_errback)
return d
def collect(self, config):
log.debug('Starting Delivery orders collect')
# TODO : cleanup job collect
ip_address = config.manageIp
if not ip_address:
log.error("%s: IP Address cannot be empty", device.id)
returnValue(None)
applicationList = []
deferreds = []
sem = DeferredSemaphore(1)
for datasource in config.datasources:
applicationNameID = datasource.params['applicationNameID']
if applicationNameID in applicationList:
continue
applicationList.append(applicationNameID)
serviceURL = datasource.params['serviceURL']
url = self.urls[datasource.datasource].format(serviceURL)
# TODO : move headers to Config properties
d = sem.run(getPage, url,
headers={
"Accept": "application/json",
"User-Agent": "Mozilla/3.0Gold",
"iv-groups": datasource.zIVGroups,
"iv-user": datasource.zIVUser,
},
)
tag = '{}_{}'.format(datasource.datasource, applicationNameID) # order_app_delivery_service_3db30547
d.addCallback(self.add_tag, tag)
deferreds.append(d)
return DeferredList(deferreds)
def __init__(self, name, sygnal, config, canonical_reg_id_store):
super(GcmPushkin, self).__init__(name, sygnal, config)
nonunderstood = set(self.cfg.keys()).difference(
self.UNDERSTOOD_CONFIG_FIELDS)
if len(nonunderstood) > 0:
logger.warning(
"The following configuration fields are not understood: %s",
nonunderstood,
)
self.http_pool = HTTPConnectionPool(reactor=sygnal.reactor)
self.max_connections = self.get_config("max_connections",
DEFAULT_MAX_CONNECTIONS)
self.connection_semaphore = DeferredSemaphore(self.max_connections)
self.http_pool.maxPersistentPerHost = self.max_connections
tls_client_options_factory = ClientTLSOptionsFactory()
self.http_agent = Agent(
reactor=sygnal.reactor,
pool=self.http_pool,
contextFactory=tls_client_options_factory,
)
self.db = sygnal.database
self.canonical_reg_id_store = canonical_reg_id_store
self.api_key = self.get_config("api_key")
if not self.api_key:
raise PushkinSetupException("No API key set in config")
def run(self, concurrency=1):
"""Ask the rack controllers to download the region's boot resources.
Report the results via the log.
:param concurrency: Limit the number of rack controllers importing at
one time to no more than `concurrency`.
"""
lock = DeferredSemaphore(concurrency)
def report(results):
message_success = (
"Rack controller (%s) has imported boot resources.")
message_failure = (
"Rack controller (%s) failed to import boot resources.")
message_disconn = (
"Rack controller (%s) did not import boot resources; it is "
"not connected to the region at this time.")
for system_id, (success, result) in zip(self.system_ids, results):
if success:
log.msg(message_success % system_id)
elif result.check(NoConnectionsAvailable):
log.msg(message_disconn % system_id)
else:
log.err(result, message_failure % system_id)
return self(lock).addCallback(report).addErrback(
log.err, "General failure syncing boot resources.")
def __init__(self, conn_str):
self.waiting = []
self.inuse = []
self.free = []
self.semaphore = DeferredSemaphore(1)
self.updateTime()
def __init__(self, queue):
self.managedPackageVersions = set()
self.certifiDirectory = tempfile.mkdtemp()
self.testSemaphore = DeferredSemaphore(tokens=32)
self.binPath = os.path.join(
os.path.split(__file__)[0], 'certifi_test.py')
self.queue = queue
self._log = logger.new(object="supervisor")
def parallel_map(iterable, fn, *args, **kwargs):
deferreds = []
parallelism_limiter = DeferredSemaphore(MAX_PARALLELISM)
for item in iterable:
d = parallelism_limiter.run(fn, item, *args, **kwargs)
deferreds.append(d)
results = yield gatherResults(deferreds)
returnValue(results)
def __init__(self, name, sygnal, config):
super(WebpushPushkin, self).__init__(name, sygnal, config)
nonunderstood = self.cfg.keys() - self.UNDERSTOOD_CONFIG_FIELDS
if nonunderstood:
logger.warning(
"The following configuration fields are not understood: %s",
nonunderstood,
)
self.http_pool = HTTPConnectionPool(reactor=sygnal.reactor)
self.max_connections = self.get_config("max_connections",
DEFAULT_MAX_CONNECTIONS)
self.connection_semaphore = DeferredSemaphore(self.max_connections)
self.http_pool.maxPersistentPerHost = self.max_connections
tls_client_options_factory = ClientTLSOptionsFactory()
# use the Sygnal global proxy configuration
proxy_url = sygnal.config.get("proxy")
self.http_agent = ProxyAgent(
reactor=sygnal.reactor,
pool=self.http_pool,
contextFactory=tls_client_options_factory,
proxy_url_str=proxy_url,
)
self.http_agent_wrapper = HttpAgentWrapper(self.http_agent)
self.allowed_endpoints = None # type: Optional[List[Pattern]]
allowed_endpoints = self.get_config("allowed_endpoints")
if allowed_endpoints:
if not isinstance(allowed_endpoints, list):
raise PushkinSetupException(
"'allowed_endpoints' should be a list or not set")
self.allowed_endpoints = list(map(glob_to_regex,
allowed_endpoints))
privkey_filename = self.get_config("vapid_private_key")
if not privkey_filename:
raise PushkinSetupException(
"'vapid_private_key' not set in config")
if not os.path.exists(privkey_filename):
raise PushkinSetupException(
"path in 'vapid_private_key' does not exist")
try:
self.vapid_private_key = Vapid.from_file(
private_key_file=privkey_filename)
except VapidException as e:
raise PushkinSetupException(
"invalid 'vapid_private_key' file") from e
self.vapid_contact_email = self.get_config("vapid_contact_email")
if not self.vapid_contact_email:
raise PushkinSetupException(
"'vapid_contact_email' not set in config")
self.ttl = self.get_config("ttl", DEFAULT_TTL)
if not isinstance(self.ttl, int):
raise PushkinSetupException("'ttl' must be an int if set")
def run(self, host, probe_func):
# Use a MultiLock with one semaphore limiting the overall
# connections and another limiting the per-host connections.
if host in self.host_locks:
multi_lock = self.host_locks[host]
else:
multi_lock = MultiLock(self.overall_semaphore,
DeferredSemaphore(PER_HOST_REQUESTS))
self.host_locks[host] = multi_lock
return multi_lock.run(probe_func)
def make_database_unpool(maxthreads=max_threads_for_database_pool):
"""Create a general non-thread-pool for database activity.
Its consumer are the old-school web application, i.e. the plain HTTP and
HTTP API services, and the WebSocket service, for the responsive web UI.
Each thread is fully connected to the database.
However, this is a :class:`ThreadUnpool`, which means that threads are not
actually pooled: a new thread is created for each task. This is ideal for
testing, to improve isolation between tests.
"""
return ThreadUnpool(DeferredSemaphore(maxthreads), ExclusivelyConnected)
def __init__(self, plan, configuration, database, plugin_service_api, artifacts_path):
self.plan = plan
self.configuration = configuration
self.database = database
self.plugin_service_api = plugin_service_api
self.artifacts_path = artifacts_path
self.id = str(uuid.uuid4())
self.state = 'CREATED'
self.plugin_configurations = []
self.semaphore = DeferredSemaphore(1)
self.plugin_sessions = []
self.delete_when_stopped = False
def query_all_nodes(nodes, max_concurrency=5, clock=reactor):
"""Queries the given nodes for their power state.
Nodes' states are reported back to the region.
:return: A deferred, which fires once all nodes have been queried,
successfully or not.
"""
semaphore = DeferredSemaphore(tokens=max_concurrency)
queries = (semaphore.run(query_node, node, clock) for node in nodes
if node["power_type"] in PowerDriverRegistry)
return DeferredList(queries, consumeErrors=True)
def __init__(self, expected_subscribers, experiment_start_delay):
self.expected_subscribers = expected_subscribers
self.experiment_start_delay = experiment_start_delay
self.parsing_semaphore = DeferredSemaphore(500)
self.connection_counter = -1
self.connections_made = []
self.connections_ready = []
self.vars_received = []
self._made_looping_call = None
self._subscriber_looping_call = None
self._subscriber_received_looping_call = None
self._timeout_delayed_call = None
class TwistedRequestDriver(HTTPRequestDriver):
# Using a connection pool enables persistent connections, so we can avoid
# the connection setup overhead when sending multiple messages to the
# server.
pool = HTTPConnectionPool(reactor, persistent=True)
# Used to control the number of concurrent requests because
# HTTPConnectionPool does not do that on its own.
# Discussed here:
# http://stackoverflow.com/questions/25552432/how-to-make-pooling-http-connection-with-twisted
sem = DeferredSemaphore(settings.PDSERVER_MAX_CONCURRENT_REQUESTS)
def receive(self, response):
"""
Receive response from twisted web client and convert it to a
PDServerResponse object.
"""
deferred = Deferred()
response.deliverBody(JSONReceiver(response, deferred))
return deferred
def request(self, method, url, body=None):
def makeRequest(ignored):
bodyProducer = None
if body is not None:
bodyProducer = FileBodyProducer(six.StringIO(body))
headers = {}
for key, value in six.iteritems(self.headers):
headers[key] = [value]
agent = Agent(reactor, pool=TwistedRequestDriver.pool)
d = agent.request(method, url, Headers(headers), bodyProducer)
d.addCallback(self.receive)
return d
def releaseSemaphore(result):
TwistedRequestDriver.sem.release()
# Forward the result to the next handler.
return result
d = TwistedRequestDriver.sem.acquire()
# Make the request once we acquire the semaphore.
d.addCallback(makeRequest)
# Release the semaphore regardless of how the request goes.
d.addBoth(releaseSemaphore)
return d
def request(self, method, uri, headers=None, bodyProducer=None):
"""
Issue a new request.
@param method: The request method to send.
@type method: C{str}
@param uri: The request URI send.
@type uri: C{str}
@param scheme: A string like C{'http'} or C{'https'} (the only two
supported values) to use to determine how to establish the
connection.
@param host: A C{str} giving the hostname which will be connected to in
order to issue a request.
@param port: An C{int} giving the port number the connection will be on.
@param path: A C{str} giving the path portion of the request URL.
@param headers: The request headers to send. If no I{Host} header is
included, one will be added based on the request URI.
@type headers: L{Headers}
@param bodyProducer: An object which will produce the request body or,
if the request body is to be empty, L{None}.
@type bodyProducer: L{IBodyProducer} provider
@return: A L{Deferred} which fires with the result of the request (a
L{Response} instance), or fails if there is a problem setting up a
connection over which to issue the request. It may also fail with
L{SchemeNotSupported} if the scheme of the given URI is not
supported.
@rtype: L{Deferred}
"""
scheme, host, port, path = _parse(uri)
if headers is None:
headers = Headers()
if not headers.hasHeader('host'):
# This is a lot of copying. It might be nice if there were a bit
# less.
headers = Headers(dict(headers.getAllRawHeaders()))
headers.addRawHeader(
'host', self._computeHostValue(scheme, host, port))
if self.persistent:
sem = self._semaphores.get((scheme, host, port))
if sem is None:
sem = DeferredSemaphore(self.maxConnectionsPerHostName)
self._semaphores[scheme, host, port] = sem
return sem.run(self._request, method, scheme, host, port, path,
headers, bodyProducer)
else:
return self._request(
method, scheme, host, port, path, headers, bodyProducer)
def main():
agent = Agent(reactor)
sem = DeferredSemaphore(5)
print "Loading IDs"
ids = getBeermeIds()
ids = ids[:100]
print "Done Loading %s IDs" % str(len(ids))
jobs = []
for id in ids:
jobs.append(sem.run(beerme_request, id, agent))
d = gatherResults(jobs)
d.addBoth(cbShutdown)
print "Starting reactor..."
reactor.run()
def __init__(self, reactor, connect_vbms_path, bundle_path, endpoint_url,
keyfile, samlfile, key, keypass, ca_cert, client_cert):
self._reactor = reactor
self._connect_vbms_path = connect_vbms_path
self._bundle_path = bundle_path
self._endpoint_url = endpoint_url
self._keyfile = keyfile
self._samlfile = samlfile
self._key = key
self._keypass = keypass
self._ca_cert = ca_cert
self._client_cert = client_cert
self._connect_vbms_semaphore = DeferredSemaphore(tokens=8)
def main():
agent = Agent(reactor)
sem = DeferredSemaphore(10)
print "Loading breweries..."
mongo = MongoClient().entities.breweries
breweries = loadBreweries(mongo)
print "Done loading breweries."
jobs = []
for brewery in breweries:
jobs.append(sem.run(socialRequest, brewery, agent, mongo))
# if len(jobs) % 50 == 0:
# print "Brewery Jobs started: %d" % len(jobs)
d = gatherResults(jobs)
d.addBoth(cbShutdown)
print "Let the Reactor BEGIN!"
reactor.run()
def _cbReqPhotoPage(self, photo_list):
def photoConnectLost(message, url):
log.err(message)
raise Exception("can't access {0!s}".format(url))
dl = []
sem = DeferredSemaphore(20)
for i in photo_list:
d = sem.run(self._agent.request, b'GET', bytes(i.get('url'), 'ascii'))
d.addCallback(self._cbGetPhotoDlLink, *[i.get('url')])
d.addErrback(photoConnectLost, *[i.get('url')])
d.addCallback(self._cbDownloadPhoto)
d.addErrback(log.err)
dl.append(d)
deferreds = DeferredList(dl, consumeErrors=True)
return deferreds
class RequestManager:
overall_semaphore = DeferredSemaphore(OVERALL_REQUESTS)
# Yes, I want a mutable class attribute because I want changes done in an
# instance to be visible in other instances as well.
host_locks = {}
def run(self, host, probe_func):
# Use a MultiLock with one semaphore limiting the overall
# connections and another limiting the per-host connections.
if host in self.host_locks:
multi_lock = self.host_locks[host]
else:
multi_lock = MultiLock(self.overall_semaphore,
DeferredSemaphore(PER_HOST_REQUESTS))
self.host_locks[host] = multi_lock
return multi_lock.run(probe_func)
def SetupProxy(self):
if len(self.ProxyOptions) == 2:
self.Interface, self.Port = self.ProxyOptions
else: # Only the port was passed
self.Port = self.ProxyOptions[0]
# If the interface was not specified listen on 127.0.0.1 only:
self.Interface = "127.0.0.1"
print "Interface=" + str(self.Interface)
print "Port=" + str(self.Port)
cprint("Setting up Proxy listener on " + self.Interface + ':' +
self.Port)
self.Port = int(self.Port)
self.Semaphore = DeferredSemaphore(1)
self.Queue = DeferredQueue
self.Proxy = Proxy(self.Requester, self.Semaphore, self.Queue)
self.Site = server.Site(self.Proxy)
reactor.listenTCP(self.Port, self.Site, interface=self.Interface)
reactor.run()
def __init__(self, name, sygnal, config, canonical_reg_id_store):
super(GcmPushkin, self).__init__(name, sygnal, config)
nonunderstood = set(self.cfg.keys()).difference(
self.UNDERSTOOD_CONFIG_FIELDS)
if len(nonunderstood) > 0:
logger.warning(
"The following configuration fields are not understood: %s",
nonunderstood,
)
self.http_pool = HTTPConnectionPool(reactor=sygnal.reactor)
self.max_connections = self.get_config("max_connections",
DEFAULT_MAX_CONNECTIONS)
self.connection_semaphore = DeferredSemaphore(self.max_connections)
self.http_pool.maxPersistentPerHost = self.max_connections
tls_client_options_factory = ClientTLSOptionsFactory()
# use the Sygnal global proxy configuration
proxy_url = sygnal.config.get("proxy")
self.http_agent = ProxyAgent(
reactor=sygnal.reactor,
pool=self.http_pool,
contextFactory=tls_client_options_factory,
proxy_url_str=proxy_url,
)
self.db = sygnal.database
self.canonical_reg_id_store = canonical_reg_id_store
self.api_key = self.get_config("api_key")
if not self.api_key:
raise PushkinSetupException("No API key set in config")
# Use the fcm_options config dictionary as a foundation for the body;
# this lets the Sygnal admin choose custom FCM options
# (e.g. content_available).
self.base_request_body: dict = self.get_config("fcm_options", {})
if not isinstance(self.base_request_body, dict):
raise PushkinSetupException(
"Config field fcm_options, if set, must be a dictionary of options"
)
def get_sempahore(operation, conf_name):
"""
Get global semaphore of given operation if configured based on conf_name.
Otherwise return None
:param str operation: Operation for which semaphore is required. Must be
same each time it is called for that operation
:param str conf_name: Semaphore is returned only if this config exists
:return: A :obj:`DeferredSemaphore` object corresponding to the operation
"""
sem = _semaphores.get(operation)
if sem is not None:
return sem
conf = config_value(conf_name)
if conf is None:
return None
_semaphores[operation] = DeferredSemaphore(conf)
return _semaphores[operation]
def trigger_convergence_groups(authenticator, region, groups,
concurrency_limit, no_error_group):
"""
Trigger convergence on given groups
:param IAuthenticator authenticator: Otter authenticator
:param str region: Region where this is running
:param list groups: List of group dicts
:param int concurrency_limit: Concurrency limit
:param bool no_error_group: If true then do not converge ERROR groups
:return: Deferred fired with None
"""
sem = DeferredSemaphore(concurrency_limit)
return gatherResults([
sem.run(trigger_convergence, authenticator, region, group,
no_error_group) for group in groups
],
consumeErrors=True).addCallback(lambda _: None)
def runDeferredCommand(self):
self.startTime = time.time()
self.log.info('%s using BBCApplicationSNMP' % self.device.id)
self.stats['totalRequests'] = len(self.device.modelledOids)
semaf = DeferredSemaphore(self.workers)
jobs = []
for oids in self.device.modelledOids:
self.stats['collectedAppOidCounter'] += oids.count('.75025.')
df = semaf.run(utils.getProcessOutputAndValue, '/usr/bin/snmpget',
self.prepareSnmpCmdArgList() + oids.split())
df.addErrback(self.handleError)
jobs.append(df)
df = gatherResults(jobs)
df.addErrback(self.handleError)
df.addCallback(self.parseOutput)
return df
def __init__(self,
crawler,
client_endpoint,
page_limit=4,
browser_options=None,
cookies_middleware=None):
super().__init__()
self._crawler = crawler
self._client_endpoint = client_endpoint
if page_limit:
self._semaphore = DeferredSemaphore(page_limit)
else:
self._semaphore = DummySemaphore()
self.browser_options = (browser_options or {})
self.cookies_mw = cookies_middleware
self._downloader = BrowserRequestDownloader(self._crawler)
self._browser = None
self._browser_init_lock = DeferredLock()
def collect(self, device, log):
log.debug('{}: Modeling collect'.format(device.id))
port = getattr(device, 'zSpringBootPort', None)
uri = getattr(device, 'zSpringBootURI', None)
ivGroups = getattr(device, 'zIVGroups', None)
ivUser = getattr(device, 'zIVUser', None)
ip_address = device.manageIp
if not ip_address:
log.error("%s: IP Address cannot be empty", device.id)
returnValue(None)
deferreds = []
sem = DeferredSemaphore(1)
# TODO: remove loop
for query in self.queries:
url = query[1].format(ip_address, port, uri)
log.debug('SBA collect url: {}'.format(url))
d = sem.run(
getPage,
url,
headers={
"Accept": "application/json",
"User-Agent": "Mozilla/3.0Gold",
"iv-groups": ivGroups,
"iv-user": ivUser,
},
)
d.addCallback(self.add_tag, '{}'.format(query[0]))
deferreds.append(d)
results = yield DeferredList(deferreds, consumeErrors=True)
for success, result in results:
if not success:
log.error('{}: {}'.format(device.id, result.getErrorMessage()))
returnValue(results)
