class Processor():
"""
Whenever mixtapeReceived is called, deferToThread is scheduled to be run as
soon as a "slot" for being run is available. There is currently 1 slot
deferToThread runs process_mixtape in another thread, and releases the
slot when its that process is done
"""
def __init__(self):
self.sem = DeferredSemaphore(1) #do one thing at a time
def mixtapeReceived(self, mixtape):
debug("Adding %s to be processed" % mixtape)
self.sem.run(deferToThread, process.process_mixtape, *mixtape)
def collect(self, config):
log.debug('Starting Delivery orders collect')
# TODO : cleanup job collect
ip_address = config.manageIp
if not ip_address:
log.error("%s: IP Address cannot be empty", device.id)
returnValue(None)
applicationList = []
deferreds = []
sem = DeferredSemaphore(1)
for datasource in config.datasources:
applicationNameID = datasource.params['applicationNameID']
if applicationNameID in applicationList:
continue
applicationList.append(applicationNameID)
serviceURL = datasource.params['serviceURL']
url = self.urls[datasource.datasource].format(serviceURL)
# TODO : move headers to Config properties
d = sem.run(getPage, url,
headers={
"Accept": "application/json",
"User-Agent": "Mozilla/3.0Gold",
"iv-groups": datasource.zIVGroups,
"iv-user": datasource.zIVUser,
},
)
tag = '{}_{}'.format(datasource.datasource, applicationNameID) # order_app_delivery_service_3db30547
d.addCallback(self.add_tag, tag)
deferreds.append(d)
return DeferredList(deferreds)
def recordVideoProcess(self, resW, resH, totalTimeSec, framerate, serverIP, piName, recordTimesList, file):
semi = DeferredSemaphore(1)
jobs = []
for runs in range(len(recordTimesList)/2):
print "recordTimes recordVideoProcess:", recordTimesList
self.writeFile("recordTimes recordVideoProcess:")
try:
startAtTime = self.calculateTimeDifference(recordTimesList.pop(0), recordTimesList.pop(0))
jobs.append(semi.run(tv.takeVideo, int(resW), int(resH), int(totalTimeSec),\
int(framerate), startAtTime, serverIP, piName, file))
except:
self.writeFile("That time was not valid. Calling next time.")
self.writeFile("len recordTimesList: " + str(len(recordTimesList)))
if len(recordTimesList)%2>0:
self.writeFile("odd number")
recordTimesList.pop(0)
self.writeFile("new len: " + str(len(recordTimesList)))
reactor.callLater(0.5, self.transport.write, "TIMEINPUTERROR {0}\n".format(piName))
continue
jobs = DeferredList(jobs)
print "Results: ", jobs.addCallback(self.getResults, piName)
# self.writeFile("Results: ", jobs.addCallback(self.getResults, piName))
jobs.addCallback(lambda _: reactor.callLater(5, reactor.stop))
def collect(self, config):
log.debug('Starting Delivery health collect')
# Runs once at Application level and once more at components level
ip_address = config.manageIp
if not ip_address:
log.error("%s: IP Address cannot be empty", device.id)
returnValue(None)
# Gather the info about applications
applicationList = []
deferreds = []
sem = DeferredSemaphore(1)
for datasource in config.datasources:
applicationComponentID = datasource.params['applicationComponentID']
if applicationComponentID in applicationList:
continue
applicationList.append(applicationComponentID)
applicationNameID = datasource.params['applicationNameID']
serviceURL = datasource.params['serviceURL']
url = self.urls[datasource.datasource].format(serviceURL)
d = sem.run(getPage, url,
headers={
"Accept": "application/json",
"User-Agent": "Mozilla/3.0Gold",
"iv-groups": datasource.zIVGroups,
"iv-user": datasource.zIVUser,
},
)
tag = '{}_{}'.format(datasource.datasource, applicationNameID)
d.addCallback(self.add_tag, tag)
deferreds.append(d)
return DeferredList(deferreds)
def trigger_convergence_groups(authenticator, region, groups,
concurrency_limit, no_error_group):
"""
Trigger convergence on given groups
:param IAuthenticator authenticator: Otter authenticator
:param str region: Region where this is running
:param list groups: List of group dicts
:param int concurrency_limit: Concurrency limit
:param bool no_error_group: If true then do not converge ERROR groups
:return: Deferred fired with None
"""
sem = DeferredSemaphore(concurrency_limit)
d = DeferredList(
[sem.run(trigger_convergence, authenticator, region, group,
no_error_group)
for group in groups],
fireOnOneCallback=False,
fireOnOneErrback=False,
consumeErrors=True)
d.addCallback(
lambda results: [(g["tenantId"], g["groupId"], f.value)
for g, (s, f) in zip(groups, results) if not s])
return d
def parallel_map(iterable, fn, *args, **kwargs):
deferreds = []
parallelism_limiter = DeferredSemaphore(MAX_PARALLELISM)
for item in iterable:
d = parallelism_limiter.run(fn, item, *args, **kwargs)
deferreds.append(d)
results = yield gatherResults(deferreds)
returnValue(results)
def parallel_map(iterable, fn, *args, **kwargs):
deferreds = []
parallelism_limiter = DeferredSemaphore(MAX_PARALLELISM)
for item in iterable:
d = parallelism_limiter.run(fn, item, *args, **kwargs)
deferreds.append(d)
results = yield gatherResults(deferreds)
returnValue(results)
def trigger_convergence_groups(authenticator, region, groups,
concurrency_limit):
"""
Trigger convergence on given groups
"""
sem = DeferredSemaphore(concurrency_limit)
return gatherResults(
[sem.run(trigger_convergence, authenticator, region, group)
for group in groups],
consumeErrors=True)
def runFiles(): semi = DeferredSemaphore(1) jobs = [] for runs in range(5): jobs.append(semi.run(collectFiles)) jobs = DeferredList(jobs) def cbFinished(ignored): print 'Finishing job' jobs.addCallback(cbFinished) return jobs
def query_all_nodes(nodes, max_concurrency=5, clock=reactor):
"""Queries the given nodes for their power state.
Nodes' states are reported back to the region.
:return: A deferred, which fires once all nodes have been queried,
successfully or not.
"""
semaphore = DeferredSemaphore(tokens=max_concurrency)
queries = (semaphore.run(query_node, node, clock) for node in nodes
if node["power_type"] in PowerDriverRegistry)
return DeferredList(queries, consumeErrors=True)
def request(self, method, uri, headers=None, bodyProducer=None):
"""
Issue a new request.
@param method: The request method to send.
@type method: C{str}
@param uri: The request URI send.
@type uri: C{str}
@param scheme: A string like C{'http'} or C{'https'} (the only two
supported values) to use to determine how to establish the
connection.
@param host: A C{str} giving the hostname which will be connected to in
order to issue a request.
@param port: An C{int} giving the port number the connection will be on.
@param path: A C{str} giving the path portion of the request URL.
@param headers: The request headers to send. If no I{Host} header is
included, one will be added based on the request URI.
@type headers: L{Headers}
@param bodyProducer: An object which will produce the request body or,
if the request body is to be empty, L{None}.
@type bodyProducer: L{IBodyProducer} provider
@return: A L{Deferred} which fires with the result of the request (a
L{Response} instance), or fails if there is a problem setting up a
connection over which to issue the request. It may also fail with
L{SchemeNotSupported} if the scheme of the given URI is not
supported.
@rtype: L{Deferred}
"""
scheme, host, port, path = _parse(uri)
if headers is None:
headers = Headers()
if not headers.hasHeader('host'):
# This is a lot of copying. It might be nice if there were a bit
# less.
headers = Headers(dict(headers.getAllRawHeaders()))
headers.addRawHeader(
'host', self._computeHostValue(scheme, host, port))
if self.persistent:
sem = self._semaphores.get((scheme, host, port))
if sem is None:
sem = DeferredSemaphore(self.maxConnectionsPerHostName)
self._semaphores[scheme, host, port] = sem
return sem.run(self._request, method, scheme, host, port, path,
headers, bodyProducer)
else:
return self._request(
method, scheme, host, port, path, headers, bodyProducer)
def collectFiles():
semaphore = DeferredSemaphore(1)
files = glob.glob('*.py')
dl = list()
for item in range(len(files)):
#Queues list of things to be sent and runs it
dl.append(semaphore.run(sendFiles, files[item]))
# convert to a DefferedList. Allows for callback call
dl = DeferredList(dl)
def cbFinished(ignored):
print 'Finishing job'
dl.addCallback(cbFinished)
return dl
def main():
agent = Agent(reactor)
sem = DeferredSemaphore(5)
print "Loading IDs"
ids = getBeermeIds()
ids = ids[:100]
print "Done Loading %s IDs" % str(len(ids))
jobs = []
for id in ids:
jobs.append(sem.run(beerme_request, id, agent))
d = gatherResults(jobs)
d.addBoth(cbShutdown)
print "Starting reactor..."
reactor.run()
def main():
agent = Agent(reactor)
sem = DeferredSemaphore(5)
print "Loading IDs"
ids = getBeermeIds()
ids = ids[:100]
print "Done Loading %s IDs" % str(len(ids))
jobs = []
for id in ids:
jobs.append(sem.run(beerme_request,id,agent))
d = gatherResults(jobs)
d.addBoth(cbShutdown)
print "Starting reactor..."
reactor.run()
def start(self):
""" get each page """
deferreds = []
sem = DeferredSemaphore(self.MAX_RUN)
for key in self.book.keys():
sleep(self.RATE_LIMIT)
d = sem.run(getPage, self.book[key])
d.addCallback(self.pageCallback, key)
d.addErrback(self.errorHandler, key)
deferreds.append(d)
dl = DeferredList(deferreds)
dl.addCallback(self.listCallback)
dl.addCallback(self.finish)
def main():
agent = Agent(reactor)
sem = DeferredSemaphore(10)
print "Loading breweries..."
mongo = MongoClient().entities.breweries
breweries = loadBreweries(mongo)
print "Done loading breweries."
jobs = []
for brewery in breweries:
jobs.append(sem.run(socialRequest, brewery, agent, mongo))
# if len(jobs) % 50 == 0:
# print "Brewery Jobs started: %d" % len(jobs)
d = gatherResults(jobs)
d.addBoth(cbShutdown)
print "Let the Reactor BEGIN!"
reactor.run()
def main():
agent = Agent(reactor)
sem = DeferredSemaphore(10)
print "Loading breweries..."
mongo = MongoClient().entities.breweries
breweries = loadBreweries(mongo)
print "Done loading breweries."
jobs = []
for brewery in breweries:
jobs.append(sem.run(socialRequest,brewery,agent,mongo))
# if len(jobs) % 50 == 0:
# print "Brewery Jobs started: %d" % len(jobs)
d = gatherResults(jobs)
d.addBoth(cbShutdown)
print "Let the Reactor BEGIN!"
reactor.run()
def trigger_convergence_groups(authenticator, region, groups, concurrency_limit, no_error_group):
"""
Trigger convergence on given groups
:param IAuthenticator authenticator: Otter authenticator
:param str region: Region where this is running
:param list groups: List of group dicts
:param int concurrency_limit: Concurrency limit
:param bool no_error_group: If true then do not converge ERROR groups
:return: Deferred fired with None
"""
sem = DeferredSemaphore(concurrency_limit)
return gatherResults(
[sem.run(trigger_convergence, authenticator, region, group, no_error_group) for group in groups],
consumeErrors=True,
).addCallback(lambda _: None)
def _cbReqPhotoPage(self, photo_list):
def photoConnectLost(message, url):
log.err(message)
raise Exception("can't access {0!s}".format(url))
dl = []
sem = DeferredSemaphore(20)
for i in photo_list:
d = sem.run(self._agent.request, b'GET', bytes(i.get('url'), 'ascii'))
d.addCallback(self._cbGetPhotoDlLink, *[i.get('url')])
d.addErrback(photoConnectLost, *[i.get('url')])
d.addCallback(self._cbDownloadPhoto)
d.addErrback(log.err)
dl.append(d)
deferreds = DeferredList(dl, consumeErrors=True)
return deferreds
def trigger_convergence_groups(authenticator, region, groups,
concurrency_limit, no_error_group):
"""
Trigger convergence on given groups
:param IAuthenticator authenticator: Otter authenticator
:param str region: Region where this is running
:param list groups: List of group dicts
:param int concurrency_limit: Concurrency limit
:param bool no_error_group: If true then do not converge ERROR groups
:return: Deferred fired with None
"""
sem = DeferredSemaphore(concurrency_limit)
return gatherResults([
sem.run(trigger_convergence, authenticator, region, group,
no_error_group) for group in groups
],
consumeErrors=True).addCallback(lambda _: None)
def run(packages, modules, other_args, reactor, limit, excludes):
sem = DeferredSemaphore(limit)
proc_argss = get_cafe_args(packages, modules, excludes)
deferreds = [
sem.run(getProcessOutputAndValue, 'cafe-runner',
other_args + proc_args, env=os.environ, reactor=reactor)
for proc_args in proc_argss]
results = yield gatherResults(deferreds, consumeErrors=True)
failed = False
for proc_args, (stdout, stderr, code) in zip(proc_argss, results):
if code == 0:
continue
failed = True
print('Error when running ', ' '.join(proc_args))
print('Stdout\n', stdout, 'Stderr\n', stderr)
if failed:
raise SystemExit('Some tests failed')
def runDeferredCommand(self):
self.startTime = time.time()
self.log.info('%s using BBCApplicationSNMP' % self.device.id)
self.stats['totalRequests'] = len(self.device.modelledOids)
semaf = DeferredSemaphore(self.workers)
jobs = []
for oids in self.device.modelledOids:
self.stats['collectedAppOidCounter'] += oids.count('.75025.')
df = semaf.run(utils.getProcessOutputAndValue, '/usr/bin/snmpget',
self.prepareSnmpCmdArgList() + oids.split())
df.addErrback(self.handleError)
jobs.append(df)
df = gatherResults(jobs)
df.addErrback(self.handleError)
df.addCallback(self.parseOutput)
return df
def moveFiles(self, virtSrcBase, virtDestBase, relFiles):
self.debug("MOVING: %r, %r, %r", virtSrcBase, virtDestBase, relFiles)
if not self._local:
raise TranscoderError("Component not properly setup yet")
def move_failed(failure, src, dest):
msg = ("Fail to move file '%s' to '%s': %s"
% (src, dest, log.getFailureMessage(failure)))
self.warning("%s", msg)
raise TranscoderError(msg, cause=failure)
def move_file(src, dest, attr=None):
self.debug("Moving file '%s' to '%s'", src, dest)
dest_dir = os.path.dirname(dest)
safe_mkdirs(dest_dir, "input file destination", attr)
d = deferToThread(shutil.move, src, dest)
d.addErrback(move_failed, src, dest)
return d
def move_files_failed(results):
first_failure = None
for ok, result in results:
if not ok:
if not first_failure:
first_failure = result
return first_failure
sem = DeferredSemaphore(1)
move_tasks = []
for file in relFiles:
source_path = virtSrcBase.append(file).localize(self._local)
dest_path = virtDestBase.append(file).localize(self._local)
source_path = os.path.realpath(source_path)
dest_path = os.path.realpath(dest_path)
d = sem.run(move_file, source_path, dest_path, self._pathAttr)
move_tasks.append(d)
dl = DeferredList(move_tasks, consumeErrors=True)
dl.addErrback(move_files_failed)
return d
def collect(self, device, log):
log.debug('{}: Modeling collect'.format(device.id))
port = getattr(device, 'zSpringBootPort', None)
uri = getattr(device, 'zSpringBootURI', None)
ivGroups = getattr(device, 'zIVGroups', None)
ivUser = getattr(device, 'zIVUser', None)
ip_address = device.manageIp
if not ip_address:
log.error("%s: IP Address cannot be empty", device.id)
returnValue(None)
deferreds = []
sem = DeferredSemaphore(1)
# TODO: remove loop
for query in self.queries:
url = query[1].format(ip_address, port, uri)
log.debug('SBA collect url: {}'.format(url))
d = sem.run(
getPage,
url,
headers={
"Accept": "application/json",
"User-Agent": "Mozilla/3.0Gold",
"iv-groups": ivGroups,
"iv-user": ivUser,
},
)
d.addCallback(self.add_tag, '{}'.format(query[0]))
deferreds.append(d)
results = yield DeferredList(deferreds, consumeErrors=True)
for success, result in results:
if not success:
log.error('{}: {}'.format(device.id, result.getErrorMessage()))
returnValue(results)
def run(packages, modules, other_args, reactor, limit, excludes):
sem = DeferredSemaphore(limit)
proc_argss = get_cafe_args(packages, modules, excludes)
deferreds = [
sem.run(getProcessOutputAndValue,
'cafe-runner',
other_args + proc_args,
env=os.environ,
reactor=reactor) for proc_args in proc_argss
]
results = yield gatherResults(deferreds, consumeErrors=True)
failed = False
for proc_args, (stdout, stderr, code) in zip(proc_argss, results):
if code == 0:
continue
failed = True
print('Error when running ', ' '.join(proc_args))
print('Stdout\n', stdout, 'Stderr\n', stderr)
if failed:
raise SystemExit('Some tests failed')
def runFiles(self): semi = DeferredSemaphore(1) jobs = [] recordTimes = "01/24/17 12:00:00 01/24/17 12:15:00 01/24/17 12:30:00" recordTimesList = [data for data in recordTimes.split()] for runs in range(len(recordTimesList)/2): print "recordTimes:", recordTimesList # recordTimeStartTime = recordTimesList.pop(0) + " " + recordTimesList.pop(0) # print "start time: ", recordTimeStartTime startAtTime = self.calculateTimeDifference(recordTimesList.pop(0), recordTimesList.pop(0)) # jobs.append(semi.run(tv.takeVideo, int(resW), int(resH), int(totalTimeSec),\ # int(framerate), startAtTime, serverIP, piName)) jobs.append(semi.run(self.sendFiles, startAtTime)) jobs = DeferredList(jobs) def cbFinished(ignored): print 'Finishing job' # reactor.callLater(0.5, self.transport.write, 'finished') jobs.addCallback(cbFinished) return jobs
def collect(self, config):
log.debug('Starting SBA JVM collect')
# TODO : cleanup job collect
ip_address = config.manageIp
if not ip_address:
log.error("%s: IP Address cannot be empty", device.id)
returnValue(None)
ds0 = config.datasources[0]
applicationList = []
deferreds = []
sem = DeferredSemaphore(1)
for metric, props in self.metrics.iteritems():
applicationNameID = ds0.params['applicationNameID'] # app_delivery_service_8df95ae5
# if applicationNameID in applicationList:
# continue
# applicationList.append(applicationNameID)
serviceURL = ds0.params['serviceURL']
# url = self.urls[datasource.datasource].format(serviceURL)
endpoint = props['endpoint']
url = '{}/management/metrics/{}'.format(serviceURL, endpoint)
log.debug('AAA url: {}'.format(url))
d = sem.run(getPage, url,
headers={
"Accept": "application/json",
"User-Agent": "Mozilla/3.0Gold",
"iv-groups": ds0.zIVGroups,
"iv-user": ds0.zIVUser,
},
)
tag = 'jvm_{}_{}'.format(metric, applicationNameID) # order_delivery_service_3db30547
d.addCallback(self.add_tag, tag)
deferreds.append(d)
return DeferredList(deferreds)
def run_deploy(self, hosts, components, commands):
try:
self.transport.initialize()
except TransportError as e:
raise DeployError("could not initialize transport: %s" % e)
def signal_handler(sig, _):
reason = SIGNAL_MESSAGES[sig]
reactor.callFromThread(self.abort, reason)
signal.signal(signal.SIGINT, signal_handler)
signal.signal(signal.SIGHUP, signal_handler)
yield self.event_bus.trigger("deploy.begin")
try:
if components:
yield self.event_bus.trigger("build.begin")
try:
# synchronize the code host with upstreams
# this will return a build token and build host for each
# component
sync_command = ["synchronize"] + components
(sync,) = yield self.process_host(
self.code_host, [sync_command])
# this is where we build up the final deploy command
# resulting from all our syncing and building
deploy_command = ["deploy"]
# collect the results of the sync per-buildhost
by_buildhost = collections.defaultdict(list)
for component, sync_info in sync.iteritems():
component_ref = component + "@" + sync_info["token"]
build_host = sync_info.get("buildhost", None)
if build_host:
by_buildhost[build_host].append(component_ref)
else:
# no build host means we just pass the sync token
# straight through as a deploy token
deploy_command.append(component_ref)
# ask each build host to build our components and return
# a deploy token
for build_host, build_refs in by_buildhost.iteritems():
build_command = ["build"] + build_refs
(tokens,) = yield self.process_host(
build_host, [build_command])
for ref in build_refs:
component, at, sync_token = ref.partition("@")
assert at == "@"
try:
deploy_ref = component + "@" + tokens[ref]
except KeyError:
raise ComponentNotBuiltError(component)
deploy_command.append(deploy_ref)
except Exception:
traceback.print_exc()
raise DeployError("unexpected error in sync/build")
else:
# inject our built-up deploy command at the beginning of
# the command list for each host
commands = [deploy_command] + commands
yield self.event_bus.trigger("build.end")
parallelism_limiter = DeferredSemaphore(tokens=self.parallel)
host_deploys = []
first_host = True
for host in hosts:
if not first_host:
for i in xrange(self.sleeptime, 0, -1):
yield self.event_bus.trigger(
"deploy.sleep", host=host, count=i)
yield sleep(1)
else:
first_host = False
deferred = parallelism_limiter.run(
self.process_host, host, commands)
deferred.addErrback(self.on_host_error)
host_deploys.append(deferred)
yield self.event_bus.trigger(
"deploy.enqueue", deploys=host_deploys)
yield DeferredList(host_deploys)
except (DeployError, AbortDeploy, TransportError) as e:
yield self.abort(str(e))
else:
yield self.event_bus.trigger("deploy.end")
class View(TabManager,iterator):
def __init__(self,controller):
self.name,self.version="rattlekekz-qt",20100806 # Diese Variablen werden vom View abgefragt
self.controller=controller
self.revision=rev
self.alert=app.alert
TabManager.__init__(self)
self.spaces=re.compile(r" {1,}")
self.urls=re.compile(r"(?=\b)((?#Protocol)(?:(?:ht|f)tp(?:s?)\:\/\/|~/|/)(?#Username:Password)(?:\w+:\w+@)?(?#Subdomains)(?:(?:[-\w]+\.)+(?#TopLevel Domains)(?:com|org|net|gov|mil|biz|info|mobi|name|aero|jobs|museum|travel|edu|pro|asia|cat|coop|int|tel|post|xxx|[a-z]{2}))(?#Port)(?::[\d]{1,5})?(?#Directories)(?:(?:(?:/(?:[-\w~!$+|.,=]|%[a-f\d]{2})+)+|/)+|#)?(?#Query)(?:(?:\?(?:[-\w~!$+|.,*:]|%[a-f\d{2}])+=(?:[-\w~!$+|.,*:=]|%[a-f\d]{2})*)(?:&(?:[-\w~!$+|.,*:]|%[a-f\d{2}])+=(?:[-\w~!$+|.,*:=]|%[a-f\d]{2})*)*)*(?#Anchor)(?:#(?:[-\w~!$+|.,*:=]|%[a-f\d]{2})*)?)(?=\b)",re.I)
self.blubb=lambda x:chr(ord(x)-43)
self.plugins={}
self._setup()
self.addTab("$login",rattlekekzLoginTab)
self.changeTab("$login")
self.main.show()
self.smilie_data=self.readSmilies()
self.loading_data=open(sys.prefix+os.sep+'share'+os.sep+'emoticons'+os.sep+'rattlekekz'+os.sep+'loading.png').read()
self.loading_image=QtGui.QImage()
self.loading_image.loadFromData(self.loading_data,"PNG")
self.images={}
self.pendingImages=[]
self.smilies={"s6":":-)",
"s4":":-(",
"s1":":-/",
"s8":"X-O",
"s7":"(-:",
"s9":"?-|",
"s10":"X-|",
"s11":"8-)",
"s2":":-D",
"s3":":-P",
"s5":";-)",
"sxmas":"o:)",
"s12":":-E",
"s13":":-G"}
self.colors={"red":"FF0000",
"blue":"0000FF",
"green":"008000",
"gray":"696969",
"cyan":"00FFFF",
"magenta":"FF00FF",
"orange":"FFA500",
"pink":"FFC0CB",
"yellow":"FFFF00",
"normal":"000000",
"normalaway":"696969",
"greenaway":"4D804D",
"blueaway":"5050E6",
"orangeaway":"E5B151",
"redaway":"E65151"}
self.imageLock = DeferredSemaphore(1)
def _setup(self):
self.main=rattlekekzMainWidget()
self.main.setWindowTitle(self.name)
self.main.setMenuBar(rattlekekzMenuBar())
self.main.setStatusBar(rattlekekzStatusBar())
self.menu=self.main.menuBar()
self.status=self.main.statusBar()
self.main.setCentralWidget(QtGui.QTabWidget())
self.tabs=self.main.centralWidget()
self.tabs.setMovable(True)
self.tabs.setTabsClosable(True)
#self.tabs.setMinimumSize(400,550)
self.main.connect(self.tabs,QtCore.SIGNAL("tabCloseRequested(int)"),self.closeTab)
self.main.connect(self.main,QtCore.SIGNAL("closed()"),self.quit)
self.main.connect(self.menu,QtCore.SIGNAL("quit()"),self.quit)
self.main.connect(self.menu,QtCore.SIGNAL("config()"),self.openConfig)
self.main.connect(self.tabs,QtCore.SIGNAL("currentChanged(int)"),self.activateTab)
self.main.connect(self.main,QtCore.SIGNAL("gotFocus()"),self.changeFocus)
def readSmilies(self):
data=[]
for i in glob(sys.prefix+os.sep+'share'+os.sep+'emoticons'+os.sep+'rattlekekz'+os.sep+'*.png'):
data.append((QtCore.QUrl("smilie://"+i.split(os.sep)[-1]),QtGui.QImage(i,"PNG")))
return data
def activateTab(self,integer):
self.ShownRoom=self.stringHandler(self.tabs.tabText(integer))
self.tabs.tabBar().setTabTextColor(integer,QtCore.Qt.black)
self.tabs.widget(integer).gotFocus()
def closeTab(self,integer):
if isinstance(self.tabs.widget(integer),rattlekekzMsgTab):
self.sendStr(self.stringHandler(self.tabs.tabText(integer)),"/part")
else:
if isinstance(self.tabs.widget(integer),rattlekekzInfoTab):
text = self.stringHandler(self.tabs.tabText(integer))
if text.startswith("whois"):
self.controller.closedWhois(text.split(" ")[1])
self.tabs.removeTab(integer)
del self.lookupRooms[0]
def changeFocus(self):
self.tabs.widget(self.getTabId(self.ShownRoom)).gotFocus()
def getRooms(self):
rooms=[]
for i in range(self.tabs.count()):
room = self.stringHandler(self.tabs.tabText(i))
if not room.startswith("#") and not room.startswith("whois: "):
rooms.append(room)
return rooms
def quit(self):
self.iterPlugins('quitConnection')
reactor.stop()
def openConfig(self):
print "STUB: implement config tab"
def deparse(self,msg):
text,format=self.controller.decode(msg)
msg=[]
for i in range(len(text)):
text[i] = self.escapeText(text[i])
if format[i] == "newline":
msg.append("<br>")
continue
if format[i] == "hline":
msg.append("<hr>")
continue
if format[i] == "imageurl":
image=self.controller.loadImage(text[i])
if image[0] == "image":
self.pendingImages.append(str(image[1]))
msg.append("<img src='image://"+str(image[1])+".jpg'>")
#try:
# image=urllib.urlretrieve(text[i])[0]
# msg.append("<img src='"+self.stringHandler(image)+"'>")
#except:
# msg.append(self.escapeText("<image url is invalid>"))
#image=urllib2.urlopen(text[i]).read()
#for y in range(self.tabs.count()):
# if isinstance(self.tabs.widget(y),rattlekekzPrivTab):
# self.tabs.widget(y).output.document().addResource(QtGui.QTextDocument.ImageResource,QtCore.QUrl("mydata://"+self.stringHandler(text[i])),QVariant(image))
# msg.append(("<img src='"+"mydata://"+self.stringHandler(text[i])+"'>"))
continue
if len(format[i]) > 1:
if format[i][0] == "ownnick":
if not "red" in format[i][1]:
color = "red"
else:
color = "green"
msg.append("<font color='#"+self.colors[color]+"'><b>"+text[i]+"</b></font>")
continue
#if text[i].isspace() or text[i]=="": # NOTE: If there are any bugs with new rooms and the roomop-message THIS could be is the reason ;)
# continue #
if text[i] == "": #
continue #
text[i]=self.urls.sub(r'<a href="\1">\1</a>',text[i])
form=format[i].split(",")
color=""
font=([],[])
for a in form:
if a in ["red", "blue", "green", "gray", "cyan", "magenta", "orange", "pink", "yellow","white","reset"]:
if a != "reset":
color=a
else:
color=""
if a == "bold":
font[0].append("<b>")
font[1].append("</b>")
if a == "italic":
font[0].append("<i>")
font[1].append("</i>")
if a == "sb":
text[i]="<img src='"+"smilie://"+self.stringHandler(text[i])+".png' />"
if a == "button":
color=""
font[0].append("<a href='button:"+text[i]+"'>")
font[1].append("</a>")
text[i] = "["+text[i]+"]"
if color != "":
msg.append("<font color='#"+self.colors[color]+"'>"+"".join(font[0])+text[i]+"".join(font[1])+"</font>")
else:
msg.append("".join(font[0])+text[i]+"".join(font[1]))
return msg
def escapeText(self,text):
text="&".join(text.split("&"))
text="<".join(text.split("<"))
text=">".join(text.split(">"))
count=self.spaces.findall(text)
text=self.spaces.split(text)
for i in range(len(text)):
if len(count) != 0:
text[i]=text[i]+" "+(len(count.pop(0))-1)*" "
else:
break
return "".join(text)
def loadedImage(self,id,image_data):
id=str(id)
image=QtGui.QImage()
image.loadFromData(image_data)
if self.images.has_key(id):
tab = self.getTab(self.images[id])
self.imageLock.run(tab.refreshImage,id,image)
else:
reactor.callLater(10,self.loadedImage,id,image_data)
def stringHandler(self,string,return_utf8=False):
if type(string) is list:
result=[]
for i in string:
if return_utf8 == False:
try:
i=str(i)
except UnicodeEncodeError:
i=unicode(i).encode("utf_8")
result.append(i)
else:
try:
i=unicode(i)
except UnicodeDecodeError:
i=str(i).decode("utf_8","replace")
result.append(i)
return result
else:
if return_utf8 == False:
try:
return str(string)
except UnicodeEncodeError:
string=unicode(string)
return string.encode("utf_8")
else:
try:
return unicode(string)
except UnicodeDecodeError:
string=str(string)
return string.decode("utf_8","replace")
def finishedReadingConfigfile(self):
pass
def receivedPreLoginData(self,rooms,array):
self.isConnected=True
self.status.showMessage("connected")
self.addTab("$login",rattlekekzLoginTab)
self.getTab("$login").receivedPreLoginData(rooms,array)
def updateRooms(self,rooms):
try:
tab=self.getTab("$login")
except:
pass
else:
tab.updateRooms(rooms)
def startConnection(self,host,port):
self.controller.startConnection(host,port)
def addRoom(self,room,tab):
tablist={"ChatRoom":rattlekekzMsgTab,
"PrivRoom":rattlekekzPrivTab,
"InfoRoom":rattlekekzInfoTab,
"WhoisRoom":rattlekekzInfoTab,
"MailRoom":rattlekekzMailTab,
"SecureRoom":rattlekekzSecureTab,
"EditRoom":rattlekekzWhoisEditTab,
"MailEditRoom":rattlekekzMailEditTab}
self.addTab(room,tablist[tab])
def newTopic(self,room,topic):
self.getTab(room).newTopic(topic)
def sendLogin(self, nick, passwd, room):
self.getTab("$login").grayOut()
self.iterPlugins('sendLogin', [nick, passwd, room])
def registerNick(self, nick, passwd, email):
self.iterPlugins('registerNick', [nick, passwd, email])
def changePassword(self, oldPassword, newPassword):
self.iterPlugins('changePassword', [oldPassword, newPassword])
def updateProfile(self, newName, newLocation, newHomepage, newHobbies, newSignature, passwd):
self.iterPlugins('updateProfile', [newName, newLocation, newHomepage, newHobbies, newSignature, passwd])
def startedConnection(self):
self.status.showMessage("connecting ...")
def connectionLost(self,reason):
self.status.showMessage(self.stringHandler("connection lost",True))
def connectionFailed(self):
self.status.showMessage("connection attempt failed")
def successLogin(self,nick,status,room,reconnected=False):
self.nickname=nick
self.addTab(room,rattlekekzMsgTab)
if not reconnected:
self.ShownRoom=room
self.changeTab(room)
try:
self.delTab("$login")
except:
pass
def successRegister(self):
self.status.showMessage("nick registered")
def successNewPassword(self):
self.status.showMessage("password changed")
def receivedProfile(self,name,location,homepage,hobbies,signature):
self.changeTab("$edit")
self.getTab("$edit").receivedProfile(name,location,homepage,hobbies,signature)
def successNewProfile(self):
self.status.showMessage("profile updated")
def securityCheck(self, infotext):
pass
def receivedPing(self,deltaPing):
self.status.showMessage(self.stringHandler("Ping: "+str(deltaPing)+" ms",True))
def printMsg(self,room,msg):
#print "<%s> %s" % (self.stringHandler(room),"".join(self.stringHandler(msg)))
msg = "".join(msg)
ids=re.findall(r"<img\s+src='image://(\d+)\.jpg'\s*>",msg)
if len(ids) != 0:
if isinstance(self.getTab(room),(rattlekekzPrivTab,rattlekekzInfoTab,rattlekekzMailTab)):
for i in ids:
self.images[i]=room
if i in self.pendingImages:
image=QtGui.QImage()
image.loadFromData(self.controller.getImage(int(i)))
self.getTab(room).addImage(i,image)
self.pendingImages.remove(i)
else:
self.getTab(room).addImage(i,self.loading_image)
self.getTab(room).addLine(msg)
def gotException(self, message):
self.status.showMessage(message)
def gotLoginException(self,message):
try:
tab = self.getTab("$login")
except:
pass
else:
tab.grayOut(False)
tab.prelogin=True
tab.gotFocus()
self.status.showMessage(message)
def listUser(self,room,users):
usercolors = self.controller.getValue("usercolors")
if usercolors == None: usercolors = True
self.getTab(room).listUser(users,usercolors)
def meJoin(self,room,background):
self.addTab(room,rattlekekzMsgTab)
if not background:
self.changeTab(room)
def mePart(self,room):
self.delTab(room)
def meGo(self,oldroom,newroom):
index = self.getTabId(oldroom)
self.addTab(newroom,rattlekekzMsgTab,index)
self.changeTab(newroom)
self.delTab(oldroom)
def newTopic(self,room,topic):
self.getTab(room).newTopic(topic)
def loggedOut(self):
pass
def fubar(self):
"""This function sends bullshit to the controller for debugging purposes"""
self.iterPlugins('sendBullshit',["".join(map(self.blubb,'_a`\x90\x8cc^b\\\\d\x8d\x8d^\x8e\x8d``\x90\x8f]]c_]b\x91b\x8dd^\x8c_\x8e\x91\x91__\x8c\x91'))])
def receivedInformation(self,info):
pass
def minorInfo(self,room,nick):
"""this method is used to determine wether a new info-tab must be opened to display e.g. a bot message"""
pre=None
if nick=="":
pre="Info:°nn°"
if not isinstance(self.getTab(self.ShownRoom),rattlekekzMsgTab):
self.addRoom("$info","InfoRoom")
self.changeTab("$info")
return (pre,"$info")
elif room=="":
return (pre,self.ShownRoom)
else:
return (pre,room)
#if isinstance(self.getTab(self.ShownRoom),self.rattlekekzMsgTab):
# self.getTab(self.ShownRoom).addLine("Info: "+self.stringHandler(message))
#else:
# self.addTab("$infos",rattlekekzInfoTab)
# self.changeTab("$infos")
# self.getTab(self.ShownRoom).addLine("Info: "+self.stringHandler(message))
def receivedWhois(self,nick,array):
title=u"whois: "+self.stringHandler(nick,True)
out = map(lambda x:"".join(self.deparse(x)), array)
try:
tab = self.getTab(title)
except:
self.addRoom(title,"WhoisRoom")
self.changeTab(title)
self.getTab(title).addWhois()
for msg in out:
self.printMsg(title,msg)
self.highlightTab(title,2)
def MailInfo(self,info):
pass
def openLinkTab(self,room,links):
room,links=self.stringHandler(room,True),self.stringHandler(links,True)
self.addTab("$links of "+room,rattlekekzInfoTab)
self.changeTab("$links of "+room)
for i in links:
self.getTab("$links of "+room).addLine('<a href="'+i+'">'+i+'</a>')
def openMailEditTab(self,receiver=""):
tabname="$mail_to"
if receiver!="":
tabname=tabname+" "+receiver
self.addTab(tabname,rattlekekzMailEditTab)
self.getTab(tabname).setContent(receiver)
self.changeTab(tabname)
def openMailTab(self):
self.addTab("$mails",rattlekekzMailTab)
self.changeTab("$mails")
def getMail(self,index):
self.iterPlugins('getMail', [index])
def refreshMaillist(self):
self.iterPlugins('refreshMaillist')
def receivedMails(self,userid,mailcount,mails):
self.openMailTab()
self.getTab("$mails").receivedMails(userid,mailcount,mails)
def printMail(self,user,date,mail):
self.openMailTab()
header = u"Mail by "+user+" from "+date+u": °np° ---begin of mail ---- °np°"
end = u"°np°---end of mail---°np°"
mail = header + self.stringHandler(mail,True) + end
msg = self.deparse(mail)
self.getTab("$mails").addLine(msg)
def sendStr(self,channel,string):
self.iterPlugins('sendStr', [channel, string])
def sendMail(self,nick,msg):
self.iterPlugins('sendMail', [nick, msg])
def timestamp(self, string):
return "<font color='#"+self.colors["green"]+"'>"+string+"</font>"
def colorizeText(self, color, text):
return "<font color='#"+self.colors[color]+"'>"+text+"</font>"
def unknownMethod(self,name):
pass
def __getattr__(self, name):
return self.unknownMethod(name)
class MapUpdater(object):
def __init__(self, mapsPath, fetchURL, deleteIfNotPresent, tfLevelSounds):
assert isinstance(mapsPath, str) and len(mapsPath)
assert isinstance(fetchURL, str) and len(fetchURL)
self.mapsPath = FilePath(mapsPath)
self.downloadTempPath = self.mapsPath.child('mapupdater')
self.fetchURL = URLPath.fromString(fetchURL)
self.deleteIfNotPresent = deleteIfNotPresent
self.tfLevelSounds = tfLevelSounds
self.semaphore = DeferredSemaphore(1)
self.downloadSemaphore = DeferredSemaphore(4)
for fp in self.downloadTempPath.globChildren('*.bsp.bz2'):
fp.remove()
def checkMaps(self, *a, **kw):
"""
Wrap self._checkMaps to prevent running multiple checks at once.
"""
return self.semaphore.run(self._checkMaps, *a, **kw)
def _checkMaps(self, forceDownloadMaps=None):
def _cb(remoteMaps):
if forceDownloadMaps:
remoteMaps = list(set(remoteMaps + forceDownloadMaps))
remoteMapsLower = [f.lower() for f in remoteMaps]
ourMaps = filter(
lambda p: not p.isdir() and p.path.endswith('.bsp'),
self.mapsPath.children())
ourMapFilenames = [p.basename().lower() + '.bz2' for p in ourMaps]
missing = []
for f in remoteMaps:
if f.lower() not in ourMapFilenames:
missing.append(f)
delete = []
for p in ourMaps:
filename = p.basename().lower() + '.bz2'
if filename not in remoteMapsLower:
delete.append(p)
if self.deleteIfNotPresent and delete:
for fp in delete:
fp.remove()
print 'Deleted {} map(s) not present at remote server:'.format(
len(delete))
print ', '.join([x.basename() for x in delete])
if missing:
print 'Fetching {} map(s)'.format(len(missing))
def _allFinished(ignored):
self.mapsPath.child('tempus_map_updater_run_once').touch()
if self.tfLevelSounds:
self.addLevelSounds(ourMaps)
print 'Now up-to-date.'
ds = []
for filename in missing:
ds.append(self.fetchMap(filename))
return gatherResults(ds).addCallback(_allFinished)
elif self.tfLevelSounds:
self.addLevelSounds(ourMaps)
return self.getMapList().addCallback(_cb)
def fetchMap(self, *a, **kw):
return self.downloadSemaphore.run(self._fetchMap, *a, **kw)
def _fetchMap(self, filename):
downloadTempPath = self.downloadTempPath
if not downloadTempPath.exists():
downloadTempPath.makedirs()
def _cb(response, fn):
tp = downloadTempPath.child(fn)
fd = tp.open('wb')
def _extracted(ignored):
extractedPath = tp.sibling(tp.basename().replace('.bz2', ''))
extractedPath.moveTo(
self.mapsPath.child(tp.basename().replace('.bz2', '')))
try:
tp.remove()
# File already gone
except OSError:
pass
print 'Finished downloading {}'.format(fn)
def _finished(ignored):
fd.close()
d = getProcessOutputAndValue(
'aunpack', (tp.path, '-X', downloadTempPath.path))
d.addErrback(log.err)
d.addCallback(_extracted)
return d
def _eb(failure):
print 'Error downloading {}:'.format(fn)
print failure.getTraceback()
fd.close()
try:
tp.remove()
# File already gone
except OSError:
pass
d = treq.collect(response, fd.write)
d.addCallback(_finished)
d.addErrback(_eb)
return d
d = treq.get(str(self.fetchURL.child(filename)))
return d.addCallback(_cb, filename)
def getMapList(self, forceDownloadMaps):
raise NotImplementedError('Subclasses must override this method.')
def addLevelSounds(self, mapPaths):
content = FilePath(
mapupdater.__file__).sibling('tf_level_sounds.txt').getContent()
added = []
for p in mapPaths:
mapName = p.basename()[:-4]
p2 = p.sibling('{}_level_sounds.txt'.format(mapName))
if p2.exists() and p2.getContent() == content:
continue
added.append(mapName)
p2.setContent(content)
if added:
print 'Added level sounds for:'
print ', '.join(added)
class Supervisor(object):
"""
The Supervisor class manages the environments of testifi, and allows
scheduling of work.
"""
def __init__(self, queue):
self.managedPackageVersions = set()
self.certifiDirectory = tempfile.mkdtemp()
self.testSemaphore = DeferredSemaphore(tokens=32)
self.binPath = os.path.join(
os.path.split(__file__)[0], 'certifi_test.py')
self.queue = queue
self._log = logger.new(object="supervisor")
@inlineCallbacks
def pollForNewReleases(self):
"""
Check whether any new certifi releases have been made.
"""
log = self._log.bind(function="pollForNewReleases")
log.msg("polling for new releases")
releases = yield certifiVersions()
for release, file in releases:
if release in self.managedPackageVersions:
log.msg("skipping existing release", release=release)
continue
# The release is new. We need to start processing it.
yield self.addNewRelease(release, file)
@inlineCallbacks
def addNewRelease(self, release, tarball_path):
"""
For each new release that we've discovered, set up processing for it.
"""
log = self._log.bind(function="addNewRelease", release=release)
log.msg("adding release")
# Download the tarball.
file_path = os.path.join(self.certifiDirectory,
'certifi-' + release + '.tar.gz')
with open(file_path, 'wb') as f:
yield downloadFile(tarball_path, f)
log.msg("tarball saved", tarball=file_path)
rc = yield getProcessValue(
distutils.spawn.find_executable('tar'),
args=['xvf', file_path, '-C', self.certifiDirectory])
assert not rc
log.msg("adding to managed versions")
self.managedPackageVersions.add(release)
@inlineCallbacks
def _testHostAgainstRelease(self, host, release):
"""
Tests a given host against a specific certifi release. Returns a
Deferred that fires with the result of the test.
"""
log = self._log.bind(function="_testHostAgainstRelease")
file_path = os.path.join(self.certifiDirectory, 'certifi-' + release)
args = [self.binPath, file_path, host]
log.msg("running test", args=args)
result = yield getProcessValue(
distutils.spawn.find_executable('pypy'),
args=args,
)
returnValue(result)
@inlineCallbacks
def testHost(self, host):
"""
Runs the certifi tests against a given host.
Returns a deferred that fires with a list of tuples: certifi release
and whether the test passed.
"""
log = self._log.bind(function="testHost", host=host)
results = []
for release in self.managedPackageVersions:
result = yield self.testSemaphore.run(self._testHostAgainstRelease,
host, release)
results.append((release, result))
log.msg("test complete", release=release, result=result)
returnValue(results)
@inlineCallbacks
def testLoop(self):
"""
Loop forever, popping tests off the queue and running them.
"""
log = self._log.bind(function="testLoop")
while True:
test = yield self.queue.get()
log = log.bind(test_id=test.id, host=test.host)
log.msg("beginning test")
try:
results = yield self.testHost(test.host)
except Exception as e:
print e
continue
log.msg("test suite complete", results=results)
class VBMSClient(object):
def __init__(self, reactor, connect_vbms_path, bundle_path, endpoint_url,
keyfile, samlfile, key, keypass, ca_cert, client_cert):
self._reactor = reactor
self._connect_vbms_path = connect_vbms_path
self._bundle_path = bundle_path
self._endpoint_url = endpoint_url
self._keyfile = keyfile
self._samlfile = samlfile
self._key = key
self._keypass = keypass
self._ca_cert = ca_cert
self._client_cert = client_cert
self._connect_vbms_semaphore = DeferredSemaphore(tokens=8)
def _path_to_ruby(self, path):
if path is None:
return "nil"
else:
return repr(path)
def _execute_connect_vbms(self, logger, request, formatter, args):
ruby_code = """#!/usr/bin/env ruby
$LOAD_PATH << '{connect_vbms_path}/src/'
require 'json'
require 'vbms'
client = VBMS::Client.new(
{endpoint_url!r},
{keyfile},
{samlfile},
{key},
{keypass!r},
{ca_cert},
{client_cert},
)
request = {request}
result = client.send(request)
STDOUT.write({formatter})
STDOUT.flush()
""".format(
connect_vbms_path=self._connect_vbms_path,
endpoint_url=self._endpoint_url,
keyfile=self._path_to_ruby(self._keyfile),
samlfile=self._path_to_ruby(self._samlfile),
key=self._path_to_ruby(self._key),
keypass=self._keypass,
ca_cert=self._path_to_ruby(self._ca_cert),
client_cert=self._path_to_ruby(self._client_cert),
request=request,
formatter=formatter,
).strip()
with tempfile.NamedTemporaryFile(suffix=".rb", delete=False) as f:
f.write(ruby_code)
st = os.stat(f.name)
os.chmod(f.name, st.st_mode | stat.S_IEXEC)
@inlineCallbacks
def run():
timer = logger.time("process.spawn")
try:
stdout, stderr, exit_code = yield getProcessOutputAndValue(
'/bin/bash', [
'-lc', '{} exec {} {}'.format(
self._bundle_path, f.name, " ".join(
map(pipes.quote, args)))
],
env=os.environ,
path=self._connect_vbms_path,
reactor=self._reactor)
finally:
timer.stop()
if exit_code != 0:
raise VBMSError(stdout, stderr, exit_code)
returnValue(stdout)
return self._connect_vbms_semaphore.run(run)
@inlineCallbacks
def get_document_types(self, logger):
response = yield self._execute_connect_vbms(
logger.bind(process="GetDocumentTypes"),
"VBMS::Requests::GetDocumentTypes.new()",
"result.map(&:to_h).to_json",
[],
)
returnValue(json.loads(response))
@inlineCallbacks
def list_documents(self, logger, file_number):
response = yield self._execute_connect_vbms(
logger.bind(process="ListDocuments"),
"VBMS::Requests::ListDocuments.new(ARGV[0])",
'result.map(&:to_h).to_json',
[file_number],
)
returnValue(json.loads(response))
def fetch_document_contents(self, logger, document_id):
return self._execute_connect_vbms(
logger.bind(process="FetchDocumentById"),
"VBMS::Requests::FetchDocumentById.new(ARGV[0])",
"result.content",
[document_id],
)
class wholeProcess():
def __init__(self):
self.semi = DeferredSemaphore(1)
#Should create a global queue list that items can be appended to.
#Then use methods to run process and append items to consecutively.
def scp(self):
print "stuff"
time.sleep(0.5)
return "done"
def globProcess(self):
print "globbing"
file = glob.glob('*.py')
dl = list()
for item in range(len(files)):
dl.append(self.semi.run(sendFiles, files[item]))
dl = DeferredList(dl)
def cbFinished(ignored):
print 'Finishing job'
dl.addCallback(cbFinished)
return dl
def manage(self):
print "managing queue"
file = glob.glob('*.py')
self.semi.run(self.sendFiles, file) #Runs command
def addToQueue(self):
self.semi.run(sendFiles, files)
#Manage whole process
def runFiles():
semi = DeferredSemaphore(1)
jobs = []
for runs in range(5):
jobs.append(semi.run(collectFiles))
jobs = DeferredList(jobs)
def cbFinished(ignored):
print 'Finishing job'
jobs.addCallback(cbFinished)
return jobs
#Glob + upload > every 45 mins run this process?
def collectFiles():
semaphore = DeferredSemaphore(1)
files = glob.glob('*.py')
dl = list()
for item in range(len(files)):
#Queues list of things to be sent and runs it
dl.append(semaphore.run(sendFiles, files[item]))
# convert to a DefferedList. Allows for callback call
dl = DeferredList(dl)
def cbFinished(ignored):
print 'Finishing job'
dl.addCallback(cbFinished)
return dl
# def runFiles(self, resW, resH, totalTimeSec, framerate, serverIP, piName, recordTimesList):
def runFiles(self):
semi = DeferredSemaphore(1)
jobs = []
recordTimes = "01/24/17 12:00:00 01/24/17 12:15:00 01/24/17 12:30:00"
recordTimesList = [data for data in recordTimes.split()]
for runs in range(len(recordTimesList)/2):
print "recordTimes:", recordTimesList
# recordTimeStartTime = recordTimesList.pop(0) + " " + recordTimesList.pop(0)
# print "start time: ", recordTimeStartTime
startAtTime = self.calculateTimeDifference(recordTimesList.pop(0), recordTimesList.pop(0))
# jobs.append(semi.run(tv.takeVideo, int(resW), int(resH), int(totalTimeSec),\
# int(framerate), startAtTime, serverIP, piName))
jobs.append(semi.run(self.sendFiles, startAtTime))
jobs = DeferredList(jobs)
def cbFinished(ignored):
print 'Finishing job'
# reactor.callLater(0.5, self.transport.write, 'finished')
jobs.addCallback(cbFinished)
return jobs
def calculateTimeDifference(self, dateToEnd, timeToEnd):
fullString = dateToEnd + " " + timeToEnd
endTime = datetime.datetime.strptime(fullString, "%x %X")
nowTime = datetime.datetime.today()
difference = endTime - nowTime
return time.time() + difference.total_seconds()
#Upload SCP files
def sendFiles(self, img):
print "sending img: ", img
time.sleep(5)
return "finished"
def run_deploy(self, hosts, components, commands):
try:
yield self.event_bus.trigger("deploy.precheck")
except AbortDeploy as e:
yield self.abort(str(e))
return
try:
self.transport.initialize()
except TransportError as e:
raise DeployError("could not initialize transport: %s" % e)
def signal_handler(sig, _):
reason = SIGNAL_MESSAGES[sig]
reactor.callFromThread(self.abort, reason)
signal.signal(signal.SIGINT, signal_handler)
signal.signal(signal.SIGHUP, signal_handler)
yield self.event_bus.trigger("deploy.begin")
try:
if components:
yield self.event_bus.trigger("build.begin")
try:
# synchronize the code host with upstreams
# this will return a build token and build host for each
# component
sync_command = ["synchronize"] + components
code_host = Host.from_hostname(self.code_host)
(sync_result,) = yield self.process_host(
code_host, [sync_command])
yield self.event_bus.trigger("build.sync",
sync_info=sync_result.result)
# this is where we build up the final deploy command
# resulting from all our syncing and building
deploy_command = ["deploy"]
# collect the results of the sync per-buildhost
by_buildhost = collections.defaultdict(list)
for component, sync_info in sync_result.result.iteritems():
component_ref = component + "@" + sync_info["token"]
build_host = sync_info.get("buildhost", None)
if build_host:
by_buildhost[build_host].append(component_ref)
else:
# no build host means we just pass the sync token
# straight through as a deploy token
deploy_command.append(component_ref)
# ask each build host to build our components and return
# a deploy token
for build_hostname, build_refs in by_buildhost.iteritems():
build_command = ["build"] + build_refs
build_host = Host.from_hostname(build_hostname)
(build_result,) = yield self.process_host(
build_host, [build_command])
for ref in build_refs:
component, at, sync_token = ref.partition("@")
assert at == "@"
try:
deploy_ref = (component + "@" +
build_result.result[ref])
except KeyError:
raise ComponentNotBuiltError(component)
deploy_command.append(deploy_ref)
# Wait until components report ready IF:
# * we are actually restarting a component
# * we aren't going --dangerously-fast
restarting_component = any(
["restart" in val for val in commands])
if restarting_component and not self.dangerously_fast:
commands.append(["wait-until-components-ready"])
except Exception:
traceback.print_exc()
raise DeployError("unexpected error in sync/build")
else:
# inject our built-up deploy command at the beginning of
# the command list for each host
commands = [deploy_command] + commands
yield self.event_bus.trigger("build.end")
parallelism_limiter = DeferredSemaphore(tokens=self.parallel)
host_deploys = []
first_host = True
for host in hosts:
if not first_host:
for i in xrange(self.sleeptime, 0, -1):
yield self.event_bus.trigger(
"deploy.sleep", host=host, count=i)
yield sleep(1)
else:
first_host = False
deferred = parallelism_limiter.run(
self.process_host, host, commands,
timeout=self.execution_timeout)
deferred.addErrback(self.on_host_error)
host_deploys.append(deferred)
yield self.event_bus.trigger(
"deploy.enqueue", deploys=host_deploys)
yield DeferredList(host_deploys)
except (DeployError, AbortDeploy, TransportError) as e:
yield self.abort(str(e))
else:
yield self.event_bus.trigger("deploy.end")
class ExperimentServiceFactory(Factory):
protocol = ExperimentServiceProto
def __init__(self, expected_subscribers, experiment_start_delay):
self.expected_subscribers = expected_subscribers
self.experiment_start_delay = experiment_start_delay
self.parsing_semaphore = DeferredSemaphore(500)
self.connection_counter = -1
self.connections_made = []
self.connections_ready = []
self.vars_received = []
self._made_looping_call = None
self._subscriber_looping_call = None
self._subscriber_received_looping_call = None
self._timeout_delayed_call = None
def buildProtocol(self, addr):
self.connection_counter += 1
return ExperimentServiceProto(self, self.connection_counter + 1)
def setConnectionMade(self, proto):
if not self._timeout_delayed_call:
self._timeout_delayed_call = reactor.callLater(EXPERIMENT_SYNC_TIMEOUT, self.onExperimentSetupTimeout)
else:
self._timeout_delayed_call.reset(EXPERIMENT_SYNC_TIMEOUT)
self.connections_made.append(proto)
if len(self.connections_made) >= self.expected_subscribers:
msg("All subscribers connected!")
if self._made_looping_call and self._made_looping_call.running:
self._made_looping_call.stop()
self.pushIdToSubscribers()
else:
if not self._made_looping_call:
self._made_looping_call = task.LoopingCall(self._print_subscribers_made)
self._made_looping_call.start(1.0)
def _print_subscribers_made(self):
if len(self.connections_made) < self.expected_subscribers:
msg("%d of %d expected subscribers connected." % (len(self.connections_made), self.expected_subscribers))
def pushIdToSubscribers(self):
for proto in self.connections_made:
self.parsing_semaphore.run(proto.sendAndWaitForReady)
def setConnectionReady(self, proto):
self._timeout_delayed_call.reset(EXPERIMENT_SYNC_TIMEOUT)
self.connections_ready.append(proto)
if len(self.connections_ready) >= self.expected_subscribers:
msg("All subscribers are ready, pushing data!")
if self._subscriber_looping_call and self._subscriber_looping_call.running:
self._subscriber_looping_call.stop()
self.pushInfoToSubscribers()
else:
if not self._subscriber_looping_call:
self._subscriber_looping_call = task.LoopingCall(self._print_subscribers_ready)
self._subscriber_looping_call.start(1.0)
def _print_subscribers_ready(self):
msg("%d of %d expected subscribers ready." % (len(self.connections_ready), self.expected_subscribers))
def pushInfoToSubscribers(self):
# Generate the json doc
vars = {}
for subscriber in self.connections_ready:
subscriber_vars = subscriber.vars.copy()
subscriber_vars['port'] = subscriber.id + 12000
subscriber_vars['host'] = subscriber.transport.getPeer().host
vars[subscriber.id] = subscriber_vars
json_vars = json.dumps(vars)
del vars
msg("Pushing a %d bytes long json doc." % len(json_vars))
# Send the json doc to the subscribers
task.cooperate(self._sendLineToAllGenerator(json_vars))
def _sendLineToAllGenerator(self, line):
for subscriber in self.connections_ready:
yield subscriber.sendLine(line)
def setConnectionReceived(self, proto):
self._timeout_delayed_call.reset(EXPERIMENT_SYNC_TIMEOUT)
self.vars_received.append(proto)
if len(self.vars_received) >= self.expected_subscribers:
msg("Data sent to all subscribers, giving the go signal in %f secs." % self.experiment_start_delay)
reactor.callLater(0, self.startExperiment)
self._timeout_delayed_call.cancel()
else:
if not self._subscriber_received_looping_call:
self._subscriber_received_looping_call = task.LoopingCall(self._print_subscribers_received)
self._subscriber_received_looping_call.start(1.0)
def _print_subscribers_received(self):
msg("%d of %d expected subscribers received the data." % (len(self.vars_received), self.expected_subscribers))
def startExperiment(self):
# Give the go signal and disconnect
msg("Starting the experiment!")
if self._subscriber_received_looping_call and self._subscriber_received_looping_call.running:
self._subscriber_received_looping_call.stop()
start_time = time() + self.experiment_start_delay
for subscriber in self.connections_ready:
# Sync the experiment start time among instances
subscriber.sendLine("go:%f" % (start_time + subscriber.vars['time_offset']))
d = task.deferLater(reactor, 5, lambda: msg("Done, disconnecting all clients."))
d.addCallback(lambda _: self.disconnectAll())
d.addCallbacks(self.onExperimentStarted, self.onExperimentStartError)
def disconnectAll(self):
reactor.runUntilCurrent()
def _disconnectAll():
for subscriber in self.connections_ready:
yield subscriber.transport.loseConnection()
task.cooperate(_disconnectAll())
def unregisterConnection(self, proto):
if proto in self.connections_ready:
self.connections_ready.remove(proto)
if proto in self.vars_received:
self.vars_received.remove(proto)
if proto.id in self.vars_received:
self.vars_received.remove(proto.id)
msg("Connection cleanly unregistered.", logLevel=logging.DEBUG)
def onExperimentStarted(self, _):
msg("Experiment started, shutting down sync server.")
reactor.callLater(0, stopReactor)
def onExperimentStartError(self, failure):
err("Failed to start experiment")
reactor.exitCode = 1
reactor.callLater(0, stopReactor)
return failure
def onExperimentSetupTimeout(self):
err("Waiting for all peers timed out, exiting.")
reactor.exitCode = 1
reactor.callLater(0, stopReactor)
def lineLengthExceeded(self, line):
err("Line length exceeded, %d bytes remain." % len(line))
class Supervisor(object):
"""
The Supervisor class manages the environments of testifi, and allows
scheduling of work.
"""
def __init__(self, queue):
self.managedPackageVersions = set()
self.certifiDirectory = tempfile.mkdtemp()
self.testSemaphore = DeferredSemaphore(tokens=32)
self.binPath = os.path.join(
os.path.split(__file__)[0], 'certifi_test.py'
)
self.queue = queue
self._log = logger.new(object="supervisor")
@inlineCallbacks
def pollForNewReleases(self):
"""
Check whether any new certifi releases have been made.
"""
log = self._log.bind(function="pollForNewReleases")
log.msg("polling for new releases")
releases = yield certifiVersions()
for release, file in releases:
if release in self.managedPackageVersions:
log.msg("skipping existing release", release=release)
continue
# The release is new. We need to start processing it.
yield self.addNewRelease(release, file)
@inlineCallbacks
def addNewRelease(self, release, tarball_path):
"""
For each new release that we've discovered, set up processing for it.
"""
log = self._log.bind(function="addNewRelease", release=release)
log.msg("adding release")
# Download the tarball.
file_path = os.path.join(
self.certifiDirectory, 'certifi-' + release + '.tar.gz'
)
with open(file_path, 'wb') as f:
yield downloadFile(tarball_path, f)
log.msg("tarball saved", tarball=file_path)
rc = yield getProcessValue(
distutils.spawn.find_executable('tar'),
args=['xvf', file_path, '-C', self.certifiDirectory]
)
assert not rc
log.msg("adding to managed versions")
self.managedPackageVersions.add(release)
@inlineCallbacks
def _testHostAgainstRelease(self, host, release):
"""
Tests a given host against a specific certifi release. Returns a
Deferred that fires with the result of the test.
"""
log = self._log.bind(function="_testHostAgainstRelease")
file_path = os.path.join(self.certifiDirectory, 'certifi-' + release)
args = [self.binPath, file_path, host]
log.msg("running test", args=args)
result = yield getProcessValue(
distutils.spawn.find_executable('pypy'),
args=args,
)
returnValue(result)
@inlineCallbacks
def testHost(self, host):
"""
Runs the certifi tests against a given host.
Returns a deferred that fires with a list of tuples: certifi release
and whether the test passed.
"""
log = self._log.bind(function="testHost", host=host)
results = []
for release in self.managedPackageVersions:
result = yield self.testSemaphore.run(
self._testHostAgainstRelease, host, release
)
results.append((release, result))
log.msg("test complete", release=release, result=result)
returnValue(results)
@inlineCallbacks
def testLoop(self):
"""
Loop forever, popping tests off the queue and running them.
"""
log = self._log.bind(function="testLoop")
while True:
test = yield self.queue.get()
log = log.bind(test_id=test.id, host=test.host)
log.msg("beginning test")
try:
results = yield self.testHost(test.host)
except Exception as e:
print e
continue
log.msg("test suite complete", results=results)
def collect(self, device, log):
log.debug('{}: Modeling collect'.format(device.id))
port = getattr(device, 'zSpringBootPort', None)
ivGroups = getattr(device, 'zIVGroups', None)
if not ivGroups:
log.error("%s: zIVGroups is not defined", device.id)
returnValue(None)
ivUser = getattr(device, 'zIVUser', None)
if not ivUser:
log.error("%s: zIVUser is not defined", device.id)
returnValue(None)
applications = device.get_SBAApplications
app_result = (True, ('apps', json.dumps(applications)))
ip_address = device.manageIp
if not ip_address:
log.error("%s: IP Address cannot be empty", device.id)
returnValue(None)
deferreds = []
sem = DeferredSemaphore(1)
for app in applications:
# {'hostingServer': 'dvb-app-l15.dev.credoc.be',
# 'mgmtURL': 'http://dvb-app-l15.dev.credoc.be:8105/delivery-service_v1/management',
# 'healthURL': 'http://dvb-app-l15.dev.credoc.be:8105/delivery-service_v1/management/health',
# 'id': 'app_Delivery Service_b0acc0ef',
# 'serviceURL': 'http://dvb-app-l15.dev.credoc.be:8105/delivery-service_v1'}
# TODO: drop obsolete getPage
d = sem.run(getPage,
app['healthURL'],
headers={
"Accept": "application/json",
"iv-groups": ivGroups,
"iv-user": ivUser,
})
d.addCallback(self.add_tag, '{}_{}'.format(app['id'], 'health'))
deferreds.append(d)
url = '{}/metrics/job'.format(app['mgmtURL'])
d = sem.run(getPage,
url,
headers={
"Accept": "application/json",
"iv-groups": ivGroups,
"iv-user": ivUser,
})
d.addCallback(self.add_tag, '{}_{}'.format(app['id'],
'metricsJob'))
deferreds.append(d)
results = yield DeferredList(deferreds, consumeErrors=True)
results.append(app_result)
for success, result in results:
if not success:
log.error('{}: {}'.format(device.id, result.getErrorMessage()))
returnValue(results)
class MapUpdater(object):
def __init__(self, mapsPath, fetchURL, deleteIfNotPresent, tfLevelSounds):
assert isinstance(mapsPath, str) and len(mapsPath)
assert isinstance(fetchURL, str) and len(fetchURL)
self.mapsPath = FilePath(mapsPath)
self.downloadTempPath = self.mapsPath.child('mapupdater')
self.fetchURL = URLPath.fromString(fetchURL)
self.deleteIfNotPresent = deleteIfNotPresent
self.tfLevelSounds = tfLevelSounds
self.semaphore = DeferredSemaphore(1)
self.downloadSemaphore = DeferredSemaphore(4)
for fp in self.downloadTempPath.globChildren('*.bsp.bz2'):
fp.remove()
def checkMaps(self, *a, **kw):
"""
Wrap self._checkMaps to prevent running multiple checks at once.
"""
return self.semaphore.run(self._checkMaps, *a, **kw)
def _checkMaps(self, forceDownloadMaps=None):
def _cb(remoteMaps):
if forceDownloadMaps:
remoteMaps = list(set(remoteMaps + forceDownloadMaps))
remoteMapsLower = [f.lower() for f in remoteMaps]
ourMaps = filter(lambda p: not p.isdir() and p.path.endswith('.bsp'),
self.mapsPath.children())
ourMapFilenames = [p.basename().lower() + '.bz2' for p in ourMaps]
missing = []
for f in remoteMaps:
if f.lower() not in ourMapFilenames:
missing.append(f)
delete = []
for p in ourMaps:
filename = p.basename().lower() + '.bz2'
if filename not in remoteMapsLower:
delete.append(p)
if self.deleteIfNotPresent and delete:
for fp in delete:
fp.remove()
print 'Deleted {} map(s) not present at remote server:'.format(len(delete))
print ', '.join([x.basename() for x in delete])
if missing:
print 'Fetching {} map(s)'.format(len(missing))
def _allFinished(ignored):
self.mapsPath.child('tempus_map_updater_run_once').touch()
if self.tfLevelSounds:
self.addLevelSounds(ourMaps)
print 'Now up-to-date.'
ds = []
for filename in missing:
ds.append(self.fetchMap(filename))
return gatherResults(ds).addCallback(_allFinished)
elif self.tfLevelSounds:
self.addLevelSounds(ourMaps)
return self.getMapList().addCallback(_cb)
def fetchMap(self, *a, **kw):
return self.downloadSemaphore.run(self._fetchMap, *a, **kw)
def _fetchMap(self, filename):
downloadTempPath = self.downloadTempPath
if not downloadTempPath.exists():
downloadTempPath.makedirs()
def _cb(response, fn):
tp = downloadTempPath.child(fn)
fd = tp.open('wb')
def _extracted(ignored):
extractedPath = tp.sibling(tp.basename().replace('.bz2', ''))
extractedPath.moveTo(
self.mapsPath.child(tp.basename().replace('.bz2', '')))
try:
tp.remove()
# File already gone
except OSError:
pass
print 'Finished downloading {}'.format(fn)
def _finished(ignored):
fd.close()
d = getProcessOutputAndValue(
'aunpack', (tp.path, '-X', downloadTempPath.path))
d.addErrback(log.err)
d.addCallback(_extracted)
return d
def _eb(failure):
print 'Error downloading {}:'.format(fn)
print failure.getTraceback()
fd.close()
try:
tp.remove()
# File already gone
except OSError:
pass
d = treq.collect(response, fd.write)
d.addCallback(_finished)
d.addErrback(_eb)
return d
d = treq.get(str(self.fetchURL.child(filename)))
return d.addCallback(_cb, filename)
def getMapList(self, forceDownloadMaps):
raise NotImplementedError('Subclasses must override this method.')
def addLevelSounds(self, mapPaths):
content = FilePath(mapupdater.__file__).sibling(
'tf_level_sounds.txt').getContent()
added = []
for p in mapPaths:
mapName = p.basename()[:-4]
p2 = p.sibling('{}_level_sounds.txt'.format(mapName))
if p2.exists() and p2.getContent() == content:
continue
added.append(mapName)
p2.setContent(content)
if added:
print 'Added level sounds for:'
print ', '.join(added)
def run_deploy(self, hosts, components, commands):
try:
yield self.event_bus.trigger("deploy.precheck")
except AbortDeploy as e:
yield self.abort(str(e))
return
try:
self.transport.initialize()
except TransportError as e:
raise DeployError("could not initialize transport: %s" % e)
def signal_handler(sig, _):
reason = SIGNAL_MESSAGES[sig]
reactor.callFromThread(self.abort, reason)
signal.signal(signal.SIGINT, signal_handler)
signal.signal(signal.SIGHUP, signal_handler)
yield self.event_bus.trigger("deploy.begin")
try:
if components:
yield self.event_bus.trigger("build.begin")
try:
# synchronize the code host with upstreams
# this will return a build token and build host for each
# component
sync_command = SynchronizeCommand(components)
code_host = Host.from_hostname(self.code_host)
(sync_result,) = yield self.process_host(
code_host, [sync_command])
yield self.event_bus.trigger("build.sync",
sync_info=sync_result.result)
# this is where we build up the final deploy command
# resulting from all our syncing and building
deploy_command = DeployCommand()
# collect the results of the sync per-buildhost
by_buildhost = collections.defaultdict(list)
for component, sync_info in sync_result.result.iteritems():
component_ref = component + "@" + sync_info["token"]
build_host = sync_info.get("buildhost", None)
if build_host:
by_buildhost[build_host].append(component_ref)
else:
# no build host means we just pass the sync token
# straight through as a deploy token
deploy_command.add_argument(component_ref)
# ask each build host to build our components and return
# a deploy token
for build_hostname, build_refs in by_buildhost.iteritems():
build_command = BuildCommand(build_refs)
build_host = Host.from_hostname(build_hostname)
(build_result,) = yield self.process_host(
build_host, [build_command])
for ref in build_refs:
component, at, sync_token = ref.partition("@")
assert at == "@"
try:
deploy_ref = (component + "@" +
build_result.result[ref])
except KeyError:
raise ComponentNotBuiltError(component)
deploy_command.add_argument(deploy_ref)
# Wait until components report ready IF:
# * we are actually restarting a component
# * we aren't going --dangerously-fast
restarting_component = any(
[isinstance(val, RestartCommand) for val in commands])
if restarting_component and not self.dangerously_fast:
commands.append(WaitUntilComponentsReadyCommand())
except Exception:
traceback.print_exc()
raise DeployError("unexpected error in sync/build")
else:
# inject our built-up deploy command at the beginning of
# the command list for each host
commands = [deploy_command] + commands
yield self.event_bus.trigger("build.end")
parallelism_limiter = DeferredSemaphore(tokens=self.parallel)
host_deploys = []
first_host = True
for host in hosts:
if not first_host:
for i in xrange(self.sleeptime, 0, -1):
yield self.event_bus.trigger(
"deploy.sleep", host=host, count=i)
yield sleep(1)
else:
first_host = False
deferred = parallelism_limiter.run(
self.process_host, host, commands,
timeout=self.execution_timeout)
deferred.addErrback(self.on_host_error)
host_deploys.append(deferred)
yield self.event_bus.trigger(
"deploy.enqueue", host=host, deferred=deferred)
yield DeferredList(host_deploys)
except (DeployError, AbortDeploy, TransportError) as e:
yield self.abort(str(e))
else:
yield self.event_bus.trigger("deploy.end")
class VBMSClient(object):
def __init__(self, reactor, connect_vbms_path, bundle_path, endpoint_url,
keyfile, samlfile, key, keypass, ca_cert, client_cert):
self._reactor = reactor
self._connect_vbms_path = connect_vbms_path
self._bundle_path = bundle_path
self._endpoint_url = endpoint_url
self._keyfile = keyfile
self._samlfile = samlfile
self._key = key
self._keypass = keypass
self._ca_cert = ca_cert
self._client_cert = client_cert
self._connect_vbms_semaphore = DeferredSemaphore(tokens=8)
def _path_to_ruby(self, path):
if path is None:
return "nil"
else:
return repr(path)
def _execute_connect_vbms(self, logger, request, formatter, args):
ruby_code = """#!/usr/bin/env ruby
$LOAD_PATH << '{connect_vbms_path}/src/'
require 'json'
require 'vbms'
client = VBMS::Client.new(
{endpoint_url!r},
{keyfile},
{samlfile},
{key},
{keypass!r},
{ca_cert},
{client_cert},
)
request = {request}
result = client.send(request)
STDOUT.write({formatter})
STDOUT.flush()
""".format(
connect_vbms_path=self._connect_vbms_path,
endpoint_url=self._endpoint_url,
keyfile=self._path_to_ruby(self._keyfile),
samlfile=self._path_to_ruby(self._samlfile),
key=self._path_to_ruby(self._key),
keypass=self._keypass,
ca_cert=self._path_to_ruby(self._ca_cert),
client_cert=self._path_to_ruby(self._client_cert),
request=request,
formatter=formatter,
).strip()
with tempfile.NamedTemporaryFile(suffix=".rb", delete=False) as f:
f.write(ruby_code)
st = os.stat(f.name)
os.chmod(f.name, st.st_mode | stat.S_IEXEC)
@inlineCallbacks
def run():
timer = logger.time("process.spawn")
try:
stdout, stderr, exit_code = yield getProcessOutputAndValue(
'/bin/bash', [
'-lc',
'{} exec {} {}'.format(
self._bundle_path,
f.name,
" ".join(map(pipes.quote, args))
)
],
env=os.environ,
path=self._connect_vbms_path,
reactor=self._reactor
)
finally:
timer.stop()
if exit_code != 0:
raise VBMSError(stdout, stderr, exit_code)
returnValue(stdout)
return self._connect_vbms_semaphore.run(run)
@inlineCallbacks
def get_document_types(self, logger):
response = yield self._execute_connect_vbms(
logger.bind(process="GetDocumentTypes"),
"VBMS::Requests::GetDocumentTypes.new()",
"result.map(&:to_h).to_json",
[],
)
returnValue(json.loads(response))
@inlineCallbacks
def list_documents(self, logger, file_number):
response = yield self._execute_connect_vbms(
logger.bind(process="ListDocuments"),
"VBMS::Requests::ListDocuments.new(ARGV[0])",
'result.map(&:to_h).to_json',
[file_number],
)
returnValue(json.loads(response))
def fetch_document_contents(self, logger, document_id):
return self._execute_connect_vbms(
logger.bind(process="FetchDocumentById"),
"VBMS::Requests::FetchDocumentById.new(ARGV[0])",
"result.content",
[document_id],
)
class LookupImportController:
def __init__(self, dbSessionCreator):
self._dbSessionCreator = dbSessionCreator
self._semaphore = DeferredSemaphore(1)
def shutdown(self):
pass
@inlineCallbacks
def importLookups(self, modelSetKey: str, coordSetKey: Optional[str],
lookupTupleType: str, lookupTuples: List,
deleteOthers: bool, updateExisting: bool):
yield self._semaphore.run(self._importInThread, modelSetKey, coordSetKey,
lookupTupleType, lookupTuples,
deleteOthers, updateExisting)
logger.debug("TODO, Notify the observable")
return True
@deferToThreadWrapWithLogger(logger)
def _importInThread(self, modelSetKey: str, coordSetKey: str, tupleType: str,
tuples,
deleteOthers: bool, updateExisting: bool):
LookupType = ORM_TUPLE_MAP[tupleType]
if LookupType == DispLineStyle:
self._convertLineStyles(tuples)
itemsByImportHash = {}
addCount = 0
updateCount = 0
deleteCount = 0
ormSession = self._dbSessionCreator()
try:
modelSet = getOrCreateModelSet(ormSession, modelSetKey)
coordSet = None
if coordSetKey:
coordSet = getOrCreateCoordSet(
ormSession, modelSetKey, coordSetKey)
all = (ormSession.query(LookupType)
.filter(LookupType.coordSetId == coordSet.id)
.all())
else:
all = (ormSession.query(LookupType)
.filter(LookupType.modelSetId == modelSet.id)
.all())
def updateFks(lookup):
if hasattr(lookup, "coordSetId"):
assert coordSet
lookup.coordSetId = coordSet.id
else:
lookup.modelSetId = modelSet.id
for lookup in all:
# Initialise
itemsByImportHash[lookup.importHash] = lookup
for lookup in tuples:
importHash = str(lookup.importHash)
# If it's an existing item, update it
if importHash in itemsByImportHash:
existing = itemsByImportHash.pop(importHash)
if updateExisting:
for fieldName in lookup.tupleFieldNames():
setattr(existing, fieldName,
getattr(lookup, fieldName))
updateFks(existing)
updateCount += 1
# If it's a new item, create it
else:
newTuple = LookupType()
for fieldName in lookup.tupleFieldNames():
if fieldName in ("id", "coordSetId", "modelSetId"):
continue
setattr(newTuple, fieldName,
getattr(lookup, fieldName))
updateFks(newTuple)
ormSession.add(newTuple)
addCount += 1
if deleteOthers:
for lookup in list(itemsByImportHash.values()):
ormSession.delete(lookup)
deleteCount += 1
try:
ormSession.commit()
except Exception as e:
ormSession.rollback()
logger.exception(e)
raise
logger.debug("Updates for %s received, Added %s, Updated %s, Deleted %s",
tupleType, addCount, updateCount, deleteCount)
except Exception as e:
logger.exception(e)
raise
finally:
ormSession.close()
@deferToThreadWrapWithLogger(logger)
def getLookups(self, modelSetKey: str, coordSetKey: Optional[str],
tupleType: str):
LookupType = ORM_TUPLE_MAP[tupleType]
ormSession = self._dbSessionCreator()
try:
modelSet = getOrCreateModelSet(ormSession, modelSetKey)
if coordSetKey:
coordSet = getOrCreateCoordSet(
ormSession, modelSetKey, coordSetKey)
all = (ormSession.query(LookupType)
.filter(LookupType.coordSetId == coordSet.id)
.all())
else:
all = (ormSession.query(LookupType)
.filter(LookupType.modelSetId == modelSet.id)
.all())
importTuples = []
ImportTuple = TUPLE_TYPES_BY_NAME[tupleType]
for ormTuple in all:
newTuple = ImportTuple()
for fieldName in newTuple.tupleFieldNames():
if fieldName == 'modelSetKey':
newTuple.modelSetKey = modelSetKey
elif fieldName == 'coordSetKey':
newTuple.coordSetKey = coordSetKey
else:
setattr(newTuple, fieldName,
getattr(ormTuple, fieldName))
importTuples.append(newTuple)
return importTuples
except Exception as e:
logger.exception(e)
finally:
ormSession.close()
def _convertLineStyles(self, importLineStyles: List[ImportDispTextStyleTuple]):
for style in importLineStyles:
dp = style.dashPattern
if dp is None:
continue
if not isinstance(dp, list):
dp = [dp]
style.dashPattern = json.dumps(dp)
class ExperimentServiceFactory(Factory):
protocol = ExperimentServiceProto
def __init__(self, expected_subscribers, experiment_start_delay):
self._logger = logging.getLogger(self.__class__.__name__)
self.expected_subscribers = expected_subscribers
self.experiment_start_delay = experiment_start_delay
self.parsing_semaphore = DeferredSemaphore(500)
self.connection_counter = -1
self.connections_made = []
self.connections_ready = []
self.vars_received = []
self._made_looping_call = None
self._subscriber_looping_call = None
self._subscriber_received_looping_call = None
self._timeout_delayed_call = None
def buildProtocol(self, addr):
self.connection_counter += 1
return ExperimentServiceProto(self, self.connection_counter + 1)
def setConnectionMade(self, proto):
if not self._timeout_delayed_call:
self._timeout_delayed_call = reactor.callLater(
EXPERIMENT_SYNC_TIMEOUT, self.onExperimentSetupTimeout)
else:
self._timeout_delayed_call.reset(EXPERIMENT_SYNC_TIMEOUT)
self.connections_made.append(proto)
if len(self.connections_made) >= self.expected_subscribers:
self._logger.info("All subscribers connected!")
if self._made_looping_call and self._made_looping_call.running:
self._made_looping_call.stop()
self.pushIdToSubscribers()
else:
if not self._made_looping_call:
self._made_looping_call = task.LoopingCall(
self._print_subscribers_made)
self._made_looping_call.start(1.0)
def _print_subscribers_made(self):
if len(self.connections_made) < self.expected_subscribers:
self._logger.info("%d of %d expected subscribers connected.",
len(self.connections_made),
self.expected_subscribers)
def pushIdToSubscribers(self):
for proto in self.connections_made:
self.parsing_semaphore.run(proto.sendAndWaitForReady)
def setConnectionReady(self, proto):
self._timeout_delayed_call.reset(EXPERIMENT_SYNC_TIMEOUT)
self.connections_ready.append(proto)
if len(self.connections_ready) >= self.expected_subscribers:
self._logger.info("All subscribers are ready, pushing data!")
if self._subscriber_looping_call and self._subscriber_looping_call.running:
self._subscriber_looping_call.stop()
self.pushInfoToSubscribers()
else:
if not self._subscriber_looping_call:
self._subscriber_looping_call = task.LoopingCall(
self._print_subscribers_ready)
self._subscriber_looping_call.start(1.0)
def _print_subscribers_ready(self):
self._logger.info("%d of %d expected subscribers ready.",
len(self.connections_ready),
self.expected_subscribers)
def pushInfoToSubscribers(self):
# Generate the json doc
vars = {}
for subscriber in self.connections_ready:
subscriber_vars = subscriber.vars.copy()
subscriber_vars['port'] = subscriber.id + 12000
subscriber_vars['host'] = subscriber.transport.getPeer().host
vars[subscriber.id] = subscriber_vars
json_vars = json.dumps(vars)
del vars
self._logger.info("Pushing a %d bytes long json doc.", len(json_vars))
# Send the json doc to the subscribers
task.cooperate(self._sendLineToAllGenerator(json_vars))
def _sendLineToAllGenerator(self, line):
for subscriber in self.connections_ready:
yield subscriber.sendLine(line)
def setConnectionReceived(self, proto):
self._timeout_delayed_call.reset(EXPERIMENT_SYNC_TIMEOUT)
self.vars_received.append(proto)
if len(self.vars_received) >= self.expected_subscribers:
self._logger.info(
"Data sent to all subscribers, giving the go signal in %f secs.",
self.experiment_start_delay)
reactor.callLater(0, self.startExperiment)
self._timeout_delayed_call.cancel()
else:
if not self._subscriber_received_looping_call:
self._subscriber_received_looping_call = task.LoopingCall(
self._print_subscribers_received)
self._subscriber_received_looping_call.start(1.0)
def _print_subscribers_received(self):
self._logger.info("%d of %d expected subscribers received the data.",
len(self.vars_received), self.expected_subscribers)
def startExperiment(self):
# Give the go signal and disconnect
self._logger.info("Starting the experiment!")
if self._subscriber_received_looping_call and self._subscriber_received_looping_call.running:
self._subscriber_received_looping_call.stop()
start_time = time() + self.experiment_start_delay
for subscriber in self.connections_ready:
# Sync the experiment start time among instances
subscriber.sendLine("go:%f" %
(start_time + subscriber.vars['time_offset']))
d = task.deferLater(
reactor, 5,
lambda: self._logger.info("Done, disconnecting all clients."))
d.addCallback(lambda _: self.disconnectAll())
d.addCallbacks(self.onExperimentStarted, self.onExperimentStartError)
def disconnectAll(self):
reactor.runUntilCurrent()
def _disconnectAll():
for subscriber in self.connections_ready:
yield subscriber.transport.loseConnection()
task.cooperate(_disconnectAll())
def unregisterConnection(self, proto):
if proto in self.connections_ready:
self.connections_ready.remove(proto)
if proto in self.vars_received:
self.vars_received.remove(proto)
if proto.id in self.vars_received:
self.vars_received.remove(proto.id)
self._logger.debug("Connection cleanly unregistered.")
def onExperimentStarted(self, _):
self._logger.info("Experiment started, shutting down sync server.")
reactor.callLater(0, stopReactor)
def onExperimentStartError(self, failure):
self._logger.error("Failed to start experiment")
reactor.exitCode = 1
reactor.callLater(0, stopReactor)
return failure
def onExperimentSetupTimeout(self):
self._logger.error("Waiting for all peers timed out, exiting.")
reactor.exitCode = 1
reactor.callLater(0, stopReactor)
def lineLengthExceeded(self, line):
self._logger.error("Line length exceeded, %d bytes remain.", len(line))