class Counter(object):
def __init__(self, initial=0):
self.atomic = AtomicInteger(initial)
# waiting is important here to ensure that
# defaultdict factories can step on each other
time.sleep(0.001)
def decrementAndGet(self):
return self.atomic.decrementAndGet()
def incrementAndGet(self):
return self.atomic.incrementAndGet()
def get(self):
return self.atomic.get()
def __repr__(self):
return "Counter<%s>" % (self.atomic.get())
class Counter(object):
def __init__(self, initial=0):
self.atomic = AtomicInteger(initial)
# waiting is important here to ensure that
# defaultdict factories can step on each other
time.sleep(0.001)
def decrementAndGet(self):
return self.atomic.decrementAndGet()
def incrementAndGet(self):
return self.atomic.incrementAndGet()
def get(self):
return self.atomic.get()
def __repr__(self):
return "Counter<%s>" % (self.atomic.get())
def fetch(self):
""" generated source for method fetch """
totalFetchDataSize = self.calcDataSize(len(self.variables))
if totalFetchDataSize == 0:
raise RuntimeException("no data to fetch")
if totalFetchDataSize > dataSizeLimitForFetch:
raise RuntimeException("exceed the max data limit for fetch")
dataSet = GridDataSet(self.meta)
latch = CountDownLatch(len(self.variables) * self.tRange.getSize() * self.zRange.getSize())
exceptions = ConcurrentLinkedQueue()
counter = AtomicInteger()
taskCount = 0
for variable in variables:
dataSet.addVariable(variable, Grid4D(buffer_, self.meta.getDataType(), self.getOrigin(), self.getShape()))
#
# * not thread-safe
#
while t < tRange.getEnd():
#
# * not thread-safe
#
#
# * not thread-safe
#
while z < zRange.getEnd():
#
# * not thread-safe
#
self.addTask(counter, data, curPos, variable, t, z, latch, exceptions)
curPos += xRange.getSize() * yRange.getSize() * meta.getDataType().getSize()
taskCount += 1
z += 1
t += 1
latch.await()
if not exceptions.isEmpty():
raise exceptions.peek()
if counter.get() != taskCount:
raise RuntimeException("not all task success")
return dataSet
currentLayerPath = os.path.join(os.path.dirname(projectPath), 'currentLayer_' + namePlugin + '.txt')
currentWrittenLayer = fc.incrementCounter(currentLayerPath, increment = nLayersAtATime)
l = AtomicInteger(currentWrittenLayer)
# fc.startThreads(elasticMontage(), wait = 1, nThreads = nThreads) /!\ it does not work I do not understand why. Probably a java6 issue because it works in other scripts in java8 ...
threads = []
for p in range(nThreads):
thread = Thread(elasticMontage)
threads.append(thread)
thread.start()
time.sleep(0.5)
for thread in threads:
thread.join()
IJ.log( namePlugin + ' layer ' + str(currentWrittenLayer))
fc.resizeDisplay(layerset)
project.save()
IJ.log('Sleeping in case the saving of the large project takes some time ...')
time.sleep(20)
# save all transforms
transformsPath = os.path.join(os.path.dirname(projectPath) , namePlugin + '_Transforms.txt')
if l.get() > nLayers-1:
fc.writeAllAffineTransforms(project,transformsPath)
fc.shouldRunAgain(namePlugin, currentWrittenLayer, nLayers, MagCFolder, project, increment = nLayersAtATime)
with open(filePathsPath,'w') as f:
for path in filePaths:
f.write(path + '\n')
# pickle.dump(filePaths,f)
else:
filePaths = []
with open(filePathsPath,'r') as f:
lines = f.readlines()
for line in lines:
filePaths.append(line.replace('\n', ''))
# filePaths = pickle.load(f)
#Create all the subfolders
downSampledEMFolder = fc.mkdir_p(os.path.join(MagCEMFolder, 'MagC_EM_' + factorString, ''))
for sectionFolderName in os.walk(EMDataFolder).next()[1]:
fc.mkdir_p(os.path.join(downSampledEMFolder, sectionFolderName))
normLocalContrastSize = MagCParameters[namePlugin]['normLocalContrastSize']
# downsample in parallel
threads = []
currentLayerPath = os.path.join(MagCEMFolder, 'currentLayer_' + namePlugin + '.txt')
currentWrittenLayer = fc.incrementCounter(currentLayerPath, increment = nTilesAtATime)
IJ.log(namePlugin + ' layer ' + str(currentWrittenLayer))
atomicI = AtomicInteger(currentWrittenLayer)
fc.startThreads(resizeAndSave, fractionCores = 0.9, wait = 0, arguments = (filePaths, atomicI))
# terminate or rerun if more tiles to be processed
time.sleep(1)
fc.shouldRunAgain(namePlugin, atomicI.get(), len(filePaths), MagCFolder, '')
class NsServerNumConcurrentRequests(UserResourceTask):
def __init__(self, user, node, streaminguri):
super(NsServerNumConcurrentRequests, self).__init__(user, node)
self.nconns = [] # Belongs exclusively to poll method
self.thread = None
self.httprq = get_http_request(self.node.ip, streaminguri,
self.node.rest_username,
self.node.rest_password)
self.no_of_connections = AtomicInteger(0)
self.no_of_open_connections = AtomicInteger(0)
self.no_of_throughput_updates = AtomicInteger(0)
self.rest = RestConnection(self.node)
def on_throughput_increase(self, throughput):
log.debug("Increasing throughput by {}".format(throughput -
self.throughput))
# Record the last throughput update
last_throughput_update = self.no_of_throughput_updates.get()
# Update the throughput
self.no_of_connections.set(throughput)
# Launch the thread
if self.thread is None:
self.thread = Thread(target=self.poll)
self.thread.start()
# Block until the update has gone (TODO add a timeout)
while self.no_of_throughput_updates.get() <= last_throughput_update:
continue
def on_throughput_decrease(self, throughput):
log.debug("Decreasing throughput by {}".format(self.throughput -
throughput))
# Record the last throughput update
last_throughput_update = self.no_of_throughput_updates.get()
# Update the throughput
self.no_of_connections.set(throughput)
if self.thread and throughput == 0:
self.thread.join()
self.thread = None
# Block until the update has gone (TODO add a timeout)
while self.no_of_throughput_updates.get() <= last_throughput_update:
continue
def get_throughput_success(self):
return self.no_of_open_connections.get()
def poll(self):
""" Repeatedly poll each connection and attempt to keep them alive """
no_of_conns = self.no_of_connections.get()
while no_of_conns > 0 or len(self.nconns) > 0:
update_nconns = no_of_conns != len(self.nconns)
if update_nconns:
# Add any new connections
for i in range(no_of_conns - len(self.nconns)):
self.nconns.append(
NonBlockingConnection(self.node.ip, 8091, self.httprq))
# Disconnect the connections that need to be closed
for conn in self.nconns[no_of_conns:]:
conn.disconnect()
# Delete the disconnected connections
del self.nconns[no_of_conns:]
# Poll and count open connections
open_count = 0
for conn in self.nconns:
if conn.poll():
open_count += 1
# Update the number of open connections
self.no_of_open_connections.set(open_count)
# Notify the main thread that the connections have been updated
if update_nconns:
self.no_of_throughput_updates.incrementAndGet()
no_of_conns = self.no_of_connections.get()
def error(self):
return self.rest._http_request(self.rest.baseUrl + "/pools/default")[1]
def expected_error(self):
return 'Limit(s) exceeded [num_concurrent_requests]'
class AbstractTimedThroughputWorker(object):
""" Send throughput across a time period. """
def __init__(self, period=60, chunks=100, throughput=0):
self.period = period # The time period over which to produce throughput
self.chunks = chunks
self.throughput = AtomicInteger(throughput)
self.throughput_success = AtomicInteger(0)
self.resume = AtomicBoolean(True)
self.curr_tick = AtomicInteger(0)
def stop(self):
self.resume.set(False)
self.thread.join()
def start(self):
self.thread = Thread(target=self.loop)
self.thread.start()
def loop(self):
while self.resume.get():
self.throughput_success.set(self.tick(self.throughput.get()))
def action(self, throughput):
""" This method fires
Args:
throughput (int): The throughput in bytes
Returns (bool): Indicating success.
"""
raise NotImplementedError("Please implement this method")
def next_tick(self, period):
""" Returns the next multiple of a time period """
curr_time = time.time()
return curr_time + (period - curr_time % period)
def tick(self, throughput):
""" Fires throughput over this time period """
# The next 60 second time period
next_tick = self.next_tick(self.period)
# Every mini_tick, we will fire a thoughput of this size
throughput_per_chunk = throughput / self.chunks
# The size of a mini_tick (e.g. 0.6 seconds)
mini_tick_period = self.period / float(self.chunks)
chunks_sent, successes = 0, 0
while time.time() < next_tick and chunks_sent < self.chunks:
if not self.resume.get():
break
# Fire action and record time taken
# time_taken, success = time_it(self.action, throughput_per_chunk)
success = self.action(throughput_per_chunk)
# Count successes
if success:
successes += 1
chunks_sent += 1
# The time remaining to reach the next mini tick
time_till_next_mini_tick = max(
0,
self.next_tick(mini_tick_period) - time.time())
# sleep to next mini tick to ensure actions happen evenly
time.sleep(time_till_next_mini_tick)
return successes * throughput_per_chunk
