def general_multiproc_fitting(run_wrapper, *args):
result_queue = Queue()
update_queue = Queue()
workers_num = get_max_workers()
optimizers = [
Process(target=run_wrapper, args=(result_queue, update_queue) + args)
for _ in range(workers_num)]
for optimizer in optimizers:
optimizer.start()
optimizers_left = workers_num
results = []
while optimizers_left > 0:
time.sleep(0.1)
if not update_queue.empty() and callback_progress is not None:
callback_progress(update_queue.get())
if not result_queue.empty():
results.append(result_queue.get())
optimizers_left -= 1
for optimizer in optimizers:
optimizer.join()
sorted_results = sorted(results, key=lambda x: x[2])
logger.debug(str(sorted_results[0]))
return sorted_results[0][1], sorted_results[0][2]
def test_inceptionresnetv2_notop():
def target(queue):
model = applications.InceptionResNetV2(weights=None, include_top=False)
queue.put(model.output_shape)
global_image_data_format = K.image_data_format()
queue = Queue()
K.set_image_data_format('channels_first')
p = Process(target=target, args=(queue,))
p.start()
p.join()
K.set_image_data_format(global_image_data_format)
assert not queue.empty(), 'Model creation failed.'
model_output_shape = queue.get_nowait()
assert model_output_shape == (None, 1536, None, None)
K.set_image_data_format('channels_last')
p = Process(target=target, args=(queue,))
p.start()
p.join()
K.set_image_data_format(global_image_data_format)
assert not queue.empty(), 'Model creation failed.'
model_output_shape = queue.get_nowait()
assert model_output_shape == (None, None, None, 1536)
def uploadBatch(self):
numUploads = 0
time.sleep(1)
if not self.queue.empty():
managedQueue = Queue()
failQueue = Queue()
while not self.queue.empty():
managedQueue.put(self.queue.get())
numUploads = numUploads + 1
time.sleep(1)
print "Starting Batch of " + str(numUploads) + " images."
numWorkers = 0
for workerCount in range(4):
UploadWorker(managedQueue,failQueue,self.myClient).start()
numWorkers = workerCount
print str(numWorkers) + " workers started."
for workerCount in range(4):
managedQueue.put(None)
numWorkers = workerCount
#self.uploadFile(self.dequeue())
time.sleep(Utility.POLL_TIME) # The .empty() method is instantaneously unreliable after emptying a queue
while not managedQueue.empty():
time.sleep(Utility.POLL_TIME)
print "Waiting for uploads to finish..."
print str(numWorkers) + " workers ended."
while not failQueue.empty():
print "Failed image upload: " + failQueue.get()
print "Batch Upload finished..."
else:
print "Queue currently empty, canceling upload."
class CommunicationQueues(object):
"""Queues to handle communication between the threads.
On the bc side, this is also a logging handler sending
log messages to the node side."""
def __init__(self):
self.bc_to_node = Queue()
self.node_to_bc = Queue()
def set(self, bc_to_node=None, node_to_bc=None, queues=None):
if bc_to_node:
self.bc_to_node = bc_to_node
return
if node_to_bc:
self.node_to_bc = node_to_bc
return
assert queues.bc_to_node
self.bc_to_node = queues.bc_to_node
assert queues.node_to_bc
self.node_to_bc = queues.node_to_bc
def empty_queues(self):
print "Emptying queues:"
while not self.bc_to_node.empty():
print "BC to node:", self.bc_to_node.get()
while not self.node_to_bc.empty():
print "Node to BC:", self.node_to_bc.get()
print "Emptying queues done."
def get_handler():
return _CommQueueHandler(self.bc_to_node)
def run():
#global admRunPath
#print vars()
#if not 'admRunPath' in vars():
# admRunPath = '.'
#else:
# print admRunPath
myTest = TestClass('setUpClass')
results = []
noresults = True
try:
myTest._setUp()
total = getattr(myTest, 'total', 0)
score = getattr(myTest, 'score', 0)
yield ["<test>" + json.dumps(["setup","setup", score , "The test environment has been setup."]),total]
except Exception:
try:
myTest.setUp()
total = getattr(myTest, 'total', 0)
score = getattr(myTest, 'score', 0)
yield ["<test>" + json.dumps(["setup","setup", score , "The test environment has been setup."]),total]
except Exception:
pass
myScore = Queue()
for name in dir(TestClass):
if not (name[:1] == '_' or name in dir(unittest.TestCase)):
attr = getattr(myTest,name)
if callable(attr):
import os
getBack = os.getcwd()
os.chdir(admRunPath)
myQ = Queue()
myProcess = Process(target=runTest, args=(myTest, name, attr, myQ))
myProcess.start()
count = 0
while myQ.empty() and count < 50:
time.sleep(0.05)
count += 1
time.sleep(0.1)
if myQ.empty():
myProcess.terminate()
yield ["<test>" + json.dumps([name, "time", 0 , "It took more then 2.5 seconds to execute this test \\n" + (attr.__doc__ or 'No suggestion')]), 2500]
noresults = False
else:
yield [myQ.get(False) , count * 50]
noresults = False
os.chdir(getBack)
if noresults:
yield ["<test>" + json.dumps(["no test","testless", 0,"There are no results. are there no tests?"]),0]
try:
myTest.tearDown()
yield ["<test>" + json.dumps(["teardown","teardown", 0, "The test environment has been teared down."]),total]
except Exception:
pass
def test_worker_processes_shuts_down_after_processing_its_maximum_number_of_messages():
"""
Test worker processes shutdown after processing maximum number of messages
"""
# Setup SQS Queue
conn = boto.connect_sqs()
queue = conn.create_queue("tester")
# Build the SQS Message
message_body = {"task": "tests.tasks.index_incrementer", "args": [], "kwargs": {"message": 23}}
message = Message()
body = json.dumps(message_body)
message.set_body(body)
# Add message to internal queue
internal_queue = Queue(3)
internal_queue.put({"queue": queue.id, "message": message, "start_time": time.time(), "timeout": 30})
internal_queue.put({"queue": queue.id, "message": message, "start_time": time.time(), "timeout": 30})
internal_queue.put({"queue": queue.id, "message": message, "start_time": time.time(), "timeout": 30})
# When I Process messages
worker = ProcessWorker(internal_queue)
worker._messages_to_process_before_shutdown = 2
# Then I return from run()
worker.run().should.be.none
# With messages still on the queue
internal_queue.empty().should.be.false
internal_queue.full().should.be.false
class Importer(object):
def __init__(self):
logging.debug("In Importer::__init__()")
self._directories = []
self.series = {}
self.changed = 0
self.queue = Queue()
self.stopCancelQueue = Queue()
self._parentConn = None
self.finished = 0
def clearData(self):
logging.debug("In Importer::clearData()")
def loadDirectory(self, directory, recursive):
logging.debug("In Importer::loadDirectory()")
self.finished = 0
prov = []
while not self.stopCancelQueue.empty():
self.stopCancelQueue.get()
self.process = Process( target=scanDirectory,
args = (directory, recursive,
self.series,
self.queue, self.stopCancelQueue))
self.process.start()
if not directory in self._directories:
prov.append(directory)
self._directories = self._directories+prov
def stop(self):
self.stopCancelQueue.put("stop")
def cancel(self):
self.stopCancelQueue.put("cancel")
def updateSeries(self):
logging.debug("In Importer::updateSeries()")
if not self.queue.empty():
key, value = self.queue.get()
if key == "finished-1":
self.finished = 1
elif key == "finished-2":
self.finished = 2
else:
self.series.update(value)
def makeImport(self, indexes):
logging.debug("In Importer::makeImport()")
self.finished = 0
while not self.queue.empty():
self.queue.get()
while not self.stopCancelQueue.empty():
self.stopCancelQueue.get()
self.process = Process(target=processImport,
args = (indexes,
self.series,
self.queue, self.stopCancelQueue))
self.process.start()
class ParallelBufferedIO:
"""
Provides asynchronous output to a file. You can call "write" and a subprocess will handle output buffering so that the
main processes does not hang while waiting for a lock.
NOTE: You MUST close this or else all hell breaks loose with subprocesses
"""
def __init__(self, path):
"""
- path - the output path of this file. Must append to.
"""
self.path = path
self.Q = Queue()
self.printing_process = Process(target=self.subprocess_loop, args=[])
self.printing_process.start()
# make sure we close on exit
signal.signal(signal.SIGINT, self.close )
# and set this to close on exiting, so that we don't hang
atexit.register(self.close)
def write(self,*args):
self.Q.put(args)
def writen(self,*args):
args = list(args)
args.extend("\n")
self.Q.put(args)
def close(self):
self.Q.put(None)
def subprocess_loop(self):
"""
An internal loop my subprocess maintains for outputting
"""
# convert to a full path and make a lock
path = os.path.realpath(self.path)
lock = FileLock(self.path)
while True:
time.sleep(DELAY_TIME)
if (not self.Q.empty()):
lock.acquire() # get the lock (or wait till we do)
with open(self.path, 'a') as o:
while not self.Q.empty(): # dump the entire queue
x = self.Q.get()
if x is None: # this is our signal we are done with input
lock.release()
return
else: # this
for xi in x: print >>o, xi,
# No newline by default now
#print >>o, "\n",
lock.release()
def run_simulations( self ):
n_cpu = cpu_count()
commands = self.get_parameters()
n_scen = len( commands )
running = 0
finished = 0
q = Queue()
print '>> You have ' + str( n_scen ) + ' simulations to run!'
print '>> You`re using ' + self.par_dict['N_CPUs'] + ' of ' + str( n_cpu ) + ' CPUs available in this machine!'
self.start = datetime.now()
print '>> Starting simulations at ' + str( self.start ) + '\n'
try:
while finished < n_scen:
while len( commands ):
if running < int( self.par_dict['N_CPUs'] ):
running += 1
if len( commands ) == 1:
p = Process( target=self.trigger_simulation, args=( commands[-1], q, ) )
commands.pop()
self.counter( '\trunning: ', running,
'\twaiting: ', len( commands ),
'\tfinished: ', finished )
p.start()
p.join()
else:
p = Process( target=self.trigger_simulation, args=( commands[-1], q, ) )
p.start()
commands.pop()
else:
if not q.empty():
q.get()
running -= 1
finished += 1
time.sleep( 1 )
self.counter( '\trunning: ', running,
'\twaiting: ', len( commands ),
'\tfinished: ', finished )
if not q.empty():
q.get()
running -= 1
finished += 1
time.sleep( 1 )
self.counter( '\trunning: ', running,
'\twaiting: ', len( commands ),
'\tfinished: ', finished )
except KeyboardInterrupt:
print '\n\n>> Ctrl+c pressed! Exiting...\n'
exit()
self.counter( '\trunning: ', running,
'\twaiting: ', len( commands ),
'\tfinished: ', finished )
print '\n\n>> The simulations have finished!'
def test_worker_processes_shuts_down_after_processing_its_max_number_of_msgs():
"""
Test worker processes shutdown after processing maximum number of messages
"""
# Setup SQS Queue
conn = boto3.client('sqs', region_name='us-east-1')
queue_url = conn.create_queue(QueueName="tester")['QueueUrl']
# Build the SQS Message
message = {
'Body': json.dumps({
'task': 'tests.tasks.index_incrementer',
'args': [],
'kwargs': {
'message': 23,
},
}),
"ReceiptHandle": "receipt-1234",
}
# Add message to internal queue
internal_queue = Queue(3)
internal_queue.put(
{
"queue": queue_url,
"message": message,
"start_time": time.time(),
"timeout": 30,
}
)
internal_queue.put(
{
"queue": queue_url,
"message": message,
"start_time": time.time(),
"timeout": 30,
}
)
internal_queue.put(
{
"queue": queue_url,
"message": message,
"start_time": time.time(),
"timeout": 30,
}
)
# When I Process messages
worker = ProcessWorker(internal_queue, INTERVAL)
worker._messages_to_process_before_shutdown = 2
# Then I return from run()
worker.run().should.be.none
# With messages still on the queue
internal_queue.empty().should.be.false
internal_queue.full().should.be.false
class Connector(object):
def __init__(self, reply_generator: ConnectorReplyGenerator, connectors_event: Event):
self._reply_generator = reply_generator
self._scheduler = None
self._thread = Thread(target=self.run)
self._write_queue = Queue()
self._read_queue = Queue()
self._frontends_event = connectors_event
self._shutdown_event = Event()
self._muted = True
def give_nlp(self, nlp):
self._reply_generator.give_nlp(nlp)
def start(self):
self._scheduler.start()
self._thread.start()
def run(self):
while not self._shutdown_event.is_set():
message = self._scheduler.recv(timeout=0.2)
if self._muted:
self._scheduler.send(None)
elif message is not None:
# Receive the message and put it in a queue
self._read_queue.put(message)
# Notify main program to wakeup and check for messages
self._frontends_event.set()
# Send the reply
reply = self._write_queue.get()
self._scheduler.send(reply)
def send(self, message: str):
self._write_queue.put(message)
def recv(self) -> Optional[ConnectorRecvMessage]:
if not self._read_queue.empty():
return self._read_queue.get()
return None
def shutdown(self):
# Shutdown event signals both our thread and process to shutdown
self._shutdown_event.set()
self._scheduler.shutdown()
self._thread.join()
def generate(self, message: str, doc: Doc=None) -> str:
return self._reply_generator.generate(message, doc)
def mute(self):
self._muted = True
def unmute(self):
self._muted = False
def empty(self):
return self._read_queue.empty()
class FileWatcher(object):
def __init__(self,collector_path,supported_files):
self._initialize_members(collector_path,supported_files)
def _initialize_members(self,collector_path,supported_files):
# initializing observer.
event_handler = NewFileEvent(self)
self._observer = Observer()
self._observer.schedule(event_handler,collector_path)
self._collector_path = collector_path
self._files_queue = Queue()
self._supported_files = supported_files
self._logger = logging.getLogger('SPOT.INGEST.WATCHER')
self._logger.info("Creating File watcher")
self._logger.info("Supported Files: {0}".format(self._supported_files))
def start(self):
self._logger.info("Watching: {0}".format(self._collector_path))
self._observer.start()
def new_file_detected(self,file):
self._logger.info("-------------------------------------- New File detected --------------------------------------")
self._logger.info("File: {0}".format(file))
# Validate the file is supported.
collected_file_parts = file.split("/")
collected_file = collected_file_parts[len(collected_file_parts) -1 ]
if (collected_file.endswith(tuple(self._supported_files)) or collected_file.startswith(tuple(self._supported_files)) ) and not ".current" in collected_file:
self._files_queue.put(file)
self._logger.info("File {0} added to the queue".format(file))
else:
self._logger.warning("File extension not supported: {0}".format(file))
self._logger.warning("File won't be ingested")
self._logger.info("------------------------------------------------------------------------------------------------")
def stop(self):
self._logger.info("Stopping File Watcher")
self._files_queue.close()
while not self._files_queue.empty(): self._files_queue.get()
self._observer.stop()
self._observer.join()
def GetNextFile(self):
return self._files_queue.get()
@property
def HasFiles(self):
return not self._files_queue.empty()
def run(self):
""" Start ProcessTask main function """
filenames = split_file_by_filenum(self.config.seedfile,
self.config.proc_num)
output_queue = Queue()
progress_queue = Queue()
processes = []
w = ProcessTask(self.config.scan_func,
self.config.pool_size,
self.config.pool_timeout)
if self.config.scan_callback:
w.callback = self.config.scan_callback
for i, filename in enumerate(filenames):
proc_name = 'Worker-{:<2d}'.format(i+1)
p = Process(name=proc_name,
target=w.run,
args=(filename, progress_queue, output_queue))
if p not in processes:
processes.append(p)
for p in processes:
p.start()
if self.config.enable_console:
monitor = ConsoleMonitor(self.config,
processes,
progress_queue,
output_queue)
monitor.run()
else:
progress = {}
task_total = count_file_linenum(self.config.seedfile)
task_num = 0
with_stream_logger = logging.getLogger('output.with.stream')
while any(p.is_alive() for p in processes):
time.sleep(0.1)
while not progress_queue.empty():
proc_name, count, task_total = progress_queue.get()
progress[proc_name] = count
task_num = sum([v for k, v in progress.items()])
while not output_queue.empty():
proc_name, output = output_queue.get()
with_stream_logger.info('{}'.format(output))
if task_num == task_total:
for _ in processes:
_.terminate()
def download_all(self, *, resume = True, thread = 1, strip_filename = False, retry = 0):
# self.__entries = sorted( self.__entries, itemgetter('id') )
print( 'Soring list...' )
self.__entries.sort(key=lambda entry: entry['id'].lower())
queued = Queue()
# resume from break point
if resume:
skip_count = 0
for entry in self.__entries:
if strip_filename:
fn = self.__file_prefix + NameUtils.strip_filename( entry['id'] ) + self.__file_ext
else:
fn = self.__file_prefix + entry['id'] + self.__file_ext
if os.path.exists( fn ):
# print( 'Skipping ' + entry['id'] + ' (' + fn + ') ...' )
skip_count += 1
else:
queued.put( entry )
print( 'Determining files to download...' )
print( 'Skipped ' + str(skip_count) + ', ', end='' )
else:
for entry in self.__entries:
queued.put( entry )
if queued.empty():
print( 'Nothing to be downloaded.' )
else:
print( str(len(self.__entries)) + ' to be downloaded...' )
# Let's go multi-thread!
pool = list()
failed_queue = Queue()
args = [queued, failed_queue, strip_filename]
while retry >= 0:
for i in range(0, thread):
t = Thread(target=self.__download_entry, args=args)
pool.append( t )
if not failed_queue.empty():
queued = failed_queue # set the queue to previously failed queue to retry downloading
failed_queue = Queue() # clear failed queue
if not retry == 0: # when retry set to 0, retry downloading infinitely
retry -= 1
else:
break
# wait the threads
for t in pool:
t.join()
def time_limit(seconds, fp, func, *args, **kwargs):
if fp:
if not hasattr(fp, 'write'):
raise TypeError("Expected 'file-like' object, got '%s'" % fp)
else:
def record(msg):
fp.write(msg)
else:
def record(msg):
return
def capture_results(msg_queue, func, *args, **kwargs):
try:
result = func(*args, **kwargs)
except Exception as e:
msg_queue.put(
"Running function '%s' resulted in exception '%s' with "
"message: '%s'\n" % (func.__name__, e.__class__.__name__, e))
# no point re-raising an exception from the subprocess, instead
# return False
return False
else:
msg_queue.put(
"Running function '%s' finished with result '%s', and"
"stack:\n%s\n" % (func.__name__, result,
traceback.format_stack()))
return result
messages = Queue()
# although creating a separate process is expensive it's the only way to
# ensure cross platform that we can cleanly terminate after timeout
p = Process(target=functools.partial(capture_results, messages, func),
args=args, kwargs=kwargs)
p.start()
p.join(seconds)
if p.is_alive():
p.terminate()
while not messages.empty():
record(messages.get())
record("Running function '%s' did not finish\n" % func.__name__)
raise TestsTimeoutException
else:
while not messages.empty():
record(messages.get())
record("Running function '%s' finished with exit code '%s'\n"
% (func.__name__, p.exitcode))
class BcastListener(Thread):
"""Broadcast listening thread"""
def __init__(self, logger, port=BCASTPORT, datagram_size=DATAGRAM_SIZE):
Thread.__init__(self)
self.port = port
self.datagram_size = datagram_size
self.actions = Queue()
self.messages = Queue()
self.lock = thread.allocate_lock()
self.logger = logger
def get_msg(self):
"""Returns one of received messages"""
if not self.messages.empty():
return self.messages.get()
else:
return None
def has_msgs(self):
"""Returns if we have new messages"""
return not self.messages.empty()
def stop(self):
"""Stops the execution"""
self.actions.put(1)
def run(self):
"""Keep listening for broadcasting messages"""
# Configura o socket
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
s.bind(('', self.port))
# configura timeout para 1 segundo
s.settimeout(1)
# configura o mecanismo de captura de tempo
while 1:
if not self.actions.empty():
self.logger.info("Finishing broadcast capture")
s.close()
return
try:
data, client_addr = s.recvfrom(self.datagram_size)
self.messages.put((data, client_addr[0]))
except socket.timeout:
#print "Timeout!"
pass
except:
self.logger.exception("Exception while handling broadcast")
def test_transfert_queue():
t1 = "testTopic"
topic = Topics()
q = Queue()
topic.process(t1,123)
topic.process(t1,456)
topic.process(t1,789)
assert q.empty()
topic.transfer(t1,q)
assert q.qsize() > 0
assert q.get() == [0, 123]
assert q.get() == [1, 456]
assert q.get() == [2, 789]
topic.process(t1,111)
topic.process(t1,222)
assert q.qsize() > 0
assert q.get() == [3, 111]
assert q.get() == [4, 222]
def initParallels():
queue = Queue()
live = Value('b', True)
ear = peers.listener(settings.config['port'], settings.config['outbound'], queue, live, settings.config['server'])
ear.daemon = True
ear.items = sync()
ear.start()
mouth = peers.propagator(settings.config['port'] + 1, live)
mouth.daemon = True
mouth.items = ear.items
mouth.start()
feedback = []
stamp = time()
while queue.empty():
if time() - 15 > stamp:
break
global ext_ip, ext_port
ext_ip = ""
ext_port = -1
try:
feedback = queue.get(False)
settings.outbound = feedback[0]
if settings.outbound is not True:
ext_ip, ext_port = feedback[1:3]
except:
safeprint("No feedback received from listener")
return live
def pullMusic(folders): """ Walk through the music folders and create song objects. Return an array """ print "Start Parsing Folders!" lock = Lock() dbQueue = Queue() # create a process for each music folder in the configuration file for folder in folders: walker = Process(target=worker.walker, args=(folder, dbQueue, lock,)) walker.start() while dbQueue.empty(): pass # create a process to enter files from the dbQueue into the database enterdb = Process(target=worker.enterDB, args=(dbQueue, lock)) enterdb.start() # wait until enterDB is finished before starting # This can be taken out later. I want complete information for testing enterdb.join() print "Done!"
def _get_output_shape(model_fn):
if K.backend() == 'cntk':
# Create model in a subprocess so that
# the memory consumed by InceptionResNetV2 will be
# released back to the system after this test
# (to deal with OOM error on CNTK backend).
# TODO: remove the use of multiprocessing from these tests
# once a memory clearing mechanism
# is implemented in the CNTK backend.
def target(queue):
model = model_fn()
queue.put(model.output_shape)
queue = Queue()
p = Process(target=target, args=(queue,))
p.start()
p.join()
# The error in a subprocess won't propagate
# to the main process, so we check if the model
# is successfully created by checking if the output shape
# has been put into the queue
assert not queue.empty(), 'Model creation failed.'
return queue.get_nowait()
else:
model = model_fn()
return model.output_shape
def parex(jobs,processor,lock=None,args=(),procn=cpu_count(),\
store_results=False):
"""
Execution of a process in parallel (using the Worker class and Python
multiprocessing interface).
"""
# create a queue to pass to workers to store the results
result_queue = None
if store_results:
result_queue = Queue()
# initialising worker processes
workers = [Worker(i,jobs,processor,result_queue,lock,args) for i in \
range(procn)]
# spawning the processes
for worker in workers:
worker.start()
# waiting for the processes to finish
for worker in workers:
worker.join()
#for i in range(procn):
# worker = Worker(i,jobs,processor,result_queue,lock,args)
# worker.start()
# collect the results off the queue
results = []
if store_results:
while not result_queue.empty(): # should be safe after everything's done
result = result_queue.get()
if result != None:
# append only meaningful results
results.append(result)
return results
def start(self):
queues = [] #the communication channels between stages
processes = [] #the processes computing each stage
resQueue = Queue() #the last queue containing the final result of the computation
#queues initialisation
for i in range(0, self.nStages - 1):
queues.append(Queue())
#stages initialisation
for i in range(0, self.nStages):
if (i == 0):
processes.insert(i, Process(target=self.functions[i], args=(self.data, queues[0],)))
elif( i == self.nStages - 1):
processes.insert(i, Process(target = self.functions[i], args=(queues[i-1], resQueue,)))
else:
processes.insert(i, Process(target = self.functions[i], args=(queues[i-1], queues[i],)))
for i in range(0, self.nStages):
processes[i].start()
for p in processes:
p.join()
result = []
while not(resQueue.empty()):
result.append(resQueue.get())
return result
class TestQueue(object):
def __init__(self, test_source_cls, test_type, tests, **kwargs):
self.queue = None
self.test_source_cls = test_source_cls
self.test_type = test_type
self.tests = tests
self.kwargs = kwargs
self.queue = None
def __enter__(self):
if not self.tests[self.test_type]:
return None
self.queue = Queue()
has_tests = self.test_source_cls.queue_tests(self.queue,
self.test_type,
self.tests,
**self.kwargs)
# There is a race condition that means sometimes we continue
# before the tests have been written to the underlying pipe.
# Polling the pipe for data here avoids that
self.queue._reader.poll(10)
assert not self.queue.empty()
return self.queue
def __exit__(self, *args, **kwargs):
if self.queue is not None:
self.queue.close()
self.queue = None
def multi_scan(self, path, max_speed):
q = Queue()
engines = list(self.engines)
running = []
results = {}
while len(engines) > 0 or len(running) > 0:
if len(engines) > 0 and len(running) < self.processes:
av_engine = engines.pop()
args = (av_engine, path, results, max_speed, q)
p = Process(target=self.scan_one, args=args)
p.start()
running.append(p)
newrunning = []
for p in list(running):
p.join(0.1)
if p.is_alive():
newrunning.append(p)
running = newrunning
results = {}
while not q.empty():
results.update(q.get())
return results
def main():
'''
Use processes and Netmiko to connect to each of the devices. Execute
'show version' on each device. Use a queue to pass the output back to the parent process.
Record the amount of time required to do this.
'''
start_time = datetime.now()
output_q = Queue(maxsize=20)
procs = []
for a_device in devices:
my_proc = Process(target=show_version_queue, args=(a_device, output_q))
my_proc.start()
procs.append(my_proc)
# Make sure all processes have finished
for a_proc in procs:
a_proc.join()
while not output_q.empty():
my_dict = output_q.get()
for k, val in my_dict.items():
print(k)
print(val)
print("\nElapsed time: " + str(datetime.now() - start_time))
def main():
"""
The main function
"""
django.setup()
device_list = NetworkDevice.objects.all()
q = Queue(maxsize=20)
starttime = time.time()
procs_list = []
for device in device_list:
device_proc = Process(target=show_version_queue, args=(device, q))
device_proc.start()
procs_list.append(device_proc)
for a_proc in procs_list:
print a_proc
a_proc.join()
while not q.empty():
my_dict = q.get()
for k, v in my_dict.iteritems():
print k
print v
finishtime = time.time()
total_elapsed_time = finishtime - starttime
print '*' * 50
print 'Overall retrieval time: %.2f seconds' % round(total_elapsed_time, 2)
print '*' * 50
class automator:
def __init__(self,keyword):
print "Initiating Room Automation System"
#store keyword
self.keyword=keyword
#setup queue
self.threadQueue = Queue()
#create threads
texterThread=Thread(target=texter.checkInbox, args=(self.threadQueue,))
audioThread=Thread(target=recordCommand.recordCommand, args=(self.threadQueue,))
#set to backround tasks
texterThread.daemon = True
audioThread.daemon = True
#start threads
texterThread.start()
audioThread.start()
def run(self):
#scan for commands
while True:
#read in all waiting commands from queue
while not self.threadQueue.empty():
#move from thread queue to command queue
self.commandQueue.add(self.threadQueue.get())
for c in self.commandQueue:
c.execute()
def _create_zones_proc(numzones, newzonequeue=None, oldzones=None, tenantqueue=None, tenant=TENANT, host=HOST):
"""
Individual zone creation process
:param numzones: Number of zones to create. Ignored if tenantqueue is not None
:param newzones: (optional) Shared queue for putting created zones into
:param oldzones: (optional) List of zones that already exist. May help avoid name collisions.
:param tenantqueue: (optional) Shared queue of (tenant, numzone) pairs. Will override numzones and tenants.
:param tenants: Tenant ID to create zones for. Ignored if tenantqueue is not None
"""
if not tenantqueue:
tenantqueue = Queue()
tenantqueue.put((tenant, numzones))
# Pop tenant and numzones from tenant queue
while not tenantqueue.empty():
# Get next tenant
try:
currtenant, currnumzones = tenantqueue.get(block=True, timeout=0.1)
print "\nTenant '{0}': Creating {1} zones...".format(
currtenant, currnumzones)
except QueueEmpty:
break
zones = set()
if oldzones:
for oldzone in oldzones:
zones.add(oldzone)
# Generate new zone name
# Add PID so multiple processes can add zones w/o collisions
# Add Tenant ID so multiple tenants can add zones w/o collisions
for zonenum in range(currnumzones):
newzone = "{0}-{1}-{2}.{3}".format(
random.choice(words),
os.getpid(),
currtenant,
random.choice(tlds),
)
while newzone in zones:
# Collision: make a random subzone of newzone and check again
newzone = "{0}.{1}".format(random.choice(words), newzone)
# Add zone
r = create_zone(newzone, tenant=currtenant, host=host)
# Check response
if r.status_code != 201:
_print_error(r.status_code, r.text)
continue
# Log successful zone creation
zones.add(newzone)
# Add new zones to shared queue
if newzonequeue:
newzonequeue.put(newzone, block=True)
def main():
register_openers()
printers = local_settings.PRINTERS
statuses = []
threads = []
queue = Queue()
for item in printers:
p = multiprocessing.Process(target=clientprog, args=(queue, item))
p.start()
threads.append(p)
for item in threads:
try:
item.join()
except:
pass
while not queue.empty():
statuses.append(queue.get())
if len(statuses) == 0:
print "Failed to connect to any printer"
return
data = {"printers": statuses, "timestamp": time.time()}
json.dump(data, open("statuses.json", "w"))
send(settings.UPLOAD_DESTINATION, "statuses.json")
class BcastSender(Thread):
"""Sends broadcast requests"""
def __init__(self, logger, port, data):
Thread.__init__(self)
self.port = port
self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.sock.bind(('', 0))
self.actions = Queue()
self.data = data
self.logger = logger
def stop():
"""Stops sending"""
self.actions.put()
def run(self):
"""Starts threading loop"""
while 1:
# TODO: add timers to exit when required
try:
if not self.actions.empty():
# exiting
return
if DEBUG:
self.logger.debug("Sending broadcasting message..")
self.sock.sendto(self.data, ('255.255.255.255', self.port))
time.sleep(1)
except:
self.logger.exception("Error sending broadcast message: %s" % sys.exc_value)
time.sleep(1)
def input_thread(q, stdin):
while True:
print('ROOT: ')
cmd = stdin.readline()
q.put(cmd)
def root(char):
assert isinstance(char, str), "Argument must be string!"
ROOT.gROOT.ProcessLine(char)
if __name__ == '__main__':
___queue___ = Queue()
___newstdin___ = os.fdopen(os.dup(sys.stdin.fileno()))
___input_p___ = Process(target=input_thread,
args=(___queue___, ___newstdin___))
___input_p___.daemon = True
___input_p___.start()
___g___ = ROOT.gSystem.ProcessEvents
try:
while 1:
if not ___queue___.empty():
___cmd___ = ___queue___.get()
try:
exec(___cmd___, globals())
except:
print(sys.exc_info())
time.sleep(0.01)
___g___()
except (KeyboardInterrupt, SystemExit):
___input_p___.terminate()
if __name__ == '__main__':
parser = OptionParser()
parser.add_option("-c",
dest='configfile',
default=sys.argv[0].replace('.py', '.conf'),
help="configuration file to use")
(options, args) = parser.parse_args()
initConfig()
sh=logging.StreamHandler(sys.stdout)
sh.setFormatter(formatter)
logger.addHandler(sh)
makeEvents()
makeAlerts()
makeAttackers()
while not logcache.empty():
try:
postingProcess=Process(target=postLogs,args=(logcache,),name="json2MozdefDemoData")
postingProcess.start()
except OSError as e:
if e.errno==35: #resource temporarily unavailable.
print(e)
pass
else:
logger.error('%r'%e)
from multiprocessing import Queue
from time import sleep
q = Queue(3)
print('empty', q.empty())
q.put(1)
q.put(2)
q.put(3, True, 3)
sleep(0.1)
print('empty', q.empty())
print('size', q.qsize())
print('full', q.full())
q.get()
print('size', q.qsize())
q.put(4, True, 4)
class Analyer:
def __init__(self):
self.__plugins = []
self.__mp = []
self.__manager = Manager()
self.__share_data = self.__manager.list()
self.__queue = Queue()
self.__status = []
self.__result = []
tmpList = os.listdir("analyze/plugins/")
for filename in tmpList:
name, ext = os.path.splitext(filename)
if (filename != "__init__.py" and ext == ".py"):
plugins_class = __import__("analyze.plugins.%s" % (name),
fromlist=[name])
instanceClass = getattr(plugins_class,
name)(self.__manager.list(), None)
self.__plugins.append([
name,
instanceClass.analysis_name(),
instanceClass.colnum_info(), True
])
def get_supported_plugins(self):
retSupported = ""
for plugins in self.__plugins:
if retSupported != "":
retSupported = retSupported + ", "
retSupported = retSupported + \
plugins[AnalyzeFieldIdx.IDX_ANALYE_NAME.value]
return retSupported
def get_plugins(self):
return self.__plugins
def set_data(self, arg_data):
self.__data = arg_data
self.__share_data = self.__manager.list(arg_data)
def get_data(self):
return self.__data
data = property(get_data, set_data)
def run(self):
self.__mp.clear()
self.__status.clear()
self.__result.clear()
i = 0
for plugins in self.__plugins:
if plugins[AnalyzeFieldIdx.IDX_IS_EXECUTE.value]:
name = plugins[AnalyzeFieldIdx.IDX_MODULE_NAME.value]
plugins_class = __import__("analyze.plugins.%s" % (name),
fromlist=[name])
self.__mp.append(
getattr(plugins_class, name)(self.__data, self.__queue))
self.__status.append([
plugins[AnalyzeFieldIdx.IDX_ANALYE_NAME.value],
Info.INFO_CALCULATING
])
i += 1
for mp in self.__mp:
mp.start()
def is_result_exist(self):
return len(self.__result)
def is_alive(self):
while any(i.is_alive() for i in self.__mp):
return True
return False
def is_queue_empty(self):
return self.__queue.empty()
def retrive_data(self):
while not self.__queue.empty():
dataItem = self.__queue.get()
if (dataItem[0] == RetriveType.DATA):
self.__result.append(dataItem[1])
elif (dataItem[0] == RetriveType.INFO):
for i in range(len(self.__status)):
if (self.__status[i][0] == dataItem[1][0]):
self.__status[i][1] = dataItem[1][1]
def get_plugins_count(self):
return len(self.__mp)
def get_status(self):
return self.__status
def get_column_info(self, arg_name):
for plugin in self.__plugins:
if plugin[AnalyzeFieldIdx.IDX_ANALYE_NAME.value] == arg_name:
return plugin[AnalyzeFieldIdx.IDX_COLUMN_INFO.value]
return None
def get_result(self):
return self.__result
def export_analyze_result(self, arg_symbol):
if not os.path.exists(DEF_EXPORT_FOLDER_NAME):
os.makedirs(DEF_EXPORT_FOLDER_NAME)
main_data = DataFrame(list(self.__data))
main_data.columns = DEF_STOCK_COULMN_NAME
result_data = []
result_data.append(main_data)
result_data += self.__result
df = pd.concat(result_data, axis=1)
filename = datetime.today().strftime(arg_symbol + "_%Y%m%d-%H%M%S.csv")
try:
df.to_csv(DEF_EXPORT_FOLDER_NAME + os.sep + filename,
sep=',',
encoding='utf-8')
except:
return None
return filename
from multiprocessing import Queue from time import sleep #创建队列 q = Queue(3) q.put(1) sleep(0.5) print(q.empty()) q.put(2) print(q.full()) q.put(3) print(q.qsize()) # q.put(4,True,3) print(q.get()) q.close() #关闭队列
class MultipleImageMultipleLabelDataLayer(caffe.Layer):
def setup_prefetch(self):
self._slots_used = Queue(self._max_queue_size)
self._slots_filled = Queue(self._max_queue_size)
global shared_mem_list
shared_mem_list = [[] for t in range(self._num_tops)]
for t in range(self._num_tops):
for c in range(self._max_queue_size):
shared_mem = sharedArray(ctypes.c_float, self._blob_counts[t])
with shared_mem.get_lock():
s = np.frombuffer(shared_mem.get_obj(), dtype=np.float32)
assert (s.size == self._blob_counts[t]), '{} {}'.format(
s.size, self._blob_counts)
shared_mem_list[t].append(shared_mem)
self._shared_mem_shape = self._data_shapes
#start the process now
self._prefetch_process_name = "data prefetcher"
self._prefetch_process_id_q = Queue(1)
self._prefetch_process = BlobFetcher(self._prefetch_process_name, self._prefetch_process_id_q,\
self._slots_used, self._slots_filled,\
self._shared_mem_shape, self._num_tops, self.get_next_minibatch_helper)
for c in range(self._max_queue_size):
self._slots_used.put(c)
self._prefetch_process.start()
self._prefetch_process_id = self._prefetch_process_id_q.get()
print 'prefetching enabled: %d' % (self._prefetch_process_id)
print 'setting up prefetcher with queue size: %d' % (
self._max_queue_size)
def cleanup():
print 'terminate BlobFetcher'
self._prefetch_process.terminate()
self._prefetch_process.join()
atexit.register(cleanup)
def check_prefetch_alive(self):
try:
os.kill(self._prefetch_process_id,
0) #not killing just poking to see if alive
except err:
#will raise exception if process is dead
#can do something more intelligent here rather than raise the same error ...
raise err
def crop_ms_augment_image(self, imPath, im_shape, should_flip_image):
if self._dataset == 'ucf101' or self._dataset == 'hmdb51':
#caffe read image automatically adds in RGB jittering
if self._dataset == 'ucf101':
orig_im = self.caffe_read_image(imPath,
shape=[1, 3, 256, 340],
immean=self._image_mean,
caffe_order=False)
else:
orig_im = self.caffe_read_image(imPath,
shape=None,
immean=self._image_mean,
caffe_order=False)
self.setup_crop_ms_aug(
im_size=[orig_im.shape[0], orig_im.shape[1]])
# tb.print_stack();namespace = globals().copy();namespace.update(locals());code.interact(local=namespace)
#multiscale
ms_ind = np.random.randint(0, len(self._crop_sizes))
crop_ind = np.random.randint(0, len(self._crop_offsets))
crop_ht = self._crop_sizes[ms_ind][0]
crop_wid = self._crop_sizes[ms_ind][1]
ht_off = self._crop_offsets[crop_ind][0]
wid_off = self._crop_offsets[crop_ind][1]
crop_im = orig_im[ht_off:(ht_off + crop_ht),
wid_off:(wid_off + crop_wid), :]
# tb.print_stack();namespace = globals().copy();namespace.update(locals());code.interact(local=namespace)
if crop_im.shape[0] != self._nw_size or crop_im.shape[
1] != self._nw_size:
aug_im = cv2.resize(crop_im, (self._nw_size, self._nw_size),
interpolation=cv2.INTER_LINEAR).astype(
np.float32)
else:
aug_im = crop_im
#flip?
if should_flip_image == True:
aug_im = aug_im[:, ::-1, :]
#caffe_order
aug_im = np.transpose(aug_im, axes=(2, 0, 1))
return aug_im
def setup_crop_ms_aug(self, im_size=None):
if self._dataset == 'ucf101' or self._dataset == 'hmdb51':
self._scale_ratios = [1.0]
self._nw_size = self._data_shapes[0][2]
if im_size is None:
self._orig_im_size = [256, 340]
else:
self._orig_im_size = im_size
height_off = int((self._orig_im_size[0] - self._nw_size) / 4.0)
wid_off = int((self._orig_im_size[1] - self._nw_size) / 4.0)
self._crop_offsets = []
self._crop_offsets.append([0, 0])
self._crop_offsets.append([0, 4 * wid_off])
self._crop_offsets.append([4 * height_off, 0])
self._crop_offsets.append([4 * height_off, 4 * wid_off])
self._crop_offsets.append([2 * height_off, 2 * wid_off])
#crop_sizes
self._crop_sizes = []
base_size = min(self._orig_im_size[0], self._orig_im_size[1])
for ii, h in enumerate(self._scale_ratios):
crop_h = int(base_size * h)
crop_h = self._nw_size if (
abs(crop_h - self._nw_size) < 3) else crop_h
for jj, w in enumerate(self._scale_ratios):
crop_w = int(base_size * w)
crop_w = self._nw_size if (
abs(crop_w - self._nw_size) < 3) else crop_w
if abs(ii - jj) <= 1:
self._crop_sizes.append([crop_h, crop_w])
def rgb_jitter_image(self, im):
assert im.shape[2] == 3
#we want a rgb or bgr image (NOT in caffe order)
assert im.dtype == np.float32
for ch in range(3):
thisRand = np.random.uniform(0.8, 1.2000000001)
im[:, :, ch] *= thisRand
shiftVal = np.random.randint(0, 6)
if np.random.randint(2) == 1:
shiftVal = -shiftVal
im += shiftVal
im = im.astype(np.uint8)
#cap values to [0,255]
im = im.astype(np.float32)
return im
def caffe_read_image(self,
imname,
shape,
immean,
caffe_order=True,
flip_image=False):
# tt = timer.Timer();
# tt.tic();
# im = cv2.imread(imname,cv2.IMREAD_COLOR);
im = np.array(Image.open(imname))
im = im[:, :, ::-1]
# print 'read', tt.toc();
if im is None:
print 'could not read image: {}'.format(imname)
#resize the image
h = im.shape[0]
w = im.shape[1]
# tt.tic();
if (shape is not None) and \
(shape[2] != h or shape[3] != w or len(im.shape)!=3 or im.shape[2]!=3):
im = scipy.misc.imresize(im,
(int(shape[2]), int(shape[3]), shape[1]),
interp='bilinear')
# print 'resize', tt.toc();
im = im.astype(np.float32)
im -= immean
#ADD RGB JITTER AUG HERE
if self._rgb_jitter_aug:
im = self.rgb_jitter_image(im)
if flip_image:
im = im[:, ::-1, :]
if caffe_order:
im = np.transpose(im, axes=(2, 0, 1))
return im
def setup(self, bottom, top):
"""Setup the KeyHDF5Layer."""
layer_params = yaml.load(self.param_str)
#new version of caffe
self._keys_file = layer_params['keys_file']
self._hdf5_files = layer_params['label_files']
self._txt_files = layer_params['txt_files']
self._image_dir = layer_params['image_dir']
self._image_mean = np.array(layer_params['bgr_mean'])
self._image_ext = layer_params['image_ext']
self._top_dtypes = layer_params['top_dtypes']
self._data_shapes = layer_params['data_shapes']
self._max_queue_size = layer_params['prefetch_size']
self._shuffle = layer_params['shuffle']
self._key_sep = '\t'
if 'sleep_time' in layer_params:
self._sleep_time = layer_params['sleep_time']
else:
self._sleep_time = 20
#initial sleep time
if self._max_queue_size > 0:
self._use_prefetch = True
else:
self._use_prefetch = False
if 'crop_ms_aug' in layer_params:
self._crop_ms_aug = layer_params['crop_ms_aug']
else:
self._crop_ms_aug = False
if 'lr_flip_aug' in layer_params:
self._lr_flip_aug = layer_params['lr_flip_aug']
else:
self._lr_flip_aug = False
if 'seed' in layer_params:
self._shuffle_seed = layer_params['seed']
else:
self._shuffle_seed = 0
if 'dataset' in layer_params:
self._dataset = layer_params['dataset']
else:
self._dataset = 'ucf101'
self._rgb_jitter_aug = False
print 'crop_ms_aug', self._crop_ms_aug
print 'lr_flip_aug', self._lr_flip_aug
print 'shuffle_seed', self._shuffle_seed
print 'rgb_jitter_image', self._rgb_jitter_aug
print 'dataset', self._dataset
if self._crop_ms_aug:
self.setup_crop_ms_aug()
#do the tops make sense
assert (len(top) == len(self._data_shapes))
assert (len(top) == len(self._top_dtypes))
for x in self._top_dtypes:
assert (x in ['im', 'lab', 'txt'])
self._num_im = self._top_dtypes.count('im')
self._num_lab = self._top_dtypes.count('lab')
self._num_txt = self._top_dtypes.count('txt')
self._num_tops = len(self._top_dtypes)
self._batch_size = self._data_shapes[0][0]
t0shape = tuple(self._data_shapes[0])
self._blob_counts = []
self._single_batch_shapes = []
for d in range(len(top)):
assert self._data_shapes[d][0] == self._batch_size
self._blob_counts.append(int(prod(self._data_shapes[d])))
top[d].reshape(*(self._data_shapes[d]))
self._single_batch_shapes.append([
1, self._data_shapes[d][1], self._data_shapes[d][2],
self._data_shapes[d][3]
])
# tb.print_stack();namespace = globals().copy();namespace.update(locals());code.interact(local=namespace)
#verify keys and labels go together
with open(self._keys_file, 'r') as fh:
self._data_keys = [
line.strip().split(self._key_sep) for line in fh
if len(line.strip()) >= 1
]
for k in self._data_keys:
assert (len(k) == self._num_tops)
#now setup labels
assert (self._num_lab == len(self._hdf5_files))
self._hdf5_fhs = []
self._txts = []
hctr = 0
tctr = 0
for t in range(self._num_tops):
fh = None
txt = None
if self._top_dtypes[t] == 'lab':
fh = h5py.File(self._hdf5_files[hctr], 'r')
for x in self._data_keys:
assert (x[t] in fh)
hctr += 1
elif self._top_dtypes[t] == 'txt':
tfh = open(self._txt_files[tctr], 'r')
txt = [
np.array(line.strip()) for line in tfh
if len(line.strip()) >= 1
]
assert (len(txt) == len(self._data_keys))
tctr += 1
self._hdf5_fhs.append(fh)
self._txts.append(txt)
assert (hctr == len(
self._hdf5_files)), '%d %d' % (hctr, len(self._hdf5_files))
assert (tctr == len(
self._txt_files)), '%d %d' % (tctr, len(self._txt_files))
if self._shuffle:
np.random.seed(self._shuffle_seed)
rndInds = np.random.permutation(len(self._data_keys))
self._data_keys = [self._data_keys[x] for x in rndInds]
for ii, t in enumerate(self._txts):
# self._txts[ii] = self._txts[ii][rndInds]; #since this is an array
if self._txts[ii] is not None:
self._txts[ii] = [self._txts[ii][x] for x in rndInds]
#fetching and prefetching
self._read_key_index = 0
self._disk_key_index = 0
if self._use_prefetch:
self.setup_prefetch()
#let prefetch thread get ahead of us
print 'sleeping to let prefetch get ahead: %d seconds' % (
self._sleep_time)
time.sleep(self._sleep_time)
def get_next_minibatch(self):
if not self._use_prefetch:
return self.get_next_minibatch_helper()
# raise ValueError('not implemented without prefetch')
#is child still alive?
while self._slots_filled.empty():
self.check_prefetch_alive()
slot = self._read_key_index % self._max_queue_size
deq_slot = self._slots_filled.get()
assert slot == deq_slot, '{} {}'.format(slot, deq_slot)
im_datas = []
for t in range(self._num_tops):
shared_mem = shared_mem_list[t][slot]
with shared_mem.get_lock():
shared_mem_arr = np.reshape(
np.frombuffer(shared_mem.get_obj(), dtype=np.float32),
self._data_shapes[t])
im_data = shared_mem_arr[...].astype(np.float32, copy=True)
#copy since we will mark this slot as used
im_datas.append(im_data)
# print 'fwd:: ', slot, im_datas[t].min(), im_datas[t].max(), im_datas[t].mean();
self._read_key_index += 1
if self._read_key_index >= len(self._data_keys):
self._read_key_index = 0
self._slots_used.put(slot)
return im_datas
def get_next_minibatch_helper(self):
# tim = timer.Timer();
# tim.tic();
im_datas = []
for c in range(self._num_tops):
im_datas.append(
np.zeros(tuple(self._data_shapes[c]), dtype=np.float32))
for b in range(self._batch_size):
if self._disk_key_index >= len(self._data_keys):
self._disk_key_index = 0
imBatchKeys = self._data_keys[self._disk_key_index]
should_flip_image = True if (
self._lr_flip_aug and np.random.randint(2) == 1) else False
for t in range(self._num_tops):
imName = imBatchKeys[t]
if self._top_dtypes[t] == 'im':
imPath = os.path.join(self._image_dir,
imName + self._image_ext)
if self._crop_ms_aug:
im_datas[t][b,...] = self.crop_ms_augment_image(imPath, self._single_batch_shapes[t],\
should_flip_image)
else:
#caffe_read_image automatically adds in RGB jittering if enabled
im_datas[t][b,...] = self.caffe_read_image(imPath, shape=self._single_batch_shapes[t], immean=self._image_mean,\
flip_image = should_flip_image)
elif self._top_dtypes[t] == 'lab':
im_datas[t][
b, ...] = self._hdf5_fhs[t][imName].value[:].reshape(
self._single_batch_shapes[t])
elif self._top_dtypes[t] == 'txt':
im_datas[t][b, ...] = self._txts[t][
self._disk_key_index].reshape(
self._single_batch_shapes[t])
else:
raise ValueError('uknown dtype: {}'.format(
self._top_dtypes[t]))
self._disk_key_index += 1
# tb.print_stack();namespace = globals().copy();namespace.update(locals());code.interact(local=namespace)
# print 'min_max:: ', im_datas.min(), im_datas.max(), im_datas.mean();
# print tim.toc()
return im_datas
def forward(self, bottom, top):
"""Get blobs and copy them into this layer's top blob vector."""
im_datas = self.get_next_minibatch()
for c in range(self._num_tops):
top[c].data[...] = im_datas[c].astype(np.float32, copy=False)
# tb.print_stack();namespace = globals().copy();namespace.update(locals());code.interact(local=namespace)
# time.sleep(2);
def backward(self, top, propagate_down, bottom):
"""This layer does not propagate gradients."""
pass
def reshape(self, bottom, top):
"""Reshaping happens during the call to forward."""
pass
print(frame.shape)
key=cv2.waitKey(1) & 0xFF
if key == ord("q"):
break
#print(type(arr))
print('Image is', np_frame)
#image.verify()
#print('Image is verified')
# Process Image
frame = imutils.resize(frame, width=400)
(fH, fW) = frame.shape[:2]
# if the input queue *is* empty, give the current frame to
# classify
if inputQueue.empty():
inputQueue.put(frame)
#inputQueue.queue.clear()
# if the output queue *is not* empty, grab the detections
if not outputQueue.empty():
detections = outputQueue.get()
# check to see if our detectios are not None (and if so, we'll
# draw the detections on the frame)
if detections is not None:
# loop over the detections
for i in np.arange(0, detections.shape[2]):
# extract the confidence (i.e., probability) associated
# with the prediction
number_threads = 8
if len(sys.argv) > 1:
number_threads = int(sys.argv[1])
allocation = gen_allocation(256, number_threads)
for i in range(16):
ps = list()
for j in range(number_threads):
p = Process(target=process_data, args=(j, i))
p.daemon = True
p.start()
ps.append(p)
for j in range(number_threads):
ps[j].join()
while correlation_queue.empty() != True:
a = correlation_queue.get()
correlations[i][a[0]] = (a[1])
final_key.append(correlations[i].index(min(correlations[i])))
print(final_key)
end = time.time()
fig, axs = plt.subplots(4, 4)
for i in range(4):
for j in range(4):
axs[i, j].plot(correlations[i * 4 + j])
axs[i, j].set_title('Byte ' + str(i * 4 + j))
plt.show()
print(end - start)
def __writeData_async_andCollect(self, startindex, outputDir):
#set tree name to use
import DeepJetCore.preprocessing
DeepJetCore.preprocessing.setTreeName(self.dataclass.treename)
from multiprocessing import Process, Queue, cpu_count, Lock
wo_queue = Queue()
writelock = Lock()
import os
thispid = str(os.getpid())
if not os.path.isfile(outputDir + '/snapshot.dc'):
self.writeToFile(outputDir + '/snapshot.dc')
tempstoragepath = '/dev/shm/' + thispid
print('creating dir ' + tempstoragepath)
os.system('mkdir -p ' + tempstoragepath)
def writeData_async(index, woq, wrlck):
import copy
from stopwatch import stopwatch
sw = stopwatch()
td = copy.deepcopy(self.dataclass)
sample = self.originRoots[index]
ramdisksample = tempstoragepath + '/' + str(
os.getpid()) + os.path.basename(sample)
def removefile():
os.system('rm -f ' + ramdisksample)
import atexit
atexit.register(removefile)
success = False
out_samplename = ''
out_sampleentries = 0
newname = os.path.basename(sample).rsplit('.', 1)[0]
newname += str(index)
if usenewformat:
newname += '.meta'
else:
newname += '.z'
newpath = os.path.abspath(outputDir + newname)
try:
fileTimeOut(sample, 120) #once available copy to ram
os.system('cp ' + sample + ' ' + ramdisksample)
td.readFromRootFile(ramdisksample, self.means, self.weighter)
#wrlck.acquire()
td.writeOut(newpath)
#wrlck.release()
print('converted and written ' + newname + ' in ',
sw.getAndReset(), ' sec -', index)
out_samplename = newname
out_sampleentries = td.nsamples
success = True
td.clear()
removefile()
woq.put((index, [success, out_samplename, out_sampleentries]))
except:
print('problem in ' + newname)
removefile()
woq.put((index, [False, out_samplename, out_sampleentries]))
raise
def __collectWriteInfo(successful, samplename, sampleentries,
outputDir):
if not successful:
raise Exception("write not successful, stopping")
import os
self.samples.append(samplename)
self.nsamples += sampleentries
self.sampleentries.append(sampleentries)
self.writeToFile(outputDir +
'/snapshot_tmp.dc') #avoid to overwrite directly
os.system('mv ' + outputDir + '/snapshot_tmp.dc ' + outputDir +
'/snapshot.dc')
processes = []
processrunning = []
processfinished = []
for i in range(startindex, len(self.originRoots)):
processes.append(
Process(target=writeData_async, args=(i, wo_queue, writelock)))
processrunning.append(False)
processfinished.append(False)
nchilds = int(cpu_count() / 2) - 2 if self.nprocs <= 0 else self.nprocs
#import os
#f 'max' in os.getenv('HOSTNAME'):
# nchilds=int(cpu_count()/3)-2
if nchilds < 1:
nchilds = 1
#nchilds=10
lastindex = startindex - 1
alldone = False
results = []
import time
try:
while not alldone:
nrunning = 0
for runs in processrunning:
if runs: nrunning += 1
for i in range(len(processes)):
if nrunning >= nchilds:
break
if processrunning[i]: continue
if processfinished[i]: continue
time.sleep(0.1)
logging.info('starting %s...' %
self.originRoots[startindex + i])
processes[i].start()
processrunning[i] = True
nrunning += 1
if not wo_queue.empty():
res = wo_queue.get()
results.append(res)
originrootindex = res[0]
logging.info('finished %s...' %
self.originRoots[originrootindex])
processfinished[originrootindex - startindex] = True
processes[originrootindex - startindex].join(5)
processrunning[originrootindex - startindex] = False
#immediately send the next
continue
for r in results:
thisidx = r[0]
if thisidx == lastindex + 1:
logging.info('>>>> collected result %d of %d' %
(thisidx + 1, len(self.originRoots)))
__collectWriteInfo(r[1][0], r[1][1], r[1][2],
outputDir)
lastindex = thisidx
if nrunning == 0 and lastindex + 1 == len(self.originRoots):
alldone = True
continue
time.sleep(0.1)
except:
os.system('rm -rf ' + tempstoragepath)
raise
os.system('rm -rf ' + tempstoragepath)
proc_list = [
Process(target=list_vms, args=(provider, queue), name='list_vms:{}'.format(provider))
for provider in providers
]
for proc in proc_list:
proc.start()
for proc in proc_list:
proc.join()
print('Done processing providers, assembling report...')
# Now pull all the results off of the queue
# Stacking the generator this way is equivalent to using list.extend instead of list.append
# Need to check queue.empty since a call to get will raise an Empty exception
output_data = []
while not queue.empty():
output_data.extend(queue.get())
header = '''## VM/Instances on providers matching:
## providers: {}
## tags: {}
'''.format(args.provider, args.tag) # don't forget trailing newline...
with open(args.outfile, 'w') as output_file:
# stdout and the outfile
output_file.write(header)
print(header)
report = tabulate(output_data,
headers=['Provider', 'VM', 'Status', 'Created On', 'Type/Hardware'],
tablefmt='orgtbl')
class checkpoint():
def __init__(self, args):
self.args = args
self.ok = True
self.log = torch.Tensor()
now = datetime.datetime.now().strftime('%Y-%m-%d-%H:%M:%S')
if not args.load:
if not args.save:
args.save = now
self.dir = os.path.join('..', 'experiment', args.save)
else:
self.dir = os.path.join('..', 'experiment', args.load)
if os.path.exists(self.dir):
self.log = torch.load(self.get_path('psnr_log.pt'))
print('Continue from epoch {}...'.format(len(self.log)))
else:
args.load = ''
if args.reset:
os.system('rm -rf ' + self.dir)
args.load = ''
os.makedirs(self.dir, exist_ok=True)
os.makedirs(self.get_path('model'), exist_ok=True)
for d in args.data_test:
os.makedirs(self.get_path('results-{}'.format(d)), exist_ok=True)
open_type = 'a' if os.path.exists(self.get_path('log.txt')) else 'w'
self.log_file = open(self.get_path('log.txt'), open_type)
with open(self.get_path('config.txt'), open_type) as f:
f.write(now + '\n\n')
for arg in vars(args):
f.write('{}: {}\n'.format(arg, getattr(args, arg)))
f.write('\n')
self.n_processes = 8
def get_path(self, *subdir):
return os.path.join(self.dir, *subdir)
def save(self, trainer, epoch, is_best=False):
trainer.model.save(self.get_path('model'), epoch, is_best=is_best)
trainer.loss.save(self.dir)
trainer.loss.plot_loss(self.dir, epoch)
self.plot_psnr(epoch)
trainer.optimizer.save(self.dir)
torch.save(self.log, self.get_path('psnr_log.pt'))
def add_log(self, log):
self.log = torch.cat([self.log, log])
def write_log(self, log, refresh=False):
print(log)
self.log_file.write(log + '\n')
if refresh:
self.log_file.close()
self.log_file = open(self.get_path('log.txt'), 'a')
def done(self):
self.log_file.close()
def plot_psnr(self, epoch):
axis = np.linspace(1, epoch, epoch)
for idx_data, d in enumerate(self.args.data_test):
label = 'SR on {}'.format(d)
fig = plt.figure()
plt.title(label)
for idx_scale, scale in enumerate(self.args.scale):
plt.plot(axis,
self.log[:, idx_data, idx_scale].numpy(),
label='Scale {}'.format(scale))
plt.legend()
plt.xlabel('Epochs')
plt.ylabel('PSNR')
plt.grid(True)
plt.savefig(self.get_path('test_{}.pdf'.format(d)))
plt.close(fig)
def begin_background(self):
self.queue = Queue()
def bg_target(queue):
while True:
if not queue.empty():
filename, tensor = queue.get()
if filename is None: break
imageio.imwrite(filename, tensor.numpy())
self.process = [
Process(target=bg_target, args=(self.queue,)) \
for _ in range(self.n_processes)
]
for p in self.process:
p.start()
def end_background(self):
for _ in range(self.n_processes):
self.queue.put((None, None))
while not self.queue.empty():
time.sleep(1)
for p in self.process:
p.join()
def save_results(self, dataset, filename, save_list, scale):
if self.args.save_results:
filename = self.get_path('results-{}'.format(dataset.dataset.name),
'{}_x{}_'.format(filename, scale))
postfix = ('SR', 'LR', 'HR')
for v, p in zip(save_list, postfix):
normalized = v[0].mul(255 / self.args.rgb_range)
tensor_cpu = normalized.byte().permute(1, 2, 0).cpu()
self.queue.put(('{}{}.png'.format(filename, p), tensor_cpu))
class MapViewer(object):
def __init__(self, system, w, h):
self.system = system
self.w = w
self.h = h
self.q_camera = Queue()
self.q_trajectory = Queue()
self.q_point = Queue()
self.q_color = Queue()
self.q_pose = Queue()
self.q_image = Queue()
self.q_old_trajectory = Queue()
self.q_old_camera = Queue()
self.q_connection_edge = Queue()
# self.q_depth = Queue()
self.view_thread = Process(target=self.view)
self.view_thread.start()
def update(self, refresh=False):
if refresh:
print('************************** refresh')
else:
cameras = []
for pose in self.system.poses:
cameras.append(pose)
if len(cameras) > 0:
self.q_camera.put(cameras)
old_cameras = []
for pose in self.system.old_poses:
cameras.append(pose)
if len(old_cameras) > 0:
self.q_old_camera.put(old_cameras)
trajectory = []
for point in self.system.trajectory:
trajectory.append(point)
if len(trajectory) > 1:
self.q_trajectory.put(trajectory)
old_trajectory = []
for point in self.system.old_trajectory:
old_trajectory.append(point)
if len(old_trajectory) > 1:
self.q_old_trajectory.put(old_trajectory)
connection_edge = []
for point in self.system.connection_edge:
connection_edge.append(point)
if len(connection_edge) > 1:
self.q_connection_edge.put(connection_edge)
points = self.system.points
if points is not None:
self.q_point.put(points)
colors = self.system.colors
if colors is not None:
self.q_color.put(colors)
image = self.system.image
if image is not None:
self.q_image.put(image)
# depth = self.system.depth
# if depth is not None:
# self.q_depth.put(depth)
if self.system.current_keyframe is not None:
self.q_pose.put(self.system.current_keyframe.pose_matrix)
def view(self):
pangolin.CreateWindowAndBind('Viewer', 1024, 768)
gl.glEnable(gl.GL_DEPTH_TEST)
gl.glEnable(gl.GL_BLEND)
gl.glBlendFunc(gl.GL_SRC_ALPHA, gl.GL_ONE_MINUS_SRC_ALPHA)
viewpoint_x = 0
viewpoint_y = -5 # -10
viewpoint_z = -10 # -0.1
viewpoint_f = 2000
camera_width = 0.02
proj = pangolin.ProjectionMatrix(1024, 768, viewpoint_f, viewpoint_f,
512, 389, 0.1, 5000)
look_view = pangolin.ModelViewLookAt(viewpoint_x, viewpoint_y,
viewpoint_z, 0, 0, 0, 0, -1, 0)
scam = pangolin.OpenGlRenderState(proj, look_view)
dcam = pangolin.CreateDisplay()
dcam.SetBounds(0.0, 1.0, 0.0, 1.0, -1024. / 768.)
dcam.SetHandler(pangolin.Handler3D(scam))
dimg = pangolin.Display('image')
dimg.SetBounds(0.0, self.h / 768., 0.0, self.w / 1024.,
float(self.w) / self.h)
dimg.SetLock(pangolin.Lock.LockLeft, pangolin.Lock.LockTop)
texture = pangolin.GlTexture(self.w, self.h, gl.GL_RGB, False, 0,
gl.GL_RGB, gl.GL_UNSIGNED_BYTE)
# ddepth = pangolin.Display('depth')
# ddepth.SetBounds(self.h / 768., self.h / 768. * 2.0, 0.0, self.w / 1024., float(self.w)/float(self.h))
# ddepth.SetLock(pangolin.Lock.LockLeft, pangolin.Lock.LockTop)
# texture_depth = pangolin.GlTexture(self.w, self.h, gl.GL_LUMINANCE , False, 0, gl.GL_LUMINANCE, gl.GL_UNSIGNED_BYTE)
cameras = []
old_cameras = []
trajectory = []
old_trajectory = []
connection_edge = []
pose = pangolin.OpenGlMatrix()
points = np.empty(shape=(0, 3))
colors = np.empty(shape=(0, 3))
# image = random_image(self.w, self.h)
image = 255 * np.ones((self.h, self.w, 3), 'uint8')
# depth = 255 * np.ones((self.h, self.w), 'uint8')
gl.glPointSize(3)
gl.glLineWidth(2)
while not pangolin.ShouldQuit():
gl.glClear(gl.GL_COLOR_BUFFER_BIT | gl.GL_DEPTH_BUFFER_BIT)
gl.glClearColor(0.75, 0.75, 0.75, 1.0)
if not self.q_pose.empty():
pose.m = self.q_pose.get()
# scam.Follow(pose, True)
dcam.Activate(scam)
gl.glLineWidth(2)
if not self.q_camera.empty():
cameras = self.q_camera.get()
gl.glColor3f(0.0, 0.0, 1.0)
if len(cameras) > 0:
pangolin.DrawCameras(cameras, camera_width)
if not self.q_old_camera.empty():
old_cameras = self.q_old_camera.get()
if len(old_cameras) > 0:
pangolin.DrawCameras(old_cameras, camera_width)
gl.glLineWidth(4)
if not self.q_trajectory.empty():
trajectory = self.q_trajectory.get()
if len(trajectory) > 1:
gl.glColor3f(0.0, 0.0, 0.0)
pangolin.DrawLine(trajectory)
if not self.q_connection_edge.empty():
connection_edge = self.q_connection_edge.get()
if len(connection_edge) > 1:
gl.glColor3f(1.0, 0.0, 0.0)
pangolin.DrawLine(connection_edge)
if not self.q_old_trajectory.empty():
old_trajectory = self.q_old_trajectory.get()
if len(old_trajectory) > 1:
gl.glColor3f(1.0, 1.0, 0.0)
pangolin.DrawLine(old_trajectory)
if not self.q_point.empty():
points = self.q_point.get()
if not self.q_color.empty():
colors = self.q_color.get()
if len(points) > 0:
pangolin.DrawPoints(points, colors)
if not self.q_image.empty():
image = self.q_image.get()
# print(image.shape, image.dtype)
# texture.Upload(image, gl.GL_RGB, gl.GL_UNSIGNED_BYTE)
# dimg.Activate()
# gl.glColor3f(1.0, 1.0, 1.0)
# texture.RenderToViewport()
# if not self.q_depth.empty():
# depth = self.q_depth.get()
# print(depth.shape, depth.dtype)
# texture_depth.Upload(depth, gl.GL_LUMINANCE, gl.GL_UNSIGNED_BYTE)
# ddepth.Activate()
# gl.glColor3f(1.0, 1.0, 1.0)
# texture_depth.RenderToViewport()
pangolin.FinishFrame()
def execute_manager(max_subprocess: int):
global server_socket
global process_dict
global total_counting_contribution, total_jobs_done, total_jobs_failed, total_updates
process_dict = {} # job_id => thread dictionary
q = Queue()
server_socket = connect_to(SERVER_IP, SERVER_PORT)
if not server_socket:
force_close_by_manager(at_start=True)
return
last_job_found = True
while True:
try:
while not q.empty():
job_id, result = q.get()
p = process_dict.pop(job_id)
p.join()
if type(result) is str:
total_jobs_done += 1
if prints:
ndprint(
f"Error occurred while calculating job {job_id}:\n{result}"
)
else:
total_jobs_done += 1
total_counting_contribution += result
msg = f'{POST_RES} {job_id} {result}'
if prints:
ndprint(msg)
send(server_socket, msg)
job_path = jobs_dir + job_id
if isfile(job_path):
remove(job_path)
if not getattr(manager_thread, "do_run"):
break
if len(process_dict) >= max_subprocess:
sleep(1)
continue
send(server_socket, GET_JOB)
response = recv(server_socket)
if response is None:
ndprint('Weird server response (conversation start).')
sleep(1)
continue
if response == NO_JOB_FOUND:
if last_job_found:
ndprint("No job available.")
last_job_found = False
sleep(5)
continue
else:
last_job_found = True
graph_name = recv(server_socket)
if graph_name is None:
ndprint('Weird server response (graph name).')
sleep(1)
continue
graph_path = graphs_dir + graph_name
job_id = recv(server_socket)
if job_id is None:
ndprint('Weird server response (job id).')
sleep(1)
continue
job_path = jobs_dir + job_id
####################
if isfile(job_path):
if job_id in process_dict:
# TODO: notify server
continue
elif not recvfile(server_socket, job_path):
ndprint('Weird server response (job file).')
sleep(1)
continue
if not isfile(graph_path):
msg = f"{GET_GRAPH} {graph_name}"
send(server_socket, msg)
if not recvfile(server_socket, graph_path):
ndprint('Weird server response (graph file).')
sleep(1)
continue
p = Process(target=compute_job,
args=(q, graph_path, job_path, job_id))
p.start()
process_dict[job_id] = p
except Exception as e:
ndprint(f"Error occurred on manager thread: {repr(e)}\n\t")
if server_socket:
server_socket.close()
server_socket = connect_to(SERVER_IP, SERVER_PORT)
if not server_socket:
force_close_by_manager()
return
last_job_found = True
#cv2.imshow("Live", live_img)
if image_proc is None:
image_s = Queue()
image_r = Queue()
image_proc = threading.Thread(target=view_image,
args=(
image_s,
image_r,
))
image_proc.daemon = True
image_beat = time.time()
image_s.put(cas)
image_s.put(image_img.copy())
image_proc.start()
if image_r.empty():
cv2.line(live_img, (0, 1), (240, 1), (0, 0, 255), 3)
else:
cv2.line(live_img, (0, 1), (240, 1), (255, 0, 0), 3)
image_beat = time.time()
img = image_r.get()
cv2.imshow("Image", img)
image_s.put(image_img.copy())
if (time.time() - image_beat) > 10:
print("image 10s")
break
cv2.imshow("Live", live_img)
if cv2.waitKey(10) >= 0:
break
def execute_cmds(binary, cmds, index, queue):
print "cmds: " + str(cmds) + " index: " + str(index)
# base case
if index == len(cmds):
# execute "processed" binary (for PIN purposes) and attach pin
pinBinary = prepare_pin_binary(binary)
pinTraceFile = tempfile.NamedTemporaryFile(delete=False)
pinTraceFile.close()
child = helpers.pexpect_spawn_process_pin(pinBinary, pinTraceFile.name)
##pinPid = helpers.attach_pin(child.pid, pinTraceFile.name)
rc = helpers.pexpect_send_multiple_inputs(child, cmds)
if rc:
print "binary died at some point (could be good -- 'exit'-like cmd -- or bad)"
if child.isalive():
child.terminate(force=True)
##os.kill(pinPid, 0)
# remove binary used for pin
os.remove(pinBinary)
# save trace
print "inputs: " + str(cmds) + " " + str(pinTraceFile.name)
print "DONE"
##queue.put({"inputs": cmds, "trace": pinTraceFile.name})
queue.append({"inputs": list(cmds), "trace": pinTraceFile.name})
return 0
# code from here to end of function tries to find arguments
# find initial commands
initCmds = cmds[0:-index]
# find_args for current command
lastCmd = cmds[index]
print "init_cmds: " + str(initCmds)
print "last_cmd: " + str(lastCmd)
global CMDARGS
try:
i = CMDARGS[index].keys()
except KeyError:
CMDARGS[index] = dict()
if lastCmd in CMDARGS[index].keys():
for arg in CMDARGS[index][lastCmd]:
cmds[index] = arg.strip()
execute_cmds(binary, cmds, index+1, queue)
else:
args = Queue()
find_input.find_args(binary, last_cmd, init_cmds, args)
while not args.empty():
arg = args.get()
print "ARG: " + str(arg)
cmds[index] = arg.strip()
try:
CMDARGS[index][lastCmd].append(arg)
except KeyError:
CMDARGS[index][lastCmd] = [ arg ]
execute_cmds(binary, list(cmds), index+1, queue)
return 0
publishers = []
# sub_buffer = JoinableQueue()
sub_buffer = Queue()
for i in range(2000):
p = Publisher(name='C' + str(i),
hostname='10.02.01.15',
port=1883,
buffer=sub_buffer)
publishers.append(p)
p.start()
# p.join()
name, delta = p.getJobDelta
print('Job : {}, completed in {} seconds'.format(name, delta))
# while True:
while not sub_buffer.empty():
ret = sub_buffer.get_nowait()
print(f'[INFO] ret:{ret}')
# time.sleep(2)
# Waiting for clients to finish job
for client in publishers:
client.join()
# name, delta = client.getJobDelta
# print('Job : {}, completed in {} seconds'.format(name, delta))
try:
ret = sub_buffer.get()
print('[INFO] ret : {}'.format(ret))
except ValueError as error:
print('[ERROR] Queue : {}'.format(error))
def dd_search_config(change_set, type_set, search_config, original_config, bitcode, original_score, div):
global search_counter
#
# partition change_set into deltas and delta inverses
#
delta_change_set = []
delta_type_set = []
delta_inv_change_set = []
delta_inv_type_set = []
div_size = int(math.ceil(float(len(change_set))/float(div)))
for i in xrange(0, len(change_set), div_size):
delta_change = []
delta_type = []
delta_inv_change = []
delta_inv_type = []
for j in xrange(0, len(change_set)):
if j >= i and j < i+div_size:
delta_change.append(change_set[j])
delta_type.append(type_set[j])
else:
delta_inv_change.append(change_set[j])
delta_inv_type.append(type_set[j])
delta_change_set.append(delta_change)
delta_type_set.append(delta_type)
delta_inv_change_set.append(delta_inv_change)
delta_inv_type_set.append(delta_inv_type)
min_score = -1
#
# iterate through all delta and inverse delta set
# record delta set that passes
#
pass_inx = -1
inv_is_better = False
#
# iterate through all deltas and delta inverses
# run in parallel
#
start_inx = search_counter
queue = Queue()
inv_queue = Queue()
workers = []
for i in xrange(0, len(delta_change_set)):
workers.append(Process(target=run_delta, args=(delta_change_set[i],
delta_type_set[i], change_set, type_set, search_config, original_config, bitcode,
original_score, start_inx+i, queue)))
for i in xrange(0, len(delta_inv_change_set)):
workers.append(Process(target=run_delta, args=(delta_inv_change_set[i],
delta_inv_type_set[i], change_set, type_set, search_config, original_config, bitcode,
original_score, start_inx+len(delta_change_set)+i, inv_queue)))
# run in parallel maximum cpu_no processes at a time
cpu_no = CPU_NO
# multiprocessing.cpu_count()
loop = int(len(workers)/cpu_no)
for i in xrange(0, loop):
for j in xrange(i*cpu_no, min((i+1)*cpu_no, len(workers))):
workers[j].start()
for j in xrange(i*cpu_no, min((i+1)*cpu_no, len(workers))):
workers[j].join()
while not queue.empty():
result = queue.get()
if min_score == -1 or result[1] < min_score:
pass_inx = result[0] - start_inx
inv_is_better = False
while not inv_queue.empty():
result = inv_queue.get()
if min_score == -1 or result[1] < min_score:
pass_inx = result[0] - start_inx - len(delta_change_set)
inv_is_better = True
search_counter += len(delta_change_set) + len(delta_inv_change_set)
#
# recursively search in pass delta or pass delta inverse
# right now keep searching for the first pass delta or
# pass delta inverse; later on we will integrate cost
# model here
#
if pass_inx != -1:
pass_change_set = delta_inv_change_set[pass_inx] if inv_is_better else delta_change_set[pass_inx]
pass_type_set = delta_inv_type_set[pass_inx] if inv_is_better else delta_type_set[pass_inx]
if len(pass_change_set) > 1:
# always reset to lowest precision
utilities.to_2nd_highest_precision(change_set, type_set)
dd_search_config(pass_change_set, pass_type_set, search_config, original_config, bitcode, original_score, 2)
else:
utilities.to_2nd_highest_precision(change_set, type_set)
utilities.to_highest_precision(pass_change_set, pass_type_set)
return
#
# stop searching when division greater than change set size
#
if div >= len(change_set):
utilities.to_highest_precision(change_set, type_set)
return
else:
dd_search_config(change_set, type_set, search_config, original_config, bitcode, original_score, 2*div)
return
def hc(self,
metric='AIC',
max_iter=100,
debug=False,
restriction=None,
whitelist=None):
"""
Greedy Hill Climbing search proceeds by choosing the move
which maximizes the increase in fitness of the
network at the current step. It continues until
it reaches a point where there does not exist any
feasible single move that increases the network fitness.
It is called "greedy" because it simply does what is
best at the current iteration only, and thus does not
look ahead to what may be better later on in the search.
For computational saving, a Priority Queue (python's heapq)
can be used to maintain the best operators and reduce the
complexity of picking the best operator from O(n^2) to O(nlogn).
This works by maintaining the heapq of operators sorted by their
delta score, and each time a move is made, we only have to recompute
the O(n) delta-scores which were affected by the move. The rest of
the operator delta-scores are not affected.
For additional computational efficiency, we can cache the
sufficient statistics for various families of distributions -
therefore, computing the mutual information for a given family
only needs to happen once.
The possible moves are the following:
- add edge
- delete edge
- invert edge
Arguments
---------
*data* : a nested numpy array
The data from which the Bayesian network
structure will be learned.
*metric* : a string
Which score metric to use.
Options:
- AIC
- BIC / MDL
- LL (log-likelihood)
*max_iter* : an integer
The maximum number of iterations of the
hill-climbing algorithm to run. Note that
the algorithm will terminate on its own if no
improvement is made in a given iteration.
*debug* : boolean
Whether to print the scores/moves of the
algorithm as its happening.
*restriction* : a list of 2-tuples
For MMHC algorithm, the list of allowable edge additions.
Returns
-------
*bn* : a BayesNet object
"""
# INITIALIZE NETWORK W/ NO EDGES
# maintain children and parents dict for fast lookups
self.c_dict = dict([(n, []) for n in self.nodes])
self.p_dict = dict([(n, []) for n in self.nodes])
self.restriction = restriction
self.whitelist = whitelist
if whitelist is None:
whitelist = []
for (u, v) in whitelist:
self.c_dict[u].append(v)
self.p_dict[v].append(u)
print("Whitelist", whitelist)
self.bn = BayesNet(self.c_dict)
# COMPUTE INITIAL LIKELIHOOD SCORE
# value_dict = dict([(n, np.unique(np_data[:,i])) for i,n in enumerate(names)])
print("Nodes:", list(self.bn.nodes()))
score = model_score(self.data, self.bn) - model_complexity(
self.bn, self.nrow, metric)
print("Initial Score:", score)
# CREATE EMPIRICAL DISTRIBUTION OBJECT FOR CACHING
#ED = EmpiricalDistribution(data,names)
_iter = 0
improvement = True
man = Manager()
mut_inf_cache = man.dict()
configs_cache = man.dict()
while improvement:
start_t = time.time()
improvement = False
max_delta = 0
max_operation = None
if debug:
print('ITERATION: ', _iter)
return_queue = Queue()
p_add = Process(target=self.test_arc_additions,
args=(configs_cache, mut_inf_cache, return_queue))
p_rem = Process(target=self.test_arc_deletions,
args=(configs_cache, mut_inf_cache, return_queue))
p_rev = Process(target=self.test_arc_reversals,
args=(configs_cache, mut_inf_cache, return_queue))
p_add.start()
p_rem.start()
p_rev.start()
p_add.join()
p_rem.join()
p_rev.join()
while not return_queue.empty():
results = return_queue.get()
if results[1] > max_delta:
max_arc = results[0]
max_delta = results[1]
max_operation = results[2]
max_qi = results[3]
### DETERMINE IF/WHERE IMPROVEMENT WAS MADE ###
if max_operation:
score += max_delta
improvement = True
u, v = max_arc
str_arc = [e for e in max_arc]
if max_operation == 'Addition':
if debug:
print("delta:", max_delta)
print('ADDING: ', str_arc, '\n')
self.p_dict[v].append(u)
self.bn.add_edge(u, v)
self.bn.F[v]["qi"] = max_qi
elif max_operation == 'Deletion':
if debug:
print("delta:", max_delta)
print('DELETING: ', str_arc, '\n')
self.p_dict[v].remove(u)
self.bn.remove_edge(u, v)
self.bn.F[v]["qi"] = max_qi
elif max_operation == 'Reversal':
if debug:
print("delta:", max_delta)
print('REVERSING: ', str_arc, '\n')
self.p_dict[v].remove(u)
self.bn.remove_edge(u, v)
self.bn.F[v]['qi'] = max_qi[1]
self.p_dict[u].append(v)
self.bn.add_edge(v, u)
self.bn.F[u]['qi'] = max_qi[0]
print("Model score:", score
) # TODO: improve so only changed elements get an update
else:
if debug:
print('No Improvement on Iter: ', _iter)
print("Time for iteration:", time.time() - start_t)
### TEST FOR MAX ITERATION ###
_iter += 1
# if _iter > max_iter:
# if debug:
# print('Max Iteration Reached')
# break
bn = BayesNet(self.c_dict)
print("Size of Cache", len(mut_inf_cache))
print("SCORE =", score)
return bn
class spiderXiaoMi(object):
def __init__(self):
self.url = "http://app.mi.com/categotyAllListApi?"
self.headers = headers
self.urlQueue = Queue()
self.parseQueue = Queue()
# URL入队列
def getUrl(self):
for page in range(20):
params = {
"page": str(page),
"categoryId": "2",
"pageSize": "30",
}
params = urllib.parse.urlencode(params)
fullUrl = self.url + params
self.urlQueue.put(fullUrl)
# get获取URL，发请求，把响应入解析队列
def getHtml(self):
while True:
if not self.urlQueue.empty():
url = self.urlQueue.get()
# 三步走
res = requests.get(url, headers=self.headers)
res.encoding = "utf-8"
html = res.text
self.parseQueue.put(html)
else:
break
# get获取html，提取并保存数据
def parseHtml(self):
while True:
if not self.parseQueue.empty():
html = self.parseQueue.get()
hList = json.loads(html)["data"]
for h in hList:
link = "http://app.mi.com/details?id=" + h["packageName"]
d = {
"name": h["displayName"],
"link": link,
}
with open("app.json", "a", encoding="utf-8") as f:
f.write(str(d) + "\n")
else:
break
def workOn(self):
# URL入队列
self.getUrl()
# 创建线程
t1List = []
# 存放所有解析线程列表
t2List = []
# 采集线程开始执行
for i in range(5):
t = Thread(target=self.getHtml)
t1List.append(t)
t.start()
# 阻塞等待回收采集线程
for j in t1List:
j.join()
# 所有解析线程开始执行
for i in range(5):
t = Thread(target=self.parseHtml)
t2List.append(t)
t.start()
# 阻塞等待回收解析线程
for j in t2List:
j.join()
def main():
global search_counter
bitcode = sys.argv[1]
search_conf_file = sys.argv[2]
original_conf_file = sys.argv[3]
#
# delete log file if exists
#
try:
os.remove("log.dd")
except OSError:
pass
#
# parsing config files
#
search_conf = json.loads(open(search_conf_file, 'r').read())
original_conf = json.loads(open(original_conf_file, 'r').read())
search_changes = search_conf["config"]
change_set = []
type_set = []
#
# record the change set
#
for search_change in search_changes:
type_vector = search_change.values()[0]["type"]
if isinstance(type_vector, list):
type_set.append(type_vector)
change_set.append(search_change.values()[0])
# get original score
utilities.to_highest_precision(change_set, type_set)
utilities.run_config(search_conf, original_conf, bitcode, search_counter)
original_score = utilities.get_dynamic_score(search_counter) * 1.05
search_counter = search_counter + 1
cpu_no = CPU_NO
# multiprocessing.cpu_count()
#
# search for valid configuration
#
print "Searching for valid configuration using delta-debugging algorithm ..."
# keep searching while the type set is not searched throughout
while not utilities.is_empty(type_set):
#
# distribute change set
#
dis_no = ((len(change_set)-1)/cpu_no)+1
queue = Queue()
workers = []
for i in xrange(cpu_no):
workers.append(Process(target=search_config_dis, args=(
change_set[i*dis_no:min((i+1)*dis_no, len(change_set))],
type_set[i*dis_no:min((i+1)*dis_no, len(type_set))],
search_conf, original_conf, bitcode, search_counter + i*dis_no*dis_no, i*dis_no, queue)))
utilities.to_highest_precision(change_set, type_set)
for w in workers:
w.start()
for w in workers:
w.join()
print len(type_set)
while not queue.empty():
inx = queue.get()
print inx
del(type_set[inx][:])
j = 0
while j < len(type_set):
if len(type_set[j]) == 0:
type_set.pop(j)
change_set.pop(j)
else:
j += 1
search_counter += cpu_no*dis_no*dis_no
search_config(change_set, type_set, search_conf, original_conf, bitcode, original_score)
# get the score of modified program
utilities.run_config(search_conf, original_conf, bitcode, search_counter)
modified_score = utilities.get_dynamic_score(search_counter)
search_counter = search_counter + 1
if modified_score <= original_score:
print "Check valid_" + bitcode + ".json for the valid configuration file"
# print valid configuration file and diff file
utilities.print_config(search_conf, "dd2_valid_" + bitcode + ".json")
utilities.print_diff(search_conf, original_conf, "dd2_diff_" + bitcode + ".json")
else:
print "No configuration is found!"
class FaceManager:
FACE_MATCH_THRESHOLD = 1.0
REPS_HARD_COPY_FILE = 'reps_hard_copy.txt'
TIME_HARD_COPY_FILE = 'cumulative_time_hard_copy.txt'
MINIMUM_TIME_SPENT_IN_ROOM = 5.0
SHOULD_LOAD_REPS = False
def __init__(self, should_load):
self.entering_faces = Queue()
self.exiting_faces = Queue()
self.faces_in_room = []
self.faces_in_room_lock = Lock()
self.total_time = 0.0
self.process = Process(target=self.run)
self.door_0_queue = Queue()
self.door_1_queue = Queue()
if self.SHOULD_LOAD_REPS:
self.load_reps_from_hard_copy()
if should_load:
self.load_total_time_from_hard_copy()
self.process.start()
def add_face_to_room(self, new_face):
self.faces_in_room_lock.acquire()
already_in_room = []
for x in self.faces_in_room:
already_in_room.append(self.faces_match(new_face[0], x[0], 0.5))
if not any(already_in_room):
self.faces_in_room.append(new_face)
self.faces_in_room_lock.release()
def output_time(self, time_spent, door, exit_time):
if time_spent > self.MINIMUM_TIME_SPENT_IN_ROOM:
if door == 0:
self.door_0_queue.put(time_spent)
else:
self.door_1_queue.put(time_spent)
self.save_total_time_to_hard_copy()
print('this face was in the room for: ' + str(time_spent) +
' seconds. they left by door ' + str(door))
def get_output_queue(self, door):
return self.door_0_queue if door == 0 else self.door_1_queue
def remove_face_from_room(self, new_face):
# remove all matching faces in the room
match, not_match = self.partition_faces(new_face)
self.faces_in_room_lock.acquire()
self.faces_in_room = not_match
self.faces_in_room_lock.release()
matching_entry_times = [face[1] for face in match]
# only output the longest time found
if len(matching_entry_times) > 0:
entry_time = max(matching_entry_times)
time_in_room = new_face[1] - entry_time
self.total_time += time_in_room
self.output_time(time_in_room, new_face[2], new_face[1])
def partition_faces(self, face):
self.faces_in_room_lock.acquire()
match = filter(lambda x: self.faces_match(x[0], face[0]),
self.faces_in_room)
not_match = filter(lambda x: not self.faces_match(x[0], face[0]),
self.faces_in_room)
self.faces_in_room_lock.release()
return match, not_match
def save_reps_to_hard_copy(self):
f = open(self.REPS_HARD_COPY_FILE, 'w')
self.faces_in_room_lock.acquire()
array = np.array(self.faces_in_room)
np.save(f, array)
self.faces_in_room_lock.release()
f.close()
def load_reps_from_hard_copy(self):
f = open(self.REPS_HARD_COPY_FILE, 'r')
array = np.load(f)
self.faces_in_room_lock.acquire()
self.faces_in_room = []
for rep in array:
self.faces_in_room.append(rep)
self.faces_in_room_lock.release()
def save_total_time_to_hard_copy(self):
f = open(self.TIME_HARD_COPY_FILE, 'w')
f.write(str(self.get_cumulative_time()))
f.close()
def load_total_time_from_hard_copy(self):
f = open(self.TIME_HARD_COPY_FILE, 'r')
self.total_time = int(f.readline().rstrip())
f.close()
def run(self):
old_num_faces = 0
while True:
if not self.entering_faces.empty():
print('handling an entering face')
self.add_face_to_room(self.entering_faces.get())
new_num_faces = self.get_num_faces()
if old_num_faces != new_num_faces:
self.save_reps_to_hard_copy()
print colored('number of faces in room: ' + str(new_num_faces),
'green')
old_num_faces = new_num_faces
if not self.exiting_faces.empty():
print('handling an exiting face')
self.remove_face_from_room(self.exiting_faces.get())
new_num_faces = self.get_num_faces()
if old_num_faces != new_num_faces:
self.save_reps_to_hard_copy()
print colored('number of faces in room: ' + str(new_num_faces),
'green')
old_num_faces = new_num_faces
def get_num_faces(self):
self.faces_in_room_lock.acquire()
num_faces_in_room = len(self.faces_in_room)
self.faces_in_room_lock.release()
return num_faces_in_room
@staticmethod
def faces_match(a, b, threshold=1.0):
if a is None or b is None:
return False
if len(a) != len(b): # Check they're same length
return False
d = a - b
dot = np.dot(d, d)
print colored('dot = ' + str(dot), 'yellow')
return dot < threshold # This number comes from openface docs
def get_cumulative_time(self):
print colored('total time requested: ' + str(self.total_time),
'yellow')
return self.total_time
##set signal handler
signal.signal(signal.SIGTERM, signal_handler)
signal.signal(signal.SIGINT, signal_handler)
logger.info("create process number:" + str(main_cfg.get_value("work_process")))
for i in range(main_cfg.get_value("work_process")):
p = main_process(subinfo_queue)
p.daemon = True
p.start()
sub_prosess.append(p)
logger.info("main loop starting...")
## main loop
while main_process_run_flg:
while not subinfo_queue.empty():
msg = subinfo_queue.get_nowait()
logger.info("sub process status:")
logger.info(msg)
#logger.info("sub process check..")
for cur_p in sub_prosess:
#check
if not cur_p.is_alive():
logger.error("vmain process is exited. exitcode: %d " %
cur_p.exitcode)
sub_prosess.remove(cur_p)
new_p = main_process(subinfo_queue)
new_p.daemon = True
new_p.start()
def main(argv):
"""
Main program method
:param argv: command line arguments
:return
"""
ap_id = '111111'
register_nodes = False
shuffle_nodes = False
duty_cycle_na = 0
node_file = "data/group1.txt"
bandit_nodes = False
algorithm = 'ucb'
test_scenario = False
max_x = MAX_X_POSITION
max_y = MAX_Y_POSITION
# Reading arguments from command line
try:
opts, args = getopt.getopt(argv, "hi:rsf:ab:t", ["id=", "file=", "help", "register", "shuffle", "bandit=", "test"])
except getopt.GetoptError:
print("main.py -i <access-point-id> [-f <file-path> -r -s -b <algorithm> -t]")
sys.exit(2)
for opt, arg in opts:
if opt == '-h':
print("main.py -i <access-point-id>\n")
print("-i <dev_id>, --id=<dev_id>\t- Specify LoRa AP hardware id")
print("-r, --register\t\t- Include end nodes registration process")
print("-s, --shuffle\t\t- Shuffle list of end nodes")
print("-f <file_path>, --file=<file_path>\t- Specify LoRa node id file")
print("-b <algorithm>, --bandit\t- Activates bandit nodes and select UCB or TS algorithm")
print("-t, --test\t- Using test scenario for developing purposes")
sys.exit(0)
elif opt in ("-i", "--id"):
ap_id = arg
elif opt in ("-r", "--regen"):
register_nodes = True
elif opt in ("-s", "--shuffle"):
shuffle_nodes = True
elif opt in ("-f", "--file"):
node_file = arg
elif opt in ("-b", "--bandit"):
bandit_nodes = True
algorithm = arg
elif opt in ("-t", "--test"):
test_scenario = True
# If there was an AP id defined
conn.connect('lora.fiit.stuba.sk', 25001)
access_point = AccessPoint(ap_id, conn)
access_point.send_setr()
if test_scenario:
node_ids = ['KmoT', 'meQy', 'meBh', 'cbun', 'ttYa']
else:
node_ids = load_nodes(node_file)
if shuffle_nodes:
random.shuffle(node_ids)
nodes = {}
processes = {}
message_queue = Queue()
emergency_queue = Queue()
num_of_nodes = 0
while True:
while not message_queue.empty() or not emergency_queue.empty():
try:
while not emergency_queue.empty():
read_reply(emergency_queue, access_point, nodes)
read_reply(message_queue, access_point, nodes)
except Exception as qe:
print(qe)
# Other nodes joins later
if num_of_nodes < len(node_ids):
node_id = node_ids[num_of_nodes]
if bandit_nodes:
node = BanditNode(node_id, algorithm, register_nodes)
else:
node = EndNode(node_id, register_nodes)
process = Process(target=node.device_routine, args=(message_queue, emergency_queue,))
process.daemon = True
process.start()
processes[node_id] = process
nodes[node_id] = node
num_of_nodes += 1
if num_of_nodes < len(node_ids):
node_id = node_ids[num_of_nodes]
if bandit_nodes:
node = BanditNode(node_id, algorithm, register_nodes)
else:
node = EndNode(node_id, register_nodes)
process = Process(target=node.device_routine, args=(message_queue, emergency_queue,))
process.daemon = True
process.start()
processes[node_id] = process
nodes[node_id] = node
num_of_nodes += 1
class Dataset():
def __init__(self, path=None):
self.num_classes = None
self.classes = None
self.images = None
self.labels = None
self.features = None
self.index_queue = None
self.queue_idx = None
self.batch_queue = None
self.is_typeB = None
if path is not None:
self.init_from_path(path)
def init_from_path(self, path):
path = os.path.expanduser(path)
_, ext = os.path.splitext(path)
if os.path.isdir(path):
self.init_from_folder(path)
elif ext == '.txt':
self.init_from_list(path)
else:
raise ValueError('Cannot initialize dataset from path: %s\n\
It should be either a folder or a .txt list file' % path)
print('%d images of %d classes loaded' %
(len(self.images), self.num_classes))
def init_from_folder(self, folder):
folder = os.path.expanduser(folder)
class_names = os.listdir(folder)
class_names.sort()
classes = []
images = []
labels = []
for label, class_name in enumerate(class_names):
classdir = os.path.join(folder, class_name)
if os.path.isdir(classdir):
images_class = os.listdir(classdir)
images_class.sort()
images_class = [
os.path.join(classdir, img) for img in images_class
]
indices_class = np.arange(len(images),
len(images) + len(images_class))
classes.append(DataClass(class_name, indices_class, label))
images.extend(images_class)
labels.extend(len(images_class) * [label])
self.classes = np.array(classes, dtype=np.object)
self.images = np.array(images, dtype=np.object)
self.labels = np.array(labels, dtype=np.int32)
self.num_classes = len(classes)
def init_from_list(self, filename):
with open(filename, 'r') as f:
lines = f.readlines()
lines = [line.strip().split(' ') for line in lines]
assert len(lines)>0, \
'List file must be in format: "fullpath(str) label(int)"'
images = [line[0] for line in lines]
if len(lines[0]) > 1:
labels = [int(line[1]) for line in lines]
else:
labels = [os.path.dirname(img) for img in images]
_, labels = np.unique(labels, return_inverse=True)
self.images = np.array(images, dtype=np.object)
self.labels = np.array(labels, dtype=np.int32)
self.init_classes()
def init_classes(self):
dict_classes = {}
classes = []
for i, label in enumerate(self.labels):
if not label in dict_classes:
dict_classes[label] = [i]
else:
dict_classes[label].append(i)
for label, indices in dict_classes.items():
classes.append(DataClass(str(label), indices, label))
self.classes = np.array(classes, dtype=np.object)
self.num_classes = len(classes)
def separate_AB(self):
assert type(self.images[0]) is str
self.is_typeB = np.zeros(len(self.images), dtype=np.bool)
for c in self.classes:
# Find the index of type A file
c.indices_A = [
i for i in c.indices if not is_typeB(self.images[i])
]
assert len(c.indices_A) >= 1, str(self.images[c.indices])
# Find the index of type B file
c.indices_B = [i for i in c.indices if is_typeB(self.images[i])]
assert len(c.indices_B) >= 1, str(self.images[c.indices])
self.is_typeB[c.indices_B] = True
print('type A images: %d type B images: %d' %
(np.sum(~self.is_typeB), np.sum(self.is_typeB)))
# Data Loading
def init_index_queue(self):
if self.index_queue is None:
self.index_queue = Queue()
index_queue = np.random.permutation(len(self.images))[:, None]
for idx in list(index_queue):
self.index_queue.put(idx)
def get_batch(self, batch_size, batch_format):
''' Get the indices from index queue and fetch the data with indices.'''
indices_batch = []
if batch_format == 'random_sample':
while len(indices_batch) < batch_size:
indices_batch.extend(
self.index_queue.get(block=True, timeout=30))
assert len(indices_batch) == batch_size
elif batch_format == 'random_pair':
assert batch_size % 2 == 0
classes = np.random.permutation(self.classes)[:batch_size // 2]
indices_batch = np.concatenate([c.random_pair() for c in classes],
axis=0)
elif batch_format == 'random_AB_pair':
assert batch_size % 2 == 0
classes = np.random.permutation(self.classes)[:batch_size // 2]
indices_batch = np.concatenate(
[c.random_AB_pair() for c in classes], axis=0)
else:
raise ValueError(
'get_batch: Unknown batch_format: {}!'.format(batch_format))
batch = {}
if len(indices_batch) > 0:
batch['images'] = self.images[indices_batch]
batch['labels'] = self.labels[indices_batch]
if self.is_typeB is not None:
batch['is_typeB'] = self.is_typeB[indices_batch]
return batch
# Multithreading preprocessing images
def start_index_queue(self):
self.index_queue = Queue()
def index_queue_worker():
while True:
if self.index_queue.empty():
self.init_index_queue()
time.sleep(0.01)
self.index_worker = Process(target=index_queue_worker)
self.index_worker.daemon = True
self.index_worker.start()
def start_batch_queue(self, config, is_training, maxsize=1, num_threads=4):
if self.index_queue is None:
self.start_index_queue()
self.batch_queue = Queue(maxsize=maxsize)
def batch_queue_worker(seed):
np.random.seed(seed)
while True:
batch = self.get_batch(config.batch_size, config.batch_format)
if batch is not None:
batch['image_paths'] = batch['images']
batch['images'] = preprocess(batch['image_paths'], config,
is_training)
self.batch_queue.put(batch)
self.batch_workers = []
for i in range(num_threads):
worker = Process(target=batch_queue_worker, args=(i, ))
worker.daemon = True
worker.start()
self.batch_workers.append(worker)
def pop_batch_queue(self):
batch = self.batch_queue.get(block=True, timeout=60)
return batch
def release_queue(self):
if self.index_queue is not None:
self.index_queue.close()
if self.batch_queue is not None:
self.batch_queue.close()
if self.index_worker is not None:
self.index_worker.terminate()
del self.index_worker
self.index_worker = None
if self.batch_workers is not None:
for w in self.batch_workers:
w.terminate()
del w
self.batch_workers = None
# -*- coding: utf-8 -*-
# @Time : 2019/5/26 22:20
# @Author : XiaTian
# @File : 8、队列.py
from multiprocessing import Queue
q = Queue(3) # 规定队列的容量，队列中适合放较小的数据，当里面不填时代表队列可以无穷存放数据
q.put('hahah') # 向队列中存放数据，存放数据数量不能超过队列的容量,队列满后再向里面存放数据程序会卡住，直到数据被取走
q.put([1, 2, 3])
q.put({'a': 123})
print(q.full()) # 判断队列是否满了
print(q.get()) # 从队列中取数据，队列取数据遵循先进先出原则。当取得数据量超过队列容量时程序会卡住，直到在次存放数据
print(q.get())
print(q.get())
print(q.empty()) # 判断队列是否空了
b = []
# Fila Queue, criar uma fila para o processamento
q = Queue()
def fatorial(n,q):
fat = n
for i in range(n-1,1,-1):
fat = fat * i
q.put(fat)
return(fat)
if __name__ == '__main__':
tempo = time.time()
# 8 Questao usando o módulo multiprocessing com 4 processos
n = int(input("Entre com o numero de processos: "))
for x in range(n):
a.append(random.randint(1,10))
for i in a:
t = multiprocessing.Process(target=fatorial,args=(i,q))
t.start()
# Esperar a threading ser concluida
t.join()
# Enquanto a fila nao estiver vazia imprima
while not q.empty():
print("Fatorial :", q.get())
tempo_final = time.time()
print("Tempo de execucao", {round(tempo - tempo_final,2)}, "segundos")
class TestRunnerManager(threading.Thread):
init_lock = threading.Lock()
def __init__(self,
suite_name,
test_queue,
test_source_cls,
browser_cls,
browser_kwargs,
executor_cls,
executor_kwargs,
stop_flag,
pause_on_unexpected=False):
"""Thread that owns a single TestRunner process and any processes required
by the TestRunner (e.g. the Firefox binary).
TestRunnerManagers are responsible for launching the browser process and the
runner process, and for logging the test progress. The actual test running
is done by the TestRunner. In particular they:
* Start the binary of the program under test
* Start the TestRunner
* Tell the TestRunner to start a test, if any
* Log that the test started
* Log the test results
* Take any remedial action required e.g. restart crashed or hung
processes
"""
self.suite_name = suite_name
self.test_queue = test_queue
self.test_source_cls = test_source_cls
self.browser_cls = browser_cls
self.browser_kwargs = browser_kwargs
self.executor_cls = executor_cls
self.executor_kwargs = executor_kwargs
self.test_source = None
self.browser = None
self.browser_pid = None
# Flags used to shut down this thread if we get a sigint
self.parent_stop_flag = stop_flag
self.child_stop_flag = multiprocessing.Event()
self.pause_on_unexpected = pause_on_unexpected
self.manager_number = next_manager_number()
self.command_queue = Queue()
self.remote_queue = Queue()
self.test_runner_proc = None
threading.Thread.__init__(self,
name="Thread-TestrunnerManager-%i" %
self.manager_number)
# This is started in the actual new thread
self.logger = None
# The test that is currently running
self.test = None
self.unexpected_count = 0
# This may not really be what we want
self.daemon = True
self.init_fail_count = 0
self.max_init_fails = 5
self.init_timer = None
self.restart_count = 0
self.max_restarts = 5
def run(self):
"""Main loop for the TestManager.
TestManagers generally receive commands from their
TestRunner updating them on the status of a test. They
may also have a stop flag set by the main thread indicating
that the manager should shut down the next time the event loop
spins."""
self.logger = structuredlog.StructuredLogger(self.suite_name)
with self.browser_cls(
self.logger,
**self.browser_kwargs) as browser, self.test_source_cls(
self.test_queue) as test_source:
self.browser = browser
self.test_source = test_source
try:
if self.init() is Stop:
return
while True:
commands = {
"init_succeeded": self.init_succeeded,
"init_failed": self.init_failed,
"test_start": self.test_start,
"test_ended": self.test_ended,
"restart_runner": self.restart_runner,
"runner_teardown": self.runner_teardown,
"log": self.log,
"error": self.error
}
try:
command, data = self.command_queue.get(True, 1)
except IOError:
if not self.should_stop():
self.logger.error("Got IOError from poll")
self.restart_count += 1
if self.restart_runner() is Stop:
break
except Empty:
command = None
if self.should_stop():
self.logger.debug("A flag was set; stopping")
break
if command is not None:
self.restart_count = 0
if commands[command](*data) is Stop:
break
else:
if not self.test_runner_proc.is_alive():
if not self.command_queue.empty():
# We got a new message so process that
continue
# If we got to here the runner presumably shut down
# unexpectedly
self.logger.info("Test runner process shut down")
if self.test is not None:
# This could happen if the test runner crashed for some other
# reason
# Need to consider the unlikely case where one test causes the
# runner process to repeatedly die
self.logger.info(
"Last test did not complete, requeueing")
self.requeue_test()
self.logger.warning(
"More tests found, but runner process died, restarting"
)
self.restart_count += 1
if self.restart_runner() is Stop:
break
finally:
self.logger.debug(
"TestRunnerManager main loop terminating, starting cleanup"
)
self.stop_runner()
self.teardown()
self.logger.debug("TestRunnerManager main loop terminated")
def should_stop(self):
return self.child_stop_flag.is_set() or self.parent_stop_flag.is_set()
def init(self):
"""Launch the browser that is being tested,
and the TestRunner process that will run the tests."""
# It seems that this lock is helpful to prevent some race that otherwise
# sometimes stops the spawned processes initalising correctly, and
# leaves this thread hung
if self.init_timer is not None:
self.init_timer.cancel()
self.logger.debug("Init called, starting browser and runner")
def init_failed():
# This is called from a seperate thread, so we send a message to the
# main loop so we get back onto the manager thread
self.logger.debug("init_failed called from timer")
if self.command_queue:
self.command_queue.put(("init_failed", ()))
else:
self.logger.debug("Setting child stop flag in init_failed")
self.child_stop_flag.set()
with self.init_lock:
# To guard against cases where we fail to connect with marionette for
# whatever reason
self.init_timer = threading.Timer(self.browser.init_timeout,
init_failed)
test_queue = self.test_source.get_queue()
if test_queue is None:
self.logger.info("No more tests")
return Stop
try:
self.init_timer.start()
self.browser.start()
self.browser_pid = self.browser.pid()
self.start_test_runner(test_queue)
except:
self.logger.warning("Failure during init %s" %
traceback.format_exc())
self.init_timer.cancel()
self.logger.error(traceback.format_exc())
succeeded = False
else:
succeeded = True
# This has to happen after the lock is released
if not succeeded:
self.init_failed()
def init_succeeded(self):
"""Callback when we have started the browser, connected via
marionette, and we are ready to start testing"""
self.logger.debug("Init succeeded")
self.init_timer.cancel()
self.init_fail_count = 0
self.start_next_test()
def init_failed(self):
"""Callback when we can't connect to the browser via
marionette for some reason"""
self.init_fail_count += 1
self.logger.warning("Init failed %i" % self.init_fail_count)
self.init_timer.cancel()
if self.init_fail_count < self.max_init_fails:
self.restart_runner()
else:
self.logger.critical(
"Test runner failed to initialise correctly; shutting down")
return Stop
def start_test_runner(self, test_queue):
# Note that we need to be careful to start the browser before the
# test runner to ensure that any state set when the browser is started
# can be passed in to the test runner.
assert self.command_queue is not None
assert self.remote_queue is not None
self.logger.info("Starting runner")
executor_browser_cls, executor_browser_kwargs = self.browser.executor_browser(
)
args = (test_queue, self.remote_queue, self.command_queue,
self.executor_cls, self.executor_kwargs, executor_browser_cls,
executor_browser_kwargs, self.child_stop_flag)
self.test_runner_proc = Process(target=start_runner,
args=args,
name="Thread-TestRunner-%i" %
self.manager_number)
self.test_runner_proc.start()
self.logger.debug("Test runner started")
def send_message(self, command, *args):
self.remote_queue.put((command, args))
def cleanup(self):
if self.init_timer is not None:
self.init_timer.cancel()
self.logger.debug("TestManager cleanup")
while True:
try:
self.logger.warning(" ".join(
map(repr, self.command_queue.get_nowait())))
except Empty:
break
while True:
try:
self.logger.warning(" ".join(
map(repr, self.remote_queue.get_nowait())))
except Empty:
break
def teardown(self):
self.logger.debug("teardown in testrunnermanager")
self.test_runner_proc = None
self.command_queue.close()
self.remote_queue.close()
self.command_queue = None
self.remote_queue = None
def ensure_runner_stopped(self):
if self.test_runner_proc is None:
return
self.test_runner_proc.join(10)
if self.test_runner_proc.is_alive():
# This might leak a file handle from the queue
self.logger.warning("Forcibly terminating runner process")
self.test_runner_proc.terminate()
self.test_runner_proc.join(10)
else:
self.logger.debug("Testrunner exited with code %i" %
self.test_runner_proc.exitcode)
def runner_teardown(self):
self.ensure_runner_stopped()
return Stop
def stop_runner(self):
"""Stop the TestRunner and the Firefox binary."""
self.logger.debug("Stopping runner")
if self.test_runner_proc is None:
return
try:
self.browser.stop()
if self.test_runner_proc.is_alive():
self.send_message("stop")
self.ensure_runner_stopped()
finally:
self.cleanup()
def start_next_test(self):
self.send_message("run_test")
def requeue_test(self):
self.test_source.requeue(self.test)
self.test = None
def test_start(self, test):
self.test = test
self.logger.test_start(test.id)
def test_ended(self, test, results):
"""Handle the end of a test.
Output the result of each subtest, and the result of the overall
harness to the logs.
"""
assert test == self.test
# Write the result of each subtest
file_result, test_results = results
subtest_unexpected = False
for result in test_results:
if test.disabled(result.name):
continue
expected = test.expected(result.name)
is_unexpected = expected != result.status
if is_unexpected:
self.unexpected_count += 1
self.logger.debug("Unexpected count in this thread %i" %
self.unexpected_count)
subtest_unexpected = True
self.logger.test_status(test.id,
result.name,
result.status,
message=result.message,
expected=expected)
# TODO: consider changing result if there is a crash dump file
# Write the result of the test harness
expected = test.expected()
status = file_result.status if file_result.status != "EXTERNAL-TIMEOUT" else "TIMEOUT"
is_unexpected = expected != status
if is_unexpected:
self.unexpected_count += 1
self.logger.debug("Unexpected count in this thread %i" %
self.unexpected_count)
if status == "CRASH":
self.browser.log_crash(process=self.browser_pid, test=test.id)
self.logger.test_end(test.id,
status,
message=file_result.message,
expected=expected)
self.test = None
if self.pause_on_unexpected and (subtest_unexpected or is_unexpected):
self.logger.info(
"Got an unexpected result, pausing until the browser exits")
self.browser.runner.process_handler.wait()
# Handle starting the next test, with a runner restart if required
if (file_result.status in ("CRASH", "EXTERNAL-TIMEOUT")
or subtest_unexpected or is_unexpected):
return self.restart_runner()
else:
return self.start_next_test()
def restart_runner(self):
"""Stop and restart the TestRunner"""
if self.restart_count >= self.max_restarts:
return Stop
self.stop_runner()
return self.init()
def log(self, action, kwargs):
getattr(self.logger, action)(**kwargs)
def error(self, message):
self.logger.error(message)
self.restart_runner()
def multi_proc():
"""
Utilisation des multiprocesseurs pour accelerer et fiabiliser la decouverte
De cette maniere les interface seront en ecoutent lors du scan actif.
1. Scan passif des interfaces et recherche active en parallele
2. Ajout des machines decouvertes, et des informations DHCP
3. Recheche des informations sur internet et scan des ports pour chaque machine decouverte
Globales utilisee : settings.DUREE, settings.LOCAL_ONLY, setting.LS_MACHINES
:return: None
"""
tools.info("#-- Recuperation des informations du PC administrateur --#")
pcadm = PcAdmin.PcAdmin()
q_machine = Queue()
q_dhcp = Queue()
process = []
tools.info("#-- Scan passif des interfaces --#")
tps = settings.DUREE
for rzo2 in pcadm.rzo['L2']:
if rzo2.adm_nic.etat:
p1 = Process(target=rzo2.sniff, args=(tps, q_machine))
process.append(p1)
p2 = Process(target=rzo2.find_dhcp, args=(q_dhcp, ))
process.append(p2)
tools.info("#-- Scan actif des reseaux L3 --#")
for rzo3 in pcadm.rzo['L3']:
p3 = Process(target=scan_l3, args=(rzo3, q_machine))
process.append(p3)
tools.info("#-- Début multi-processing --#")
tools.multip_exec(process)
tools.info("#-- Fin de la phase de decouverte des voisins --#")
# Ajout des machines decouvertes, vidange de la queue machine
while not q_machine.empty():
val = q_machine.get()
try:
rzo = pcadm.contains_rzo(val[0])
except ValueError:
continue
tools.create_neigh(rzo, val[1], val[2])
# Ajout des informations DHCP, vidange de la queue dhcp
while not q_dhcp.empty():
val = q_dhcp.get()
try:
rzo = pcadm.contains_rzo(val[0])
except ValueError:
continue
rzo.dhcp_info = val[1]
# Recherche des informations sur Internet
process = []
q_inet = Queue()
if not settings.LOCAL_ONLY:
tools.info("#-- Recherche informations sur Internet --#")
for rzo3 in pcadm.rzo['L3']:
p4 = Process(target=rzo3.internet_info, args=(q_inet, ))
process.append(p4)
q_port = Queue()
if settings.PORT_SCAN:
tools.info("#-- Ports scan des machines decouvertes --#")
for machine in [
ordi for ordi in settings.LS_MACHINES
if not isinstance(ordi, PcAdmin.PcAdmin)
]:
p5 = Process(target=machine.host_scan, args=(q_port, ))
process.append(p5)
tools.multip_exec(process)
# Ajout des informations internet
while not q_inet.empty():
val = q_inet.get()
try:
rzo = pcadm.contains_rzo(val[0])
except ValueError:
continue
rzo.pubip = val[1]
rzo.geoloc = val[2]
rzo.route = val[3]
# Ajout des resulats du port scan
while not q_port.empty():
val = q_port.get()
qma = val[0]
qip = str(val[1].ip())
qp = [val[2], val[3]]
for machine in [
ordi for ordi in settings.LS_MACHINES
if not isinstance(ordi, PcAdmin.PcAdmin)
]:
if qma == machine:
adresse = machine.addr_from_ip(qip)
if adresse:
adresse.l_port = qp
break
return pcadm
# There are two columns,
# First column is the name of the genome file
# Second column is what the genome should have been
l = l.rstrip().split(',')
correction_names[l[1]] = l[0]
print(correction_names)
# Setup the queue
procQueue = Queue()
for ref_name in list(ref_grid.keys()):
procQueue.put(ref_name)
# Divde all the files into groups corresponding to number of cores.
coreCnt = 0
proc = []
while not procQueue.empty():
if coreCnt < coreNum:
ref_name = procQueue.get(
) # should be ref_name, which is a string
p = Process(target=process_one_reference,
args=(A, ref_name, ref_grid[ref_name],
correction_names))
p.start()
proc.append(p)
coreCnt += 1
sleep(5) # put 10 sec in between calling new threads
else: # all 16 cores are used so wait until things finish
for p in proc:
p.join()
proc = []
coreCnt = 0
class InsightWorker:
""" Worker that collects data from the Github API and stores it in our database
task: most recent task the broker added to the worker's queue
child: current process of the queue being ran
queue: queue of tasks to be fulfilled
config: holds info like api keys, descriptions, and database connection strings
"""
def __init__(self, config, task=None):
self.config = config
logging.basicConfig(filename='worker_{}.log'.format(
self.config['id'].split('.')[len(self.config['id'].split('.')) -
1]),
filemode='w',
level=logging.INFO)
logging.info('Worker (PID: {}) initializing...'.format(str(
os.getpid())))
self._task = task
self._child = None
self._queue = Queue()
self.db = None
self.tool_source = 'Insight Worker'
self.tool_version = '0.0.2' # See __init__.py
self.data_source = 'Augur API'
self.refresh = True
self.send_insights = True
self.finishing_task = False
self.anomaly_days = self.config['anomaly_days']
self.training_days = self.config['training_days']
self.confidence = self.config['confidence_interval'] / 100
logging.info("Worker initializing...")
specs = {
"id": self.config['id'],
"location": self.config['location'],
"qualifications": [{
"given": [["git_url"]],
"models": ["insights"]
}],
"config": [self.config]
}
self.metric_results_counter = 0
self.insight_results_counter = 0
"""
Connect to GHTorrent
:param dbstr: The [database string](http://docs.sqlalchemy.org/en/latest/core/engines.html) to connect to the GHTorrent database
"""
self.DB_STR = 'postgresql://{}:{}@{}:{}/{}'.format(
self.config['user'], self.config['password'], self.config['host'],
self.config['port'], self.config['database'])
dbschema = 'augur_data' # Searches left-to-right
self.db = s.create_engine(
self.DB_STR,
poolclass=s.pool.NullPool,
connect_args={'options': '-csearch_path={}'.format(dbschema)})
helper_schema = 'augur_operations'
self.helper_db = s.create_engine(
self.DB_STR,
poolclass=s.pool.NullPool,
connect_args={'options': '-csearch_path={}'.format(helper_schema)})
# produce our own MetaData object
metadata = MetaData()
helper_metadata = MetaData()
# we can reflect it ourselves from a database, using options
# such as 'only' to limit what tables we look at...
metadata.reflect(self.db,
only=[
'chaoss_metric_status', 'repo_insights',
'repo_insights_records'
])
helper_metadata.reflect(self.helper_db,
only=['worker_history', 'worker_job'])
# we can then produce a set of mappings from this MetaData.
Base = automap_base(metadata=metadata)
HelperBase = automap_base(metadata=helper_metadata)
# calling prepare() just sets up mapped classes and relationships.
Base.prepare()
HelperBase.prepare()
# mapped classes are ready
self.chaoss_metric_status_table = Base.classes[
'chaoss_metric_status'].__table__
self.repo_insights_table = Base.classes['repo_insights'].__table__
self.repo_insights_records_table = Base.classes[
'repo_insights_records'].__table__
self.history_table = HelperBase.classes.worker_history.__table__
self.job_table = HelperBase.classes.worker_job.__table__
connected = False
for i in range(5):
try:
logging.info("attempt {}".format(i))
if i > 0:
time.sleep(10)
requests.post('http://{}:{}/api/unstable/workers'.format(
self.config['broker_host'], self.config['broker_port']),
json=specs)
logging.info("Connection to the broker was successful")
connected = True
break
except requests.exceptions.ConnectionError:
logging.error('Cannot connect to the broker. Trying again...')
if not connected:
sys.exit(
'Could not connect to the broker after 5 attempts! Quitting...'
)
def update_config(self, config):
""" Method to update config and set a default
"""
self.config = {
'database_connection_string':
'psql://{}:5432/augur'.format(self.config['broker_host']),
"display_name":
"",
"description":
"",
"required":
1,
"type":
"string"
}
self.config.update(config)
@property
def task(self):
""" Property that is returned when the worker's current task is referenced
"""
return self._task
@task.setter
def task(self, value):
""" entry point for the broker to add a task to the queue
Adds this task to the queue, and calls method to process queue
"""
repo_git = value['given']['git_url']
""" Query all repos """
repoUrlSQL = s.sql.text("""
SELECT repo_id, repo_group_id FROM repo WHERE repo_git = '{}'
""".format(repo_git))
rs = pd.read_sql(repoUrlSQL, self.db, params={})
try:
self._queue.put({
"git_url": repo_git,
"repo_id": int(rs.iloc[0]["repo_id"]),
"repo_group_id": int(rs.iloc[0]["repo_group_id"]),
"job_type": value['job_type']
})
except Exception as e:
logging.info("that repo is not in our database, {}".format(e))
if self._queue.empty():
if 'github.com' in repo_git:
self._task = value
self.run()
def cancel(self):
""" Delete/cancel current task
"""
self._task = None
def run(self):
""" Kicks off the processing of the queue if it is not already being processed
Gets run whenever a new task is added
"""
logging.info("Running...\n")
self._child = Process(target=self.collect, args=())
self._child.start()
def collect(self):
""" Function to process each entry in the worker's task queue
Determines what action to take based off the message type
"""
while True:
time.sleep(2)
if not self._queue.empty():
message = self._queue.get()
# else:
# break
self.discover_insights(message)
def discover_insights(self, entry_info):
""" Data collection function
Query the github api for contributors and issues (not yet implemented)
"""
# Update table of endpoints before we query them all
logging.info(
"Discovering insights for task with entry info: {}".format(
entry_info))
self.record_model_process(entry_info, 'insights')
# Set the endpoints we want to discover insights for
endpoints = [{
'cm_info': "issues-new"
}, {
'cm_info': "code-changes"
}, {
'cm_info': "code-changes-lines"
}, {
'cm_info': 'reviews'
}]
""""""
""" For when we want all endpoints """
# """ Query all endpoints """
# endpointSQL = s.sql.text("""
# SELECT * FROM chaoss_metric_status WHERE cm_source = 'augur_db'
# """)
# for endpoint in pd.read_sql(endpointSQL, self.db, params={}).to_records():
# endpoints.append(endpoint)
""""""
# If we are discovering insights for a group vs repo, the base url will change
if 'repo_group_id' in entry_info and 'repo_id' not in entry_info:
base_url = 'http://{}:{}/api/unstable/repo-groups/{}/'.format(
self.config['broker_host'], self.config['broker_port'],
entry_info['repo_group_id'])
else:
base_url = 'http://{}:{}/api/unstable/repo-groups/9999/repos/{}/'.format(
self.config['broker_host'], self.config['broker_port'],
entry_info['repo_id'])
# Hit and discover insights for every endpoint we care about
for endpoint in endpoints:
# Hit endpoint
url = base_url + endpoint['cm_info']
logging.info("Hitting endpoint: " + url + "\n")
r = requests.get(url=url)
data = r.json()
def is_unique_key(key):
""" Helper method used to find which keys we want to analyze in each data point """
return 'date' not in key and key != 'repo_group_id' and key != 'repo_id' and key != 'repo_name' and key != 'rg_name'
# Filter out keys that we do not want to analyze (e.g. repo_id)
raw_values = {}
unique_keys = None
if len(data) > 0:
try:
unique_keys = list(filter(is_unique_key, data[0].keys()))
except Exception as e:
logging.info(
"Length bigger than 0 but cannot get 0th element? : {}, {}"
.format(data, e))
else:
logging.info(
"Endpoint with url: {} returned an empty response. Moving on to next endpoint.\n"
.format(url))
continue
# num issues, issue comments, num commits, num pr, comments pr
logging.info(
"Found the following unique keys for this endpoint: {}".format(
unique_keys))
date_filtered_data = []
i = 0
not_timeseries = False
begin_date = datetime.datetime.now()
# Subtract configurable amount of time
begin_date = begin_date - datetime.timedelta(
days=self.training_days)
begin_date = begin_date.strftime('%Y-%m-%d')
for dict in data:
try:
if dict['date'] > begin_date:
date_filtered_data = data[i:]
logging.info(
"data {} days ago date found: {}, {}".format(
self.training_days, dict['date'], begin_date))
break
except:
logging.info(
"Endpoint {} is not a timeseries, moving to next".
format(endpoint))
not_timeseries = True
break
i += 1
if not_timeseries:
continue
date_found_index = None
date_found = False
x = 0
begin_date = datetime.datetime.now() - datetime.timedelta(
days=self.anomaly_days)
for dict in date_filtered_data:
dict_date = datetime.datetime.strptime(
dict['date'],
'%Y-%m-%dT%H:%M:%S.%fZ') #2018-08-20T00:00:00.000Z
if dict_date > begin_date and not date_found:
date_found = True
date_found_index = x
logging.info(
"raw values within {} days ago date found: {}, {}".
format(self.anomaly_days, dict['date'], begin_date))
x += 1
for key in unique_keys:
try:
trash = int(dict[key]) * 2 + 1
raw_values[key].append(int(dict[key]))
except:
try:
trash = int(dict[key]) * 2 + 1
raw_values[key] = [int(dict[key])]
except:
logging.info(
"Key: {} is non-numerical, moving to next key."
.format(key))
for key in raw_values.keys():
if len(raw_values[key]) > 0:
mean, lower, upper = self.confidence_interval(
raw_values[key], confidence=self.confidence)
logging.info("Upper: {}, middle: {}, lower: {}".format(
upper, mean, lower))
i = 0
discovery_index = None
insight = False
max_difference = 0
score = 0
date_filtered_raw_values = []
date_filtered_raw_values = date_filtered_data[
date_found_index:]
logging.info(
"Raw values: {}".format(date_filtered_raw_values))
for dict in date_filtered_raw_values:
if (dict[key] > upper
and dict[key] - upper > max_difference) or (
dict[key] < lower
and lower - dict[key] > max_difference):
logging.info(
"Band breached at {}. Marking discovery. dict: {}, key: {}, mean: {}"
.format(i, dict, key, mean))
max_difference = max(dict[key] - upper,
lower - dict[key])
score = abs(dict[key] - mean) / mean * 100
insight = True
discovery_index = i
i += 1
if insight and 'date' in data[0]:
### INSIGHT DISCOVERED ###
# Check if new insight has a better score than other insights in its place, use result
# to determine if we continue in the insertion process (0 for no insertion, 1 for record
# insertion, 2 for record and insight data points insertion)
instructions = self.clear_insight(
entry_info['repo_id'], score, endpoint['cm_info'],
key)
# self.clear_insight(entry_info['repo_id'], score, endpoint['cm_info'] + ' ({})'.format(key))
# Use result from clearing function to determine if we need to insert the record
if instructions['record']:
# Insert record in records table and send record to slack bot
record = {
'repo_id':
int(entry_info['repo_id']),
'ri_metric':
endpoint['cm_info'],
'ri_field':
key,
'ri_value':
date_filtered_raw_values[discovery_index]
[key], #date_filtered_raw_values[j][key],
'ri_date':
date_filtered_raw_values[discovery_index]
['date'], #date_filtered_raw_values[j]['date'],
'ri_score':
score,
'ri_detection_method':
'{} confidence interval'.format(
self.confidence),
"tool_source":
self.tool_source,
"tool_version":
self.tool_version,
"data_source":
self.data_source
}
result = self.db.execute(
self.repo_insights_records_table.insert(
).values(record))
logging.info(
"Primary key inserted into the repo_insights_records table: {}"
.format(result.inserted_primary_key))
self.insight_results_counter += 1
# Send insight to Jonah for slack bot
self.send_insight(
record,
abs(date_filtered_raw_values[discovery_index]
[key] - mean))
# Use result from clearing function to determine if we still need to insert the insight
if instructions['insight']:
j = 0
logging.info(
"Starting j: {}, discovery_index: {}, data: {}"
.format(j, discovery_index,
date_filtered_data[j]))
for tuple in date_filtered_raw_values:
try:
data_point = {
'repo_id':
int(entry_info['repo_id']),
'ri_metric':
endpoint['cm_info'],
'ri_field':
key,
'ri_value':
tuple[
key], #date_filtered_raw_values[j][key],
'ri_date':
tuple[
'date'], #date_filtered_raw_values[j]['date'],
'ri_fresh':
0 if j < discovery_index else 1,
'ri_score':
score,
'ri_detection_method':
'{} confidence interval'.format(
self.confidence),
"tool_source":
self.tool_source,
"tool_version":
self.tool_version,
"data_source":
self.data_source
}
result = self.db.execute(
self.repo_insights_table.insert(
).values(data_point))
logging.info(
"Primary key inserted into the repo_insights table: "
+ str(result.inserted_primary_key))
logging.info(
"Inserted data point for endpoint: {}\n"
.format(endpoint['cm_info']))
j += 1
logging.info(
"incremented j: {}, discovery_index: {}, data: {}"
.format(j, discovery_index,
date_filtered_data[j]))
except Exception as e:
logging.info(
"error occurred while storing datapoint: {}"
.format(repr(e)))
break
else:
logging.info(
"Key: {} has empty raw_values, should not have key here"
.format(key))
self.register_task_completion(entry_info, "insights")
def record_model_process(self, entry_info, model):
task_history = {
"repo_id": entry_info['repo_id'],
"worker": self.config['id'],
"job_model": model,
"oauth_id": self.config['zombie_id'],
"timestamp": datetime.datetime.now(),
"status": "Stopped",
"total_results": self.insight_results_counter
}
if self.finishing_task:
result = self.helper_db.execute(self.history_table.update().where(
self.history_table.c.history_id == self.history_id).values(
task_history))
else:
result = self.helper_db.execute(
self.history_table.insert().values(task_history))
logging.info("Record incomplete history tuple: {}".format(
result.inserted_primary_key))
self.history_id = int(result.inserted_primary_key[0])
def register_task_completion(self, entry_info, model):
# Task to send back to broker
task_completed = {
'worker_id': self.config['id'],
'job_type': entry_info['job_type'],
'repo_id': entry_info['repo_id'],
'git_url': entry_info['git_url']
}
# Add to history table
task_history = {
"repo_id": entry_info['repo_id'],
"worker": self.config['id'],
"job_model": model,
"oauth_id": self.config['zombie_id'],
"timestamp": datetime.datetime.now(),
"status": "Success",
"total_results": self.insight_results_counter
}
self.helper_db.execute(self.history_table.update().where(
self.history_table.c.history_id == self.history_id).values(
task_history))
logging.info("Recorded job completion for: " + str(task_completed) +
"\n")
# Update job process table
updated_job = {
"since_id_str": entry_info['repo_id'],
"last_count": self.insight_results_counter,
"last_run": datetime.datetime.now(),
"analysis_state": 0
}
self.helper_db.execute(self.job_table.update().where(
self.job_table.c.job_model == model).values(updated_job))
logging.info("Update job process for model: " + model + "\n")
# Notify broker of completion
logging.info("Telling broker we completed task: " +
str(task_completed) + "\n\n" + "This task inserted: " +
str(self.insight_results_counter) + " tuples.\n\n")
requests.post('http://{}:{}/api/unstable/completed_task'.format(
self.config['broker_host'], self.config['broker_port']),
json=task_completed)
# Reset results counter for next task
self.insight_results_counter = 0
def send_insight(self, insight, units_from_mean):
try:
repoSQL = s.sql.text("""
SELECT repo_git, rg_name
FROM repo, repo_groups
WHERE repo_id = {}
""".format(insight['repo_id']))
repo = pd.read_sql(repoSQL, self.db, params={}).iloc[0]
begin_date = datetime.datetime.now() - datetime.timedelta(
days=self.anomaly_days)
dict_date = datetime.datetime.strptime(
insight['ri_date'],
'%Y-%m-%dT%H:%M:%S.%fZ') #2018-08-20T00:00:00.000Z
# logging.info("about to send to jonah")
if dict_date > begin_date and self.send_insights:
logging.info(
"Insight less than {} days ago date found: {}\n\nSending to Jonah..."
.format(self.anomaly_days, insight))
to_send = {
'insight': True,
# 'rg_name': repo['rg_name'],
'repo_git': repo['repo_git'],
'value': insight[
'ri_value'], # y-value of data point that is the anomaly
'date': insight[
'ri_date'], # date of data point that is the anomaly
'field': insight[
'ri_field'], # name of the field from the endpoint that the anomaly was detected on
'metric': insight[
'ri_metric'], # name of the metric the anomaly was detected on
'units_from_mean': units_from_mean,
'detection_method': insight['ri_detection_method']
}
requests.post(
'https://fgrmv7bswc.execute-api.us-east-2.amazonaws.com/dev/insight-event',
json=to_send)
except Exception as e:
logging.info("sending insight to jonah failed: {}".format(e))
def clear_insight(self, repo_id, new_score, new_metric, new_field):
logging.info("Checking if insight slots filled...")
# Dict that will be returned that instructs the rest of the worker where the insight insertion is
# needed (determined by if this new insights score is higher than already stored ones)
insertion_directions = {'record': False, 'insight': False}
# Query current record for this
recordSQL = s.sql.text("""
SELECT ri_metric, repo_id, ri_score, ri_field
FROM repo_insights_records
WHERE repo_id = {}
AND ri_metric = '{}'
AND ri_field = '{}'
ORDER BY ri_score DESC
""".format(repo_id, new_metric, new_field))
rec = json.loads(
pd.read_sql(recordSQL, self.db,
params={}).to_json(orient='records'))
logging.info("recordsql: {}, \n{}".format(recordSQL, rec))
# If new score is higher, continue with deletion
if len(rec) > 0:
if new_score > rec[0]['ri_score'] or self.refresh:
insertion_directions['record'] = True
for record in rec:
logging.info(
"Refresh is on or Insight record found with a greater score than current slot filled for "
"repo {} metric {} new score {}, old score {}".format(
repo_id, record['ri_metric'], new_score,
record['ri_score']))
deleteSQL = """
DELETE
FROM
repo_insights_records I
WHERE
repo_id = {}
AND ri_metric = '{}'
AND ri_field = '{}'
""".format(record['repo_id'], record['ri_metric'],
record['ri_field'])
try:
result = self.db.execute(deleteSQL)
except Exception as e:
logging.info(
"Error occured deleting insight slot: {}".format(
e))
else:
insertion_directions['record'] = True
# Query current insights and rank by score
num_insights_per_repo = 2
insightSQL = s.sql.text("""
SELECT distinct(ri_metric),repo_id, ri_score
FROM repo_insights
WHERE repo_id = {}
ORDER BY ri_score ASC
""".format(repo_id))
ins = json.loads(
pd.read_sql(insightSQL, self.db,
params={}).to_json(orient='records'))
logging.info("This repos insights: {}".format(ins))
# Determine if inisghts need to be deleted based on if there are more insights than we want stored,
# or if the current insights have a lower score
num_insights = len(ins)
to_delete = []
for insight in ins:
insight['ri_score'] = insight['ri_score'] if insight[
'ri_score'] else 0.0
logging.info("{}, {}, {}, {}".format(insight['ri_metric'],
new_metric,
insight['ri_score'],
num_insights_per_repo))
if (insight['ri_score'] < new_score
and num_insights >= num_insights_per_repo
) or num_insights > num_insights_per_repo or (
insight['ri_metric'] == new_metric and self.refresh):
num_insights -= 1
to_delete.append(insight)
logging.info(
"condition met, new len: {}, insight score: {}, new_score: {}"
.format(num_insights, insight['ri_score'], new_score))
# After psuedo-deletion, determine if insertion of the new insight is needed
if num_insights < num_insights_per_repo:
insertion_directions['insight'] = True
# Delete all insights marked for deletion
for insight in to_delete:
logging.info(
"insight found with a greater score than current slots filled for repo {} new score {}, old score {}"
.format(repo_id, new_score, insight['ri_score']))
deleteSQL = """
DELETE
FROM
repo_insights I
WHERE
repo_id = {}
AND ri_metric = '{}'
""".format(insight['repo_id'], insight['ri_metric'])
try:
result = self.db.execute(deleteSQL)
except Exception as e:
logging.info(
"Error occured deleting insight slot: {}".format(e))
return insertion_directions
def confidence_interval(self, data, timeperiod='week', confidence=.95):
""" Method to find high activity issues in the past specified timeperiod """
a = 1.0 * np.array(data)
logging.info("np array: {}".format(a))
n = len(a)
m, se = np.mean(a), scipy.stats.sem(a)
logging.info("Mean: {}, standard error: {}".format(m, se))
h = se * scipy.stats.t.ppf((1 + confidence) / 2., n - 1)
logging.info("H: {}".format(h))
return m, m - h, m + h
def update_metrics(self):
logging.info(
"Preparing to update metrics ...\n\n" +
"Hitting endpoint: http://{}:{}/api/unstable/metrics/status ...\n".
format(self.config['broker_host'], self.config['broker_port']))
r = requests.get(url='http://{}:{}/api/unstable/metrics/status'.format(
self.config['broker_host'], self.config['broker_port']))
data = r.json()
active_metrics = [
metric for metric in data
if metric['backend_status'] == 'implemented'
]
# Duplicate checking ...
need_insertion = self.filter_duplicates(
{'cm_api_endpoint_repo': "endpoint"}, ['chaoss_metric_status'],
active_metrics)
logging.info("Count of contributors needing insertion: " +
str(len(need_insertion)) + "\n")
for metric in need_insertion:
tuple = {
"cm_group": metric['group'],
"cm_source": metric['data_source'],
"cm_type": metric['metric_type'],
"cm_backend_status": metric['backend_status'],
"cm_frontend_status": metric['frontend_status'],
"cm_defined":
True if metric['is_defined'] == 'true' else False,
"cm_api_endpoint_repo": metric['endpoint'],
"cm_api_endpoint_rg": None,
"cm_info": metric['display_name'],
"cm_working_group": metric['group'],
"cm_info": metric['tag'],
"tool_source": self.tool_source,
"tool_version": self.tool_version,
"data_source": metric['data_source']
}
# Commit metric insertion to the chaoss metrics table
result = self.db.execute(
self.chaoss_metric_status_table.insert().values(tuple))
logging.info("Primary key inserted into the metrics table: " +
str(result.inserted_primary_key))
self.metric_results_counter += 1
logging.info("Inserted metric: " + metric['display_name'] + "\n")
def filter_duplicates(self, cols, tables, og_data):
need_insertion = []
table_str = tables[0]
del tables[0]
for table in tables:
table_str += ", " + table
for col in cols.keys():
colSQL = s.sql.text("""
SELECT {} FROM {}
""".format(col, table_str))
values = pd.read_sql(colSQL, self.db, params={})
for obj in og_data:
if values.isin([obj[cols[col]]]).any().any():
logging.info("value of tuple exists: " +
str(obj[cols[col]]) + "\n")
elif obj not in need_insertion:
need_insertion.append(obj)
logging.info(
"While filtering duplicates, we reduced the data size from " +
str(len(og_data)) + " to " + str(len(need_insertion)) + "\n")
return need_insertion
