def createSubproccesses(self):
# Haven't decided yet whether we should force Kwola to always write to disc or spool in memory
# using /tmp. The following lines switch between the two approaches
# self.batchDirectory = tempfile.mkdtemp(dir=getKwolaUserDataDirectory("batches"))
self.batchDirectory = tempfile.mkdtemp()
self.subProcessCommandQueues = []
self.subProcessBatchResultQueues = []
self.subProcesses = []
for subprocessIndex in range(self.config['training_batch_prep_subprocesses']):
subProcessCommandQueue = multiprocessing.Queue()
subProcessBatchResultQueue = multiprocessing.Queue()
subProcess = multiprocessing.Process(target=TrainingManager.prepareAndLoadBatchesSubprocess, args=(self.configDir, self.batchDirectory, subProcessCommandQueue, subProcessBatchResultQueue, subprocessIndex, self.applicationId))
subProcess.start()
atexit.register(lambda: subProcess.terminate())
self.subProcessCommandQueues.append(subProcessCommandQueue)
self.subProcessBatchResultQueues.append(subProcessBatchResultQueue)
self.subProcesses.append(subProcess)
for queue in self.subProcessBatchResultQueues:
readyState = queue.get()
if readyState == "error":
raise Exception("Error occurred during batch prep sub process initiation.")
def start_queues(self):
self.in_queue = billiard.Queue(maxsize=self.config.q_multipler *
self.config.n_readers)
self.out_queue = billiard.Queue(maxsize=self.config.q_multipler *
self.config.n_readers)
self.malloc_queue = billiard.Queue(maxsize=self.config.q_multipler *
self.config.n_readers)
for _ in range(self.config.n_generators):
self.multicast_queues.append(
billiard.Queue(maxsize=self.config.q_multipler *
self.config.n_readers))
def createTestingSubprocesses(self):
self.subProcesses = []
for n in range(self.config['testing_subprocess_pool_size']):
subProcessCommandQueue = multiprocessing.Queue()
subProcessResultQueue = multiprocessing.Queue()
preloadTraceFiles = [file for fileList in self.executionSessionTraceLocalPickleFiles for file in fileList]
subProcess = multiprocessing.Process(target=TestingStepManager.predictedActionSubProcess, args=(self.configDir, self.shouldBeRandom, subProcessCommandQueue, subProcessResultQueue, preloadTraceFiles))
subProcess.start()
atexit.register(lambda: subProcess.terminate())
self.subProcesses.append((subProcessCommandQueue, subProcessResultQueue, subProcess))
def restartOneTestingSubprocess(self):
subProcessCommandQueue, subProcessResultQueue, subProcess = self.subProcesses.pop(0)
subProcessCommandQueue.put("quit")
subProcess.terminate()
subProcessCommandQueue = multiprocessing.Queue()
subProcessResultQueue = multiprocessing.Queue()
preloadTraceFiles = [file for fileList in self.executionSessionTraceLocalPickleFiles for file in fileList]
subProcess = multiprocessing.Process(target=TestingStepManager.predictedActionSubProcess, args=(self.configDir, self.shouldBeRandom, subProcessCommandQueue, subProcessResultQueue, preloadTraceFiles))
subProcess.start()
atexit.register(lambda: subProcess.terminate())
self.subProcesses.append((subProcessCommandQueue, subProcessResultQueue, subProcess))
def parallel_crawl(product, marketplace):
if product.phrases == None:
keywords_and_phrases = product.keywords
else:
keywords_and_phrases = product.keywords + product.phrases
# Define an output queue
output_queue = mp.Queue()
# Setup a list of processes that we want to run
processes = [
mp.Process(target=begin_crawl,
args=(product, marketplace, keyword, 0, output_queue))
for keyword in keywords_and_phrases
]
#intial_time = time()
# Run processes
for p in processes:
p.start()
# Exit the completed processes
for p in processes:
p.join()
# Get process results from the output queue
results = [output_queue.get() for p in processes]
#final_time = time()
return dict(pair for d in results for pair in d.items())
def __init__(self, blocking=True, db_path=None, ncpu=1):
"""
Init function
Parameter
---------
blocking: bool
determines whether join() blocks or not
db_path: str
the string to a LevelDB for command persistence
"""
self.__blocking = blocking
self.__broker_queue = mp.Queue()
self.__job_queue = mp.JoinableQueue()
self.__pending_dict = mp.Manager().dict()
self.__results_queue = mp.Queue()
self.__results_queue_worker = mp.Queue()
if db_path is None:
tmp_db = NamedTemporaryFile(delete=False,
dir=os.getcwd(),
suffix=".db")
tmp_db.close()
self.__is_temp_db = True
self.__db_path = tmp_db.name
else:
self.__is_temp_db = False
self.__db_path = db_path
self.__broker = _Broker(self.__broker_queue,
self.__job_queue,
self.__results_queue,
self.__results_queue_worker,
self.__pending_dict,
db_path=self.__db_path)
self.__broker.daemon = False
self.__broker.start()
self.__worker = []
for i in range(ncpu):
p = _Worker(self.__broker_queue, self.__job_queue,
self.__results_queue_worker)
p.daemon = False
self.__worker.append(p)
p.start()
def __init__(self, config, plugins=None):
if plugins is None:
self.plugins = []
else:
self.plugins = plugins
self.plugins = [
JSRewriter(config),
HTMLRewriter(config)
] + self.plugins
self.config = config
self.commandQueue = multiprocessing.Queue()
self.resultQueue = multiprocessing.Queue()
self.proxyProcess = multiprocessing.Process(target=self.runProxyServerSubprocess, args=(self.config, self.commandQueue, self.resultQueue, pickle.dumps(self.plugins)))
self.proxyProcess.start()
# Wait for the result indicating that the proxy process is ready
self.port = self.resultQueue.get()
time.sleep(0.1)
def run(self):
tasks = mp.Queue()
results = mp.JoinableQueue()
interim = []
args = (tasks, results)
# n_procs = min(mp.cpu_count(), len(self._videos))
n_procs = mp.cpu_count()
all_jobs = []
for video_gt_pair in self._videos:
gt = video_gt_pair[0]
fp = video_gt_pair[1]
for func in self._funcs:
func_name = func[0]
func_ptr = func[1]
base_params = {
'gt_path': gt,
'video_path': fp,
'metric_func': func_ptr,
'init': False
}
for classifier in self._classifiers:
params = base_params.copy()
params.update(self._experiment_args)
params['classifier'] = classifier(metric=func_name)
log.info("Params ({}): {}".format(id(params), params))
all_jobs.append((params, self._experiment))
for job in all_jobs:
tasks.put(job)
for _ in range(n_procs):
p = mp.Process(target=train_classifier, args=args).start()
for _ in range(len(all_jobs)):
interim.append(results.get())
results.task_done()
for _ in range(n_procs):
tasks.put(None)
results.join()
tasks.close()
results.close()
return interim
def __init__(self, depth, threading):
# Logger already setup by config, just get an instance
logobj = logging.getLogger('eventgen')
from eventgenconfig import EventgenAdapter
adapter = EventgenAdapter(logobj, {
'module': 'Queue',
'sample': 'null'
})
self.logger = adapter
# logger.info("Creating Queue of depth %d, threading %s" % (depth, threading))
if threading == 'thread':
self.q = PQueue.Queue(depth)
else:
self.q = multiprocessing.Queue(depth)
self.depth = depth
def get_meta(ds_path):
"""
This function is a wrapper for the get_gdal metadata because if there is a database diconnection there is no obvious
way to clean up and free those resources therefore it is put on a separate process and if it fails it can just be
tried again.
This is using GDAL 2.2.4 this should be checked again to see if it can be simplified in a later version.
:param ds_path: String: Path to dataset
:return: Metadata dict
driver: Short name of GDAL driver for dataset
is_raster: True if dataset is a raster type
nodata: NODATA value for all bands if all bands have the same one, otherwise None (raster sets only)
"""
multiprocess_queue = billiard.Queue()
proc = billiard.Process(target=get_gdal_metadata, daemon=True, args=(ds_path, multiprocess_queue,))
proc.start()
proc.join()
return multiprocess_queue.get()
def run(self):
globals()['threadrun'] = self.real_run
cProfile.runctx("threadrun()", globals(), locals(), "prof_putter")
def real_run(self):
while True:
l = []
# for i in xrange(20000):
# l.append('2014-01-05 23:07:08 WINDBAG Event 1 of 100000')
l = [
'2014-01-05 23:07:08 WINDBAG Event 1 of 100000'
for i in xrange(20000)
]
try:
self.q.put(l, block=True, timeout=1.0)
except Queue.Full:
pass
if __name__ == '__main__':
print 'running!'
q = multiprocessing.Queue(20)
g = Getter(q)
# g.daemon = True
p = Putter(q)
# p.daemon = True
p.start()
g.start()
def execute(self):
"""See base_runner.execute()."""
import platform
# In OS X the multiprocessing module is horribly broken, but a fixed
# version has been released as the 'billiard' module
if platform.system() == 'Darwin':
import billiard as multiprocessing
multiprocessing.forking_enable(0)
else:
import multiprocessing
from libqtopensesame.misc import process, _
from libopensesame import misc
self._workspace_globals = {}
if os.name == u'nt' or (sys.platform == u'darwin' \
and not hasattr(sys,"frozen")):
# Under Windows and OSX, the multiprocess runner assumes that there
# is a file called `opensesame.py` or `opensesame.pyc`. If this file
# does not exist, try to copy it from the main script
# (`opensesame`). If this fails, provide an informative error
# message.
os_folder = misc.opensesame_folder()
# misc.opensesame_folder() doesn't work for OSX and returns None then,
# so determine OpenSesame's rootdir in another way
if os_folder is None:
os_folder = os.path.dirname(
os.path.abspath(sys.modules['__main__'].__file__))
if not os.path.exists(os.path.join(os_folder, u'opensesame.pyc')) \
and not os.path.exists(os.path.join(os_folder, u'opensesame.py')):
import shutil
try:
shutil.copyfile(os.path.join(os_folder, u'opensesame'),
os.path.join(os_folder, u'opensesame.py'))
except Exception as e:
return osexception(_(
u'Failed to copy `opensesame` to `opensesame.py`, which is required for the multiprocess runner. Please copy the file manually, or select a different runner under Preferences.'
),
exception=e)
self.channel = multiprocessing.Queue()
try:
self.exp_process = process.ExperimentProcess(
self.experiment, self.channel)
except Exception as e:
return osexception(_(u'Failed to initialize experiment process'),
exception=e)
# Start process!
self.exp_process.start()
# Wait for experiment to finish.
# Listen for incoming messages in the meantime.
while self.exp_process.is_alive() or not self.channel.empty():
# We need to process the GUI. To make the GUI feel more responsive
# during pauses, we refresh the GUI more often when paused.
QtGui.QApplication.processEvents()
if self.paused:
for i in range(25):
time.sleep(.01)
QtGui.QApplication.processEvents()
# Make sure None is not printed. Ugly hack for a bug in the Queue
# class?
self.console.suppress_stdout()
# Wait for messages. Will throw Exception if no message is received
# before timeout.
try:
msg = self.channel.get(True, 0.05)
except:
continue
# Restore connection to stdout
self.console.capture_stdout()
if isinstance(msg, basestring):
sys.stdout.write(safe_decode(msg, errors=u'ignore'))
continue
# Capture exceptions
if isinstance(msg, Exception):
return msg
# The workspace globals are sent as a dict. A special __pause__ key
# indicates whether the experiment should be paused or resumed.
if isinstance(msg, dict):
self._workspace_globals = msg
if u'__heartbeat__' in msg:
self.console.set_workspace_globals(msg)
self.main_window.extension_manager.fire(u'heartbeat')
elif u'__pause__' in msg:
if msg[u'__pause__']:
self.pause()
else:
self.resume()
continue
# Anything that is not a string, not an Exception, and not None is
# unexpected
return osexception(
u"Illegal message type received from child process: %s (%s)" \
% (msg, type(msg)))
# Return None if experiment finished without problems
return None
def execute(self):
"""See base_runner.execute()."""
import platform
# In OS X the multiprocessing module is horribly broken, but a fixed
# version has been released as the 'billiard' module
if platform.system() == 'Darwin':
import billiard as multiprocessing
multiprocessing.forking_enable(0)
else:
import multiprocessing
from libqtopensesame.misc import process, _
from libopensesame import misc, debug
from StringIO import StringIO
if os.name == u'nt':
# Under Windows, the multiprocess runner assumes that there is a
# file called `opensesame.py` or `opensesame.pyc`. If this file does
# not exist, try to copy it from the main script (`opensesame`). If
# this fails, provide an informative error message.
os_folder = misc.opensesame_folder()
if not os.path.exists(os.path.join(os_folder, u'opensesame.pyc')) \
and not os.path.exists(os.path.join(os_folder, u'opensesame.py')):
import shutil
try:
shutil.copyfile(os.path.join(os_folder, u'opensesame'), \
os.path.join(os_folder, u'opensesame.py'))
except Exception as e:
return osexception( \
_(u'Failed to copy `opensesame` to `opensesame.py`, which is required for the multiprocess runner. Please copy the file manually, or select a different runner under Preferences.'), exception=e)
self.channel = multiprocessing.Queue()
self.exp_process = process.ExperimentProcess(self.experiment, \
self.channel)
# Start process!
self.exp_process.start()
# Variables used for ugly hack to suppress 'None' print by Queue.get()
_stdout = sys.stdout
_pit = StringIO()
# Wait for experiment to finish.
# Listen for incoming messages in the meantime.
while self.exp_process.is_alive() or not self.channel.empty():
QtGui.QApplication.processEvents()
# Make sure None is not printed. Ugly hack for a bug in the Queue
# class?
sys.stdout = _pit
# Wait for messages. Will throw Exception if no message is received
# before timeout.
try:
msg = self.channel.get(True, 0.05)
except:
msg = None
# Restore connection to stdout
sys.stdout = _stdout
# For standard print statements
if isinstance(msg, basestring):
sys.stdout.write(msg)
# Errors arrive as a tuple with (Error object, traceback)
elif isinstance(msg, Exception):
return msg
# Anything that is not a string, not an Exception, and not None is
# unexpected
elif msg != None:
return osexception( \
u"Illegal message type received from child process: %s (%s)" \
% (msg, type(msg)))
# Return None if experiment finished without problems
return None
def fill_template(
template_name,
context,
output_format='odt',
options=None,
separate_process=True,
):
"""Fill a document with data and convert it to the requested format.
Returns an absolute path to the generated file.
Supported output format:
Text documents: doc, docx, fodt, html, odt, ott, pdf, txt, xhtml, png
Spreadsheets: csv, fods, html, ods, ots, pdf, xhtml, xls, xlsx, png
Presentations: fodp, html, odg, odp, otp, pdf, potm, pot, pptx, pps,
ppt, svg, swf, xhtml, png
Drawings: fodg, html, odg, pdf, svg, swf, xhtml, png
More on filter options,
https://wiki.openoffice.org/wiki/Documentation/DevGuide/Spreadsheets/Filter_Options # noqa: E501
:param template_name: the path to template, in OpenDocument format
:param context: the context to be used to inject content
:param output_format: the output format
:param options: value of filterOptions in libreofficekit
:param separate_process: allow LO to
:return:
"""
if not isinstance(context, Context):
context = Context(context)
context['output_format'] = output_format
source_file = find_template_file(template_name)
source_extension = os.path.splitext(source_file)[1]
source = zipfile.ZipFile(source_file, 'r')
dest_file = NamedTemporaryFile(delete=False, suffix=source_extension)
dest = zipfile.ZipFile(dest_file, 'w')
manifest_data = ''
for name in source.namelist():
data = source.read(name)
if name.endswith('.xml'):
data = smart_str(data)
if any(name.endswith(file) for file in ('content.xml', 'styles.xml')):
template = Template(fix_inline_tags(data))
data = template.render(context)
elif name == 'META-INF/manifest.xml':
manifest_data = data[:-20] # Cut off the closing </manifest> tag
continue # We will append it at the very end
dest.writestr(name, smart_bytes(data))
for _, image in context.dicts[0].get(IMAGES_CONTEXT_KEY, {}).items():
filename = os.path.basename(image.name)
ext = os.path.splitext(filename)[1][1:]
manifest_data += (
'<manifest:file-entry '
'manifest:media-type="image/%(ext)s" '
'manifest:full-path="Pictures/%(filename)s"/>\n') % locals()
image.open()
dest.writestr('Pictures/%s' % filename, image.read())
image.close()
manifest_data += '</manifest:manifest>'
dest.writestr('META-INF/manifest.xml', manifest_data)
source.close()
dest.close()
if source_extension[1:] != output_format:
if separate_process:
results = multiprocessing.Queue()
converter = multiprocessing.Process(
target=_convert_file,
args=(str(dest_file.name), output_format, results, options),
)
converter.start()
return results.get()
else:
return _convert_file(
filename=str(dest_file.name),
format=output_format,
options=options,
)
else:
return dest_file.name
def execute(self):
"""See base_runner.execute()."""
import platform
if platform.system() == 'Darwin' and \
sys.version_info < (3,4):
# In OS X the multiprocessing module is horribly broken,
# for python 2.7 but a fixed version has been released
# as the 'billiard' module
import billiard as multiprocessing
else:
import multiprocessing
from libqtopensesame.misc import process, _
self._workspace_globals = {}
self.channel = multiprocessing.Queue()
try:
self.exp_process = process.ExperimentProcess(
self.experiment, self.channel)
except Exception as e:
return osexception(_(u'Failed to initialize experiment process'),
exception=e)
# Start process!
self.exp_process.start()
# Wait for experiment to finish.
# Listen for incoming messages in the meantime.
finished = False
while self.exp_process.is_alive() or not self.channel.empty():
# We need to process the GUI. To make the GUI feel more responsive
# during pauses, we refresh the GUI more often when paused.
QtWidgets.QApplication.processEvents()
if self.paused:
for i in range(25):
time.sleep(.01)
QtWidgets.QApplication.processEvents()
# Make sure None is not printed. Ugly hack for a bug in the Queue
# class?
self.console.suppress_stdout()
# Wait for messages. Will throw Exception if no message is received
# before timeout.
try:
msg = self.channel.get(True, 0.05)
except:
continue
# Restore connection to stdout
self.console.capture_stdout()
if isinstance(msg, basestring):
sys.stdout.write(safe_decode(msg, errors=u'ignore'))
continue
# Capture exceptions
if isinstance(msg, Exception):
return msg
# The workspace globals are sent as a dict. A special __pause__ key
# indicates whether the experiment should be paused or resumed.
if isinstance(msg, dict):
self._workspace_globals = msg
if u'__heartbeat__' in msg:
self.console.set_workspace_globals(msg)
self.main_window.extension_manager.fire(u'heartbeat')
elif u'__pause__' in msg:
if msg[u'__pause__']:
self.pause()
else:
self.resume()
elif u'__finished__' in msg:
finished = True
continue
# Anything that is not a string, not an Exception, and not None is
# unexpected
return osexception(
u"Illegal message type received from child process: %s (%s)" \
% (msg, type(msg)))
if not finished:
return osexception(
u'Python seems to have crashed. This should not '
u'happen. If Python crashes often, please report it on the '
u'OpenSesame forum.')
def run_exp_for_all_classifiers(save_dir=DIR_CLASSIFIERS, parallel=True):
"""
Runs all the saved classifiers that are located in save_dir.
parallel, if True, will use the multiprocessing module to run
multiple experiments at the same time.
At present, however, this is broken due to the way in which Python
processes match up to C-lib extensions. In this case, OpenCV just
kinda dies when processing is attempted in this manner.
Currently investigating a fix -- until then, just run linear or
via threads.
"""
classifiers = EXPClassifierHandler.get_all_saved_classifiers(
DIR_CLASSIFIERS)
classifiers = [x for x in classifiers if not x.endswith(".csv")]
if len(classifiers) == 0:
log.info("No more experiments to run, exiting.")
return
if parallel:
videos_to_classifiers = {}
for c in classifiers:
clf = load_saved_classifier(save_dir + c)
file_name = clf.video_path.split("/")[-1]
if file_name not in videos_to_classifiers:
videos_to_classifiers[file_name] = []
clfid = (clf.identifier, c)
videos_to_classifiers[file_name].append(clfid)
# So now we've mapped video_file: [classifiers], multiproc by k
tasks = mp.Queue()
results = mp.JoinableQueue()
interim = []
args = (tasks, results, save_dir)
n_procs = min(mp.cpu_count(), len(videos_to_classifiers.keys()))
for k in videos_to_classifiers.keys():
these_classifiers = videos_to_classifiers[k]
tasks.put(these_classifiers)
delegator = EXPClassifierHandler.run_exp_from_mp_queue
for _ in range(n_procs):
p = mp.Process(target=delegator, args=args).start()
for _ in range(len(videos_to_classifiers.keys())):
interim.append(results.get())
results.task_done()
for _ in range(n_procs):
tasks.put(None)
results.join()
tasks.close()
results.close()
else:
for c in classifiers:
EXPClassifierHandler.run_exp_for_classifier(c, save_dir)
# Maybe by the time we get here more will be waiting... keep going
EXPClassifierHandler.run_exp_for_all_classifiers(save_dir, parallel)
