class Logger(object):
def __init__(self, logfilepath):
try:
os.remove(logfilepath)
except OSError:
pass
self.logfilepath = logfilepath
self.logq = SimpleQueue()
self.tags = ''
self.num_tags = 0
def add_tag(self, tag):
#self.log("adding tag {}".format(tag))
self.num_tags += 1
if self.tags != '':
self.tags = self.tags + '.' + tag
else:
self.tags = tag
def remove_tag(self):
#self.log("removing tag")
tags = self.tags.split('.')
self.tags = ".".join(tags[:-1])
self.num_tags -= 1
def get_tag_part(self):
if self.tags != '':
return self.tags + ": "
else:
return ''
def log(self, message, start_group=None, end_group=None):
assert(type(message)==str)
self.logq.put(" "*self.num_tags*4 + self.get_tag_part() + message + '\n')
def getlog(self):
return self.logq.get()
def getlogs(self, n=None):
logs = []
if n == None:
while not self.logq.empty():
logs.append(self.getlog())
else:
assert(type(n)==int)
while not (self.logq.empty() or len(logs) == n):
logs.append(self.getlog())
return logs
def write_to_file(self):
# mode 'a' for append
with open(self.logfilepath, 'a') as f:
f.writelines(self.getlogs())
def test_simple_queue():
q = SimpleQueue()
input_ = [1, 2, 3, 4, 5, 6]
from_iterable(consumers.to_simple_queue(q), input_)
for i in input_:
o = q.get()
assert o == i
assert q.empty()
def _wait_for_updates(change_notifier: SimpleQueue):
"""
该方法会阻塞线程等待不断从change_notifier取出内容
:param change_notifier:
:return:
"""
# sentinels, timeout,
# wait(sentinels, timeout=timeout)
while not change_notifier.empty():
res = change_notifier.get()
logging.debug(f"got signal, content: {res}")
def QuiverPlotter(num): data_q = SimpleQueue() plot = Process(target=quiverPlotter,args=(data_q,num)) plot.start() try: while True: data = (yield) if data_q.empty() == False: continue data_q.put(data) except GeneratorExit: plot.join()
def Plotter3D(plots,scale):
data_q = SimpleQueue()
plot = Process(target=plotter3D,args=(data_q,plots,scale))
plot.start()
data = {}
try:
while True:
data.update((yield))
if data_q.empty() == False:
continue
data_q.put(data)
except GeneratorExit:
pass
class StatusTracker(object):
def __init__(self):
self.logq = SimpleQueue()
self.history = []
def put(self, msg):
assert(type(msg)==str)
self.logq.put(msg)
def flushq(self):
while not self.logq.empty():
self.history.append(self.logq.get())
self.prune_history()
def prune_history(self):
self.history = self.history[-100:]
def launch_graph_plot():
q = SimpleQueue()
Pyro4.config.HOST="10.1.1.2"
daemon = Pyro4.Daemon()
ns = Pyro4.locateNS()
p = Process(target=_launch_daemon, args=(daemon, q,))
p.start()
graph_plot = GraphPlotPanel()
while True:
if not q.empty():
item = q.get()
if item[0] == 'time':
print "got queue:", item
graph_plot.set_time(item[1])
elif item[0] == 'vertex_color':
pass
graph_plot.run()
fpsClock.tick(60)
def DensityPlotter(num,size): # num = size/scale range = [[-size,size],[-size,size]] data_q = SimpleQueue() plot = Process(target=imagedraw,args=(data_q,num)) plot.start() while True: x = (yield) if data_q.empty() == False: continue hist,_,_ = np.histogram2d(x[:,0],x[:,1],bins=num,range=range) avg = np.average(hist) hist = (hist - avg)/avg data_q.put(hist.astype(np.float32))
class RangerControlServer(HTTPServer):
def __init__(self, fm):
self.fm = fm
self.queue = SimpleQueue()
self.goDie = False
HTTPServer.__init__(self, ("127.0.0.1", 5964), RangerControlHandler)
def start(self):
self.thread = threading.Thread(target=self.process)
self.thread.start()
def stop(self):
self.shutdown()
def process(self):
self.serve_forever()
def check_messages(self):
if self.queue.empty():
return None
return self.queue.get()
def act_on_messages(self):
msg = self.check_messages()
if msg == None:
return False
action, arg = msg
match = re.match(r"/cdtab-(\S+)", action)
if match != None:
tab = match.group(1)
if not (tab in self.fm.tabs):
self.fm.tab_open(tab, arg)
else:
self.fm.tabs[tab].enter_dir(arg)
elif action == "/cd":
self.fm.enter_dir(arg)
elif action == "/cdfirst":
first_tab = self.fm._get_tab_list()[0]
self.fm.tabs[first_tab].enter_dir(arg)
else:
self.fm.notify("Unknown server command", bad=True)
return True
class RangerControlServer(HTTPServer):
def __init__(self, fm):
self.fm = fm
self.queue = SimpleQueue()
self.goDie = False
HTTPServer.__init__(self, ('127.0.0.1', 5964), RangerControlHandler)
def start(self):
self.thread = threading.Thread(target=self.process)
self.thread.start()
def stop(self):
self.shutdown()
def process(self):
self.serve_forever()
def check_messages(self):
if self.queue.empty():
return None
return self.queue.get()
def act_on_messages(self):
msg = self.check_messages()
if msg == None: return False
action, arg = msg
match = re.match(r"/cdtab-(\S+)", action)
if match != None:
tab = match.group(1)
if not (tab in self.fm.tabs):
self.fm.tab_open(tab, arg)
else:
self.fm.tabs[tab].enter_dir(arg)
elif action == "/cd":
self.fm.enter_dir(arg)
elif action == "/cdfirst":
first_tab = self.fm._get_tab_list()[0]
self.fm.tabs[first_tab].enter_dir(arg)
else:
self.fm.notify("Unknown server command", bad=True)
return True
class LinePlotter:
def __init__(self,*args,**kwargs):
self.data_q = SimpleQueue()
self.data = {}
self.plot = LinePlotterProcess(self.data_q)
self.plot.add_plot(*args,**kwargs)
def show(self):
self.plot.start()
def add_plot(self,*args,**kwargs):
self.plot.add_plot(*args,**kwargs)
def send(self,data):
if data == GeneratorExit:
self.plot.join()
self.data.update(data)
if self.data_q.empty() != False:
self.data_q.put(data)
def run_clients(options, db_table_set):
# Spawn one client for each db.table
exit_event = multiprocessing.Event()
processes = []
error_queue = SimpleQueue()
interrupt_event = multiprocessing.Event()
stream_semaphore = multiprocessing.BoundedSemaphore(options["clients"])
signal.signal(signal.SIGINT, lambda a, b: abort_export(a, b, exit_event, interrupt_event))
try:
progress_info = []
for db, table in db_table_set:
progress_info.append((multiprocessing.Value(ctypes.c_longlong, -1),
multiprocessing.Value(ctypes.c_longlong, 0)))
processes.append(multiprocessing.Process(target=export_table,
args=(options["host"],
options["port"],
options["auth_key"],
db, table,
options["directory_partial"],
options["fields"],
options["format"],
error_queue,
progress_info[-1],
stream_semaphore,
exit_event)))
processes[-1].start()
# Wait for all tables to finish
while len(processes) > 0:
time.sleep(0.1)
if not error_queue.empty():
exit_event.set() # Stop rather immediately if an error occurs
processes = [process for process in processes if process.is_alive()]
update_progress(progress_info)
# If we were successful, make sure 100% progress is reported
# (rows could have been deleted which would result in being done at less than 100%)
if error_queue.empty() and not interrupt_event.is_set():
print_progress(1.0)
# Continue past the progress output line and print total rows processed
def plural(num, text):
return "%d %s%s" % (num, text, "" if num == 1 else "s")
print("")
print("%s exported from %s" % (plural(sum([info[0].value for info in progress_info]), "row"),
plural(len(db_table_set), "table")))
finally:
signal.signal(signal.SIGINT, signal.SIG_DFL)
if interrupt_event.is_set():
raise RuntimeError("Interrupted")
if not error_queue.empty():
# multiprocessing queues don't handling tracebacks, so they've already been stringified in the queue
while not error_queue.empty():
error = error_queue.get()
print("%s" % error[1], file=sys.stderr)
if options["debug"]:
print("%s traceback: %s" % (error[0].__name__, error[2]), file=sys.stderr)
raise RuntimeError("Errors occurred during export")
def run_clients(options, workingDir, db_table_set):
# Spawn one client for each db.table, up to options.clients at a time
exit_event = multiprocessing.Event()
processes = []
if six.PY3:
ctx = multiprocessing.get_context(multiprocessing.get_start_method())
error_queue = SimpleQueue(ctx=ctx)
else:
error_queue = SimpleQueue()
interrupt_event = multiprocessing.Event()
sindex_counter = multiprocessing.Value(ctypes.c_longlong, 0)
hook_counter = multiprocessing.Value(ctypes.c_longlong, 0)
signal.signal(signal.SIGINT,
lambda a, b: abort_export(a, b, exit_event, interrupt_event))
errors = []
try:
progress_info = []
arg_lists = []
for db, table in db_table_set:
tableSize = int(
options.retryQuery(
"count",
query.db(db).table(table).info()
['doc_count_estimates'].sum()))
progress_info.append(
(multiprocessing.Value(ctypes.c_longlong, 0),
multiprocessing.Value(ctypes.c_longlong, tableSize)))
arg_lists.append((
db,
table,
workingDir,
options,
error_queue,
progress_info[-1],
sindex_counter,
hook_counter,
exit_event,
))
# Wait for all tables to finish
while processes or arg_lists:
time.sleep(0.1)
while not error_queue.empty():
exit_event.set() # Stop immediately if an error occurs
errors.append(error_queue.get())
processes = [
process for process in processes if process.is_alive()
]
if len(processes) < options.clients and len(arg_lists) > 0:
new_process = multiprocessing.Process(target=export_table,
args=arg_lists.pop(0))
new_process.start()
processes.append(new_process)
update_progress(progress_info, options)
# If we were successful, make sure 100% progress is reported
# (rows could have been deleted which would result in being done at less than 100%)
if len(errors
) == 0 and not interrupt_event.is_set() and not options.quiet:
utils_common.print_progress(1.0, indent=4)
# Continue past the progress output line and print total rows processed
def plural(num, text, plural_text):
return "%d %s" % (num, text if num == 1 else plural_text)
if not options.quiet:
print(
"\n %s exported from %s, with %s, and %s" %
(plural(sum([max(0, info[0].value)
for info in progress_info]), "row",
"rows"), plural(len(db_table_set), "table", "tables"),
plural(sindex_counter.value, "secondary index",
"secondary indexes"),
plural(hook_counter.value, "hook function",
"hook functions")))
finally:
signal.signal(signal.SIGINT, signal.SIG_DFL)
if interrupt_event.is_set():
raise RuntimeError("Interrupted")
if len(errors) != 0:
# multiprocessing queues don't handle tracebacks, so they've already been stringified in the queue
for error in errors:
print("%s" % error[1], file=sys.stderr)
if options.debug:
print("%s traceback: %s" % (error[0].__name__, error[2]),
file=sys.stderr)
raise RuntimeError("Errors occurred during export")
settings_dict[CamPrm.framerate_free] = 80
cmd_queue = SimpleQueue()
img_queue = SimpleQueue()
status_queue = SimpleQueue()
vid_writer_process =\
VideoProcess(img_queue,cmd_queue,status_queue)
cmd_queue.put((Command.new_settings, settings_dict))
#Run on current process
vid_writer_process.start()
#while 1:
# if not status_queue.empty():
# cmd,data = status_queue.get()
# print cmd
# if cmd == Command.camera_connected:
# break
cmd_queue.put((Command.record, ('test.avi', bounds)))
time_start = time.time()
timeout = 20
while time.time() < time_start + timeout:
if not img_queue.empty():
img = img_queue.get()
if not status_queue.empty():
status, data = status_queue.get()
cmd_queue.put((Command.terminate, None))
time.sleep(3)
vid_writer_process.terminate()
class AsyncScanner(object):
""" Class to derive all the scanner classes from.
To implement a scanner you have to override:
update_str_last_scanned()
Use try-finally to call terminate, if not processes will be
hanging in the background
"""
def __init__(self, data_structure, processes, scan_function, init_args,
_mp_init_function):
""" Init the scanner.
data_structure is a world.DataSet
processes is the number of child processes to use
scan_function is the function to use for scanning
init_args are the arguments passed to the init function
_mp_init_function is the function used to init the child processes
"""
assert(isinstance(data_structure, world.DataSet))
self.data_structure = data_structure
self.list_files_to_scan = data_structure._get_list()
self.processes = processes
self.scan_function = scan_function
# Queue used by processes to pass results
self.queue = SimpleQueue()
init_args.update({'queue': self.queue})
# NOTE TO SELF: initargs doesn't handle kwargs, only args!
# Pass a dict with all the args
self.pool = multiprocessing.Pool(processes=processes,
initializer=_mp_init_function,
initargs=(init_args,))
# TODO: make this automatic amount
# Recommended time to sleep between polls for results
self.SCAN_START_SLEEP_TIME = 0.001
self.SCAN_MIN_SLEEP_TIME = 1e-6
self.SCAN_MAX_SLEEP_TIME = 0.1
self.scan_sleep_time = self.SCAN_START_SLEEP_TIME
self.queries_without_results = 0
self.last_time = time()
self.MIN_QUERY_NUM = 1
self.MAX_QUERY_NUM = 5
# Holds a friendly string with the name of the last file scanned
self._str_last_scanned = None
def scan(self):
""" Launch the child processes and scan all the files. """
logging.debug("########################################################")
logging.debug("########################################################")
logging.debug("Starting scan in: " + str(self))
logging.debug("########################################################")
logging.debug("########################################################")
total_files = len(self.data_structure)
# Tests indicate that smaller amount of jobs per worker make all type
# of scans faster
jobs_per_worker = 5
#jobs_per_worker = max(1, total_files // self.processes
self._results = self.pool.map_async(self.scan_function,
self.list_files_to_scan,
jobs_per_worker)
# No more tasks to the pool, exit the processes once the tasks are done
self.pool.close()
# See method
self._str_last_scanned = ""
def get_last_result(self):
""" Return results of last file scanned. """
q = self.queue
ds = self.data_structure
if not q.empty():
d = q.get()
if isinstance(d, tuple):
self.raise_child_exception(d)
# Copy it to the father process
ds._replace_in_data_structure(d)
self.update_str_last_scanned(d)
# Got result! Reset it!
self.queries_without_results = 0
return d
else:
# Count amount of queries without result
self.queries_without_results += 1
return None
def terminate(self):
""" Terminate the pool, this will exit no matter what.
"""
self.pool.terminate()
def raise_child_exception(self, exception_tuple):
""" Raises a ChildProcessException using the info
contained in the tuple returned by the child process. """
e = exception_tuple
raise ChildProcessException(e[0], e[1][0], e[1][1], e[1][2])
def update_str_last_scanned(self):
""" Updates the string that represents the last file scanned. """
raise NotImplemented
def sleep(self):
""" Sleep waiting for results.
This method will sleep less when results arrive faster and
more when they arrive slower.
"""
# If the query number is outside of our range...
if not ((self.queries_without_results < self.MAX_QUERY_NUM) &
(self.queries_without_results > self.MIN_QUERY_NUM)):
# ... increase or decrease it to optimize queries
if (self.queries_without_results < self.MIN_QUERY_NUM):
self.scan_sleep_time *= 0.5
elif (self.queries_without_results > self.MAX_QUERY_NUM):
self.scan_sleep_time *= 2.0
# and don't go farther than max/min
if self.scan_sleep_time > self.SCAN_MAX_SLEEP_TIME:
logging.debug("Setting sleep time to MAX")
self.scan_sleep_time = self.SCAN_MAX_SLEEP_TIME
elif self.scan_sleep_time < self.SCAN_MIN_SLEEP_TIME:
logging.debug("Setting sleep time to MIN")
self.scan_sleep_time = self.SCAN_MIN_SLEEP_TIME
# Log how it's going
logging.debug("")
logging.debug("Nº of queries without result: " + str(self.queries_without_results))
logging.debug("Current sleep time: " + str(self.scan_sleep_time))
logging.debug("Time between calls to sleep(): " + str(time() - self.last_time))
self.last_time = time()
# Sleep, let the other processes do their job
sleep(self.scan_sleep_time)
@property
def str_last_scanned(self):
""" A friendly string with last scanned thing. """
return self._str_last_scanned if self._str_last_scanned \
else "Scanning..."
@property
def finished(self):
""" Finished the operation. The queue could have elements """
return self._results.ready() and self.queue.empty()
@property
def results(self):
""" Yield all the results from the scan.
This is the simpler method to control the scanning process,
but also the most sloppy. If you want to closely control the
scan process (for example cancel the process in the middle,
whatever is happening) use get_last_result().
for result in scanner.results:
# do things
"""
q = self.queue
T = self.SCAN_WAIT_TIME
while not q.empty() or not self.finished:
sleep(T)
if not q.empty():
d = q.get()
if isinstance(d, tuple):
self.raise_child_exception(d)
# Overwrite it in the data dict
self.replace_in_data_structure(d)
yield d
def __len__(self):
return len(self.data_structure)
def spawn_import_clients(options, files_info):
# Spawn one reader process for each db.table, as well as many client processes
task_queue = SimpleQueue()
error_queue = SimpleQueue()
exit_event = multiprocessing.Event()
interrupt_event = multiprocessing.Event()
errors = []
reader_procs = []
client_procs = []
parent_pid = os.getpid()
signal.signal(signal.SIGINT, lambda a, b: abort_import(a, b, parent_pid, exit_event, task_queue, client_procs, interrupt_event))
try:
progress_info = []
rows_written = multiprocessing.Value(ctypes.c_longlong, 0)
for i in xrange(options["clients"]):
client_procs.append(multiprocessing.Process(target=client_process,
args=(options["host"],
options["port"],
options["auth_key"],
task_queue,
error_queue,
rows_written,
options["force"],
options["durability"])))
client_procs[-1].start()
for file_info in files_info:
progress_info.append((multiprocessing.Value(ctypes.c_longlong, -1), # Current lines/bytes processed
multiprocessing.Value(ctypes.c_longlong, 0))) # Total lines/bytes to process
reader_procs.append(multiprocessing.Process(target=table_reader,
args=(options,
file_info,
task_queue,
error_queue,
progress_info[-1],
exit_event)))
reader_procs[-1].start()
# Wait for all reader processes to finish - hooray, polling
while len(reader_procs) > 0:
time.sleep(0.1)
# If an error has occurred, exit out early
if not error_queue.empty():
exit_event.set()
reader_procs = [proc for proc in reader_procs if proc.is_alive()]
update_progress(progress_info)
# Wait for all clients to finish
alive_clients = sum([client.is_alive() for client in client_procs])
for i in xrange(alive_clients):
task_queue.put("exit")
while len(client_procs) > 0:
time.sleep(0.1)
client_procs = [client for client in client_procs if client.is_alive()]
# If we were successful, make sure 100% progress is reported
if error_queue.empty() and not interrupt_event.is_set():
print_progress(1.0)
def plural(num, text):
return "%d %s%s" % (num, text, "" if num == 1 else "s")
# Continue past the progress output line
print("")
print("%s imported in %s" % (plural(rows_written.value, "row"),
plural(len(files_info), "table")))
finally:
signal.signal(signal.SIGINT, signal.SIG_DFL)
if interrupt_event.is_set():
raise RuntimeError("Interrupted")
if not task_queue.empty():
error_queue.put((RuntimeError, RuntimeError("Error: Items remaining in the task queue"), None))
if not error_queue.empty():
# multiprocessing queues don't handling tracebacks, so they've already been stringified in the queue
while not error_queue.empty():
error = error_queue.get()
print("%s" % error[1], file=sys.stderr)
if options["debug"]:
print("%s traceback: %s" % (error[0].__name__, error[2]), file=sys.stderr)
if len(error) == 4:
print("In file: %s" % error[3], file=sys.stderr)
raise RuntimeError("Errors occurred during import")
class AsyncScanner(object):
""" Class to derive all the scanner classes from.
To implement a scanner you have to override:
update_str_last_scanned()
Use try-finally to call terminate, if not processes will be
hanging in the background
"""
def __init__(self, data_structure, processes, scan_function, init_args, _mp_init_function):
""" Init the scanner.
data_structure is a world.DataSet
processes is the number of child processes to use
scan_function is the function to use for scanning
init_args are the arguments passed to the init function
_mp_init_function is the function used to init the child processes
"""
assert (isinstance(data_structure, world.DataSet))
self.data_structure = data_structure
self.list_files_to_scan = data_structure._get_list()
self.processes = processes
self.scan_function = scan_function
# Queue used by processes to pass results
self.queue = SimpleQueue()
init_args.update({'queue': self.queue})
# NOTE TO SELF: initargs doesn't handle kwargs, only args!
# Pass a dict with all the args
self.pool = multiprocessing.Pool(processes=processes, initializer=_mp_init_function, initargs=(init_args,))
# TODO: make this automatic amount
# Recommended time to sleep between polls for results
self.SCAN_START_SLEEP_TIME = 0.001
self.SCAN_MIN_SLEEP_TIME = 1e-6
self.SCAN_MAX_SLEEP_TIME = 0.1
self.scan_sleep_time = self.SCAN_START_SLEEP_TIME
self.queries_without_results = 0
self.last_time = time()
self.MIN_QUERY_NUM = 1
self.MAX_QUERY_NUM = 5
# Holds a friendly string with the name of the last file scanned
self._str_last_scanned = None
def scan(self):
""" Launch the child processes and scan all the files. """
logging.debug("########################################################")
logging.debug("########################################################")
logging.debug("Starting scan in: " + str(self))
logging.debug("########################################################")
logging.debug("########################################################")
total_files = len(self.data_structure)
# Tests indicate that smaller amount of jobs per worker make all type
# of scans faster
jobs_per_worker = 5
# jobs_per_worker = max(1, total_files // self.processes
self._results = self.pool.map_async(self.scan_function, self.list_files_to_scan, jobs_per_worker)
# No more tasks to the pool, exit the processes once the tasks are done
self.pool.close()
# See method
self._str_last_scanned = ""
def get_last_result(self):
""" Return results of last file scanned. """
q = self.queue
ds = self.data_structure
if not q.empty():
d = q.get()
if isinstance(d, tuple):
self.raise_child_exception(d)
# Copy it to the father process
ds._replace_in_data_structure(d)
self.update_str_last_scanned(d)
# Got result! Reset it!
self.queries_without_results = 0
return d
else:
# Count amount of queries without result
self.queries_without_results += 1
return None
def terminate(self):
""" Terminate the pool, this will exit no matter what.
"""
self.pool.terminate()
def raise_child_exception(self, exception_tuple):
""" Raises a ChildProcessException using the info
contained in the tuple returned by the child process. """
e = exception_tuple
raise ChildProcessException(e[0], e[1][0], e[1][1], e[1][2])
def update_str_last_scanned(self):
""" Updates the string that represents the last file scanned. """
raise NotImplemented
def sleep(self):
""" Sleep waiting for results.
This method will sleep less when results arrive faster and
more when they arrive slower.
"""
# If the query number is outside of our range...
if not ((self.queries_without_results < self.MAX_QUERY_NUM) & (self.queries_without_results > self.MIN_QUERY_NUM)):
# ... increase or decrease it to optimize queries
if (self.queries_without_results < self.MIN_QUERY_NUM):
self.scan_sleep_time *= 0.5
elif (self.queries_without_results > self.MAX_QUERY_NUM):
self.scan_sleep_time *= 2.0
# and don't go farther than max/min
if self.scan_sleep_time > self.SCAN_MAX_SLEEP_TIME:
logging.debug("Setting sleep time to MAX")
self.scan_sleep_time = self.SCAN_MAX_SLEEP_TIME
elif self.scan_sleep_time < self.SCAN_MIN_SLEEP_TIME:
logging.debug("Setting sleep time to MIN")
self.scan_sleep_time = self.SCAN_MIN_SLEEP_TIME
# Log how it's going
logging.debug("")
logging.debug("Nº of queries without result: " + str(self.queries_without_results))
logging.debug("Current sleep time: " + str(self.scan_sleep_time))
logging.debug("Time between calls to sleep(): " + str(time() - self.last_time))
self.last_time = time()
# Sleep, let the other processes do their job
sleep(self.scan_sleep_time)
@property
def str_last_scanned(self):
""" A friendly string with last scanned thing. """
return self._str_last_scanned if self._str_last_scanned \
else "Scanning..."
@property
def finished(self):
""" Finished the operation. The queue could have elements """
return self._results.ready() and self.queue.empty()
@property
def results(self):
""" Yield all the results from the scan.
This is the simpler method to control the scanning process,
but also the most sloppy. If you want to closely control the
scan process (for example cancel the process in the middle,
whatever is happening) use get_last_result().
for result in scanner.results:
# do things
"""
q = self.queue
T = self.SCAN_WAIT_TIME
while not q.empty() or not self.finished:
sleep(T)
if not q.empty():
d = q.get()
if isinstance(d, tuple):
self.raise_child_exception(d)
# Overwrite it in the data dict
self.replace_in_data_structure(d)
yield d
def __len__(self):
return len(self.data_structure)
def run_clients(options, db_table_set):
# Spawn one client for each db.table
exit_event = multiprocessing.Event()
processes = []
error_queue = SimpleQueue()
interrupt_event = multiprocessing.Event()
stream_semaphore = multiprocessing.BoundedSemaphore(options["clients"])
signal.signal(signal.SIGINT,
lambda a, b: abort_export(a, b, exit_event, interrupt_event))
try:
progress_info = []
for db, table in db_table_set:
progress_info.append((multiprocessing.Value(ctypes.c_longlong, -1),
multiprocessing.Value(ctypes.c_longlong, 0)))
processes.append(
multiprocessing.Process(
target=export_table,
args=(options["host"], options["port"],
options["auth_key"], db, table,
options["directory_partial"], options["fields"],
options["format"], error_queue, progress_info[-1],
stream_semaphore, exit_event)))
processes[-1].start()
# Wait for all tables to finish
while len(processes) > 0:
time.sleep(0.1)
if not error_queue.empty():
exit_event.set() # Stop rather immediately if an error occurs
processes = [
process for process in processes if process.is_alive()
]
update_progress(progress_info)
# If we were successful, make sure 100% progress is reported
# (rows could have been deleted which would result in being done at less than 100%)
if error_queue.empty() and not interrupt_event.is_set():
print_progress(1.0)
# Continue past the progress output line and print total rows processed
def plural(num, text):
return "%d %s%s" % (num, text, "" if num == 1 else "s")
print("")
print("%s exported from %s" %
(plural(sum([info[0].value for info in progress_info]),
"row"), plural(len(db_table_set), "table")))
finally:
signal.signal(signal.SIGINT, signal.SIG_DFL)
if interrupt_event.is_set():
raise RuntimeError("Interrupted")
if not error_queue.empty():
# multiprocessing queues don't handling tracebacks, so they've already been stringified in the queue
while not error_queue.empty():
error = error_queue.get()
print("%s" % error[1], file=sys.stderr)
if options["debug"]:
print("%s traceback: %s" % (error[0].__name__, error[2]),
file=sys.stderr)
raise RuntimeError("Errors occurred during export")
def spawn_import_clients(options, files_info):
# Spawn one reader process for each db.table, as well as many client processes
task_queue = SimpleQueue()
error_queue = SimpleQueue()
exit_event = multiprocessing.Event()
interrupt_event = multiprocessing.Event()
errors = []
reader_procs = []
client_procs = []
parent_pid = os.getpid()
signal.signal(
signal.SIGINT,
lambda a, b: abort_import(a, b, parent_pid, exit_event, task_queue,
client_procs, interrupt_event))
try:
progress_info = []
rows_written = multiprocessing.Value(ctypes.c_longlong, 0)
for i in xrange(options["clients"]):
client_procs.append(
multiprocessing.Process(target=client_process,
args=(options["host"], options["port"],
options["auth_key"], task_queue,
error_queue, rows_written,
options["force"],
options["durability"])))
client_procs[-1].start()
for file_info in files_info:
progress_info.append((
multiprocessing.Value(ctypes.c_longlong,
-1), # Current lines/bytes processed
multiprocessing.Value(ctypes.c_longlong,
0))) # Total lines/bytes to process
reader_procs.append(
multiprocessing.Process(target=table_reader,
args=(options, file_info, task_queue,
error_queue, progress_info[-1],
exit_event)))
reader_procs[-1].start()
# Wait for all reader processes to finish - hooray, polling
while len(reader_procs) > 0:
time.sleep(0.1)
# If an error has occurred, exit out early
if not error_queue.empty():
exit_event.set()
reader_procs = [proc for proc in reader_procs if proc.is_alive()]
update_progress(progress_info)
# Wait for all clients to finish
alive_clients = sum([client.is_alive() for client in client_procs])
for i in xrange(alive_clients):
task_queue.put("exit")
while len(client_procs) > 0:
time.sleep(0.1)
client_procs = [
client for client in client_procs if client.is_alive()
]
# If we were successful, make sure 100% progress is reported
if error_queue.empty() and not interrupt_event.is_set():
print_progress(1.0)
def plural(num, text):
return "%d %s%s" % (num, text, "" if num == 1 else "s")
# Continue past the progress output line
print("")
print("%s imported in %s" % (plural(
rows_written.value, "row"), plural(len(files_info), "table")))
finally:
signal.signal(signal.SIGINT, signal.SIG_DFL)
if interrupt_event.is_set():
raise RuntimeError("Interrupted")
if not task_queue.empty():
error_queue.put(
(RuntimeError,
RuntimeError("Error: Items remaining in the task queue"), None))
if not error_queue.empty():
# multiprocessing queues don't handling tracebacks, so they've already been stringified in the queue
while not error_queue.empty():
error = error_queue.get()
print("%s" % error[1], file=sys.stderr)
if options["debug"]:
print("%s traceback: %s" % (error[0].__name__, error[2]),
file=sys.stderr)
if len(error) == 4:
print("In file: %s" % error[3], file=sys.stderr)
raise RuntimeError("Errors occurred during import")
