def main():
timePipe, sigPipe = Pipe()
q = Queue()
clock = Process(target=tickTock, args=(timePipe, ))
testSignal = Process(target=signal, args=(q, sigPipe, [1]))
testSignal.start()
while True:
print(q.get())
def main(param):
KILLING = False
KILLING2 = False
original_sigint_handler = signal.signal(signal.SIGINT, signal.SIG_IGN)
data = setParam(param=param)
if __name__ == "__main__":
printParam(data)
logger.critical(
f"######################### NEW JOB: {data.folder} #########################"
)
# file_handler_job = logging.FileHandler(os.path.join(data.outputDir, 'log'))
# file_handler_job.setLevel(logging.INFO)
# file_handler_job.setFormatter(formatter)
# logger.addHandler(file_handler_job)
processes = []
p = Process(target=processMaster, args=(data, ), name="MASTER")
p.start()
processes.append(p)
logger.info('[i] MASTER started')
for n in range(data.workers):
p = Process(target=processWorker, args=(data, ), name="WORKER")
p.start()
processes.append(p)
logger.info(f'[i] WORKER {n+1}/{data.workers} started')
p = Process(target=processSink, args=(data, ), name="SINK")
p.start()
processes.append(p)
logger.info(f'[i] Sink started')
ksObj = KillSwitch(data)
def killswitch(a, b):
ksObj()
ksObj()
signal.signal(signal.SIGINT, killswitch)
emails, nmbPgsScraped = Producer(data).workYouBastard()
logger.info("[i] Finalising, \t")
for p in processes:
p.terminate()
logger.info('[i] Done')
return emails, nmbPgsScraped
def add_export(self):
# on MacOS we will not use os.fork, elsewhere this does nothing.
forking_enable(0)
logger.debug("Adding new export.")
should_terminate = RawValue(c_bool, False)
frames_to_export = RawValue(c_int, 0)
current_frame = RawValue(c_int, 0)
data_dir = self.data_dir
start_frame = self.start_frame.value
end_frame = self.end_frame.value
plugins = []
# Here we make clones of every plugin that supports it.
# So it runs in the current config when we lauch the exporter.
for p in self.g_pool.plugins:
try:
p_initializer = p.get_class_name(), p.get_init_dict()
plugins.append(p_initializer)
except AttributeError:
pass
out_file_path = verify_out_file_path(self.rec_name.value,
self.data_dir)
process = Process(target=export,
args=(should_terminate, frames_to_export,
current_frame, data_dir, start_frame,
end_frame, plugins, out_file_path))
process.should_terminate = should_terminate
process.frames_to_export = frames_to_export
process.current_frame = current_frame
process.out_file_path = out_file_path
self.new_export = process
def clone_with_timeout(src: str, dest: str, clone_func: Callable[[], None],
timeout: float) -> None:
"""Clone a repository with timeout.
Args:
src: clone source
dest: clone destination
clone_func: callable that does the actual cloning
timeout: timeout in seconds
"""
errors: Queue = Queue()
process = Process(target=_clone_task, args=(clone_func, errors))
process.start()
process.join(timeout)
if process.is_alive():
process.terminate()
# Give it literally a second (in successive steps of 0.1 second),
# then kill it.
# Can't use `process.join(1)` here, billiard appears to be bugged
# https://github.com/celery/billiard/issues/270
killed = False
for _ in range(10):
time.sleep(0.1)
if not process.is_alive():
break
else:
killed = True
os.kill(process.pid, signal.SIGKILL)
raise CloneTimeout(src, timeout, killed)
if not errors.empty():
raise CloneFailure(src, dest, errors.get())
def crawl(*args, **kwargs):
crawler = CrawlerProcess(get_project_settings())
crawler.crawl(*args, **kwargs)
process = Process(target=crawler.start)
process.start()
process.join()
crawler.stop()
def display_graph(self, graphdef):
'''display a graph'''
if 'mestate' in globals():
self.mestate.console.write("Expression: %s\n" %
' '.join(graphdef.expression.split()))
else:
self.mestate.child_pipe_send_console.send(
"Expression: %s\n" % ' '.join(graphdef.expression.split()))
#mestate.mlog.reduce_by_flightmodes(mestate.flightmode_selections)
#setup the graph, then pass to a new process and display
self.mg = grapher.MavGraph()
self.mg.set_marker(self.mestate.settings.marker)
self.mg.set_condition(self.mestate.settings.condition)
self.mg.set_xaxis(self.mestate.settings.xaxis)
self.mg.set_linestyle(self.mestate.settings.linestyle)
self.mg.set_show_flightmode(self.mestate.settings.show_flightmode)
self.mg.set_legend(self.mestate.settings.legend)
self.mg.add_mav(self.mestate.mlog)
for f in graphdef.expression.split():
self.mg.add_field(f)
self.mg.process(self.mestate.flightmode_selections,
self.mestate.mlog._flightmodes)
self.lenmavlist = len(self.mg.mav_list)
if platform.system() == 'Darwin':
forking_enable(False)
#Important - mg.mav_list is the full logfile and can be very large in size
#To avoid slowdowns in Windows (which copies the vars to the new process)
#We need to empty this var when we're finished with it
self.mg.mav_list = []
child = Process(target=self.mg.show, args=[
self.lenmavlist,
])
child.start()
self.mestate.mlog.rewind()
def crawl(self, origin_oj, username):
p = Process(target=self._crawl, args=[
origin_oj,
username,
])
p.start()
p.join()
def _fork_and_submit_job(self, job):
parent_pipe, child_pipe = Pipe()
try:
p = Process(target=self._submit_job_to_lsf,
args=(child_pipe, parent_pipe, job,))
p.start()
except:
parent_pipe.close()
raise
finally:
child_pipe.close()
try:
p.join()
result = parent_pipe.recv()
if isinstance(result, basestring):
raise SubmitError(result)
except EOFError:
raise SubmitError('Unknown exception submitting job')
finally:
parent_pipe.close()
return result
def _reflash(self, path):
"""
this will execute the upgrade operation in another process
because the SSH connection may hang indefinitely while reflashing
and would block the program; setting a timeout to `exec_command`
doesn't seem to take effect so at least we can stop the process
using `subprocess.join(timeout=self.UPGRADE_TIMEOUT)`
"""
def upgrade(conn, path, timeout):
conn.connect()
conn.exec_command('sysupgrade -v -c {0}'.format(path),
timeout=timeout)
conn.close()
subprocess = Process(
target=upgrade,
args=[self, path, self.UPGRADE_TIMEOUT]
)
subprocess.start()
self.log('Upgrade operation in progress...')
subprocess.join(timeout=self.UPGRADE_TIMEOUT)
self.log('SSH connection closed, will wait {0} seconds before '
'attempting to reconnect...'.format(self.SLEEP_TIME))
sleep(self.SLEEP_TIME)
# kill the subprocess if it has hanged
if subprocess.is_alive():
subprocess.terminate()
subprocess.join()
def fetch_models(channel_name, tuple_type, authorized_types, input_models,
directory):
models = []
exceptions = []
# Close django connection to force each Process to create its own as
# django orm connection is not fork safe https://code.djangoproject.com/ticket/20562
from django import db
db.connections.close_all()
for input_model in input_models:
args = (channel_name, tuple_type, authorized_types, input_model,
directory)
proc = Process(target=fetch_model, args=args)
models.append((proc, args))
proc.start()
for proc, args in models:
proc.join()
if proc.exitcode != 0:
exceptions.append(Exception(f'fetch model failed for args {args}'))
# Close django old connections to avoid potential leak
db.close_old_connections()
if exceptions:
raise Exception(exceptions)
def screenshot(
task,
url,
width=settings.BASE_WEBPAGE_PREVIEW_WIDTH,
height=settings.BASE_WEBPAGE_PREVIEW_HEIGHT,
lifetime=settings.BASE_WEBPAGE_PREVIEW_LIFETIME,
):
url_id = sha256()
url_id.update(url.encode("utf-8"))
url_id.update(bytes(width))
url_id.update(bytes(height))
key = url_id.hexdigest()
logger.info(f"Screenshot for {url} @ {width}x{height}: {key}")
if key in cache:
logger.info(f"Found {key} in cache.")
return key
logger.info(f"Locking {key}")
lock = cache.lock(key)
lock.acquire()
logger.info("Starting WebEngineScreenshot app")
parent_conn, child_conn = Pipe()
p = Process(target=WebpageTasks.worker, args=(url, width, height, child_conn))
p.start()
image = parent_conn.recv()
p.join()
if not image:
logger.info("WebEngineScreenshot app returned nothing")
return None
logger.info("Writing WebEngineScreenshot app result to cache")
cache.set(key, image, timeout=lifetime)
logger.info("Removing WebEngineScreenshot app singleton")
return key
def main():
# To assign camera by name: put string(s) in list
# Parse command line arguments
parser = argparse.ArgumentParser(description='GUI for gaze tracking and pupillometry')
parser.add_argument('-eye', dest='eye_file', type=str, help="Work with existing video recording, instead of live feed", default='')
parser.add_argument('-world', dest='world_file', type=str, help="Work with existing video recording, instead of live feed", default='')
args = parser.parse_args()
# to use a pre-recorded video.
# Use a string to specify the path to your video file as demonstrated below
if args.eye_file == '':
eye_src = ["UI154xLE-M", "USB Camera-B4.09.24.1", "FaceTime Camera (Built-in)", "Microsoft", "6000","Integrated Camera"]
# to assign cameras directly, using integers as demonstrated below
# eye_src = 1
else:
# print "Using provide file: %s" % args.filename
eye_src = args.eye_file
if args.world_file == '':
world_src = ["Logitech Camera","(046d:081d)","C510","B525", "C525","C615","C920","C930e"]
# to assign cameras directly, using integers as demonstrated below
# world_src = 0
else:
world_src = args.world_file
# Camera video size in pixels (width,height)
eye_size = (260,216) #(1280,1024)
world_size = (640,480)
# on MacOS we will not use os.fork, elsewhere this does nothing.
forking_enable(0)
# Create and initialize IPC
g_pool = Temp()
g_pool.pupil_queue = Queue()
g_pool.eye_rx, g_pool.eye_tx = Pipe(False)
g_pool.quit = RawValue(c_bool,0)
# this value will be substracted form the capture timestamp
g_pool.timebase = RawValue(c_double,0)
# make some constants avaiable
g_pool.user_dir = user_dir
g_pool.rec_dir = rec_dir
g_pool.version = version
g_pool.app = 'capture'
# set up subprocesses
p_eye = Process(target=eye, args=(g_pool,eye_src,eye_size))
# Spawn subprocess:
p_eye.start()
if platform.system() == 'Linux':
# We need to give the camera driver some time before requesting another camera.
sleep(0.5)
world(g_pool,world_src,world_size)
# Exit / clean-up
p_eye.join()
def main():
# To assign camera by name: put string(s) in list
eye_cam_names = [
"USB 2.0 Camera", "Microsoft", "6000", "Integrated Camera",
"HD USB Camera"
]
world_src = [
"Logitech Camera", "(046d:081d)", "C510", "B525", "C525", "C615",
"C920", "C930e"
]
eye_src = (eye_cam_names,
0), (eye_cam_names, 1
) #first match for eye0 and second match for eye1
# to assign cameras directly, using integers as demonstrated below
# eye_src = 4 , 5 #second arg will be ignored for monocular eye trackers
# world_src = 1
# to use a pre-recorded video.
# Use a string to specify the path to your video file as demonstrated below
# eye_src = '/Users/mkassner/Downloads/000/eye0.mkv' , '/Users/mkassner/Downloads/eye.avi'
# world_src = "/Users/mkassner/Downloads/000/world.mkv"
# Camera video size in pixels (width,height)
eye_size = (640, 480)
world_size = (1280, 720)
# on MacOS we will not use os.fork, elsewhere this does nothing.
forking_enable(0)
#g_pool holds variables. Only if added here they are shared across processes.
g_pool = Global_Container()
# Create and initialize IPC
g_pool.pupil_queue = Queue()
g_pool.quit = Value(c_bool, 0)
g_pool.timebase = Value(c_double, 0)
g_pool.eye_tx = []
# make some constants avaiable
g_pool.user_dir = user_dir
g_pool.version = get_version(version_file)
g_pool.app = 'capture'
g_pool.binocular = binocular
p_eye = []
for eye_id in range(1 + 1 * binocular):
rx, tx = Pipe(False)
p_eye += [
Process(target=eye,
args=(g_pool, eye_src[eye_id], eye_size, rx, eye_id))
]
g_pool.eye_tx += [tx]
p_eye[-1].start()
world(g_pool, world_src, world_size)
# Exit / clean-up
for p in p_eye:
p.join()
def start_process(self, command=None, billiard=False, *args, **kwargs):
from .models import ExportTask
from ..tasks.export_tasks import TaskStates
if billiard:
proc = Process(daemon=False, *args, **kwargs)
proc.start()
self.store_pid(pid=proc.pid)
proc.join()
self.exitcode = proc.exitcode
else:
proc = subprocess.Popen(command, **kwargs)
(self.stdout, self.stderr) = proc.communicate()
self.store_pid(pid=proc.pid)
self.exitcode = proc.wait()
# We need to close the existing connection because the logger could be using a forked process which,
# will be invalid and throw an error.
connection.close()
export_task = ExportTask.objects.get(uid=self.task_uid)
if export_task.status == TaskStates.CANCELED.value:
from ..tasks.exceptions import CancelException
raise CancelException(
task_name=export_task.export_provider_task.name,
user_name=export_task.cancel_user.username)
def _reflash(self, path):
"""
this will execute the upgrade operation in another process
because the SSH connection may hang indefinitely while reflashing
and would block the program; setting a timeout to `exec_command`
doesn't seem to take effect on some OpenWRT versions
so at least we can stop the process using
`subprocess.join(timeout=self.UPGRADE_TIMEOUT)`
"""
self.disconnect()
command = self.get_upgrade_command(path)
def upgrade(conn, path, timeout):
conn.connect()
conn.exec_command(command, timeout=timeout)
conn.disconnect()
subprocess = Process(target=upgrade,
args=[self, path, self.UPGRADE_TIMEOUT])
subprocess.start()
self.log('Upgrade operation in progress...')
subprocess.join(timeout=self.UPGRADE_TIMEOUT)
self.log(
f'SSH connection closed, will wait {self.RECONNECT_DELAY} seconds before '
'attempting to reconnect...')
sleep(self.RECONNECT_DELAY)
# kill the subprocess if it has hanged
if subprocess.is_alive():
subprocess.terminate()
subprocess.join()
def cmd_save(args):
'''save a graph'''
child = Process(target=save_process,
args=[
mestate.last_graph, mestate.child_pipe_send_console,
mestate.child_pipe_send_graph, mestate.status.msgs
])
child.start()
def telnet_client(server_id, port):
key = 'server-{0}-pid'.format(server_id)
pid = cache.get(key)
if pid and pid in psutil.get_pid_list():
os.kill(pid, signal.SIGTERM)
p = Process(target=TelnetClient, args=(port, 'uptee', server_id))
p.start()
cache.set(key, p.pid)
p.join()
def test_set_pdeathsig(self):
return_pid = Value('i')
p = Process(target=parent_task, args=(return_pid, ))
p.start()
sleep(3) # wait for setting pdeathsig
p.terminate()
sleep(3) # wait for process termination
with pytest.raises(psutil.NoSuchProcess):
proc = psutil.Process(return_pid.value)
def crawl(self, oj, solution_id, problem_id, language, code, username,
password):
p = Process(target=self._crawl,
args=[
oj, solution_id, problem_id, language, code, username,
password
])
p.start()
p.join()
def crawl(self, spider):
queue = Queue()
self.queue = Queue()
self.process = Process(target=self._crawl, args=(queue, spider))
self.process.start()
write_in_a_file('.crawl 1', {'process': self.process, 'process-pid': self.process and self.process.pid, 'queue': self.queue.qsize()}, "t.txt")
self.process.join()
write_in_a_file('.crawl 2', {'process': self.process,
'process-pid': self.process and self.process.pid, 'queue': self.queue.qsize()}, "t.txt")
def screenshot_endpoint(
self,
ip_address=None,
port=None,
hostname=None,
use_ssl=False,
use_sni=False,
path="/",
in_separate_process=False,
):
"""
Take a screenshot of the given endpoint, save it to a local temporary file, and return the local
file path.
:param ip_address: The IP address where the web service resides.
:param port: The port where the web service resides.
:param hostname: The hostname to request.
:param use_ssl: Whether or not to use SSL to request the endpoint.
:param use_sni: Whether or not the endpoint uses SNI.
:param path: The path of the resource to screenshot.
:param in_separate_process: Whether or not to take the screenshot in a separate process. This is to
address the incredibly long time that the Selenium webdriver can take when it hangs.
:return: A tuple containing (1) the local file path where the screenshot was saved and (2) whether or not
the screenshot was taken successfully.
"""
logger.debug(
"Now attempting to take a screenshot of the web service at %s:%s (%s). Hostname is %s, SNI support is %s."
% (ip_address, port, "using SSL" if use_ssl else "plain HTTP",
hostname, use_sni))
self.__set_endpoint(
ip_address=ip_address,
port=port,
hostname=hostname,
use_ssl=use_ssl,
use_sni=use_sni,
path=path,
)
self._output_file_path = self.get_temporary_file_path()
if in_separate_process:
process = Process(target=self.__take_screenshot)
try:
process.start()
process.join(config.selenium_screenshot_delay +
config.inspection_screenshot_join_timeout)
except IOError as e:
if e.errno == errno.EINTR:
logger.warning("Interrupted system call error received.")
else:
raise e
finally:
if process.is_alive():
print("PROCESS IS ALIVE - PID IS %s" % (process.pid, ))
os.kill(process.pid, signal.SIGTERM)
else:
self.__take_screenshot()
return self.output_file_path, FilesystemHelper.does_file_exist(
self.output_file_path)
def init_marker_cacher(self):
forking_enable(0) #for MacOs only
from marker_detector_cacher import fill_cache
visited_list = [False if x == False else True for x in self.cache]
video_file_path = os.path.join(self.g_pool.rec_dir,'world.avi')
self.cache_queue = Queue()
self.cacher_seek_idx = Value(c_int,0)
self.cacher_run = Value(c_bool,True)
self.cacher = Process(target=fill_cache, args=(visited_list,video_file_path,self.cache_queue,self.cacher_seek_idx,self.cacher_run))
self.cacher.start()
def main():
ProcessList = []
for i in range(0, 100):
ProcessList.append(Process(target=f, args=(i, )))
for p in ProcessList:
p.start()
sleep(5)
for p in ProcessList:
p.join()
print("main finished")
def init_marker_cacher(self):
forking_enable(0) #for MacOs only
from marker_detector_cacher import fill_cache
visited_list = [False if x == False else True for x in self.cache]
video_file_path = self.g_pool.capture.src
timestamps = self.g_pool.capture.timestamps
self.cache_queue = Queue()
self.cacher_seek_idx = Value('i',0)
self.cacher_run = Value(c_bool,True)
self.cacher = Process(target=fill_cache, args=(visited_list,video_file_path,timestamps,self.cache_queue,self.cacher_seek_idx,self.cacher_run,self.min_marker_perimeter_cacher))
self.cacher.start()
def test_set_pdeathsig(self):
success = "done"
q = Queue()
p = Process(target=parent_task, args=(q, success))
p.start()
child_proc = psutil.Process(q.get(timeout=3))
try:
p.terminate()
assert q.get(timeout=3) == success
finally:
child_proc.terminate()
def __init__(self, fun, args, postprocess, job):
""" Build multiprocessing queues and start worker. """
super(LongCalculation, self).__init__(job, "Cancel", 0, 0)
self.setModal(True)
self.input = Queue()
self.output = Queue()
self.input.put((fun, args, postprocess))
self.proc = Process(target=worker, args=(self.input, self.output))
self.proc.start()
self.timer = QTimer()
self.timer.timeout.connect(self.update)
self.timer.start(10)
def start_launch_servide_process(ld):
"""Starts a Launch Service process. To be called from subclasses.
Args:
ld : LaunchDescription obj.
"""
# Create the LauchService and feed the LaunchDescription obj. to it.
ls = LaunchService()
ls.include_launch_description(ld)
p = Process(target=ls.run)
p.daemon = True #The daemon process is terminated automatically before the main program exits, to avoid leaving orphaned processes running
p.start()
def run_player(self, graphdef):
if 'mestate' in globals():
self.mestate.console.write("Running Player...")
else:
self.mestate.child_pipe_send_console.send("Running Player...")
self.player = player.MavPlay()
self.player.add_mav(self.mlog)
if platform.system() == 'Darwin':
forking_enable(False)
child = Process(target=self.player.run)
child.start()
def run_crawler_process(params: dict) -> Process:
"""
Start scrapy spider from a separate process
:param dict params: scrapy spider parameters
:return: process instance
"""
process = Process(
target=run_crawler,
args=(params, ),
)
process.start()
return process
def run_crawler(params):
"""
:param params:
:return: process instance
"""
process = Process(
target=crawler_process,
args=(params, ),
)
process.start()
return process
