def _fork_and_submit_job(self, job): parent_pipe, child_pipe = Pipe() try: p = Process(target=self._submit_job_to_lsf, args=(child_pipe, parent_pipe, job,)) p.start() except: parent_pipe.close() raise finally: child_pipe.close() try: p.join() result = parent_pipe.recv() if isinstance(result, basestring): raise SubmitError(result) except EOFError: raise SubmitError('Unknown exception submitting job') finally: parent_pipe.close() return result
def crawl(self, origin_oj): p = Process( target=self._crawl, args=[origin_oj] ) p.start() p.join()
def __init__(self, spider): Process.__init__(self) settings = get_project_settings() self.crawler = Crawler(settings) self.crawler.configure() self.crawler.signals.connect(reactor.stop, signal=signals.spider_closed) self.spider = spider
def clone_with_timeout(src: str, dest: str, clone_func: Callable[[], None], timeout: float) -> None: """Clone a repository with timeout. Args: src: clone source dest: clone destination clone_func: callable that does the actual cloning timeout: timeout in seconds """ errors: Queue = Queue() process = Process(target=_clone_task, args=(clone_func, errors)) process.start() process.join(timeout) if process.is_alive(): process.terminate() # Give it literally a second (in successive steps of 0.1 second), # then kill it. # Can't use `process.join(1)` here, billiard appears to be bugged # https://github.com/celery/billiard/issues/270 killed = False for _ in range(10): time.sleep(0.1) if not process.is_alive(): break else: killed = True os.kill(process.pid, signal.SIGKILL) raise CloneTimeout(src, timeout, killed) if not errors.empty(): raise CloneFailure(src, dest, errors.get())
def add_export(self): # on MacOS we will not use os.fork, elsewhere this does nothing. forking_enable(0) logger.debug("Adding new export.") should_terminate = RawValue(c_bool,False) frames_to_export = RawValue(c_int,0) current_frame = RawValue(c_int,0) data_dir = self.data_dir start_frame= self.start_frame.value end_frame= self.end_frame.value plugins = [] # Here we make clones of every plugin that supports it. # So it runs in the current config when we lauch the exporter. for p in self.g_pool.plugins: try: p_initializer = p.get_class_name(),p.get_init_dict() plugins.append(p_initializer) except AttributeError: pass out_file_path=verify_out_file_path(self.rec_name.value,self.data_dir) process = Process(target=export, args=(should_terminate,frames_to_export,current_frame, data_dir,start_frame,end_frame,plugins,out_file_path)) process.should_terminate = should_terminate process.frames_to_export = frames_to_export process.current_frame = current_frame process.out_file_path = out_file_path self.new_export = process
def display_graph(self, graphdef): '''display a graph''' if 'mestate' in globals(): self.mestate.console.write("Expression: %s\n" % ' '.join(graphdef.expression.split())) else: self.mestate.child_pipe_send_console.send("Expression: %s\n" % ' '.join(graphdef.expression.split())) #mestate.mlog.reduce_by_flightmodes(mestate.flightmode_selections) #setup the graph, then pass to a new process and display self.mg = grapher.MavGraph() self.mg.set_marker(self.mestate.settings.marker) self.mg.set_condition(self.mestate.settings.condition) self.mg.set_xaxis(self.mestate.settings.xaxis) self.mg.set_linestyle(self.mestate.settings.linestyle) self.mg.set_show_flightmode(self.mestate.settings.show_flightmode) self.mg.set_legend(self.mestate.settings.legend) self.mg.add_mav(self.mestate.mlog) for f in graphdef.expression.split(): self.mg.add_field(f) self.mg.process(self.mestate.flightmode_selections, self.mestate.mlog._flightmodes) self.lenmavlist = len(self.mg.mav_list) if platform.system() == 'Darwin': forking_enable(False) #Important - mg.mav_list is the full logfile and can be very large in size #To avoid slowdowns in Windows (which copies the vars to the new process) #We need to empty this var when we're finished with it self.mg.mav_list = [] child = Process(target=self.mg.show, args=[self.lenmavlist, ]) child.start() self.mestate.mlog.rewind()
def main(): # To assign camera by name: put string(s) in list # Parse command line arguments parser = argparse.ArgumentParser(description='GUI for gaze tracking and pupillometry') parser.add_argument('-eye', dest='eye_file', type=str, help="Work with existing video recording, instead of live feed", default='') parser.add_argument('-world', dest='world_file', type=str, help="Work with existing video recording, instead of live feed", default='') args = parser.parse_args() # to use a pre-recorded video. # Use a string to specify the path to your video file as demonstrated below if args.eye_file == '': eye_src = ["UI154xLE-M", "USB Camera-B4.09.24.1", "FaceTime Camera (Built-in)", "Microsoft", "6000","Integrated Camera"] # to assign cameras directly, using integers as demonstrated below # eye_src = 1 else: # print "Using provide file: %s" % args.filename eye_src = args.eye_file if args.world_file == '': world_src = ["Logitech Camera","(046d:081d)","C510","B525", "C525","C615","C920","C930e"] # to assign cameras directly, using integers as demonstrated below # world_src = 0 else: world_src = args.world_file # Camera video size in pixels (width,height) eye_size = (260,216) #(1280,1024) world_size = (640,480) # on MacOS we will not use os.fork, elsewhere this does nothing. forking_enable(0) # Create and initialize IPC g_pool = Temp() g_pool.pupil_queue = Queue() g_pool.eye_rx, g_pool.eye_tx = Pipe(False) g_pool.quit = RawValue(c_bool,0) # this value will be substracted form the capture timestamp g_pool.timebase = RawValue(c_double,0) # make some constants avaiable g_pool.user_dir = user_dir g_pool.rec_dir = rec_dir g_pool.version = version g_pool.app = 'capture' # set up subprocesses p_eye = Process(target=eye, args=(g_pool,eye_src,eye_size)) # Spawn subprocess: p_eye.start() if platform.system() == 'Linux': # We need to give the camera driver some time before requesting another camera. sleep(0.5) world(g_pool,world_src,world_size) # Exit / clean-up p_eye.join()
def __init__(self, spider): Process.__init__(self) self.crawler = Crawler(spider.__class__, get_project_settings()) # self.crawler.configure() self.crawler.signals.connect(reactor.stop, signal=signals.spider_closed) self.spider = spider
def display_graph(self, graphdef): '''display a graph''' if 'mestate' in globals(): self.mestate.console.write("Expression: %s\n" % ' '.join(graphdef.expression.split())) else: self.mestate.child_pipe_send_console.send( "Expression: %s\n" % ' '.join(graphdef.expression.split())) #mestate.mlog.reduce_by_flightmodes(mestate.flightmode_selections) #setup the graph, then pass to a new process and display self.mg = grapher.MavGraph() self.mg.set_marker(self.mestate.settings.marker) self.mg.set_condition(self.mestate.settings.condition) self.mg.set_xaxis(self.mestate.settings.xaxis) self.mg.set_linestyle(self.mestate.settings.linestyle) self.mg.set_show_flightmode(self.mestate.settings.show_flightmode) self.mg.set_legend(self.mestate.settings.legend) self.mg.add_mav(self.mestate.mlog) for f in graphdef.expression.split(): self.mg.add_field(f) self.mg.process(self.mestate.flightmode_selections, self.mestate.mlog._flightmodes) self.lenmavlist = len(self.mg.mav_list) if platform.system() == 'Darwin': forking_enable(False) #Important - mg.mav_list is the full logfile and can be very large in size #To avoid slowdowns in Windows (which copies the vars to the new process) #We need to empty this var when we're finished with it self.mg.mav_list = [] child = Process(target=self.mg.show, args=[ self.lenmavlist, ]) child.start() self.mestate.mlog.rewind()
def crawl( self, origin_oj, solution_id, problem_id, language, code, username, nickname, password): p = Process( target=self._crawl, args=[ origin_oj, solution_id, problem_id, language, code, username, nickname, password ] ) p.start() p.join()
def __init__(self, spider): Process.__init__(self) setting = Settings() setting.setmodule(s) self.crawler = Crawler(setting) self.crawler.configure() self.crawler.signals.connect(reactor.stop, signal=signals.spider_closed) self.spider = spider
def main(): timePipe, sigPipe = Pipe() q = Queue() clock = Process(target=tickTock, args=(timePipe, )) testSignal = Process(target=signal, args=(q, sigPipe, [1])) testSignal.start() while True: print(q.get())
def check_user_presence(): for device in Device.objects.all(): lock_id = 'scan-device-lock-{}'.format(device.id) if acquire_lock(lock_id): t = Process(target=presence.scan_device, args=(device, lock_id)) t.start() return "ok"
def cmd_save(args): '''save a graph''' child = Process(target=save_process, args=[ mestate.last_graph, mestate.child_pipe_send_console, mestate.child_pipe_send_graph, mestate.status.msgs ]) child.start()
def test_set_pdeathsig(self): return_pid = Value('i') p = Process(target=parent_task, args=(return_pid,)) p.start() sleep(3) # wait for setting pdeathsig p.terminate() sleep(3) # wait for process termination with pytest.raises(psutil.NoSuchProcess): proc = psutil.Process(return_pid.value)
def crawl(self, spider): queue = Queue() self.queue = Queue() self.process = Process(target=self._crawl, args=(queue, spider)) self.process.start() write_in_a_file('.crawl 1', {'process': self.process, 'process-pid': self.process and self.process.pid, 'queue': self.queue.qsize()}, "t.txt") self.process.join() write_in_a_file('.crawl 2', {'process': self.process, 'process-pid': self.process and self.process.pid, 'queue': self.queue.qsize()}, "t.txt")
def telnet_client(server_id, port): key = 'server-{0}-pid'.format(server_id) pid = cache.get(key) if pid and pid in psutil.get_pid_list(): os.kill(pid, signal.SIGTERM) p = Process(target=TelnetClient, args=(port, 'uptee', server_id)) p.start() cache.set(key, p.pid) p.join()
def __init__(self, spider): Process.__init__(self) settings = get_project_settings() self.crawler = Crawler(settings) if not hasattr(project, 'crawler'): self.crawler.install() self.crawler.configure() self.crawler.signals.connect(reactor.stop, signal=signals.spider_closed) self.spider = spider
def crawl(self, number, passwd): p = Process( target=self._crawl, args=[ number, passwd ] ) p.start() p.join()
def init_marker_cacher(self): forking_enable(0) #for MacOs only from marker_detector_cacher import fill_cache visited_list = [False if x == False else True for x in self.cache] video_file_path = os.path.join(self.g_pool.rec_dir,'world.avi') self.cache_queue = Queue() self.cacher_seek_idx = Value(c_int,0) self.cacher_run = Value(c_bool,True) self.cacher = Process(target=fill_cache, args=(visited_list,video_file_path,self.cache_queue,self.cacher_seek_idx,self.cacher_run)) self.cacher.start()
def __init__(self, spider): Process.__init__(self) setting = Settings() setting.setmodule(settings,1) self.crawler = Crawler(setting) if not hasattr(project, 'crawler'): self.crawler.configure() self.crawler.signals.connect(reactor.stop, signal = signals.spider_closed) self.spider = spider
class NewBilliardMPWorker(CommonWorker): def __init__(self, callback=None): super(NewBilliardMPWorker).__init__(callback) def start(self): self.task = Process(target=common_run, daemon=True) self.task.start() def stop(self): pass
def crawl(self, oj, run_id): p = Process( target=self._crawl, args=[ oj, run_id ] ) p.start() p.join()
def crawl(self, origin_oj, username): p = Process( target=self._crawl, args=[ origin_oj, username, ] ) p.start() p.join()
def __init__(self, spider,key_word,crawl_num,n_crawls): Process.__init__(self) settings = get_project_settings() self.spider = spider self.crawler = Crawler(spider.__class__, settings) # self.crawler.configure() self.crawler.signals.connect(reactor.stop, signal=signals.spider_closed) self.n_crawls = n_crawls self.crawl_num = crawl_num self.key_word = key_word
def crawl(self, oj, username, password): p = Process( target=self._crawl, args=[ oj, username, password ] ) p.start() p.join()
def init_marker_cacher(self): forking_enable(0) #for MacOs only from marker_detector_cacher import fill_cache visited_list = [False if x == False else True for x in self.cache] video_file_path = self.g_pool.capture.src timestamps = self.g_pool.capture.timestamps self.cache_queue = Queue() self.cacher_seek_idx = Value('i',0) self.cacher_run = Value(c_bool,True) self.cacher = Process(target=fill_cache, args=(visited_list,video_file_path,timestamps,self.cache_queue,self.cacher_seek_idx,self.cacher_run,self.min_marker_perimeter_cacher)) self.cacher.start()
def main(): # To assign camera by name: put string(s) in list eye_cam_names = ["USB 2.0 Camera","Microsoft", "6000","Integrated Camera","HD USB Camera"] world_src = ["Logitech Camera","(046d:081d)","C510","B525", "C525","C615","C920","C930e"] eye_src = (eye_cam_names,0),(eye_cam_names,1) #first match for eye0 and second match for eye1 # to assign cameras directly, using integers as demonstrated below # eye_src = 4 , 5 #second arg will be ignored for monocular eye trackers # world_src = 1 # to use a pre-recorded video. # Use a string to specify the path to your video file as demonstrated below # eye_src = '/Users/mkassner/Downloads/000/eye0.mkv' , '/Users/mkassner/Downloads/eye.avi' # world_src = "/Users/mkassner/Downloads/000/world.mkv" # Camera video size in pixels (width,height) eye_size = (640,480) world_size = (1280,720) # on MacOS we will not use os.fork, elsewhere this does nothing. forking_enable(0) #g_pool holds variables. Only if added here they are shared across processes. g_pool = Global_Container() # Create and initialize IPC g_pool.pupil_queue = Queue() g_pool.quit = Value(c_bool,0) g_pool.timebase = Value(c_double,0) g_pool.eye_tx = [] # make some constants avaiable g_pool.user_dir = user_dir g_pool.version = get_version(version_file) g_pool.app = 'capture' g_pool.binocular = binocular p_eye = [] for eye_id in range(1+1*binocular): rx,tx = Pipe(False) p_eye += [Process(target=eye, args=(g_pool,eye_src[eye_id],eye_size,rx,eye_id))] g_pool.eye_tx += [tx] p_eye[-1].start() p_world = Process(target=world,args=(g_pool,world_src,world_size)) # world(g_pool,world_src,world_size) p_world.start() p_world.join() # Exit / clean-up for p in p_eye: p.join()
def __init__(self, fun, args, postprocess, job): """ Build multiprocessing queues and start worker. """ super(LongCalculation, self).__init__(job, "Cancel", 0, 0) self.setModal(True) self.input = Queue() self.output = Queue() self.input.put((fun, args, postprocess)) self.proc = Process(target=worker, args=(self.input, self.output)) self.proc.start() self.timer = QTimer() self.timer.timeout.connect(self.update) self.timer.start(10)
def run_player(self, graphdef): if 'mestate' in globals(): self.mestate.console.write("Running Player...") else: self.mestate.child_pipe_send_console.send("Running Player...") self.player = player.MavPlay() self.player.add_mav(self.mlog) if platform.system() == 'Darwin': forking_enable(False) child = Process(target=self.player.run) child.start()
def start_launch_servide_process(ld): """Starts a Launch Service process. To be called from subclasses. Args: ld : LaunchDescription obj. """ # Create the LauchService and feed the LaunchDescription obj. to it. ls = LaunchService() ls.include_launch_description(ld) p = Process(target=ls.run) p.daemon = True #The daemon process is terminated automatically before the main program exits, to avoid leaving orphaned processes running p.start()
class CrawlerScript(): def __init__(self): self.process = None self.items = [] self._count = 0 self.queue = None self._init_signals() def _init_signals(self): dispatcher.connect(self._so, signals.spider_opened) dispatcher.connect(self._item_scraped, signals.item_scraped) dispatcher.connect(self._sc, signals.spider_closed) def _so(self): write_in_a_file('spider_opened 1', {'open': 'open!', 'x': self.x, 'process': self.process, 'process-pid': self.process and self.process.pid}, "t.txt") def _sc(self): write_in_a_file('spider_closed', {'scraped items': len(self.items)}, "t.txt") def _item_scraped(self, item, **kwargs): self._count = self._count + 1 write_in_a_file('item scraped', {'count':self._count, 'item': item, 'kwargs':kwargs, 'process': self.process, 'process-pid': self.process and self.process.pid}, "t.txt") self.items.append(item) self.queue.put_nowait(item) def _crawl(self, queue, spider): crawler = CrawlerProcess(get_project_settings()) crawler.crawl(spider) write_in_a_file('signals', {'signals': dir(signals)}, 'task.txt') write_in_a_file('._crawl start', {'process': self.process, 'process-pid': self.process and self.process.pid, 'db': dir(db), 'db.connection': dir(db.connection)}, "t.txt") print(dir(db.connection)) db.connection.close() crawler.start() crawler.stop() write_in_a_file('._crawl ended 1', {'qsize': self.queue.qsize() }, "t.txt") queue.put_nowait(self.items) write_in_a_file('._crawlended after q 2', {'qsize': queue.qsize()}, "t.txt") def crawl(self, spider): queue = Queue() self.queue = Queue() self.process = Process(target=self._crawl, args=(queue, spider)) self.process.start() write_in_a_file('.crawl 1', {'process': self.process, 'process-pid': self.process and self.process.pid, 'queue': self.queue.qsize()}, "t.txt") self.process.join() write_in_a_file('.crawl 2', {'process': self.process, 'process-pid': self.process and self.process.pid, 'queue': self.queue.qsize()}, "t.txt")
def run_crawler_process(params: dict) -> Process: """ Start scrapy spider from a separate process :param dict params: scrapy spider parameters :return: process instance """ process = Process( target=run_crawler, args=(params, ), ) process.start() return process
def run_crawler(params): """ :param params: :return: process instance """ process = Process( target=crawler_process, args=(params, ), ) process.start() return process
def __init__(self, current_dt, server, topic): Process.__init__(self) self.current_dt = current_dt self.server = server self.topic = topic self.crawler = Crawler(VIXSpiderSpider, settings={'USER_AGENT': user_agent}) self.crawler.signals.connect(reactor.stop, signal=scrapy_signals.spider_closed)
def start_process(self, command=None, billiard=False, *args, **kwargs): from .models import ExportTask from ..tasks.export_tasks import TaskStates if billiard: proc = Process(daemon=False, *args, **kwargs) proc.start() self.store_pid(pid=proc.pid) proc.join() self.exitcode = proc.exitcode else: proc = subprocess.Popen(command, **kwargs) (self.stdout, self.stderr) = proc.communicate() self.store_pid(pid=proc.pid) self.exitcode = proc.wait() # We need to close the existing connection because the logger could be using a forked process which, # will be invalid and throw an error. connection.close() export_task = ExportTask.objects.get(uid=self.task_uid) if export_task.status == TaskStates.CANCELED.value: from ..tasks.exceptions import CancelException raise CancelException( task_name=export_task.export_provider_task.name, user_name=export_task.cancel_user.username)
def main(): # To assign camera by name: put string(s) in list eye_src = ["Microsoft", "6000","Integrated Camera"] world_src = ["Logitech Camera","(046d:081d)","C510","B525", "C525","C615","C920","C930e"] # to assign cameras directly, using integers as demonstrated below # eye_src = 1 # world_src = 0 # to use a pre-recorded video. # Use a string to specify the path to your video file as demonstrated below # eye_src = '/Users/mkassner/Pupil/datasets/p1-left/frames/test.avi' # world_src = "/Users/mkassner/Desktop/2014_01_21/000/world.avi" # Camera video size in pixels (width,height) eye_size = (640,360) world_size = (1280,720) # on MacOS we will not use os.fork, elsewhere this does nothing. forking_enable(0) # Create and initialize IPC g_pool = Temp() g_pool.pupil_queue = Queue() g_pool.eye_rx, g_pool.eye_tx = Pipe(False) g_pool.quit = RawValue(c_bool,0) # this value will be substracted form the capture timestamp g_pool.timebase = RawValue(c_double,0) # make some constants avaiable g_pool.user_dir = user_dir g_pool.rec_dir = rec_dir g_pool.version = version g_pool.app = 'capture' # set up subprocesses p_eye = Process(target=eye, args=(g_pool,eye_src,eye_size)) # Spawn subprocess: p_eye.start() if platform.system() == 'Linux': # We need to give the camera driver some time before requesting another camera. sleep(0.5) world(g_pool,world_src,world_size) # Exit / clean-up p_eye.join()
def cmd_map(args): '''map command''' from mavflightview import mavflightview_mav, mavflightview_options #mestate.mlog.reduce_by_flightmodes(mestate.flightmode_selections) #setup and process the map options = mavflightview_options() options.condition = mestate.settings.condition options._flightmodes = mestate.mlog._flightmodes options.show_flightmode_legend = mestate.settings.show_flightmode if len(args) > 0: options.types = ','.join(args) [path, wp, fen, used_flightmodes, mav_type] = mavflightview_mav(mestate.mlog, options, mestate.flightmode_selections) child = Process(target=map_process, args=[path, wp, fen, used_flightmodes, mav_type, options]) child.start() mestate.mlog.rewind()
def run(self, jobs): '''Start the Scrapy engine, and execute all jobs. Parms: jobs ([Job]) - one or more Job objects to be processed. ''' if not isinstance(jobs, collections.Iterable): jobs = [jobs] self.validate(jobs) p = Process(target=self._crawl, args=[jobs]) p.start() return p
def startLaunchServiceProcess(launchDesc): """Starts a Launch Service process. To be called from subclasses. Args: launchDesc : LaunchDescription obj. """ # Create the LauchService and feed the LaunchDescription obj. to it. launchService = LaunchService() launchService.include_launch_description(launchDesc) process = Process(target=launchService.run) # The daemon process is terminated automatically before the main program exits, # to avoid leaving orphaned processes running process.daemon = True process.start() return process
def main(): # To assign camera by name: put string(s) in list eye_cam_names = [ "USB 2.0 Camera", "Microsoft", "6000", "Integrated Camera", "HD USB Camera" ] world_src = [ "Logitech Camera", "(046d:081d)", "C510", "B525", "C525", "C615", "C920", "C930e" ] eye_src = (eye_cam_names, 0), (eye_cam_names, 1 ) #first match for eye0 and second match for eye1 # to assign cameras directly, using integers as demonstrated below # eye_src = 4 , 5 #second arg will be ignored for monocular eye trackers # world_src = 1 # to use a pre-recorded video. # Use a string to specify the path to your video file as demonstrated below # eye_src = '/Users/mkassner/Downloads/000/eye0.mkv' , '/Users/mkassner/Downloads/eye.avi' # world_src = "/Users/mkassner/Downloads/000/world.mkv" # Camera video size in pixels (width,height) eye_size = (640, 480) world_size = (1280, 720) # on MacOS we will not use os.fork, elsewhere this does nothing. forking_enable(0) #g_pool holds variables. Only if added here they are shared across processes. g_pool = Global_Container() # Create and initialize IPC g_pool.pupil_queue = Queue() g_pool.quit = Value(c_bool, 0) g_pool.timebase = Value(c_double, 0) g_pool.eye_tx = [] # make some constants avaiable g_pool.user_dir = user_dir g_pool.version = get_version(version_file) g_pool.app = 'capture' g_pool.binocular = binocular p_eye = [] for eye_id in range(1 + 1 * binocular): rx, tx = Pipe(False) p_eye += [ Process(target=eye, args=(g_pool, eye_src[eye_id], eye_size, rx, eye_id)) ] g_pool.eye_tx += [tx] p_eye[-1].start() world(g_pool, world_src, world_size) # Exit / clean-up for p in p_eye: p.join()
def __init__(self, title='MAVProxy: console'): if platform.system() == 'Darwin': forking_enable(False) textconsole.SimpleConsole.__init__(self) self.title = title self.menu_callback = None self.parent_pipe_recv, self.child_pipe_send = Pipe(duplex=False) self.child_pipe_recv, self.parent_pipe_send = Pipe(duplex=False) self.close_event = Event() self.close_event.clear() self.child = Process(target=self.child_task) self.child.start() self.child_pipe_send.close() self.child_pipe_recv.close() t = threading.Thread(target=self.watch_thread) t.daemon = True t.start()
def __init__(self, countries, importance, event_list, current_dt, server, topic): Process.__init__(self) self.countries = countries self.importance = importance self.event_list = event_list self.current_dt = current_dt self.server = server self.topic = topic self.crawler = Crawler(EconomicIndicatorsSpiderSpider, settings={'USER_AGENT': user_agent}) self.crawler.signals.connect(reactor.stop, signal=scrapy_signals.spider_closed)
class CrawlerScript(): def __init__(self): self.crawler = None self.process = None self.items = [] def _so(self): write_in_a_file('spider_opened 1', {}, "t.txt") def _sc(self): write_in_a_file('spider_closed', {'scraped items': len(self.items)}, "t.txt") def _so2(self): write_in_a_file('spider_opened 2', {}, "t.txt") def _item_scraped(self, item): write_in_a_file('item scraped', {'item': item}, "t.txt") self.items.append(item) def _crawl(self, queue, spider): self.crawler = CrawlerProcess(get_project_settings()) self.crawler.crawl(spider) dispatcher.connect(self._item_scraped, signals.item_scraped) dispatcher.connect(self._so, signals.spider_opened) dispatcher.connect(self._so2, signals.spider_opened) dispatcher.connect(self._sc, signals.spider_closed) write_in_a_file('crawler start', {'db': dir(db), 'db.connection': dir(db.connection)}, "t.txt") print(dir(db.connection)) db.connection.close() self.crawler.start() self.crawler.stop() write_in_a_file('crawler ended', {'qsize': queue.qsize() }, "t.txt") queue.put(self.items) def crawl(self, spider): queue = Queue() self.process = Process(target=self._crawl, args=(queue, spider,)) self.process.start() write_in_a_file('crawler started', {'crawler': dir(self.crawler)}, "t.txt") return self.process, queue#p.join()
def main(): # To assign camera by name: put string(s) in list eye_src = ["Microsoft", "6000","Integrated Camera"] world_src = ["Logitech Camera","B525", "C525","C615","C920","C930e"] # to assign cameras directly, using integers as demonstrated below # eye_src = 1 # world_src = 0 # to use a pre-recorded video. # Use a string to specify the path to your video file as demonstrated below # eye_src = "/Users/mkassner/Pupil/datasets/eye2_fieldtest/eye 10.avi" # world_src = "/Users/mkassner/Downloads/2013_10_22_M25/000/world.avi" # Camera video size in pixels (width,height) eye_size = (640,360) world_size = (1280,720) # on MacOS we will not use os.fork, elsewhere this does nothing. forking_enable(0) # Create and initialize IPC g_pool = Temp() g_pool.pupil_queue = Queue() g_pool.eye_rx, g_pool.eye_tx = Pipe(False) g_pool.quit = RawValue(c_bool,0) # make some constants avaiable g_pool.user_dir = user_dir g_pool.rec_dir = rec_dir g_pool.version = version # set up subprocesses p_eye = Process(target=eye, args=(g_pool,eye_src,eye_size)) # Spawn subprocess: p_eye.start() # On Linux, we need to give the camera driver some time before requesting another camera. sleep(0.5) # On MacOS cameras using MJPG compression (world camera) need to run in the main process. world(g_pool,world_src,world_size) # Exit / clean-up p_eye.join()
def main(): ProcessList = [] for i in range(0, 100): ProcessList.append(Process(target=f, args=(i, ))) for p in ProcessList: p.start() sleep(5) for p in ProcessList: p.join() print("main finished")
def init_marker_cacher(self): forking_enable(0) #for MacOs only from marker_detector_cacher import fill_cache visited_list = [False if x == False else True for x in self.cache] video_file_path = self.g_pool.capture.src self.cache_queue = Queue() self.cacher_seek_idx = Value('i',0) self.cacher_run = Value(c_bool,True) self.cacher = Process(target=fill_cache, args=(visited_list,video_file_path,self.cache_queue,self.cacher_seek_idx,self.cacher_run,self.min_marker_perimeter)) self.cacher.start()
def crawl_endpoint_to_file( self, ip_address=None, port=None, hostname=None, use_ssl=False, use_sni=False, start_urls=[], in_separate_process=True, ): """ Start crawling the given endpoint using the given list of URLs and write the results to a local file. :param ip_address: The IP address to crawl. :param port: The port where the application resides. :param hostname: The hostname to submit alongside all requests to the remote endpoint. :param use_ssl: Whether or not to use SSL to connect to the remote web service. :param use_sni: Whether or not to use SNI to connect to the remote web service. :param start_urls: A list of URLs to start crawling from. :param in_separate_process: Whether or not to spawn off a separate process for the crawl. This enables us to call this method multiple times in the same process, as a Twisted reactor can only be started and stopped once per process. :return: A tuple containing (1) the string containing the local file path where crawling results are stored and (2) a ScrapyResultWrapper configured to process the contents of the file. """ temp_file_path = FilesystemHelper.get_temporary_file_path() local_file_path = "%s-%s-%s:%s" % (temp_file_path, self.bot_name, ip_address, port) spider_kwargs = { "input_ip_address": ip_address, "input_start_urls": start_urls, "input_file_path": local_file_path, "input_hostname": hostname, "input_use_ssl": use_ssl, "input_use_sni": use_sni, "input_port": port, } pipeline_settings = self.__get_local_storage_item_pipeline() requested_hostname = hostname if hostname is not None else ip_address settings = self.get_scrapy_settings(item_pipeline=pipeline_settings, hostname=requested_hostname) crawling_config = { "spider_kwargs": spider_kwargs, "settings": settings, } if in_separate_process: process = Process(target=self.__crawl, kwargs=crawling_config) process.start() process.join() process.terminate() else: self.__crawl(**crawling_config) return local_file_path, ScrapyResultWrapper.from_file(local_file_path)
def screenshot_endpoint( self, ip_address=None, port=None, hostname=None, use_ssl=False, use_sni=False, path="/", in_separate_process=False, ): """ Take a screenshot of the given endpoint, save it to a local temporary file, and return the local file path. :param ip_address: The IP address where the web service resides. :param port: The port where the web service resides. :param hostname: The hostname to request. :param use_ssl: Whether or not to use SSL to request the endpoint. :param use_sni: Whether or not the endpoint uses SNI. :param path: The path of the resource to screenshot. :param in_separate_process: Whether or not to take the screenshot in a separate process. This is to address the incredibly long time that the Selenium webdriver can take when it hangs. :return: A tuple containing (1) the local file path where the screenshot was saved and (2) whether or not the screenshot was taken successfully. """ logger.debug( "Now attempting to take a screenshot of the web service at %s:%s (%s). Hostname is %s, SNI support is %s." % (ip_address, port, "using SSL" if use_ssl else "plain HTTP", hostname, use_sni)) self.__set_endpoint( ip_address=ip_address, port=port, hostname=hostname, use_ssl=use_ssl, use_sni=use_sni, path=path, ) self._output_file_path = self.get_temporary_file_path() if in_separate_process: process = Process(target=self.__take_screenshot) try: process.start() process.join(config.selenium_screenshot_delay + config.inspection_screenshot_join_timeout) except IOError as e: if e.errno == errno.EINTR: logger.warning("Interrupted system call error received.") else: raise e finally: if process.is_alive(): print("PROCESS IS ALIVE - PID IS %s" % (process.pid, )) os.kill(process.pid, signal.SIGTERM) else: self.__take_screenshot() return self.output_file_path, FilesystemHelper.does_file_exist( self.output_file_path)
def add_export(self): # on MacOS we will not use os.fork, elsewhere this does nothing. forking_enable(0) logger.debug("Adding new export.") should_terminate = RawValue(c_bool, False) frames_to_export = RawValue(c_int, 0) current_frame = RawValue(c_int, 0) data_dir = self.data_dir start_frame = self.start_frame.value end_frame = self.end_frame.value plugins = [] # Here we make clones of every plugin that supports it. # So it runs in the current config when we lauch the exporter. for p in self.g_pool.plugins: try: p_initializer = p.get_class_name(), p.get_init_dict() plugins.append(p_initializer) except AttributeError: pass out_file_path = verify_out_file_path(self.rec_name.value, self.data_dir) process = Process(target=export, args=(should_terminate, frames_to_export, current_frame, data_dir, start_frame, end_frame, plugins, out_file_path)) process.should_terminate = should_terminate process.frames_to_export = frames_to_export process.current_frame = current_frame process.out_file_path = out_file_path self.new_export = process
def add_exports(self): # on MacOS we will not use os.fork, elsewhere this does nothing. forking_enable(0) outfiles = set() for d in self.new_exports: logger.debug("Adding new export.") should_terminate = RawValue(c_bool,False) frames_to_export = RawValue(c_int,0) current_frame = RawValue(c_int,0) start_frame = None end_frame = None data_dir = d plugins = [] # Here we make clones of every plugin that supports it. # So it runs in the current config when we lauch the exporter. for p in self.g_pool.plugins: try: p_initializer = p.get_class_name(),p.get_init_dict() plugins.append(p_initializer) except AttributeError: pass #make a unique name created from rec_session and dir name rec_session, rec_dir = data_dir.rsplit(os.path.sep,2)[1:] out_name = rec_session+"_"+rec_dir+".avi" out_file_path = os.path.join(self.destination_dir.value,out_name) if out_file_path in outfiles: logger.error("This export setting would try to save %s at least twice please rename dirs to prevent this. Skipping File"%out_file_path) else: outfiles.add(out_file_path) logger.info("Exporting to: %s"%out_file_path) process = Process(target=export, args=(should_terminate,frames_to_export,current_frame, data_dir,start_frame,end_frame,plugins,out_file_path)) process.should_terminate = should_terminate process.frames_to_export = frames_to_export process.current_frame = current_frame process.out_file_path = out_file_path self.exports.append(process)
def main(): # IPC pupil_queue = Queue() timebase = Value(c_double, 0) cmd_world_end, cmd_launcher_end = Pipe() com0 = Pipe(True) eyes_are_alive = Value(c_bool, 0), Value(c_bool, 0) com1 = Pipe(True) com_world_ends = com0[0], com1[0] com_eye_ends = com0[1], com1[1] p_world = Process( target=world, args=( pupil_queue, timebase, cmd_world_end, com_world_ends, eyes_are_alive, user_dir, app_version, video_sources["world"], ), ) p_world.start() while True: # block and listen for commands from world process. cmd = cmd_launcher_end.recv() if cmd == "Exit": break else: eye_id = cmd p_eye = Process( target=eye, args=( pupil_queue, timebase, com_eye_ends[eye_id], eyes_are_alive[eye_id], user_dir, app_version, eye_id, video_sources["eye%s" % eye_id], ), ) p_eye.start() for p in active_children(): p.join() logger.debug("Laucher exit")
def run(self, jobs): '''Start the Scrapy engine, and execute all jobs. Return consolidated results in a single list. Parms: jobs ([Job]) - one or more Job objects to be processed. Returns: List of objects yielded by the spiders after all jobs have run. ''' if not isinstance(jobs, collections.Iterable): jobs = [jobs] self.validate(jobs) p = Process(target=self._crawl, args=[jobs]) p.start() p.join() p.terminate() return self.results.get()
def test_value(self, raw=False): if raw: values = [RawValue(code, value) for code, value, _ in self.codes_values] else: values = [Value(code, value) for code, value, _ in self.codes_values] for sv, cv in zip(values, self.codes_values): assert sv.value == cv[1] proc = Process(target=self._test, args=(values,)) proc.daemon = True proc.start() proc.join() for sv, cv in zip(values, self.codes_values): assert sv.value == cv[2]
def start_process(self, command=None, billiard=False, *args, **kwargs): from eventkit_cloud.tasks.models import ExportTaskRecord from eventkit_cloud.tasks import TaskStates if billiard: proc = Process(daemon=False, *args, **kwargs) proc.start() self.store_pid(pid=proc.pid) proc.join() self.exitcode = proc.exitcode else: proc = subprocess.Popen(command, **kwargs) (self.stdout, self.stderr) = proc.communicate() self.store_pid(pid=proc.pid) self.exitcode = proc.wait() # We need to close the existing connection because the logger could be using a forked process which, # will be invalid and throw an error. connection.close() export_task = ExportTaskRecord.objects.filter(uid=self.task_uid).first() if export_task and export_task.status == TaskStates.CANCELED.value: from eventkit_cloud.tasks.exceptions import CancelException raise CancelException(task_name=export_task.export_provider_task.name, user_name=export_task.cancel_user.username)
def __init__(self, spider): Process.__init__(self) self.spider = spider