Пример #1
0
 def __init__(self, threads=1000, initialize_glance_store=False):
     os.umask(0o27)  # ensure files are created with the correct privileges
     self._logger = logging.getLogger("eventlet.wsgi.server")
     self.threads = threads
     self.children = set()
     self.stale_children = set()
     self.running = True
     # NOTE(abhishek): Allows us to only re-initialize glance_store when
     # the API's configuration reloads.
     self.initialize_glance_store = initialize_glance_store
     self.pgid = os.getpid()
     try:
         # NOTE(flaper87): Make sure this process
         # runs in its own process group.
         os.setpgid(self.pgid, self.pgid)
     except OSError:
         # NOTE(flaper87): When running glance-control,
         # (glance's functional tests, for example)
         # setpgid fails with EPERM as glance-control
         # creates a fresh session, of which the newly
         # launched service becomes the leader (session
         # leaders may not change process groups)
         #
         # Running glance-(api|registry) is safe and
         # shouldn't raise any error here.
         self.pgid = 0
Пример #2
0
 def __start_pgrp(self):
     # start a dedicated dummy process, having its own process
     # group, in order to group all processes handled by this
     # conductor
     ppid = os.getpid()
     pid = os.fork()
     if pid == 0:
         os.setpgid(0, 0)
         os.chdir("/")
         os.umask(0)
         maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
         if (maxfd == resource.RLIM_INFINITY):
             maxfd = MAXFD
         for fd in range(0, maxfd):
             try: os.close(fd)
             except OSError: pass
         while True:
             # poll each second that my parent is still alive. If
             # not, die. Optionnaly kill all instanciated childs.
             if os.getppid() != ppid:
                 if configuration['kill_childs_at_end']:
                     os.killpg(0, signal.SIGTERM)
                 os._exit(0)
             time.sleep(1)
     else:
         return pid
Пример #3
0
        def _setupChild(self, *args, **kwargs):
            Process._setupChild(self, *args, **kwargs)

            # this will cause the child to be the leader of its own process group;
            # it's also spelled setpgrp() on BSD, but this spelling seems to work
            # everywhere
            os.setpgid(0, 0)
Пример #4
0
def _set_process_group(parent_process_group_id):
    """Set group_id of the child process to the group_id of the parent process.

    This way when you delete the parent process you also delete all the children.
    """
    child_process_id = os.getpid()
    os.setpgid(child_process_id, parent_process_group_id)
Пример #5
0
    def launch(self, pgid, infile, outfile, errfile, fg):
        # TODO exception handling

        pid = os.fork()
        
        if pid == 0:
            if _terminal.interactive:
                pid = os.getpid()
                if pgid == 0:
                    pgid = pid
                os.setpgid(pid, pgid)
                if fg:
                   _terminal.grab_control(pgid)
                default_signals()
            if infile != sys.stdin.fileno():
                os.dup2(infile, sys.stdin.fileno())
            if outfile != sys.stdout.fileno():
                os.dup2(outfile, sys.stdout.fileno())
            if errfile != sys.stderr.fileno():
                os.dup2(errfile, sys.stderr.fileno())

            try:
                self._launch()
            except Exception as e:
                print('pysh: {}: {}'.format(self.argv[0], str(e)), file=sys.stderr)
            sys.exit(127)
        return pid
Пример #6
0
 def __init__(self, threads=1000, workers=0):
     os.umask(0o27)  # ensure files are created with the correct privileges
     self._logger = logging.getLogger("eventlet.wsgi.server")
     self._wsgi_logger = loggers.WritableLogger(self._logger)
     self.threads = threads
     self.children = set()
     self.stale_children = set()
     self.running = True
     self.pgid = os.getpid()
     self.workers = workers
     try:
         # NOTE(flaper87): Make sure this process
         # runs in its own process group.
         os.setpgid(self.pgid, self.pgid)
     except OSError:
         # NOTE(flaper87): When running searchlight-control,
         # (searchlight's functional tests, for example)
         # setpgid fails with EPERM as searchlight-control
         # creates a fresh session, of which the newly
         # launched service becomes the leader (session
         # leaders may not change process groups)
         #
         # Running searchlight-api is safe and
         # shouldn't raise any error here.
         self.pgid = 0
Пример #7
0
def main(argv):
  # Process argv
  if len(argv) <= 1:
    print "\nUsage: python flawless_daemon.py FLAWLESS_CONFIG_PATH [PID_FILE_PATH]"
    print "  FLAWLESS_CONFIG_PATH - The path to the flawless.cfg config you want to use"
    print "  PID_FILE_PATH - (optional) The path you want to for the PID lock file\n"
    return
  flawless_cfg_path = os.path.abspath(argv[1])
  pid_file_path = os.path.abspath(argv[2]) if len(argv) == 3 else None

  # Initialize context
  context = daemon.DaemonContext()
  context.detach_process = True
  context.working_directory = '.'

  # Setup logging of output
  pid = os.getpid()
  filename = "flawless-%d.ERROR" % pid
  error_log = open(filename, "w+", 1)
  context.stdout = error_log
  context.stderr = error_log

  # Setup PID file
  if pid_file_path:
    context.pidfile = daemon.pidlockfile.TimeoutPIDLockFile(pid_file_path, acquire_timeout=2)

  try:
    with context:
      os.setpgid(0, os.getpid())
      flawless.server.server.serve(flawless_cfg_path)
  except lockfile.LockTimeout:
    sys.stderr.write("Error: Couldn't acquire lock on %s. Exiting.\n" % pid_file_path)
    sys.exit(1)
Пример #8
0
def startNewTripGPS(whichTrip):
	##Need to halt myGpsPipe, then move "CURRENT.txt" into the current trip we are working on.
	#First must stop GPS pipe process
	p= subprocess.Popen(['ps','-A'],stdout=subprocess.PIPE)
	out,err=p.communicate()

	for line in out.splitlines():
		if 'myGpsPipe' in line:
			pid=int(line.strip().split(' ')[0])
			print 'myGpsPipe on pid='+str(pid)+',should kill'
			os.kill(pid,9)			#command is "kill -9 pid"
	
	#Process is killed, need to call nmeaLocation.py to parse for time,date,locations.
	os.system('/home/root/Documents/beagle-bone.git/NMEA/nmeaLocation.py')
	#Waits for return, then we can move the CURRENT.txt into the trip folder we want.
	mvCommand='mv '+boneGPSpath+'PARSED.txt '+boneGPSpath+str(whichTrip)+'.txt'
	rmCommand='rm '+boneGPSpath+'CURRENT.txt'
	os.system(mvCommand)
	os.system(rmCommand)
	filep=boneGPSpath+'.info'
	OF=open(filep,'w')
	OF.write(str(whichTrip))		#Which trip actually represents how many trips are there.
	OF.close()						

	#Restart myGpsPipe script.
	pid=os.fork()
	if pid==0:
		os.setpgid(0,0)
		args=['/home/root/Documents/beagle-bone.git/NMEA/myGpsPipe.py','']
		os.execv(args[0],args)
Пример #9
0
def demonize():
    notify_parent = NotifyParent()

    if os.fork():
        # In the command-line parent. Wait for child status.
        status = notify_parent.read_and_close()

        if status.startswith("success"):
            raise SystemExit(0)
        else:
            raise SystemExit(1)

    # In the child, which is the flup server.
    try:
        # Create a new session with this process as the group leader
        try:
            setsid = os.setsid
        except AttributeError:
            os.setpgid(0, 0)
        else:
            setsid()
    except:
        notify_parent.failure()
        raise
    return notify_parent
Пример #10
0
    def _fork(self, path, href, text):
        pid = os.fork()

        # Parents can now bail.
        if pid:
            return pid

        # Make sure that we quote href such that malicious URLs like
        # "http://example.com & rm -rf ~/" won't be interpreted by the shell.

        href = shlex.quote(href)

        # A lot of programs don't appreciate
        # having their fds closed, so instead
        # we dup them to /dev/null.

        fd = os.open("/dev/null", os.O_RDWR)
        os.dup2(fd, sys.stderr.fileno())

        if not text:
            os.setpgid(os.getpid(), os.getpid())
            os.dup2(fd, sys.stdout.fileno())
            os.dup2(fd, sys.stdin.fileno())

        if "%u" in path:
            path = path.replace("%u", href)
        elif href:
            path = path + " " + href

        os.execv("/bin/sh", ["/bin/sh", "-c", path])

        # Just in case.
        sys.exit(0)
Пример #11
0
    def pipeline_child(self):
        # child process must close STDIN, otherwise it gets SIGTSTP
        devnull = os.open('/dev/null', os.O_RDONLY)
        os.dup2(devnull, 0)

        signal.signal(signal.SIGCHLD, self.sigchld)
        signal.signal(signal.SIGINT, self.sigint)

        os.setpgid(0, 0)

        try:
            # set up pipeline
            pipeline = self.pipeline( )
            last_fd = devnull
            children = []

            # spawn the child processes
            for x in range(len(pipeline)):
                if x == len(pipeline) - 1:
                    new_stdout = None
                    next_fd = None
                else:
                    # make pipe for stdout
                    (next_fd, new_stdout) = os.pipe( )
                
                try:
                    child_pid = os.fork( )

                    if child_pid:
                        children.append(child_pid)
                        os.close(last_fd)
                        last_fd = next_fd
                        if new_stdout is not None:
                            os.close(new_stdout)

                    else:
                        # child process - redirect stdin and stdout then go go go
                        os.dup2(last_fd, 0)
                        if new_stdout is not None:
                            os.dup2(new_stdout, 1)
                        self.exec_with_args(pipeline[x][0], pipeline[x][1])

                except OSError:
                    raise


            status = 0
            while len(children) > 0:
                try:
                    (pid, status) = os.wait( )
                    children.remove(pid)
                except OSError, e:
                    if e.errno == 4:
                        # interrupted system call - just try again...
                        pass
                    else:
                        # something dire happened so re-raise it
                        raise

            os._exit(status) # not 100% perfect...
Пример #12
0
    def create_zygote(self):
        """"Create a new zygote"""
        # read the basepath symlink
        realbase = os.path.realpath(self.basepath)

        pid = os.fork()
        if pid:
            log.info('started zygote %d pointed at base %r', pid, realbase)
            z = self.zygote_collection.add_zygote(pid, realbase, self.io_loop)
            self.current_zygote = z
            self.io_loop.add_callback(self.transition_idle_workers)
            return z
        else:
            # Try to clean up some of the file descriptors and whatnot that
            # exist in the parent before continuing. Strictly speaking, this
            # isn't necessary, but it seems good to remove these resources
            # if they're not needed in the child.
            del self.io_loop
            close_fds(self.sock.fileno())
            signal.signal(signal.SIGHUP, signal.SIG_DFL)

            # Make the zygote a process group leader
            os.setpgid(os.getpid(), os.getpid())
            # create the zygote
            z = ZygoteWorker(self.sock, realbase, self.module, self.application_args)
            z.loop()
Пример #13
0
    def do_run(self, output):
        self.pid = os.fork()

        if self.pid == 0:
            # Child

            os.setpgid(0, 0)

            os.dup2(output.fileno(), 1)
            os.dup2(output.fileno(), 2)

            os.execv(self.executable, [self.executable] + self.args)

            os._exit(255)

        # Parent

        while True:
            if self.deadline and unixtime_now() > self.deadline:
                self.timeout_handler()

            if self.poll():
                break

            sleep(1)
Пример #14
0
 def test_get_pids_for_pgrp(self):
     """Smoke test for get_pids_for_pgrp()."""
     # Move this process to a dedicated process group
     os.setpgid(0, self.pid)
     pids = get_pids_for_pgrp(self.pid)
     self.assertNotEqual(pids, [])
     self.assertIn(self.pid, pids)
Пример #15
0
def setPgid():
    """
    preexec_fn for Popen to set subprocess pgid
    
    """

    os.setpgid( os.getpid(), 0 )
Пример #16
0
def startOOAndConnect():
    """
    Start OO in child process and connect to them.
    Return OO component context object.
    """
    pid = os.fork()
    if not pid:
      pid = os.getpid()
      os.setpgid(pid,pid)
      os.system("soffice -headless -norestore '-accept=socket,host=localhost,port=%s;urp;' &" % OOCONV_PORT)
      while 1:
          time.sleep(1)
    else:
        limit = time.time() + OOCONV_MAX_STARTUP_TIME
        ctx = None
        context = uno.getComponentContext()
        resolver=context.ServiceManager.createInstanceWithContext(
            "com.sun.star.bridge.UnoUrlResolver", context)

        while time.time() < limit:
            try:
                ctx = resolver.resolve(
                    "uno:socket,host=localhost,port=%s;urp;StarOffice.ComponentContext" % OOCONV_PORT)
                break
            except:
                pass
            time.sleep(5)
        if ctx is None:
            pgid = os.getpgid(pid)
            os.killpg(pgid, signal.SIGTERM)                        
            return None, None
        return ctx, pid
Пример #17
0
    def start_daemon(self):
        pidfile = self.conf_dir + "/pid"
        if os.path.exists(pidfile) and os.path.isfile(pidfile):
            try:
                pf = open(pidfile, "a+")
                fcntl.flock(pf.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
                fcntl.flock(pf.fileno(), fcntl.LOCK_UN)
                pf.close()
            except IOError as e:
                if e.errno == errno.EAGAIN:
                    # If we failed to get a lock, then the daemon is running
                    # and we're done.
                    return

        pid = os.fork()
        if not pid:
            # Shutup any log output before canto-daemon sets up it's log
            # (particularly the error that one is already running)

            fd = os.open("/dev/null", os.O_RDWR)
            os.dup2(fd, sys.stderr.fileno())

            os.setpgid(os.getpid(), os.getpid())
            os.execve("/bin/sh",
                     ["/bin/sh", "-c", "canto-daemon -D " + self.conf_dir],
                     os.environ)

            # Should never get here, but just in case.
            sys.exit(-1)

        while not os.path.exists(self.socket_path):
            time.sleep(0.1)

        return pid
 def _start_stat_server(self):
     stat_server_path = os.path.join(test_manager.CODE_DEST_DIR, 'stat_server/entry_point.py')
     stat_server_proc_args = ['python', stat_server_path]
     stat_server_proc = subprocess.Popen(stat_server_proc_args)
     stat_server_pid = stat_server_proc.pid
     os.setpgid(stat_server_pid, stat_server_pid)
     time.sleep(5)
     return stat_server_pid
Пример #19
0
 def _stop_worker(self, worker_id):
     proc = self.processes.pop(worker_id, None)
     if proc:
         try:
             os.setpgid(proc.pid, proc.pid)
         except OSError:
             return
         proc.terminate()
Пример #20
0
def posix_setpgid(space, pid, pgrp):
    """ posix_setpgid - Set process group id for job control """
    try:
        os.setpgid(pid, pgrp)
        return space.newbool(True)
    except OSError, e:
        space.set_errno(e.errno)
        return space.newbool(False)
Пример #21
0
 def make_process_independent():
   processId = os.getpid()
   if processId > 0:
     try:
       os.setpgid(processId, processId)
     except OSError, e:
       print_warning_msg('setpgid({0}, {0}) failed - {1}'.format(pidJava, str(e)))
       pass
Пример #22
0
def setpgid(space, pid, pgrp):
    """ setpgid(pid, pgrp)

    Call the system call setpgid().
    """
    try:
        os.setpgid(pid, pgrp)
    except OSError, e:
        raise wrap_oserror(space, e)
Пример #23
0
	def start(self):
		"""Create the subprocess. Calls pre_fork() and forks.
		The parent then calls parent_post_fork() and returns,
		while the child calls child_post_fork() and then
		child_run()."""
		
		assert self.child is None

		stderr_r = stderr_w = None

		try:
			self.pre_fork()
			stderr_r, stderr_w = os.pipe()
			child = os.fork()
		except:
			if stderr_r: os.close(stderr_r)
			if stderr_w: os.close(stderr_w)
			self.start_error()
			raise

		if child == 0:
			# This is the child process
			try:
				try:
					os.setpgid(0, 0)  # Start a new process group
					os.close(stderr_r)

					if stderr_w != 2:
						os.dup2(stderr_w, 2)
						os.close(stderr_w)

					self.child_post_fork()
					self.child_run()
					raise Exception('child_run() returned!')
				except:
					import traceback
					traceback.print_exc()
			finally:
				os._exit(1)
			assert 0

		self.child = child

		# This is the parent process
		os.close(stderr_w)
		self.err_from_child = stderr_r

		import gobject
		if not hasattr(gobject, 'io_add_watch'):
			self.tag = g.input_add_full(self.err_from_child,
					g.gdk.INPUT_READ, self._got_errors)
		else:
			self.tag = gobject.io_add_watch(self.err_from_child,
					gobject.IO_IN | gobject.IO_HUP | gobject.IO_ERR,
					self._got_errors)

		self.parent_post_fork()
Пример #24
0
 def restore_input_output(self):
     if self.__old_stdout is not None:
         sys.stdout.flush()
         # now we reset stdout to be the whatever it was before
         sys.stdout = self.__old_stdout
     if self.__old_stdin is not None:
         sys.stdin = self.__old_stdin
     if self.__old_pgid is not None:
         os.setpgid(0, self.__old_pgid)
Пример #25
0
 def switch_pgid(self):
     try:
         if os.getpgrp() != os.tcgetpgrp(0):
             self.__old_pgid = os.getpgrp()
             os.setpgid(0, os.tcgetpgrp(0))
         else:
             self.__old_pgid = None
     except OSError:
         self.__old_pgid = None
Пример #26
0
    def start(self, application, default_port):
        """
        Run a WSGI server with the given application.

        :param application: The application to be run in the WSGI server
        :param default_port: Port to bind to if none is specified in conf
        """
        pgid = os.getpid()
        try:
            # NOTE(flaper87): Make sure this process
            # runs in its own process group.
            os.setpgid(pgid, pgid)
        except OSError:
            # NOTE(flaper87): When running glance-control,
            # (glance's functional tests, for example)
            # setpgid fails with EPERM as glance-control
            # creates a fresh session, of which the newly
            # launched service becomes the leader (session
            # leaders may not change process groups)
            #
            # Running glance-(api|registry) is safe and
            # shouldn't raise any error here.
            pgid = 0

        def kill_children(*args):
            """Kills the entire process group."""
            signal.signal(signal.SIGTERM, signal.SIG_IGN)
            signal.signal(signal.SIGINT, signal.SIG_IGN)
            self.running = False
            os.killpg(pgid, signal.SIGTERM)

        def hup(*args):
            """
            Shuts down the server, but allows running requests to complete
            """
            signal.signal(signal.SIGHUP, signal.SIG_IGN)
            self.running = False

        self.application = application
        self.sock = get_socket(default_port)

        os.umask(0o27)  # ensure files are created with the correct privileges
        self._logger = logging.getLogger("eventlet.wsgi.server")
        self._wsgi_logger = logging.WritableLogger(self._logger)

        if CONF.workers == 0:
            # Useful for profiling, test, debug etc.
            self.pool = self.create_pool()
            self.pool.spawn_n(self._single_run, self.application, self.sock)
            return
        else:
            LOG.info(_LI("Starting %d workers") % CONF.workers)
            signal.signal(signal.SIGTERM, kill_children)
            signal.signal(signal.SIGINT, kill_children)
            signal.signal(signal.SIGHUP, hup)
            while len(self.children) < CONF.workers:
                self.run_child()
Пример #27
0
    def _InitializeChild(self):

        # Put the child into its own process group.  This step is
        # performed in both the parent and the child; therefore both
        # processes can safely assume that the creation of the process
        # group has taken place.
        if self.__UseSeparateProcessGroupForChild():
            os.setpgid(0, 0)

        super(TimeoutExecutable, self)._InitializeChild()
Пример #28
0
 def does_stuff():
     os.setpgid(0, 0)     # become its own separated process group
     rsignal.pypysig_setflag(signal.SIGUSR1)
     os.killpg(os.getpgrp(), signal.SIGUSR1)
     rsignal.pypysig_ignore(signal.SIGUSR1)
     while True:
         n = rsignal.pypysig_poll()
         if n < 0 or n == signal.SIGUSR1:
             break
     return n
Пример #29
0
def execWithRedirect(command, argv, stdin = 0, stdout = 1, stderr = 2,
                searchPath = 0, root = '/', newPgrp = 0, ignoreTermSigs = 0):
    stdin = getfd(stdin)
    if stdout == stderr:
        stdout = getfd(stdout)
        stderr = stdout
    else:
        stdout = getfd(stdout)
        stderr = getfd(stderr)

    childpid = os.fork()
    if (not childpid):
        if (root and root != '/'):
            os.chroot(root)
            os.chdir("/")

        if ignoreTermSigs:
            signal.signal(signal.SIGTSTP, signal.SIG_IGN)
            signal.signal(signal.SIGINT, signal.SIG_IGN)

        if type(stdin) == type("a"):
            stdin = os.open(stdin, os.O_RDONLY)
        if type(stdout) == type("a"):
            stdout = os.open(stdout, os.O_RDWR)
        if type(stderr) == type("a"):
            stderr = os.open(stderr, os.O_RDWR)

        if stdin != 0:
            os.dup2(stdin, 0)
            os.close(stdin)
        if stdout != 1:
            os.dup2(stdout, 1)
            if stdout != stderr:
                os.close(stdout)
        if stderr != 2:
            os.dup2(stderr, 2)
            os.close(stderr)

        if (searchPath):
            os.execvp(command, argv)
        else:
            os.execv(command, argv)

        sys.exit(1)

    if newPgrp:
        os.setpgid(childpid, childpid)
        oldPgrp = os.tcgetpgrp(0)
        os.tcsetpgrp(0, childpid)

    status = -1
    try:
        (pid, status) = os.waitpid(childpid, 0)
    except OSError, (errno, msg):
        print __name__, "waitpid:", msg
 def make_process_independent():
   if IS_FOREGROUND: # upstart script is not able to track process from different pgid.
     return
   
   processId = os.getpid()
   if processId > 0:
     try:
       os.setpgid(processId, processId)
     except OSError, e:
       print_warning_msg('setpgid({0}, {0}) failed - {1}'.format(pidJava, str(e)))
       pass
Пример #31
0
 def preexec_fn(self):
     os.setpgid(0, 0)
    n_input=4,
    n_output=nb_output,
    wrapper_file=program_wrapper,
    hosts=hosts,
    cleanup=cleanup,
    files_to_send=[program],
    tmpdir=tmpdir,
    user_data=data)

if test_analytical:
    dist_func.set_separate_workdir(False)

if 'win' not in sys.platform:
    # change group pid in order to avoid wrapper_launcher destroying parent process
    # when interrupting
    os.setpgid(0, 0)

model = ot.NumericalMathFunction(dist_func)

# create sample
inS = ot.NumericalSample(sample_size, 4)
if not make_error:
    F = 2
else:
    F = 666
for i in range(sample_size):
    inS[i, 0] = i + 1
    inS[i, 1] = F
    inS[i, 2] = work_time
    inS[i, 3] = nb_output
Пример #33
0
def manager():
    # Create a new process group to corral our children
    os.setpgid(0, 0)

    # Create a listening socket on the AF_INET loopback interface
    listen_sock = socket.socket(AF_INET, SOCK_STREAM)
    listen_sock.bind(('127.0.0.1', 0))
    listen_sock.listen(max(1024, SOMAXCONN))
    listen_host, listen_port = listen_sock.getsockname()

    # re-open stdin/stdout in 'wb' mode
    stdin_bin = os.fdopen(sys.stdin.fileno(), 'rb', 4)
    stdout_bin = os.fdopen(sys.stdout.fileno(), 'wb', 4)
    write_int(listen_port, stdout_bin)
    stdout_bin.flush()

    def shutdown(code):
        signal.signal(SIGTERM, SIG_DFL)
        # Send SIGHUP to notify workers of shutdown
        os.kill(0, SIGHUP)
        exit(code)

    def handle_sigterm(*args):
        shutdown(1)

    signal.signal(SIGTERM, handle_sigterm)  # Gracefully exit on SIGTERM
    signal.signal(SIGHUP, SIG_IGN)  # Don't die on SIGHUP
    signal.signal(SIGCHLD, SIG_IGN)

    reuse = os.environ.get("SPARK_REUSE_WORKER")

    # Initialization complete
    try:
        while True:
            try:
                ready_fds = select.select([0, listen_sock], [], [], 1)[0]
            except select.error as ex:
                if ex[0] == EINTR:
                    continue
                else:
                    raise

            if 0 in ready_fds:
                try:
                    worker_pid = read_int(stdin_bin)
                except EOFError:
                    # Spark told us to exit by closing stdin
                    shutdown(0)
                try:
                    os.kill(worker_pid, signal.SIGKILL)
                except OSError:
                    pass  # process already died

            if listen_sock in ready_fds:
                try:
                    sock, _ = listen_sock.accept()
                except OSError as e:
                    if e.errno == EINTR:
                        continue
                    raise

                # Launch a worker process
                try:
                    pid = os.fork()
                except OSError as e:
                    if e.errno in (EAGAIN, EINTR):
                        time.sleep(1)
                        pid = os.fork()  # error here will shutdown daemon
                    else:
                        outfile = sock.makefile(mode='wb')
                        write_int(e.errno,
                                  outfile)  # Signal that the fork failed
                        outfile.flush()
                        outfile.close()
                        sock.close()
                        continue

                if pid == 0:
                    # in child process
                    listen_sock.close()
                    try:
                        # Acknowledge that the fork was successful
                        outfile = sock.makefile(mode="wb")
                        write_int(os.getpid(), outfile)
                        outfile.flush()
                        outfile.close()
                        authenticated = False
                        while True:
                            code = worker(sock, authenticated)
                            if code == 0:
                                authenticated = True
                            if not reuse or code:
                                # wait for closing
                                try:
                                    while sock.recv(1024):
                                        pass
                                except Exception:
                                    pass
                                break
                            gc.collect()
                    except:
                        traceback.print_exc()
                        os._exit(1)
                    else:
                        os._exit(0)
                else:
                    sock.close()

    finally:
        shutdown(1)
Пример #34
0
def main():
    if len(sys.argv) < 5:
        print(
            json.dumps({
                "failed":
                True,
                "msg":
                "usage: async_wrapper <jid> <time_limit> <modulescript> <argsfile> [-preserve_tmp]  "
                "Humans, do not call directly!"
            }))
        sys.exit(1)

    jid = "%s.%d" % (sys.argv[1], os.getpid())
    time_limit = sys.argv[2]
    wrapped_module = sys.argv[3]
    argsfile = sys.argv[4]
    if '-tmp-' not in os.path.dirname(wrapped_module):
        preserve_tmp = True
    elif len(sys.argv) > 5:
        preserve_tmp = sys.argv[5] == '-preserve_tmp'
    else:
        preserve_tmp = False
    # consider underscore as no argsfile so we can support passing of additional positional parameters
    if argsfile != '_':
        cmd = "%s %s" % (wrapped_module, argsfile)
    else:
        cmd = wrapped_module
    step = 5

    async_dir = os.environ.get('ASSIBLE_ASYNC_DIR', '~/.assible_async')

    # setup job output directory
    jobdir = os.path.expanduser(async_dir)
    job_path = os.path.join(jobdir, jid)

    try:
        _make_temp_dir(jobdir)
    except Exception as e:
        print(
            json.dumps({
                "failed":
                1,
                "msg":
                "could not create: %s - %s" % (jobdir, to_text(e)),
                "exception":
                to_text(traceback.format_exc()),
            }))
        sys.exit(1)

    # immediately exit this process, leaving an orphaned process
    # running which immediately forks a supervisory timing process

    try:
        pid = os.fork()
        if pid:
            # Notify the overlord that the async process started

            # we need to not return immediately such that the launched command has an attempt
            # to initialize PRIOR to assible trying to clean up the launch directory (and argsfile)
            # this probably could be done with some IPC later.  Modules should always read
            # the argsfile at the very first start of their execution anyway

            # close off notifier handle in grandparent, probably unnecessary as
            # this process doesn't hang around long enough
            ipc_notifier.close()

            # allow waiting up to 2.5 seconds in total should be long enough for worst
            # loaded environment in practice.
            retries = 25
            while retries > 0:
                if ipc_watcher.poll(0.1):
                    break
                else:
                    retries = retries - 1
                    continue

            notice("Return async_wrapper task started.")
            print(
                json.dumps({
                    "started": 1,
                    "finished": 0,
                    "assible_job_id": jid,
                    "results_file": job_path,
                    "_assible_suppress_tmpdir_delete": not preserve_tmp
                }))
            sys.stdout.flush()
            sys.exit(0)
        else:
            # The actual wrapper process

            # close off the receiving end of the pipe from child process
            ipc_watcher.close()

            # Daemonize, so we keep on running
            daemonize_self()

            # we are now daemonized, create a supervisory process
            notice("Starting module and watcher")

            sub_pid = os.fork()
            if sub_pid:
                # close off inherited pipe handles
                ipc_watcher.close()
                ipc_notifier.close()

                # the parent stops the process after the time limit
                remaining = int(time_limit)

                # set the child process group id to kill all children
                os.setpgid(sub_pid, sub_pid)

                notice("Start watching %s (%s)" % (sub_pid, remaining))
                time.sleep(step)
                while os.waitpid(sub_pid, os.WNOHANG) == (0, 0):
                    notice("%s still running (%s)" % (sub_pid, remaining))
                    time.sleep(step)
                    remaining = remaining - step
                    if remaining <= 0:
                        notice("Now killing %s" % (sub_pid))
                        os.killpg(sub_pid, signal.SIGKILL)
                        notice("Sent kill to group %s " % sub_pid)
                        time.sleep(1)
                        if not preserve_tmp:
                            shutil.rmtree(os.path.dirname(wrapped_module),
                                          True)
                        sys.exit(0)
                notice("Done in kid B.")
                if not preserve_tmp:
                    shutil.rmtree(os.path.dirname(wrapped_module), True)
                sys.exit(0)
            else:
                # the child process runs the actual module
                notice("Start module (%s)" % os.getpid())
                _run_module(cmd, jid, job_path)
                notice("Module complete (%s)" % os.getpid())
                sys.exit(0)

    except SystemExit:
        # On python2.4, SystemExit is a subclass of Exception.
        # This block makes python2.4 behave the same as python2.5+
        raise

    except Exception:
        e = sys.exc_info()[1]
        notice("error: %s" % e)
        print(json.dumps({"failed": True, "msg": "FATAL ERROR: %s" % e}))
        sys.exit(1)
Пример #35
0
    #   SEM_FAILCRITICALERRORS|
    #   SEM_NOGPFAULTERRORBOX|
    #   SEM_NOOPENFILEERRORBOX
    # ).
    #
    # For more information, see:
    # https://msdn.microsoft.com/en-us/library/windows/desktop/ms680621.aspx
    ctypes.windll.kernel32.SetErrorMode(0x0001 | 0x0002 | 0x8000)
    # gevent.subprocess has special import logic. This symbol is definitely there
    # on Windows.
    # pylint: disable=no-member
    EXTRA_KWARGS = {'creationflags': subprocess.CREATE_NEW_PROCESS_GROUP}
else:
    # Non-Windows platforms implement close_fds in a safe way.
    CLOSE_FDS = True
    EXTRA_KWARGS = {'preexec_fn': lambda: os.setpgid(0, 0)}


class SubprocessStepRunner(StepRunner):
    """Responsible for actually running steps as subprocesses, filtering their
  output into a stream."""
    def isabs(self, _name_tokens, path):
        return os.path.isabs(path)

    def isdir(self, _name_tokens, path):
        return os.path.isdir(path)

    def access(self, _name_tokens, path, mode):
        return os.access(path, mode)

    @staticmethod
Пример #36
0
 def xonsh_preexec_fn():
     """Preexec function bound to a pipeline group."""
     os.setpgid(0, pipeline_group)
     signal.signal(signal.SIGTSTP, default_signal_pauser)
Пример #37
0
 def setpgid_preexec_fn():
     os.setpgid(0, 0)
     if real_preexec_fn:
         apply(real_preexec_fn)
Пример #38
0
    def start(self):
        """
        Use multiple processes to parse and generate tasks for the
        DAGs in parallel. By processing them in separate processes,
        we can get parallelism and isolation from potentially harmful
        user code.
        """

        self.register_exit_signals()

        # Start a new process group
        os.setpgid(0, 0)

        self.log.info("Processing files using up to %s processes at a time ", self._parallelism)
        self.log.info("Process each file at most once every %s seconds", self._file_process_interval)
        self.log.info(
            "Checking for new files in %s every %s seconds", self._dag_directory, self.dag_dir_list_interval
        )

        # In sync mode we want timeout=None -- wait forever until a message is received
        if self._async_mode:
            poll_time = 0.0
        else:
            poll_time = None

        # Used to track how long it takes us to get once around every file in the DAG folder.
        self._parsing_start_time = timezone.utcnow()
        while True:
            loop_start_time = time.time()

            # pylint: disable=no-else-break
            if self._signal_conn.poll(poll_time):
                agent_signal = self._signal_conn.recv()
                self.log.debug("Received %s signal from DagFileProcessorAgent", agent_signal)
                if agent_signal == DagParsingSignal.TERMINATE_MANAGER:
                    self.terminate()
                    break
                elif agent_signal == DagParsingSignal.END_MANAGER:
                    self.end()
                    sys.exit(os.EX_OK)
                elif agent_signal == DagParsingSignal.AGENT_HEARTBEAT:
                    # continue the loop to parse dags
                    pass
            elif not self._async_mode:
                # In "sync" mode we don't want to parse the DAGs until we
                # are told to (as that would open another connection to the
                # SQLite DB which isn't a good practice
                continue
            # pylint: enable=no-else-break
            self._refresh_dag_dir()
            self._find_zombies()  # pylint: disable=no-value-for-parameter

            self._kill_timed_out_processors()
            simple_dags = self.collect_results()

            # Generate more file paths to process if we processed all the files
            # already.
            if not self._file_path_queue:
                self.emit_metrics()
                self.prepare_file_path_queue()

            self.start_new_processes()

            # Update number of loop iteration.
            self._num_run += 1

            for simple_dag in simple_dags:
                self._signal_conn.send(simple_dag)

            if not self._async_mode:
                self.log.debug(
                    "Waiting for processors to finish since we're using sqlite")
                # Wait until the running DAG processors are finished before
                # sending a DagParsingStat message back. This means the Agent
                # can tell we've got to the end of this iteration when it sees
                # this type of message
                self.wait_until_finished()

                # Collect anything else that has finished, but don't kick off any more processors
                simple_dags = self.collect_results()
                for simple_dag in simple_dags:
                    self._signal_conn.send(simple_dag)

            self._print_stat()

            all_files_processed = all(self.get_last_finish_time(x) is not None for x in self.file_paths)
            max_runs_reached = self.max_runs_reached()

            dag_parsing_stat = DagParsingStat(self._file_paths,
                                              max_runs_reached,
                                              all_files_processed,
                                              )
            self._signal_conn.send(dag_parsing_stat)

            if max_runs_reached:
                self.log.info("Exiting dag parsing loop as all files "
                              "have been processed %s times", self._max_runs)
                break

            if self._async_mode:
                loop_duration = time.time() - loop_start_time
                if loop_duration < 1:
                    poll_time = 1 - loop_duration
                else:
                    poll_time = 0.0
Пример #39
0
def setpgid_preexec_fn():
    os.setpgid(0, 0)
Пример #40
0
def main() -> None:
    setup_logger("resotoworker")
    # Try to run in a new process group and
    # ignore if not possible for whatever reason
    try:
        os.setpgid(0, 0)
    except Exception:
        pass

    resotolib.proc.parent_pid = os.getpid()

    arg_parser = ArgumentParser(
        description="resoto worker",
        env_args_prefix="RESOTOWORKER_",
    )
    add_args(arg_parser)
    jwt_add_args(arg_parser)
    logging_add_args(arg_parser)
    core_add_args(arg_parser)
    Config.add_args(arg_parser)
    TLSData.add_args(arg_parser)

    # Find resoto Plugins in the resoto.plugins module
    plugin_loader = PluginLoader()
    plugin_loader.add_plugin_args(arg_parser)

    # At this point the CLI, all Plugins as well as the WebServer have
    # added their args to the arg parser
    arg_parser.parse_args()

    try:
        wait_for_resotocore(resotocore.http_uri)
    except TimeoutError as e:
        log.fatal(f"Failed to connect to resotocore: {e}")
        sys.exit(1)

    tls_data = None
    if resotocore.is_secure:
        tls_data = TLSData(
            common_name=ArgumentParser.args.subscriber_id,
            resotocore_uri=resotocore.http_uri,
        )
        tls_data.start()
    config = Config(
        ArgumentParser.args.subscriber_id,
        resotocore_uri=resotocore.http_uri,
        tls_data=tls_data,
    )
    add_config(config)
    plugin_loader.add_plugin_config(config)
    config.load_config()

    def send_request(request: requests.Request) -> requests.Response:
        prepared = request.prepare()
        s = requests.Session()
        verify = None
        if tls_data:
            verify = tls_data.verify
        return s.send(request=prepared, verify=verify)

    core = Resotocore(send_request, config)

    collector = Collector(core.send_to_resotocore, config)

    # Handle Ctrl+c and other means of termination/shutdown
    resotolib.proc.initializer()
    add_event_listener(EventType.SHUTDOWN, shutdown, blocking=False)

    # Try to increase nofile and nproc limits
    increase_limits()

    web_server_args = {}
    if tls_data:
        web_server_args = {
            "ssl_cert": tls_data.cert_path,
            "ssl_key": tls_data.key_path,
        }
    web_server = WebServer(
        WebApp(mountpoint=Config.resotoworker.web_path),
        web_host=Config.resotoworker.web_host,
        web_port=Config.resotoworker.web_port,
        **web_server_args,
    )
    web_server.daemon = True
    web_server.start()

    core_actions = CoreActions(
        identifier=f"{ArgumentParser.args.subscriber_id}-collector",
        resotocore_uri=resotocore.http_uri,
        resotocore_ws_uri=resotocore.ws_uri,
        actions={
            "collect": {
                "timeout": Config.resotoworker.timeout,
                "wait_for_completion": True,
            },
            "cleanup": {
                "timeout": Config.resotoworker.timeout,
                "wait_for_completion": True,
            },
        },
        message_processor=partial(core_actions_processor, plugin_loader, tls_data, collector),
        tls_data=tls_data,
    )

    task_queue_filter = {}
    if len(Config.resotoworker.collector) > 0:
        task_queue_filter = {"cloud": list(Config.resotoworker.collector)}
    core_tasks = CoreTasks(
        identifier=f"{ArgumentParser.args.subscriber_id}-tagger",
        resotocore_ws_uri=resotocore.ws_uri,
        tasks=["tag"],
        task_queue_filter=task_queue_filter,
        message_processor=core_tag_tasks_processor,
        tls_data=tls_data,
    )
    core_actions.start()
    core_tasks.start()

    for Plugin in plugin_loader.plugins(PluginType.ACTION):
        try:
            log.debug(f"Starting action plugin {Plugin}")
            plugin = Plugin(tls_data=tls_data)
            plugin.start()
        except Exception as e:
            log.exception(f"Caught unhandled persistent Plugin exception {e}")

    # We wait for the shutdown Event to be set() and then end the program
    # While doing so we print the list of active threads once per 15 minutes
    shutdown_event.wait()
    web_server.shutdown()
    time.sleep(1)  # everything gets 1000ms to shutdown gracefully before we force it
    resotolib.proc.kill_children(resotolib.proc.SIGTERM, ensure_death=True)
    log.info("Shutdown complete")
    os._exit(0)
Пример #41
0
    def run(self,
            scale=None,
            ra=None,
            dec=None,
            radius=5.0,
            replace=False,
            timeout=None,
            verbose=False,
            extension=None,
            center=False,
            downsample=None):

        solve_field = [
            self.astrometry_bin + '/solve-field', '-D', self.odir,
            '--no-plots', '--no-fits2fits'
        ]

        if scale is not None:
            scale_low = scale * (1 - self.scale_relative_error)
            scale_high = scale * (1 + self.scale_relative_error)
            solve_field.append('-u')
            solve_field.append('app')
            solve_field.append('-L')
            solve_field.append(str(scale_low))
            solve_field.append('-H')
            solve_field.append(str(scale_high))

        if ra is not None and dec is not None:
            solve_field.append('--ra')
            solve_field.append(str(ra))
            solve_field.append('--dec')
            solve_field.append(str(dec))
            solve_field.append('--radius')
            solve_field.append(str(radius))

        if self.use_sextractor == True:
            solve_field.append('--use-sextractor')
            solve_field.append('--sextractor-path')
            solve_field.append(self.sextractor_bin)

        if extension is not None:
            solve_field.append('-6')
            solve_field.append(extension)

        if center:
            solve_field.append('--crpix-center')

        if downsample is not None:
            solve_field.append('-z')
            solve_field.append(downsample)

        solve_field.append(self.infpath)

        if verbose:
            print 'running', ' '.join(solve_field)

        proc = subprocess.Popen(solve_field,
                                stdout=subprocess.PIPE,
                                stderr=subprocess.PIPE,
                                preexec_fn=lambda: os.setpgid(0, 0))

        if timeout is not None:

            def __term_proc(sig, stack):
                os.killpg(os.getpgid(proc.pid), signal.SIGKILL)
                if verbose:
                    print 'killing process, as timeout was reached'

            signal.signal(signal.SIGALRM, __term_proc)
            signal.alarm(timeout)

        radecline = re.compile(
            'Field center: \(RA H:M:S, Dec D:M:S\) = \(([^,]*),(.*)\).')

        ret = None

        while True:
            try:
                a = proc.stdout.readline()
            except IOError:
                break
            if a == '':
                break
            if verbose:
                print a,
            match = radecline.match(a)
            if match:
                ret = [dms.parse(match.group(1)), dms.parse(match.group(2))]
        if replace and ret is not None:
            shutil.move(self.odir + '/input.new', self.fits_file)

        shutil.rmtree(self.odir)
        return ret
Пример #42
0
 parent.setblocking(0)
 setCloseOnExec(parent)
 child.setblocking(0)
 setCloseOnExec(child)
 try:
     pid = os.fork()
 except OSError, e:
     if e[0] in (errno.EAGAIN, errno.ENOMEM):
         return False # Can't fork anymore.
     raise
 if not pid:
     # Child
     child.close()
     # Put child into its own process group.
     pid = os.getpid()
     os.setpgid(pid, pid)
     # Restore signal handlers.
     self._restoreSignalHandlers()
     # Close copies of child sockets.
     for f in [x['file'] for x in self._children.values()
               if x['file'] is not None]:
         f.close()
     self._children = {}
     try:
         # Enter main loop.
         self._child(sock, parent)
     except KeyboardInterrupt:
         pass
     sys.exit(0)
 else:
     # Parent
Пример #43
0
 def setpgidfn():
     os.setpgid(0, 0)
Пример #44
0
def manager():
    # Create a new process group to corral our children
    os.setpgid(0, 0)

    # Create a listening socket on the AF_INET loopback interface
    listen_sock = socket.socket(AF_INET, SOCK_STREAM)
    listen_sock.bind(('127.0.0.1', 0))
    listen_sock.listen(max(1024, SOMAXCONN))
    listen_host, listen_port = listen_sock.getsockname()

    # re-open stdin/stdout in 'wb' mode
    stdin_bin = os.fdopen(sys.stdin.fileno(), 'rb', 4)
    stdout_bin = os.fdopen(sys.stdout.fileno(), 'wb', 4)
    write_int(listen_port, stdout_bin)
    stdout_bin.flush()

    def shutdown(code):
        signal.signal(SIGTERM, SIG_DFL)
        # Send SIGHUP to notify workers of shutdown
        os.kill(0, SIGHUP)
        sys.exit(code)

    def handle_sigterm(*args):
        shutdown(1)

    signal.signal(SIGTERM, handle_sigterm)  # Gracefully exit on SIGTERM
    signal.signal(SIGHUP, SIG_IGN)  # Don't die on SIGHUP
    signal.signal(SIGCHLD, SIG_IGN)

    reuse = os.environ.get("PY_WORKER_REUSE")

    # Initialization complete
    try:
        while True:
            try:
                ready_fds = select.select([0, listen_sock], [], [], 1)[0]
            except select.error as ex:
                if ex[0] == EINTR:
                    continue
                else:
                    raise

            if 0 in ready_fds:
                try:
                    worker_pid = read_int(stdin_bin)
                except EOFError:
                    # Spark told us to exit by closing stdin
                    shutdown(0)
                try:
                    os.kill(worker_pid, signal.SIGKILL)
                except OSError:
                    pass  # process already died

            if listen_sock in ready_fds:
                try:
                    sock, _ = listen_sock.accept()
                except OSError as e:
                    if e.errno == EINTR:
                        continue
                    raise

                # Launch a worker process
                try:
                    pid = os.fork()
                except OSError as e:
                    if e.errno in (EAGAIN, EINTR):
                        time.sleep(1)
                        pid = os.fork()  # error here will shutdown daemon
                    else:
                        outfile = sock.makefile(mode='wb')
                        write_int(e.errno,
                                  outfile)  # Signal that the fork failed
                        outfile.flush()
                        outfile.close()
                        sock.close()
                        continue

                if pid == 0:
                    # in child process
                    listen_sock.close()

                    # It should close the standard input in the child process so that
                    # Python native function executions stay intact.
                    #
                    # Note that if we just close the standard input (file descriptor 0),
                    # the lowest file descriptor (file descriptor 0) will be allocated,
                    # later when other file descriptors should happen to open.
                    #
                    # Therefore, here we redirects it to '/dev/null' by duplicating
                    # another file descriptor for '/dev/null' to the standard input (0).
                    # See SPARK-26175.
                    devnull = open(os.devnull, 'r')
                    os.dup2(devnull.fileno(), 0)
                    devnull.close()

                    try:
                        # Acknowledge that the fork was successful
                        outfile = sock.makefile(mode="wb")
                        write_int(os.getpid(), outfile)
                        outfile.flush()
                        outfile.close()
                        while True:
                            code = worker(sock)
                            if not reuse or code:
                                # wait for closing
                                try:
                                    while sock.recv(1024):
                                        pass
                                except Exception:
                                    pass
                                break
                            gc.collect()
                    except:
                        traceback.print_exc()
                        os._exit(1)
                    else:
                        os._exit(0)
                else:
                    sock.close()

    finally:
        shutdown(1)
Пример #45
0
def main() -> None:
    setup_logger("resotoworker")
    # Try to run in a new process group and
    # ignore if not possible for whatever reason
    try:
        os.setpgid(0, 0)
    except Exception:
        pass

    resotolib.signal.parent_pid = os.getpid()

    # Add cli args
    # The following double parsing of cli args is done so that when
    # a user specifies e.g. `--collector aws --help`  they would
    # no longer be shown cli args for other collectors like gcp.
    collector_arg_parser = ArgumentParser(
        description="resoto worker",
        env_args_prefix="RESOTOWORKER_",
        add_help=False,
        add_machine_help=False,
    )
    PluginLoader.add_args(collector_arg_parser)
    (args, _) = collector_arg_parser.parse_known_args()
    ArgumentParser.args = args

    arg_parser = ArgumentParser(
        description="resoto worker",
        env_args_prefix="RESOTOWORKER_",
    )
    jwt_add_args(arg_parser)
    logging_add_args(arg_parser)
    graph_add_args(arg_parser)
    collect_add_args(arg_parser)
    cleanup_add_args(arg_parser)
    core_add_args(arg_parser)
    resotocore_add_args(arg_parser)
    CoreActions.add_args(arg_parser)
    WebApp.add_args(arg_parser)
    PluginLoader.add_args(arg_parser)
    event_add_args(arg_parser)
    add_args(arg_parser)

    # Find resoto Plugins in the resoto.plugins module
    plugin_loader = PluginLoader()
    plugin_loader.add_plugin_args(arg_parser)

    # At this point the CLI, all Plugins as well as the WebServer have
    # added their args to the arg parser
    arg_parser.parse_args()

    # Handle Ctrl+c and other means of termination/shutdown
    resotolib.signal.initializer()
    add_event_listener(EventType.SHUTDOWN, shutdown, blocking=False)

    # Try to increase nofile and nproc limits
    increase_limits()

    web_server = WebServer(WebApp())
    web_server.daemon = True
    web_server.start()

    core_actions = CoreActions(
        identifier=f"{ArgumentParser.args.resotocore_subscriber_id}-collect_cleanup",
        resotocore_uri=ArgumentParser.args.resotocore_uri,
        resotocore_ws_uri=ArgumentParser.args.resotocore_ws_uri,
        actions={
            "collect": {
                "timeout": ArgumentParser.args.timeout,
                "wait_for_completion": True,
            },
            "cleanup": {
                "timeout": ArgumentParser.args.timeout,
                "wait_for_completion": True,
            },
        },
        message_processor=partial(
            core_actions_processor, plugin_loader.plugins(PluginType.COLLECTOR)
        ),
    )

    task_queue_filter = {}
    if ArgumentParser.args.collector and len(ArgumentParser.args.collector) > 0:
        task_queue_filter = {"cloud": list(ArgumentParser.args.collector)}
    core_tasks = CoreTasks(
        identifier="workerd-tasks",
        resotocore_ws_uri=ArgumentParser.args.resotocore_ws_uri,
        tasks=["tag"],
        task_queue_filter=task_queue_filter,
        message_processor=core_tag_tasks_processor,
    )
    core_actions.start()
    core_tasks.start()

    for Plugin in plugin_loader.plugins(PluginType.ACTION):
        try:
            log.debug(f"Starting action plugin {Plugin}")
            plugin = Plugin()
            plugin.start()
        except Exception as e:
            log.exception(f"Caught unhandled persistent Plugin exception {e}")

    # We wait for the shutdown Event to be set() and then end the program
    # While doing so we print the list of active threads once per 15 minutes
    shutdown_event.wait()
    web_server.shutdown()
    time.sleep(1)  # everything gets 1000ms to shutdown gracefully before we force it
    resotolib.signal.kill_children(resotolib.signal.SIGTERM, ensure_death=True)
    log.info("Shutdown complete")
    os._exit(0)
Пример #46
0
assert isinstance(os.getpid(), int)

# unix
if "win" not in sys.platform:
    assert isinstance(os.getegid(), int)
    assert isinstance(os.getgid(), int)
    assert isinstance(os.getsid(os.getpid()), int)
    assert isinstance(os.getuid(), int)
    assert isinstance(os.geteuid(), int)
    assert isinstance(os.getppid(), int)
    assert isinstance(os.getpgid(os.getpid()), int)

    if os.getuid() != 0:
        assert_raises(PermissionError, lambda: os.setgid(42))
        assert_raises(PermissionError, lambda: os.setegid(42))
        assert_raises(PermissionError, lambda: os.setpgid(os.getpid(), 42))
        assert_raises(PermissionError, lambda: os.setuid(42))
        assert_raises(PermissionError, lambda: os.seteuid(42))
        assert_raises(PermissionError, lambda: os.setreuid(42, 42))
        assert_raises(PermissionError, lambda: os.setresuid(42, 42, 42))

    # pty
    a, b = os.openpty()
    assert isinstance(a, int)
    assert isinstance(b, int)
    assert isinstance(os.ttyname(b), str)
    assert_raises(OSError, lambda: os.ttyname(9999))
    os.close(b)
    os.close(a)

    # os.get_blocking, os.set_blocking
Пример #47
0
        else:
            # The actual wrapper process

            # Daemonize, so we keep on running
            daemonize_self()

            # we are now daemonized, create a supervisory process
            notice("Starting module and watcher")

            sub_pid = os.fork()
            if sub_pid:
                # the parent stops the process after the time limit
                remaining = int(time_limit)

                # set the child process group id to kill all children
                os.setpgid(sub_pid, sub_pid)

                notice("Start watching %s (%s)" % (sub_pid, remaining))
                time.sleep(step)
                while os.waitpid(sub_pid, os.WNOHANG) == (0, 0):
                    notice("%s still running (%s)" % (sub_pid, remaining))
                    time.sleep(step)
                    remaining = remaining - step
                    if remaining <= 0:
                        notice("Now killing %s" % (sub_pid))
                        os.killpg(sub_pid, signal.SIGKILL)
                        notice("Sent kill to group %s" % sub_pid)
                        time.sleep(1)
                        sys.exit(0)
                notice("Done in kid B.")
                sys.exit(0)
Пример #48
0
 def run(self):
     # We do not want our worker to receive the signals our parent (master)
     # gets. Therefore move it into an own process group.
     os.setpgid(0, 0)
     super().run()
def execute():
    """
    子プロセスを作成してコマンドを実行する。

    子プロセスは自プロセスをプロセスリーダーとした、
    新しいプロセスグループを作成する。

    Return:
    [Int] 実行したコマンドの終了ステータス。

    Exception:
    [RuntimeError] 実行時に何らかのエラーが発生した場合。
    """
    try:
        global logger

        child_pid = os.fork()
        if child_pid == 0:
            # child process

            try:
                # 親プロセスから受け継いだ設定をデフォルトに戻す。
                for sig in (signal.SIGINT, signal.SIGQUIT, signal.SIGTERM):
                    signal.signal(sig, signal.SIG_DFL)

                # 新しいプロセスグループを作成する。
                # このプロセスグループでTimeManagerに登録される。
                os.setpgid(0, 0)

                # 空き時間がない場合は、コマンドを実行しない。
                if not is_unoccupied_avail():
                    logger.info('No unoccupied sched found, skip.')
                    os._exit(0)

                global args
                schedule = compose_schedule_str(args)
                if schedule == None:
                    logger.info('No story found, skip.')
                    os._exit(0)

                cmd = u'tm unoccupied | tm set - | xargs -i%% ffmpeg -loglevel quiet -i %% -vn -acodec copy pipe:1.ts | mplayer -vo null -msglevel all=0 -cache 256 -af volume=5 -; tm terminate'

                # logger.debug('[CMD] %s' % cmd)

                # 実行する。
                DEVNULL = open(os.devnull, 'wb')
                process = subprocess.Popen([cmd.encode('utf-8')],
                                           stdin=subprocess.PIPE,
                                           stderr=DEVNULL,
                                           shell=True)
                process.stdin.write((schedule + u'\n').encode('utf-8'))
                process.stdin.flush()
                process.stdin.close()
                process.wait()

                os._exit(process.returncode)

            except Exception, e:
                logger.exception(e)
                os._exit(1)
        else:
Пример #50
0
 def become_group_leader():
     os.setpgid(0, 0)
Пример #51
0
    def _run(self, lock=None):
        # Python threads loose the cwd
        os.chdir(self.cwd)

        # retrocompatibility: support ${aaa:=.} variable format
        def replace_perl_variables(m):
            vname = m.group(1)
            vdefault = m.group(2)
            if vname in os.environ:
                return "$" + vname
            else:
                return vdefault

        self.args = re.sub(r"\${(\w+):=([^}]*)}", replace_perl_variables,
                           self.args)

        # replace bash environment variables ($THINGS) to their values
        self.args = expandvars2(self.args)

        if re.match("^mkfile ", self.args) is not None:
            self._cmd_mkfile(self.args)
            if lock is not None:
                lock.release()
            return

        if re.match("^cd ", self.args) is not None:
            self._cmd_cd(self.args)
            if lock is not None:
                lock.release()
            return

        if TeshState().wrapper is not None:
            self.timeout *= 20
            self.args = TeshState().wrapper + self.args
        elif re.match(".*smpirun.*", self.args) is not None:
            self.args = "sh " + self.args
        if TeshState().jenkins and self.timeout is not None:
            self.timeout *= 10

        self.args += TeshState().args_suffix

        logs = list()
        logs.append(
            "[{file}:{number}] {args}".format(file=FileReader().filename,
                                              number=self.linenumber,
                                              args=self.args))

        args = shlex.split(self.args)

        global running_pids
        local_pid = None
        global return_code

        try:
            preexec_function = None
            if not isWindows():
                preexec_function = lambda: os.setpgid(0, 0)
            proc = subprocess.Popen(args,
                                    bufsize=1,
                                    stdin=subprocess.PIPE,
                                    stdout=subprocess.PIPE,
                                    stderr=subprocess.STDOUT,
                                    universal_newlines=True,
                                    preexec_fn=preexec_function)
            if not isWindows():
                local_pid = proc.pid
                running_pids.append(local_pid)
        except PermissionError:
            logs.append(
                "[{file}:{number}] Cannot start '{cmd}': The binary is not executable."
                .format(file=FileReader().filename,
                        number=self.linenumber,
                        cmd=args[0]))
            logs.append("[{file}:{number}] Current dir: {dir}".format(
                file=FileReader().filename,
                number=self.linenumber,
                dir=os.getcwd()))
            return_code = max(3, return_code)
            print('\n'.join(logs))
            return
        except NotADirectoryError:
            logs.append(
                "[{file}:{number}] Cannot start '{cmd}': The path to binary does not exist."
                .format(file=FileReader().filename,
                        number=self.linenumber,
                        cmd=args[0]))
            logs.append("[{file}:{number}] Current dir: {dir}".format(
                file=FileReader().filename,
                number=self.linenumber,
                dir=os.getcwd()))
            return_code = max(3, return_code)
            print('\n'.join(logs))
            return
        except FileNotFoundError:
            logs.append(
                "[{file}:{number}] Cannot start '{cmd}': File not found.".
                format(file=FileReader().filename,
                       number=self.linenumber,
                       cmd=args[0]))
            return_code = max(3, return_code)
            print('\n'.join(logs))
            return
        except OSError as osE:
            if osE.errno == 8:
                osE.strerror += "\nOSError: [Errno 8] Executed scripts should start with shebang line (like #!/usr/bin/env sh)"
            raise osE

        cmdName = FileReader().filename + ":" + str(self.linenumber)
        try:
            (stdout_data,
             stderr_data) = proc.communicate("\n".join(self.input_pipe),
                                             self.timeout)
            local_pid = None
            timeout_reached = False
        except subprocess.TimeoutExpired:
            timeout_reached = True
            logs.append(
                "Test suite `{file}': NOK (<{cmd}> timeout after {timeout} sec)"
                .format(file=FileReader().filename,
                        cmd=cmdName,
                        timeout=self.timeout))
            running_pids.remove(local_pid)
            kill_process_group(local_pid)
            # Try to get the output of the timeout process, to help in debugging.
            try:
                (stdout_data, stderr_data) = proc.communicate(timeout=1)
            except subprocess.TimeoutExpired:
                logs.append(
                    "[{file}:{number}] Could not retrieve output. Killing the process group failed?"
                    .format(file=FileReader().filename,
                            number=self.linenumber))
                return_code = max(3, return_code)
                print('\n'.join(logs))
                return

        if self.output_display:
            logs.append(str(stdout_data))

        # remove text colors
        ansi_escape = re.compile(r'\x1b[^m]*m')
        stdout_data = ansi_escape.sub('', stdout_data)

        if self.ignore_output:
            logs.append("(ignoring the output of <{cmd}> as requested)".format(
                cmd=cmdName))
        else:
            stdouta = stdout_data.split("\n")
            while len(stdouta) > 0 and stdouta[-1] == "":
                del stdouta[-1]
            stdouta = self.remove_ignored_lines(stdouta)
            stdcpy = stdouta[:]

            # Mimic the "sort" bash command, which is case unsensitive.
            if self.sort == 0:
                stdouta.sort(key=lambda x: x.lower())
                self.output_pipe_stdout.sort(key=lambda x: x.lower())
            elif self.sort > 0:
                stdouta.sort(key=lambda x: x[:self.sort].lower())
                self.output_pipe_stdout.sort(
                    key=lambda x: x[:self.sort].lower())

            diff = list(
                difflib.unified_diff(self.output_pipe_stdout,
                                     stdouta,
                                     lineterm="",
                                     fromfile='expected',
                                     tofile='obtained'))
            if len(diff) > 0:
                logs.append("Output of <{cmd}> mismatch:".format(cmd=cmdName))
                if self.sort >= 0:  # If sorted, truncate the diff output and show the unsorted version
                    difflen = 0
                    for line in diff:
                        if difflen < 50:
                            print(line)
                        difflen += 1
                    if difflen > 50:
                        logs.append("(diff truncated after 50 lines)")
                    logs.append("Unsorted observed output:\n")
                    for line in stdcpy:
                        logs.append(line)
                else:  # If not sorted, just display the diff
                    for line in diff:
                        logs.append(line)

                logs.append(
                    "Test suite `{file}': NOK (<{cmd}> output mismatch)".
                    format(file=FileReader().filename, cmd=cmdName))
                if lock is not None:
                    lock.release()
                if TeshState().keep:
                    f = open('obtained', 'w')
                    obtained = stdout_data.split("\n")
                    while len(obtained) > 0 and obtained[-1] == "":
                        del obtained[-1]
                    obtained = self.remove_ignored_lines(obtained)
                    for line in obtained:
                        f.write("> " + line + "\n")
                    f.close()
                    logs.append(
                        "Obtained output kept as requested: {path}".format(
                            path=os.path.abspath("obtained")))
                return_code = max(2, return_code)
                print('\n'.join(logs))
                return

        if timeout_reached:
            return_code = max(3, return_code)
            print('\n'.join(logs))
            return

        if proc.returncode != self.expect_return:
            if proc.returncode >= 0:
                logs.append(
                    "Test suite `{file}': NOK (<{cmd}> returned code {code})".
                    format(file=FileReader().filename,
                           cmd=cmdName,
                           code=proc.returncode))
                if lock is not None:
                    lock.release()
                return_code = max(2, return_code)
                print('\n'.join(logs))
                return
            else:
                logs.append(
                    "Test suite `{file}': NOK (<{cmd}> got signal {sig})".
                    format(file=FileReader().filename,
                           cmd=cmdName,
                           sig=SIGNALS_TO_NAMES_DICT[-proc.returncode]))
                if lock is not None:
                    lock.release()
                return_code = max(max(-proc.returncode, 1), return_code)
                print('\n'.join(logs))
                return

        if lock is not None:
            lock.release()

        print('\n'.join(logs))
Пример #52
0
def check_output(args,
                 valid_return_codes=(0, ),
                 timeout=600,
                 dots=True,
                 display_error=True,
                 shell=False,
                 return_stderr=False,
                 env=None,
                 cwd=None):
    """
    Runs the given command in a subprocess, raising ProcessError if it
    fails.  Returns stdout as a string on success.

    Parameters
    ----------
    valid_return_codes : list, optional
        A list of return codes to ignore. Defaults to only ignoring zero.
        Setting to None ignores all return codes.

    timeout : number, optional
        Kill the process if it lasts longer than `timeout` seconds.

    dots : bool, optional
        If `True` (default) write a dot to the console to show
        progress as the subprocess outputs content.  May also be
        a callback function to call (with no arguments) to indicate
        progress.

    display_error : bool, optional
        If `True` (default) display the stdout and stderr of the
        subprocess when the subprocess returns an error code.

    shell : bool, optional
        If `True`, run the command through the shell.  Default is
        `False`.

    return_stderr : bool, optional
        If `True`, return both the (stdout, stderr, errcode) as a
        tuple.

    env : dict, optional
        Specify environment variables for the subprocess.

    cwd : str, optional
        Specify the current working directory to use when running the
        process.
    """
    def get_content(header=None):
        content = []
        if header is not None:
            content.append(header)
        content.extend(
            ['STDOUT -------->', stdout[:-1], 'STDERR -------->', stderr[:-1]])

        return '\n'.join(content)

    if isinstance(args, six.string_types):
        args = [args]

    log.debug("Running '{0}'".format(' '.join(args)))

    kwargs = dict(shell=shell,
                  env=env,
                  cwd=cwd,
                  stdout=subprocess.PIPE,
                  stderr=subprocess.PIPE)
    if WIN:
        kwargs['close_fds'] = False
        kwargs['creationflags'] = subprocess.CREATE_NEW_PROCESS_GROUP
    else:
        kwargs['close_fds'] = True
        posix = getattr(os, 'setpgid', None)
        if posix:
            # Run the subprocess in a separate process group, so that we
            # can kill it and all child processes it spawns e.g. on
            # timeouts. Note that subprocess.Popen will wait until exec()
            # before returning in parent process, so there is no race
            # condition in setting the process group vs. calls to os.killpg
            kwargs['preexec_fn'] = lambda: os.setpgid(0, 0)

    proc = subprocess.Popen(args, **kwargs)

    last_dot_time = time.time()
    stdout_chunks = []
    stderr_chunks = []
    is_timeout = False

    if WIN:
        start_time = [time.time()]
        was_timeout = [False]

        def stdout_reader_run():
            while True:
                c = proc.stdout.read(1)
                if not c:
                    break
                start_time[0] = time.time()
                stdout_chunks.append(c)

        def stderr_reader_run():
            while True:
                c = proc.stderr.read(1)
                if not c:
                    break
                start_time[0] = time.time()
                stderr_chunks.append(c)

        def watcher_run():
            while proc.returncode is None:
                time.sleep(0.1)
                if time.time() - start_time[0] > timeout:
                    was_timeout[0] = True
                    proc.send_signal(signal.CTRL_BREAK_EVENT)

        watcher = threading.Thread(target=watcher_run)
        watcher.start()

        stdout_reader = threading.Thread(target=stdout_reader_run)
        stdout_reader.start()

        stderr_reader = threading.Thread(target=stderr_reader_run)
        stderr_reader.start()

        try:
            proc.wait()
        finally:
            if proc.returncode is None:
                proc.terminate()
                proc.wait()
            watcher.join()
            stderr_reader.join()
            stdout_reader.join()

            proc.stdout.close()
            proc.stderr.close()

        is_timeout = was_timeout[0]
    else:
        try:
            if posix:
                # Forward signals related to Ctrl-Z handling; the child
                # process is in a separate process group so it won't receive
                # these automatically from the terminal
                def sig_forward(signum, frame):
                    _killpg_safe(proc.pid, signum)
                    if signum == signal.SIGTSTP:
                        os.kill(os.getpid(), signal.SIGSTOP)

                signal.signal(signal.SIGTSTP, sig_forward)
                signal.signal(signal.SIGCONT, sig_forward)

            fds = {
                proc.stdout.fileno(): stdout_chunks,
                proc.stderr.fileno(): stderr_chunks
            }

            while proc.poll() is None:
                try:
                    rlist, wlist, xlist = select.select(
                        list(fds.keys()), [], [], timeout)
                except select.error as err:
                    if err.args[0] == errno.EINTR:
                        # interrupted by signal handler; try again
                        continue
                    raise

                if len(rlist) == 0:
                    # We got a timeout
                    is_timeout = True
                    break
                for f in rlist:
                    output = os.read(f, PIPE_BUF)
                    fds[f].append(output)
                if dots and time.time() - last_dot_time > 0.5:
                    if dots is True:
                        log.dot()
                    elif dots:
                        dots()
                    last_dot_time = time.time()
        finally:
            if posix:
                # Restore signal handlers
                signal.signal(signal.SIGTSTP, signal.SIG_DFL)
                signal.signal(signal.SIGCONT, signal.SIG_DFL)

            if proc.returncode is None:
                # Timeout or another exceptional condition occurred, and
                # the program is still running.
                if posix:
                    # Terminate the whole process group
                    _killpg_safe(proc.pid, signal.SIGTERM)

                    for j in range(10):
                        time.sleep(0.1)
                        if proc.poll() is not None:
                            break
                    else:
                        # Didn't terminate within 1 sec, so kill it
                        _killpg_safe(proc.pid, signal.SIGKILL)
                else:
                    proc.terminate()
                proc.wait()

        proc.stdout.flush()
        proc.stderr.flush()

        stdout_chunks.append(proc.stdout.read())
        stderr_chunks.append(proc.stderr.read())

        proc.stdout.close()
        proc.stderr.close()

    stdout = b''.join(stdout_chunks)
    stderr = b''.join(stderr_chunks)

    stdout = stdout.decode('utf-8', 'replace')
    stderr = stderr.decode('utf-8', 'replace')

    if is_timeout:
        retcode = TIMEOUT_RETCODE
    else:
        retcode = proc.returncode

    if valid_return_codes is not None and retcode not in valid_return_codes:
        header = 'Error running {0}'.format(' '.join(args))
        if display_error:
            log.error(get_content(header))
        else:
            if log.is_debug_enabled():
                log.debug(get_content(header))
        raise ProcessError(args, retcode, stdout, stderr)
    elif log.is_debug_enabled():
        log.debug(get_content())

    if return_stderr:
        return (stdout, stderr, retcode)
    else:
        return stdout
Пример #53
0
 def preexec():
     signal.signal(signal.SIGCHLD, old_sigchld)
     os.setpgid(0, 0)
Пример #54
0
 def ProcessGroup():
   if sys.platform != 'win32':
     # On non-windows platforms, start a new process group so that we can
     # be certain we bring down Chrome on a timeout.
     os.setpgid(0, 0)
Пример #55
0
 def subprocess_setup():
     signal.signal(signal.SIGPIPE, signal.SIG_DFL)
     os.setpgid(0, 0)
Пример #56
0
    def __init__(self, options):
        options.build = False
        self.worker_type = "user"
        self.state = "waiting"
        self.debug = options.debug

        os.chdir(sisyphus_dir)

        self.isBuilder = False
        self.uploadBuild = False
        self.zombie_time = 99

        uname = os.uname()
        self.os_name = uname[0]
        self.hostname = uname[1]
        self.os_version = uname[2]
        self.cpu_name = uname[-1]

        if self.os_name.find("Linux") != -1:
            self.os_name = "Linux"
            self.os_id = 'linux'
            self.os_version = re.search('([0-9]+\.[0-9]+\.[0-9]+).*',
                                        self.os_version).group(1)
            os.environ["MOZ_X_SYNC"] = "1"
        elif self.os_name.find("Darwin") != -1:
            self.os_name = "Mac OS X"
            self.os_id = 'darwin'
            proc = subprocess.Popen(
                ["sw_vers"],
                preexec_fn=lambda: os.setpgid(
                    0, 0),  # make the process its own process group
                stdout=subprocess.PIPE,
                stderr=subprocess.PIPE)
            stdout, stderr = proc.communicate()
            lines = stdout.split('\n')
            self.os_version = re.search('ProductVersion:\t([0-9]+\.[0-9]+).*',
                                        lines[1]).group(1)
        elif self.os_name.find("CYGWIN") != -1:
            self.os_name = "Windows NT"
            self.os_id = 'nt'
            proc = subprocess.Popen(
                ["cygcheck", "-s"],
                preexec_fn=lambda: os.setpgid(
                    0, 0),  # make the process its own process group
                stdout=subprocess.PIPE,
                stderr=subprocess.PIPE)
            stdout, stderr = proc.communicate()
            lines = stdout.split('\r\n')
            self.os_version = re.search('.* Ver ([^ ]*) .*', lines[4]).group(1)
        else:
            raise Exception("invalid os_name: %s" % (os_name))

        if self.os_name == "Windows NT":
            if "PROCESSOR_ARCHITEW6432" in os.environ and os.environ[
                    "PROCESSOR_ARCHITEW6432"]:
                self.cpu_name = "x86_64"
            else:
                self.cpu_name = "x86"
        else:
            bits = platform.architecture()[0]
            if self.cpu_name == "i386" or self.cpu_name == "i686":
                if bits == "32bit":
                    self.cpu_name = "x86"
                elif bits == "64bit":
                    self.cpu_name = "x86_64"
            elif self.cpu_name == 'Power Macintosh':
                self.cpu_name = 'ppc'

        # self.build_cpu_name is the cpu type where the builds that are to be run were created.
        if options.processor_type:
            if options.processor_type == 'intel32' or options.processor_type == 'amd32':
                self.build_cpu_name = 'x86'
            elif options.processor_type == 'intel64' or options.processor_type == 'amd64':
                self.build_cpu_name = 'x86_64'
            else:
                self.build_cpu_name = options.processor_type
        else:
            self.build_cpu_name = self.cpu_name

        # Create a dictionary builddata for the Branch (product, branch, version) build data
        # and three attributes for the current product, branch, buildtype
        # which will serve to control the "build_row" property which will perform lookups
        # into the builddata dictionary

        self.product = None
        self.branch = None
        self.buildtype = None
        self.builddata = {}

        branches_rows = models.Branch.objects.all()

        if len(branches_rows) == 0:
            raise Exception('Branch table is empty.')

        for branch_row in branches_rows:

            self.product = branch_row.product
            self.branch = branch_row.branch
            self.buildtype = branch_row.buildtype

            if self.product not in self.builddata:
                self.builddata[self.product] = {}

            if self.branch not in self.builddata[self.product]:
                self.builddata[self.product][self.branch] = {}

            if self.buildtype not in self.builddata[self.product][self.branch]:
                self.builddata[self.product][self.branch][self.buildtype] = {}

            self.build_id = "%s_%s_%s_%s_%s" % (
                self.product, self.branch, self.buildtype,
                self.os_name.replace(' ', '_'), self.build_cpu_name)

            # Treat missing build as fatal and let exception propagate
            self.build_row = models.Build.objects.get(build_id=self.build_id)
Пример #57
0
def setPgid():
    """
    preexec_fn for Popen to set subprocess pgid
    
    """
    os.setpgid(os.getpid(), 0)
Пример #58
0
    def run(self):
        got_handler = None
        file_handlers = []

        for handler in HANDLERS:
            # If there's no handler defined, we don't care if it's a match
            if "handler" not in handler:
                log.error("No handler binary defined for: %s" % handler)
                continue

            if "regex" in handler and handler["regex"] == True:
                handler['regex'] = True
            else:
                handler['regex'] = False

            if "match-url" in handler:
                got_handler = self.try_handler(handler, "url", self.href)
                if got_handler:
                    break
            elif "match-file" in handler:
                file_handlers.append(handler)
            else:
                log.error("No match-url or match-file in handler %s" % handler)
        else:
            # We didn't find a matching URL handler.

            # No file_handlers present, don't bother to download, create
            # a default browser handler.

            if file_handlers:
                try:
                    tmpnam = self.grab_it()
                except Exception as e:
                    log.error("Couldn't download file: %s" % e)

                    # If we couldn't get the file, skip all of these
                    file_handlers = []
                else:
                    try:
                        fileoutput = subprocess.check_output("file %s" % shlex.quote(tmpnam), shell=True)
                        fileoutput = fileoutput.decode()
                    except Exception as e:
                        log.error("Couldn't get file output: %s" % e)

                        # If we couldn't get the `file` output, also skip
                        file_handlers = []

            for f_handler in file_handlers:
                log.debug("f_handler: %s", f_handler)
                got_handler = self.try_handler(f_handler, "file", fileoutput)
                if got_handler:
                    self.href = tmpnam
                    break
            else:
                conf = self.base_obj.callbacks["get_conf"]()
                got_handler = { 
                        "handler" : conf["browser"]["path"],
                        "text" : conf["browser"]["text"]
                }

        # Okay, so at this point we have self.href, which is either the URL or
        # the temporary file path, and got_handler telling us what to invoke an
        # how.

        log.info("Opening %s with %s" % (self.href, got_handler["handler"]))

        # Make sure that we quote href such that malicious URLs like
        # "http://example.com & rm -rf ~/" won't be interpreted by the shell.

        href = shlex.quote(self.href)

        pause = False
        if "pause" in got_handler and got_handler["pause"]:
            self.base_obj.callbacks["pause_interface"]()
            pause = True

        path = got_handler["handler"]
        if "%u" in path:
            path = path.replace("%u", href)
        elif href:
            path = path + " " + href

        pid = os.fork()

        if not pid:
            # A lot of programs don't appreciate having their fds closed, so
            # instead we dup them to /dev/null.

            fd = os.open("/dev/null", os.O_RDWR)
            os.dup2(fd, sys.stderr.fileno())

            if not pause:
                os.setpgid(os.getpid(), os.getpid())
                os.dup2(fd, sys.stdout.fileno())
                os.dup2(fd, sys.stdin.fileno())

            os.execv("/bin/sh", ["/bin/sh", "-c", path])

            # Just in case.
            sys.exit(0)

        # Parent process only cares if we should wait for the process to finish

        elif pause:
            os.waitpid(pid, 0)
            self.base_obj.callbacks["unpause_interface"]()
Пример #59
0
 def _create_new_process_group(cls):
     """Creates a new process group for the calling process."""
     os.setpgid(os.getpid(), os.getpid())
Пример #60
0
    def run(self,
            scale=None,
            ra=None,
            dec=None,
            radius=5.0,
            replace=False,
            timeout=None,
            verbose=False,
            extension=None,
            center=False,
            downsample=None,
            wrkr=None):
        # '--no-verify: if not specified the output is different
        solve_field = [
            self.astrometry_bin + '/solve-field',
            '-D',
            self.odir,
            '--no-plots',
            '--no-fits2fits',
            '--no-verify',
        ]

        if scale is not None:
            scale_low = scale * (1 - self.scale_relative_error)
            scale_high = scale * (1 + self.scale_relative_error)
            solve_field.append('-u')
            solve_field.append('app')
            solve_field.append('-L')
            solve_field.append(str(scale_low))
            solve_field.append('-H')
            solve_field.append(str(scale_high))

        if ra is not None and dec is not None:
            solve_field.append('--ra')
            solve_field.append(str(ra))
            solve_field.append('--dec')
            solve_field.append(str(dec))
            solve_field.append('--radius')
            solve_field.append(str(radius))

        if self.use_sextractor == True:
            solve_field.append('--use-sextractor')
            solve_field.append('--sextractor-path')
            solve_field.append(self.sextractor_bin)

        if extension is not None:
            solve_field.append('-6')
            solve_field.append(extension)

        if center:
            solve_field.append('--crpix-center')

        if downsample is not None:
            solve_field.append('-z')
            solve_field.append(downsample)

        solve_field.append(self.infpath)

        if verbose:
            self.lg.info('running', ' '.join(solve_field))

        proc = subprocess.Popen(solve_field,
                                stdout=subprocess.PIPE,
                                stderr=subprocess.PIPE,
                                preexec_fn=lambda: os.setpgid(0, 0))

        if timeout is not None:

            def __term_proc(sig, stack):
                global lg  # not nice
                os.killpg(os.getpgid(proc.pid), signal.SIGKILL)
                if verbose:
                    self.lg.warn('killing process, as timeout was reached')
                self.lg.error(
                    '{}: time out: {} sec reached, closing down'.format(
                        wrkr, timeout))

            signal.signal(signal.SIGALRM, __term_proc)
            signal.alarm(timeout)  # timeout in seconds
        #radecline=re.compile('Field center: \(RA H:M:S, Dec D:M:S\) = \(([^,]*),(.*)\).')
        # Field center: (RA,Dec) = (88.550385, -64.468266) deg.
        # now:
        #                RA,Dec = (96.3752,-35.0503), pixel scale 1.70751 arcsec/pix.

        radecline = re.compile(
            'Field center: \(RA,Dec\) = \(([^,]*),(.*)\).*?')

        ret = None

        while True:
            try:
                a = proc.stdout.readline().decode('utf-8')
            except IOError:
                break
            if a == '':
                break

            if verbose:
                self.lg.debug(a)
            match = radecline.match(a)
            if match:
                ret = [float(match.group(1)), float(match.group(2))]
        if replace and ret is not None:
            shutil.move(self.odir + '/input.new', self.fits_file)

        shutil.rmtree(self.odir)
        return ret