def _run_post_scripts(self): for s in kickstart.get_post_scripts(self.ks): (fd, path) = tempfile.mkstemp(prefix = "ks-script-", dir = self._instroot + "/tmp") os.write(fd, s.script.encode("utf-8")) os.close(fd) os.chmod(path, 0o700) env = self._get_post_scripts_env(s.inChroot) if not s.inChroot: env["INSTALL_ROOT"] = self._instroot preexec = None script = path else: preexec = self._chroot script = "/tmp/" + os.path.basename(path) try: subprocess.check_call([s.interp, script], preexec_fn = preexec, env = env) except OSError as e: raise CreatorError("Failed to execute %%post script " "with '%s' : %s" % (s.interp, e.strerror)) except subprocess.CalledProcessError as err: if s.errorOnFail: raise CreatorError("%%post script failed with code %d " % err.returncode) logging.warning("ignoring %%post failure (code %d)" % err.returncode) finally: os.unlink(path)
def create(obj, dummy_eng): #FIXME change share tmp directory from invenio.config import CFG_TMPSHAREDDIR from invenio.legacy.bibsched.bibtask import task_low_level_submission, \ bibtask_allocate_sequenceid d = Deposition(obj) sip = d.get_latest_sip(sealed=False) sip.seal() tmp_file_fd, tmp_file_path = mkstemp( prefix="webdeposit-%s-%s" % (d.id, sip.uuid), suffix='.xml', dir=CFG_TMPSHAREDDIR, ) os.write(tmp_file_fd, sip.package) os.close(tmp_file_fd) # Trick to have access to task_sequence_id in subsequent tasks. d.workflow_object.task_sequence_id = bibtask_allocate_sequenceid() task_id = task_low_level_submission( 'bibupload', 'webdeposit', '-r' if 'recid' in sip.metadata else '-i', tmp_file_path, '-I', str(d.workflow_object.task_sequence_id) ) sip.task_ids.append(task_id) d.update()
def install_nvm(options): print '---------- Installing NVM ---------' check_run_quick('sudo chmod 775 /usr/local') check_run_quick('sudo mkdir -m 777 -p /usr/local/node /usr/local/nvm') result = check_fetch( 'https://raw.githubusercontent.com/creationix/nvm/{nvm_version}/install.sh' .format(nvm_version=NVM_VERSION)) fd, temp = tempfile.mkstemp() os.write(fd, result.content) os.close(fd) try: run_and_monitor( 'bash -c "NVM_DIR=/usr/local/nvm source {temp}"'.format(temp=temp)) finally: os.remove(temp) # curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.26.0/install.sh | NVM_DIR=/usr/local/nvm bash check_run_and_monitor('sudo bash -c "cat > /etc/profile.d/nvm.sh"', input=__NVM_SCRIPT) print '---------- Installing Node {version} ---------'.format( version=NODE_VERSION) run_and_monitor('bash -c "source /etc/profile.d/nvm.sh' '; nvm install {version}' '; nvm alias default {version}"' .format(version=NODE_VERSION))
def test_runcmd_redirects_stdin_from_file(self): fd, filename = tempfile.mkstemp() os.write(fd, 'foobar') os.lseek(fd, 0, os.SEEK_SET) self.assertEqual(cliapp.runcmd_unchecked(['cat'], stdin=fd), (0, 'foobar', '')) os.close(fd)
def save_data(data, output_filename): """ Save data to file. If the file already exists, the function will not save the data and return 1 Parameters ---------- data : str String containing the data you wish to write out to a file output_filename : str Path (full or relative) to the file you will save the data into. Returns ------- out : int Return 0 if the data was saved successfully. Return 1 if the file already exists. Hint ---- Check out the os module for determining whether a file exists already. """ if not os.path.exists(output_filename): # output_file doesn't currently exist fd = os.open(output_filename,os.O_WRONLY|os.O_CREAT) os.write(fd,data) os.close(fd) return 0 else: return 1
def _load_master_key(self): """Load the master key from file, or create one if not available.""" # TODO(jamielennox): This is but one way that a key file could be # stored. This can be pluggable later for storing/fetching keys from # better locations. mkey = None try: with open(CONF.crypto.master_key_file, 'r') as f: mkey = base64.b64decode(f.read()) except IOError as e: if e.errno == errno.ENOENT: flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL mkey = self.crypto.new_key(self.KEY_SIZE) f = None try: f = os.open(CONF.crypto.master_key_file, flags, 0o600) os.write(f, base64.b64encode(mkey)) except Exception as x: _logger.warn('Failed to read master key initially: %s', e) _logger.warn('Failed to create new master key: %s', x) raise x finally: if f: os.close(f) else: # the file could be unreadable due to bad permissions # so just pop up whatever error comes raise e return mkey
def child(pipeout): zzz = 0 while True: time.sleep(1 if zzz == 0 else zzz) msg = ('Spam %03d' % zzz).encode() os.write(pipeout, msg) zzz = (zzz+1) % 5
def run_tests(disk_file): auditor_worker = auditor.AuditorWorker(self.conf, self.logger, self.rcache, self.devices) data = '0' * 1024 etag = md5() with disk_file.create() as writer: writer.write(data) etag.update(data) etag = etag.hexdigest() timestamp = str(normalize_timestamp(time.time())) metadata = { 'ETag': etag, 'X-Timestamp': timestamp, 'Content-Length': str(os.fstat(writer._fd).st_size), } writer.put(metadata) pre_quarantines = auditor_worker.quarantines auditor_worker.object_audit( AuditLocation(disk_file._datadir, 'sda', '0')) self.assertEquals(auditor_worker.quarantines, pre_quarantines) os.write(writer._fd, 'extra_data') auditor_worker.object_audit( AuditLocation(disk_file._datadir, 'sda', '0')) self.assertEquals(auditor_worker.quarantines, pre_quarantines + 1)
def _write(root, path, buf, offset, fh): f_path = full_path(root, path) vnfs_ops = VNFSOperations(root) file_name = vnfs_ops.vnfs_get_file_name(f_path) #if file_name == "action": if file_name in special_files and special_files[file_name]+'_write' in globals(): try: nf_config = get_nf_config(vnfs_ops, f_path) # call the custom write function logger.info('Writing to ' + file_name + ' in ' + nf_config['nf_instance_name'] + '@' + nf_config['host']) ret_str = globals()[special_files[file_name]+'_write'](vnfs_ops._hypervisor, nf_config, buf.rstrip("\n")) except errors.HypervisorError, ex: logger.debug('raised OSErro ' + str(ex.errno)) raise OSError(ex.errno, os.strerror(ex.errno)) logger.info('Successfully wrote ' + file_name + ' in ' + nf_config['nf_instance_name'] + '@' + nf_config['host']) #if buf.rstrip("\n") == "activate": # try: # vnfs_ops.vnfs_deploy_nf(nf_path) # except errors.VNFCreateError: # #raise OSError(errno.EBUSY, os.strerror(errno.EBUSY)) # raise OSError(747, 'Cannot create VNF') #elif buf.rstrip("\n") == "stop": # vnfs_ops.vnfs_stop_vnf(nf_path) #elif buf.rstrip("\n") == "start": # vnfs_ops.vnfs_start_vnf(nf_path) #elif buf.rstrip("\n") == "destroy": # vnfs_ops.vnfs_destroy_vnf(nf_path) os.lseek(fh, offset, os.SEEK_SET) os.write(fh, buf.rstrip("\n")) return len(buf)
def main(argv): if len(argv) > 1 and argv[1] == "-n": # no passphrase passphrase = None else: # first read a line containing the passphrase passphrase = string.strip(sys.stdin.readline()) # fork with pty pid,master = pty.fork() assert pid != -1 if pid == 0: # child. run ssh os.execvp("ssh", ssh) else: # parent. talk to child. s = parent(master, passphrase) # ensure child is gone cleanup(pid) # write whatever we get from child os.write(1, s) # wait for child to disappear qid,status = os.wait() assert pid == qid if os.WIFEXITED(status): # child normally exited. forward its status os._exit(os.WEXITSTATUS(status)) else: # child was killed. return 255 os._exit(255)
def create_signed_cert(self, ou, san="IP:127.0.0.1,IP:::1,DNS:localhost"): print_ok("generating {0}.key, {0}.crt, {0}.p12".format(ou)) fd, openssl_config = mkstemp(dir='.') os.write(fd, "extendedKeyUsage=clientAuth,serverAuth\n".encode('utf-8')) os.write(fd, "subjectAltName = {0}".format(san).encode('utf-8')) call("openssl genrsa -out {0}.key 1024".format(ou), shell=True, stderr=FNULL) call( "openssl req -new -key {0}.key -out {0}.csr -subj /C=US/ST=CA/O=ghostunnel/OU={0}".format(ou), shell=True, stderr=FNULL) call("chmod 600 {0}.key".format(ou), shell=True) call( "openssl x509 -req -in {0}.csr -CA {1}.crt -CAkey {1}.key -CAcreateserial -out {0}_temp.crt -days 5 -extfile {2}".format( ou, self.name, openssl_config), shell=True, stderr=FNULL) call( "openssl pkcs12 -export -out {0}_temp.p12 -in {0}_temp.crt -inkey {0}.key -password pass:"******"{0}_temp.crt".format(ou), "{0}.crt".format(ou)) os.rename("{0}_temp.p12".format(ou), "{0}.p12".format(ou)) os.close(fd) os.remove(openssl_config) self.leaf_certs.append(ou)
def _create_cookie(self, timefunc=time.time): lockfd = self._get_lock() cookies = self._get_cookies(timefunc) cookie_id = 1 for tpl in cookies: if int(tpl[0]) >= cookie_id: cookie_id = int(tpl[0]) + 1 cookie = hexlify(os.urandom(24)) cookies.append( (str(cookie_id), str(int(timefunc())), cookie) ) for c in cookies: os.write(lockfd, ' '.join(c).encode('ascii') + b'\n') os.close(lockfd) if os.geteuid() == 0: os.chown(self.lock_file, self.uid, self.gid) os.rename(self.lock_file, self.cookie_file) self.cookieId = cookie_id self.cookie = cookie
def Run(self, args): """Initializes the driver.""" self.SyncTransactionLog() # This will raise if the signature is bad. args.driver.Verify(config_lib.CONFIG["Client.driver_signing_public_key"]) if args.force_reload: try: self.UninstallDriver(None, args.driver_name, delete_file=False) except Exception as e: # pylint: disable=broad-except logging.debug("Error uninstalling driver: %s", e) path_handle, path_name = tempfile.mkstemp(suffix=".sys") try: # TODO(user): Ensure we have lock here, no races logging.info("Writing driver to %s", path_name) # Note permissions default to global read, user only write. try: os.write(path_handle, args.driver.data) finally: os.close(path_handle) self.InstallDriver(path_name, args.driver_name, args.driver_display_name) finally: os.unlink(path_name)
def write_to_tempfile(content, path=None, suffix="", prefix="tmp"): """Create temporary file or use existing file. This util is needed for creating temporary file with specified content, suffix and prefix. If path is not None, it will be used for writing content. If the path doesn't exist it'll be created. :param content: content for temporary file. :param path: same as parameter 'dir' for mkstemp :param suffix: same as parameter 'suffix' for mkstemp :param prefix: same as parameter 'prefix' for mkstemp For example: it can be used in database tests for creating configuration files. """ if path: ensure_tree(path) (fd, path) = tempfile.mkstemp(suffix=suffix, dir=path, prefix=prefix) try: os.write(fd, content) finally: os.close(fd) return path
def run_once(self): super(simple_base, self).run_once() self.loginfo("Starting background docker command, timeout %s seconds", self.config["docker_timeout"]) attach_in_pipe_r, attach_in_pipe_w = os.pipe() self.sub_stuff["file_desc"].append(attach_in_pipe_r) self.sub_stuff["file_desc"].append(attach_in_pipe_w) self.sub_stuff["subargs_a"].append(self.sub_stuff["rand_name"]) dkrcmd = AsyncDockerCmd(self, "attach", self.sub_stuff["subargs_a"], verbose=True) # Runs in background self.sub_stuff["cmd_attach"] = dkrcmd self.sub_stuff["cmdresult_attach"] = dkrcmd.execute(attach_in_pipe_r) self.wait_interactive_cmd() self.logdebug("Before input should be ignored: %s", dkrcmd.cmdresult) # This input should be ignored. os.write(self.sub_stuff["run_in_pipe_w"], self.config["interactive_cmd_run"] + "\n") self.logdebug("Before input should be passed: %s", dkrcmd.cmdresult) # This input should be passed to container. os.write(attach_in_pipe_w, self.config["interactive_cmd_attach"] + "\n") self.wait_interactive_cmd() self.logdebug("After input was passsed: %s", dkrcmd.cmdresult)
def execute(tty, command, timeout=TIMEOUT, prompts=PROMPT, rm_esc_seq=True): """ Execute command in the remote shell and return it's output. Raise an Exception on timeout and on recognized error messages. tty -- file descriptor of the conrolling pseudoterminal of the remote shell command -- command to be executed timeout -- timeout applied when trying to read one(!) character prompts -- list of prompts expected at the end ot the output, shouldn't be set if connetcion was established with rlogin rm_esc_seq -- if True, remove graphical escape sequences from output """ #Send command: os.write(tty, command + os.linesep); #Read the echo of the command: (When command is long, echo has additional #" \r" or similar in it) repr(expect(tty, [command.replace(" ", "")], " \r\n", timeout)); #Read the output of the command: output = expect(tty, prompts, timeout=timeout); #Strip the prompt from the output: for prompt in prompts: output = output.replace(prompt, ""); #Remove escape sequences if necessary: #Escape secuences are: \033[NN;NN;NNm if rm_esc_seq: output = re.sub("(\033)\\[.*?m", "", output); return output.strip();
def rlogout(tty, timeout=TIMEOUT): """ End remote session. """ os.write(tty, "exit" + os.linesep); expect(tty, ["closed."], timeout=timeout);
def setDelay(self, device, value): #REP_DELAY if self.getDeviceAttribute(device, 'enabled') == True: print "[InputDevice] setDelay for device '%s' to %d ms" % (device,value) event = struct.pack('iihhi', 0, 0, 0x14, 0x00, int(value)) fd = os.open("/dev/input/" + device, os.O_RDWR) os.write(fd, event) os.close(fd)
def _sendstr(self, str): r""" Send a string to the pexpect interface, autorestarting the expect interface if anything goes wrong. INPUT: - ``str`` - a string EXAMPLES: We illustrate this function using the R interface:: sage: r._sendstr('a <- 10;\n') sage: r.eval('a') '[1] 10' We illustrate using the singular interface:: sage: singular._sendstr('int i = 5;') sage: singular('i') 5 """ if self._expect is None: self._start() try: os.write(self._expect.child_fd, str) except OSError: self._crash_msg() self.quit() self._sendstr(str)
def start_gdb(gdb_path, gdb_commands, gdb_flags=None): """Start gdb in the background and block until it finishes. Args: gdb_path: Path of the gdb binary. gdb_commands: Contents of GDB script to run. gdb_flags: List of flags to append to gdb command. """ # Windows disallows opening the file while it's open for writing. gdb_script_fd, gdb_script_path = tempfile.mkstemp() os.write(gdb_script_fd, gdb_commands) os.close(gdb_script_fd) gdb_args = [gdb_path, "-x", gdb_script_path] + (gdb_flags or []) kwargs = {} if sys.platform.startswith("win"): kwargs["creationflags"] = subprocess.CREATE_NEW_CONSOLE gdb_process = subprocess.Popen(gdb_args, **kwargs) while gdb_process.returncode is None: try: gdb_process.communicate() except KeyboardInterrupt: pass os.unlink(gdb_script_path)
def process_IAC(self, sock, cmd, option): """ Read in and parse IAC commands as passed by telnetlib. SB/SE commands are stored in sbdataq, and passed in w/ a command of SE. """ if cmd == DO: if option == TM: # timing mark - send WILL into outgoing stream os.write(self.remote, IAC + WILL + TM) else: pass elif cmd == IP: # interrupt process os.write(self.local, chr(ord('C') & 0x1F)) elif cmd == SB: pass elif cmd == SE: option = self.sbdataq[0] if option == NAWS: # negotiate window size. cols = ord(self.sbdataq[1]) rows = ord(self.sbdataq[2]) s = struct.pack('HHHH', rows, cols, 0, 0) fcntl.ioctl(self.local, termios.TIOCSWINSZ, s) elif cmd == DONT: pass else: pass
def __init__(self, data): fd, fname = tempfile.mkstemp() gzd = GzipFile(mode='r', fileobj=StringIO(b64decode(data))) os.write(fd, gzd.read()) os.close(fd) gzd.close() self.name = fname
def cosimTimeZero(): wt = int(os.environ['MYHDL_TO_PIPE']) rf = int(os.environ['MYHDL_FROM_PIPE']) buf = "TO 01 " for s, w in zip(fromSignames, fromSizes): buf += "%s %s " % (s, w) os.write(wt, to_bytes(buf))
def publish_from_path(self, path, content=None): """ Gets filename and content for a path, attempts to create directory if necessary, writes to file. """ fn, directory, fngz = self.get_filename_from_path(path) if not content: content = self.get_content_from_path(path) if not os.path.exists(directory): try: os.makedirs(directory) except: raise StaticGeneratorException('Could not create the directory: %s' % directory) try: f, tmpname = tempfile.mkstemp(dir=directory) os.write(f, content) os.close(f) os.chmod(tmpname, stat.S_IREAD | stat.S_IWRITE | stat.S_IWUSR | stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH) os.rename(tmpname, fn) try: tmpnamegz = '%s.gz' % tmpname f = gzip.open(tmpnamegz, 'wb') f.write(content) f.close() os.chmod(tmpnamegz, stat.S_IREAD | stat.S_IWRITE | stat.S_IWUSR | stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH) os.rename(tmpnamegz, fngz) except: raise StaticGeneratorException('Could not create the file: %s' % fngz) except: raise StaticGeneratorException('Could not create the file: %s' % fn)
def testContiunePollingOnSubscriberError(self): def cleanUp(_): w.stop() def fileDeleted(_): d = defer.Deferred() d.addCallback(cleanUp) reactor.callLater(0, d.callback, _) return d def fileChanged(_): os.remove(tempFile) raise Exception("This exception shouldn't be raisen") fd, tempFile = tempfile.mkstemp() w = watcher.FilesWatcher([tempFile], 0.001) w.subscribe(fileChanged = fileChanged) d = defer.Deferred() d.addCallback(fileDeleted) w.subscribe(fileDeleted = d.callback) w.start() os.write(fd, "test") os.close(fd) return d
def set_(package, question, type, value, *extra): ''' Set answers to debconf questions for a package. CLI Example: .. code-block:: bash salt '*' debconf.set <package> <question> <type> <value> [<value> ...] ''' if extra: value = ' '.join((value,) + tuple(extra)) fd_, fname = salt.utils.mkstemp(prefix="salt-", close_fd=False) line = "{0} {1} {2} {3}".format(package, question, type, value) os.write(fd_, line) os.close(fd_) _set_file(fname) os.unlink(fname) return True
def _exec(self, data): """Starts the registered function as a second process This will fork and start the registered function on a second process. This shouldn't block on anything. It merely starts then returns. Args: data: The data that should be written to stdin on the sub process. Returns: Nothing. """ logging.warning('Executing process: %s' % self._description) try: p = MinimalSubprocess(self._description, data, timeout=self._timeout) p.fork_exec(self._run_func, self._uid, self._gid) self._lock.acquire() self._processes.append(p) self._lock.release() if default_ping_fd is not None: os.write(default_ping_fd, '\0') except UnknownUserError: logging.error('%s: Unable to find user %s', self._description, self._uid) except UnknownGroupError: logging.error('%s: Unable to find group %s', self._description, self._gid)
def cosimToSignalVals(): wt = int(os.environ['MYHDL_TO_PIPE']) rf = int(os.environ['MYHDL_FROM_PIPE']) buf = "FROM 00 " for s, w in zip(fromSignames, fromSizes): buf += "%s %s " % (s, w) os.write(wt, to_bytes(buf)) os.read(rf, MAXLINE) buf = "TO 00 " for s, w in zip(toSignames, toSizes): buf += "%s %s " % (s, w) os.write(wt, to_bytes(buf)) os.read(rf, MAXLINE) os.write(wt, b"START") os.read(rf, MAXLINE) buf = "0 " for s, v in zip(toSignames, toVals): buf += s buf += " " buf += hex(v)[2:] buf += " " os.write(wt, to_bytes(buf)) os.read(rf, MAXLINE) buf = "0 " for s, v in zip(toSignames, toXVals): buf += s buf += " " buf += v buf += " " os.write(wt, to_bytes(buf))
def do_edit(given_cl, current_cl, cl_file_path): if given_cl.is_unspecified(): # Show an editor if CL not specified on the command-line tmp_fd, tmp_path = tempfile.mkstemp(prefix='appspot-', suffix='.txt') os.write(tmp_fd, editable_change(current_cl)) os.close(tmp_fd) retcode = subprocess.call( '%s %s' % (os.getenv('VISUAL', os.getenv('EDITOR', 'vi')), commands.mkarg(tmp_path)), shell=True) try: if retcode < 0: raise Exception('editor closed with signal %s' % -retcode) elif retcode > 0: raise Exception('editor exited with error value %s' % retcode) edited_cl = parse_change(open(tmp_path).read()) finally: os.remove(tmp_path) if edited_cl.is_unspecified(): print >>sys.stderr, 'cancelled edit' return edited_cl.merge_into(current_cl) else: given_cl.merge_into(current_cl) out = open(cl_file_path, 'w') out.write(editable_change(current_cl)) out.close()
def _call(self, request, async_object): """Ensure there's an active connection and put the request in the queue if there is. Returns False if the call short circuits due to AUTH_FAILED, CLOSED, EXPIRED_SESSION or CONNECTING state. """ if self._state == KeeperState.AUTH_FAILED: async_object.set_exception(AuthFailedError()) return False elif self._state == KeeperState.CLOSED: async_object.set_exception(ConnectionClosedError( "Connection has been closed")) return False elif self._state in (KeeperState.EXPIRED_SESSION, KeeperState.CONNECTING): async_object.set_exception(SessionExpiredError()) return False self._queue.append((request, async_object)) # wake the connection, guarding against a race with close() write_pipe = self._connection._write_pipe if write_pipe is None: async_object.set_exception(ConnectionClosedError( "Connection has been closed")) try: os.write(write_pipe, b'\0') except: async_object.set_exception(ConnectionClosedError( "Connection has been closed"))
def write(self, string): f = self.obtain() os.write(f.fileno(), string)
def oswritebytes(fd, obj): os.write(fd, tobytes(obj))
def spawn( cls: Type[PtyProcessType], spawn_command: List[str], echo: bool = True, rows: int = 80, cols: int = 256, ) -> PtyProcessType: """ Start the given command in a child process in a pseudo terminal. This does all the fork/exec type of stuff for a pty, and returns an instance of PtyProcess. For some devices setting terminal width strictly in the operating system (the actual network operating system) does not seem to be sufficient by itself for setting terminal length or width -- so we have optional values for rows/cols that can be passed here as well. Args: spawn_command: command to execute with arguments (if applicable), as a list echo: enable/disable echo -- defaults to True, should be left as True for "normal" scrapli operations, optionally disable for scrapli_netconf operations. rows: integer number of rows for ptyprocess "window" cols: integer number of cols for ptyprocess "window" Returns: PtyProcessType: instantiated PtyProcess object Raises: ScrapliValueError: if no ssh binary found on PATH Exception: IOError - if unable to set window size of child process Exception: OSError - if unable to spawn command in child process IOError: failing to reset window size exception: if we get an exception decoding output """ # Note that it is difficult for this method to fail. # You cannot detect if the child process cannot start. # So the only way you can tell if the child process started # or not is to try to read from the file descriptor. If you get # EOF immediately then it means that the child is already dead. # That may not necessarily be bad because you may have spawned a child # that performs some task; creates no stdout output; and then dies. import fcntl import pty import resource import termios from pty import CHILD, STDIN_FILENO spawn_executable = which(spawn_command[0]) if spawn_executable is None: raise ScrapliValueError("ssh executable not found!") spawn_command[0] = spawn_executable # [issue #119] To prevent the case where exec fails and the user is # stuck interacting with a python child process instead of whatever # was expected, we implement the solution from # http://stackoverflow.com/a/3703179 to pass the exception to the # parent process # [issue #119] 1. Before forking, open a pipe in the parent process. exec_err_pipe_read, exec_err_pipe_write = os.pipe() pid, fd = pty.fork() # Some platforms must call setwinsize() and setecho() from the # child process, and others from the master process. We do both, # allowing IOError for either. if pid == CHILD: try: _setwinsize(fd=STDIN_FILENO, rows=rows, cols=cols) except IOError as err: if err.args[0] not in (errno.EINVAL, errno.ENOTTY): raise # disable echo if requested if echo is False: try: _setecho(STDIN_FILENO, False) except (IOError, termios.error) as err: if err.args[0] not in (errno.EINVAL, errno.ENOTTY): raise # [issue #119] 3. The child closes the reading end and sets the # close-on-exec flag for the writing end. os.close(exec_err_pipe_read) fcntl.fcntl(exec_err_pipe_write, fcntl.F_SETFD, fcntl.FD_CLOEXEC) # Do not allow child to inherit open file descriptors from parent, # with the exception of the exec_err_pipe_write of the pipe. # Impose ceiling on max_fd: AIX bugfix for users with unlimited # nofiles where resource.RLIMIT_NOFILE is 2^63-1 and os.closerange() # occasionally raises out of range error max_fd = min(1048576, resource.getrlimit(resource.RLIMIT_NOFILE)[0]) pass_fds = sorted({exec_err_pipe_write}) for pair in zip([2] + pass_fds, pass_fds + [max_fd]): os.closerange(pair[0] + 1, pair[1]) try: os.execv(spawn_executable, spawn_command) except OSError as err: # [issue #119] 5. If exec fails, the child writes the error # code back to the parent using the pipe, then exits. tosend = f"OSError:{err.errno}:{str(err)}".encode() os.write(exec_err_pipe_write, tosend) os.close(exec_err_pipe_write) os._exit(os.EX_OSERR) # Parent inst = cls(pid, fd) # [issue #119] 2. After forking, the parent closes the writing end # of the pipe and reads from the reading end. os.close(exec_err_pipe_write) exec_err_data = os.read(exec_err_pipe_read, 4096) os.close(exec_err_pipe_read) # [issue #119] 6. The parent reads eof (a zero-length read) if the # child successfully performed exec, since close-on-exec made # successful exec close the writing end of the pipe. Or, if exec # failed, the parent reads the error code and can proceed # accordingly. Either way, the parent blocks until the child calls # exec. if len(exec_err_data) != 0: try: errclass, errno_s, errmsg = exec_err_data.split(b":", 2) exctype = getattr(builtins, errclass.decode("ascii"), Exception) exception = exctype(errmsg.decode("utf-8", "replace")) if exctype is OSError: exception.errno = int(errno_s) except Exception: raise Exception("Subprocess failed, got bad error data: %r" % exec_err_data) else: raise exception try: inst.setwinsize(rows=rows, cols=cols) except IOError as err: if err.args[0] not in (errno.EINVAL, errno.ENOTTY, errno.ENXIO): raise return inst
def makelock(info: str, pathname: str, checkdeadlock: bool = True) -> "Optional[int]": """Try to make a lock at given path. Write info inside it. Stale non-symlink or symlink locks are removed automatically. Symlink locks are only used by legacy code, or by the new code temporarily to prevent issues running together with the old code. Return file descriptor on success. The file descriptor must be kept for the lock to be effective. Raise EAGAIN, likely caused by another process holding the lock. Raise EEXIST or ELOOP, likely caused by another legacy hg process holding the lock. Can also raise other errors or those errors for other causes. Callers should convert errors to error.LockHeld or error.LockUnavailable. """ # This is a bit complex, since it aims to support old lock code where the # lock file is removed when the lock is released. The simpler version # where the lock file does not get unlinked when releasing the lock is: # # # Open the file. Create on demand. Fail if it's a symlink. # fd = os.open(pathname, os.O_CREAT | os.O_RDWR | os.O_NOFOLLOW | O_CLOEXEC) # try: # fcntl.flock(fd, fcntl.LOCK_NB | fcntl.LOCK_EX) # os.write(fd, info) # except (OSError, IOError): # os.close(fd) # raise # else: # return fd # # With "unlink" on release, the above simple logic can break in this way: # # [process 1] got fd. # [process 2] got fd pointing to a same file. # [process 1] .... release lock. file unlinked. # [process 2] flock on fd. (broken lock - file was gone) # # A direct fix is to use O_EXCL to make sure the file is created by the # current process, then use "flock". That means there needs to be a way to # remove stale lock, and that is not easy. A naive check and delete can # break subtly: # # [process 1] to check stale lock - got fd. # [process 2] ... release lock. file unlinked. # [process 1] flock taken, decided to remove file. # [process 3] create a new lock. # [process 1] unlink lock file. (wrong - removed the wrong lock) # # Instead of figuring out how to handle all corner cases carefully, we just # always lock the parent directory when doing "racy" write operations # (creating a lock, or removing a stale lock). So they become "atomic" and # safe. There are 2 kinds of write operations that can happen without # taking the directory lock: # # - Legacy symlink lock creation or deletion. The new code errors out # when it saw a symlink lock (os.open(..., O_NOFOLLOW) and os.rename). # So they play well with each other. # - Unlinking lock file when when releasing. The release logic is holding # the flock. So it knows nobody else has the lock. Therefore it can do # the unlink without extra locking. dirname = os.path.dirname(pathname) if checkdeadlock and pathname in _processlocks: raise error.ProgrammingError( "deadlock: %s was locked in the same process" % pathname) with _locked(dirname or "."): # Check and remove stale lock try: fd = os.open(pathname, os.O_RDONLY | os.O_NOFOLLOW | O_CLOEXEC) except (OSError, IOError) as ex: # ELOOP: symlink lock. Check if it's stale. if ex.errno == errno.ELOOP: oldinfo = os.readlink(pathname) if _issymlinklockstale(oldinfo, info): os.unlink(pathname) elif ex.errno != errno.ENOENT: raise else: try: # Use fcntl to test stale lock fcntl.flock(fd, fcntl.LOCK_NB | fcntl.LOCK_EX) os.unlink(pathname) except (OSError, IOError) as ex: # EAGAIN: lock taken - return directly # ENOENT: lock removed already - continue if ex.errno != errno.ENOENT: raise finally: os.close(fd) # Create symlink placeholder. Make sure the file replaced by # "os.rename" can only be this symlink. This avoids race condition # when legacy code creates the symlink lock without locking the # parent directory. # # This is basically the legacy lock logic. placeholdercreated = False try: os.symlink(info, pathname) placeholdercreated = True except (IOError, OSError) as ex: if ex.errno == errno.EEXIST: raise except AttributeError: pass if not placeholdercreated: # No symlink support. Suboptimal. Create a placeholder by using an # empty file. Other legacy process might see a "malformed lock" # temporarily. New processes won't see this because both "readlock" # and "islocked" take the directory lock. fd = os.open(pathname, os.O_CREAT | os.O_WRONLY | os.O_EXCL | O_CLOEXEC) os.close(fd) infobytes = encodeutf8(info) try: # Create new lock. # # mkstemp sets FD_CLOEXEC automatically. For thread-safety. Threads # used here (progress, profiling, Winodws update worker) do not fork. # So it's fine to not patch `os.open` here. fd, tmppath = tempfile.mkstemp(prefix="makelock", dir=dirname) try: os.fchmod(fd, 0o664) fcntl.flock(fd, fcntl.LOCK_NB | fcntl.LOCK_EX) os.write(fd, infobytes) os.rename(tmppath, pathname) _processlocks[pathname] = fd return fd except Exception: unlink(tmppath) os.close(fd) raise except Exception: # Remove the placeholder unlink(pathname) raise
elif thisarg == 'write_flash': write_addr = sys.argv.pop(0) binary = sys.argv.pop(0) elif thisarg: cmdline = cmdline + [thisarg] cmdline = cmdline + ['write_flash'] if write_option: cmdline = cmdline + [write_option] cmdline = cmdline + ['--flash_size', 'detect'] cmdline = cmdline + [write_addr, binary] erase_file = '' if erase_addr: # Generate temporary empty (0xff) file eraser = tempfile.mkstemp() erase_file = eraser[1] os.write(eraser[0], bytearray([0xff] * int(erase_len, 0))) os.close(eraser[0]) cmdline = cmdline + [erase_addr, erase_file] try: esptool.main(cmdline) except esptool.FatalError as e: sys.stderr.write('\nA fatal esptool.py error occurred: %s' % e) finally: if erase_file: os.remove(erase_file) if any(sys.exc_info()): sys.exit(2)
def prout(s): os.write(WiredTigerTestCase._dupout, s + '\n')
def flush(self): if self.writable: os.write(self._fd, self.buffer.getvalue()) self.buffer.truncate(0), self.buffer.seek(0)
def rpython_print_newline(): import os os.write(1, "\n")
def process(self, file_struct, directory=None, strict_ownership=1): # Older servers will not return directories; if filetype is missing, # assume file if file_struct.get('filetype') == 'directory': if directory is None: directory = "" return None, utils.mkdir_p(directory + file_struct['path']) if directory: directory += os.path.split(file_struct['path'])[0] if file_struct.get('filetype') == 'symlink': if 'symlink' not in file_struct: raise Exception("Missing key symlink") (fullpath, dirs_created, fd) = maketemp(prefix=".rhn-cfg-tmp", directory=directory) os.close(fd) os.unlink(fullpath) os.symlink(file_struct['symlink'], fullpath) return fullpath, dirs_created for k in self.file_struct_fields.keys(): if k not in file_struct: # XXX raise Exception("Missing key %s" % k) encoding = '' if 'encoding' in file_struct: encoding = file_struct['encoding'] contents = file_struct['file_contents'] if contents and (encoding == 'base64'): contents = decodestring(bstr(contents)) delim_start = file_struct['delim_start'] delim_end = file_struct['delim_end'] if ('checksum' in file_struct and 'checksum_type' in file_struct and 'verify_contents' in file_struct and file_struct['verify_contents']): if file_struct['checksum'] != utils.getContentChecksum( file_struct['checksum_type'], contents): raise Exception( "Corrupt file received: Content checksums do not match!") elif ('md5sum' in file_struct and 'verify_contents' in file_struct and file_struct['verify_contents']): if file_struct['md5sum'] != utils.getContentChecksum( 'md5', contents): raise Exception( "Corrupt file received: Content checksums do not match!") elif ('verify_contents' in file_struct and file_struct['verify_contents']): raise Exception( "Corrupt file received: missing checksum information!") (fullpath, dirs_created, fd) = maketemp(prefix=".rhn-cfg-tmp", directory=directory) try: os.write(fd, bstr(contents)) except Exception: raise finally: os.close(fd) # try to set mtime and ctime of the file to # the last modified time on the server if 'modified' in file_struct: try: modified = xmlrpc_time(file_struct['modified'].value) epoch_time = time.mktime(modified) os.utime(fullpath, (epoch_time, epoch_time)) except (ValueError, AttributeError): # we can't parse modified time pass return fullpath, dirs_created
def preprocess(path): with open(path, "rb") as fh: lines = [] it = iter(fh) for l in it: # zstd.h includes <stddef.h>, which is also included by cffi's # boilerplate. This can lead to duplicate declarations. So we strip # this include from the preprocessor invocation. # # The same things happens for including zstd.h, so give it the same # treatment. # # We define ZSTD_STATIC_LINKING_ONLY, which is redundant with the inline # #define in zstdmt_compress.h and results in a compiler warning. So drop # the inline #define. if l.startswith(( b"#include <stddef.h>", b'#include "zstd.h"', b"#define ZSTD_STATIC_LINKING_ONLY", )): continue # The preprocessor environment on Windows doesn't define include # paths, so the #include of limits.h fails. We work around this # by removing that import and defining INT_MAX ourselves. This is # a bit hacky. But it gets the job done. # TODO make limits.h work on Windows so we ensure INT_MAX is # correct. if l.startswith(b"#include <limits.h>"): l = b"#define INT_MAX 2147483647\n" # ZSTDLIB_API may not be defined if we dropped zstd.h. It isn't # important so just filter it out. if l.startswith(b"ZSTDLIB_API"): l = l[len(b"ZSTDLIB_API "):] lines.append(l) fd, input_file = tempfile.mkstemp(suffix=".h") os.write(fd, b"".join(lines)) os.close(fd) try: env = dict(os.environ) # cffi attempts to decode source as ascii. And the preprocessor # may insert non-ascii for some annotations. So try to force # ascii output via LC_ALL. env["LC_ALL"] = "C" if getattr(compiler, "_paths", None): env["PATH"] = compiler._paths process = subprocess.Popen(args + [input_file], stdout=subprocess.PIPE, env=env) output = process.communicate()[0] ret = process.poll() if ret: raise Exception("preprocessor exited with error") return output finally: os.unlink(input_file)
def write(self, str): os.write(self.fd, str)
def rpython_print_item(s): import os os.write(1, s)
def write(self, s): os.write(self.fd, s)
def daemonize(pidfile): try: pid = os.fork() if pid > 0: # exit first parent os._exit(0) except OSError as exc: raise error.ControlPlaneError('ERROR: fork #1 failed: %s' % exc) # decouple from parent environment try: os.chdir('/') os.setsid() except OSError: pass os.umask(0) # do second fork try: pid = os.fork() if pid > 0: # exit from second parent os._exit(0) except OSError as exc: raise error.ControlPlaneError('ERROR: fork #2 failed: %s' % exc) def signal_cb(s, f): raise KeyboardInterrupt for s in (signal.SIGTERM, signal.SIGINT, signal.SIGHUP, signal.SIGQUIT): signal.signal(s, signal_cb) # write pidfile def atexit_cb(): try: if pidfile: os.remove(pidfile) except OSError: pass atexit.register(atexit_cb) try: if pidfile: fd, nm = tempfile.mkstemp(dir=os.path.dirname(pidfile)) os.write(fd, ('%d\n' % os.getpid()).encode('utf-8')) os.close(fd) os.rename(nm, pidfile) except Exception as exc: raise error.ControlPlaneError('Failed to create PID file %s: %s' % (pidfile, exc)) # redirect standard file descriptors sys.stdout.flush() sys.stderr.flush() si = open(os.devnull, 'r') so = open(os.devnull, 'a+') se = open(os.devnull, 'a+') os.dup2(si.fileno(), sys.stdin.fileno()) os.dup2(so.fileno(), sys.stdout.fileno()) os.dup2(se.fileno(), sys.stderr.fileno())
def pdf_from_html(html: str, header_html: str = None, footer_html: str = None, wkhtmltopdf_filename: str = None, wkhtmltopdf_options: Dict[str, Any] = None, output_path: str = None, debug: bool = False) -> Union[bytes, bool]: """ Takes HTML and either: - returns a PDF (as a binary object in memory), if output_path is None - creates a PDF in the file specified by output_path Uses wkhtmltopdf (with pdfkit) - faster than xhtml2pdf - tables not buggy like Weasyprint - however, doesn't support CSS Paged Media, so we have the header_html and footer_html options to allow you to pass appropriate HTML content to serve as the header/footer (rather than passing it within the main HTML). """ # Customized for this Django site wkhtmltopdf_filename = wkhtmltopdf_filename or settings.WKHTMLTOPDF_FILENAME # noqa if wkhtmltopdf_options is None: wkhtmltopdf_options = settings.WKHTMLTOPDF_OPTIONS else: wkhtmltopdf_options = merge_two_dicts(settings.WKHTMLTOPDF_OPTIONS, wkhtmltopdf_options) # Generic if not wkhtmltopdf_filename: config = None else: if FIX_PDFKIT_ENCODING_BUG: config = pdfkit.configuration( wkhtmltopdf=wkhtmltopdf_filename.encode('utf-8')) # the bug is that pdfkit.pdfkit.PDFKit.__init__ will attempt to # decode the string in its configuration object; # https://github.com/JazzCore/python-pdfkit/issues/32 else: config = pdfkit.configuration(wkhtmltopdf=wkhtmltopdf_filename) # Temporary files that a subprocess can read: # http://stackoverflow.com/questions/15169101 # wkhtmltopdf requires its HTML files to have ".html" extensions: # http://stackoverflow.com/questions/5776125 h_filename = None f_filename = None try: if header_html: h_fd, h_filename = tempfile.mkstemp(suffix='.html') os.write(h_fd, header_html.encode('utf8')) os.close(h_fd) wkhtmltopdf_options["header-html"] = h_filename if footer_html: f_fd, f_filename = tempfile.mkstemp(suffix='.html') os.write(f_fd, footer_html.encode('utf8')) os.close(f_fd) wkhtmltopdf_options["footer-html"] = f_filename if debug: log.critical("wkhtmltopdf_options: " + repr(wkhtmltopdf_options)) kit = pdfkit.pdfkit.PDFKit(html, 'string', configuration=config, options=wkhtmltopdf_options) return kit.to_pdf(path=output_path) # ... if path is None, will return the PDF # ... if path is specified, will return True # https://github.com/JazzCore/python-pdfkit/blob/master/pdfkit/pdfkit.py # noqa finally: if h_filename: os.remove(h_filename) if f_filename: os.remove(f_filename)
else: if len(argv) > 2: mpd_print( 1, "when using gdb, pass cmd-line args to user pgms via the 'run' cmd" ) exit(-1) gdb_args = argv[1] gdb_info = Popen4('gdb -q %s' % (gdb_args), 0) gdbPid = gdb_info.pid # print "PID=%d GDBPID=%d" % (getpid(),gdbPid) ; stdout.flush() gdb_sin = gdb_info.tochild gdb_sin_fileno = gdb_sin.fileno() gdb_sout_serr = gdb_info.fromchild gdb_sout_serr_fileno = gdb_sout_serr.fileno() write(gdb_sin_fileno, 'set prompt (gdb)\\n\n') gdb_line = gdb_sout_serr.readline() # check if gdb reports any errors if findall(r'.*: No such file or directory.', gdb_line) != []: print gdb_line, stdout.flush() exit(-1) mpd_print(0000, "LINE1=|%s|" % (gdb_line.rstrip())) write(gdb_sin_fileno, 'set confirm off\n') gdb_line = gdb_sout_serr.readline() mpd_print(0000, "LINE2=|%s|" % (gdb_line.rstrip())) write(gdb_sin_fileno, 'handle SIGUSR1 nostop noprint\n') gdb_line = gdb_sout_serr.readline() mpd_print(0000, "LINE3=|%s|" % (gdb_line.rstrip())) write(gdb_sin_fileno, 'handle SIGPIPE nostop noprint\n') gdb_line = gdb_sout_serr.readline()
def call_from_executor(self, callback): self._calls_from_executor.append(callback) if self._schedule_pipe: os.write(self._schedule_pipe[1], b'x')
def _load_urllib(self, filename, kwargs): '''(internal) Loading a network file. First download it, save it to a temporary file, and pass it to _load_local().''' if PY2: import urllib2 as urllib_request def gettype(info): return info.gettype() else: import urllib.request as urllib_request def gettype(info): return info.get_content_type() proto = filename.split(':', 1)[0] if proto == 'smb': try: # note: it's important to load SMBHandler every time # otherwise the data is occasionally not loaded from smb.SMBHandler import SMBHandler except ImportError: Logger.warning( 'Loader: can not load PySMB: make sure it is installed') return import tempfile data = fd = _out_osfd = None try: _out_filename = '' if proto == 'smb': # read from samba shares fd = urllib_request.build_opener(SMBHandler).open(filename) else: # read from internet request = urllib_request.Request(filename) if Config.has_option('network', 'useragent'): useragent = Config.get('network', 'useragent') if useragent: request.add_header('User-Agent', useragent) opener = urllib_request.build_opener() fd = opener.open(request) if '#.' in filename: # allow extension override from URL fragment suffix = '.' + filename.split('#.')[-1] else: ctype = gettype(fd.info()) suffix = mimetypes.guess_extension(ctype) suffix = LoaderBase.EXT_ALIAS.get(suffix, suffix) if not suffix: # strip query string and split on path parts = filename.split('?')[0].split('/')[1:] while len(parts) > 1 and not parts[0]: # strip out blanks from '//' parts = parts[1:] if len(parts) > 1 and '.' in parts[-1]: # we don't want '.com', '.net', etc. as the extension suffix = '.' + parts[-1].split('.')[-1] _out_osfd, _out_filename = tempfile.mkstemp(prefix='kivyloader', suffix=suffix) idata = fd.read() fd.close() fd = None # write to local filename write(_out_osfd, idata) close(_out_osfd) _out_osfd = None # load data data = self._load_local(_out_filename, kwargs) # FIXME create a clean API for that for imdata in data._data: imdata.source = filename except Exception as ex: Logger.exception('Loader: Failed to load image <%s>' % filename) # close file when remote file not found or download error try: if _out_osfd: close(_out_osfd) except OSError: pass # update client for c_filename, client in self._client[:]: if filename != c_filename: continue # got one client to update client.image = self.error_image client.dispatch('on_error', error=ex) self._client.remove((c_filename, client)) return self.error_image finally: if fd: fd.close() if _out_osfd: close(_out_osfd) if _out_filename != '': unlink(_out_filename) return data
def __init__(self, conn, opts): XpraClientBase.__init__(self, opts) self.start_time = time.time() self._window_to_id = {} self._id_to_window = {} title = opts.title if opts.title_suffix is not None: title = "@title@ %s" % opts.title_suffix self.title = title self.readonly = opts.readonly self.session_name = opts.session_name self.compression_level = opts.compression_level self.auto_refresh_delay = opts.auto_refresh_delay self.max_bandwidth = opts.max_bandwidth if self.max_bandwidth > 0.0 and self.jpegquality == 0: """ jpegquality was not set, use a better start value """ self.jpegquality = 50 self.server_capabilities = {} self.mmap_enabled = False self.server_start_time = -1 self.server_platform = "" self.server_actual_desktop_size = None self.server_desktop_size = None self.server_randr = False self.pixel_counter = maxdeque(maxlen=100) self.server_latency = maxdeque(maxlen=100) self.server_load = None self.client_latency = maxdeque(maxlen=100) self.toggle_cursors_bell_notify = False self.bell_enabled = True self.cursors_enabled = True self.notifications_enabled = True self.clipboard_enabled = False self.mmap = None self.mmap_token = None self.mmap_file = None self.mmap_size = 0 self._client_extras = ClientExtras(self, opts) self.clipboard_enabled = not self.readonly and opts.clipboard and self._client_extras.supports_clipboard( ) self.supports_mmap = opts.mmap and ( "rgb24" in ENCODINGS) and self._client_extras.supports_mmap() if self.supports_mmap: try: import mmap import tempfile import uuid from stat import S_IRUSR, S_IWUSR, S_IRGRP, S_IWGRP mmap_dir = os.getenv("TMPDIR", "/tmp") if not os.path.exists(mmap_dir): raise Exception("TMPDIR %s does not exist!" % mmap_dir) #create the mmap file, the mkstemp that is called via NamedTemporaryFile ensures #that the file is readable and writable only by the creating user ID temp = tempfile.NamedTemporaryFile(prefix="xpra.", suffix=".mmap", dir=mmap_dir) #keep a reference to it so it does not disappear! self._mmap_temp_file = temp self.mmap_file = temp.name fd = temp.file.fileno() #set the group permissions and gid if the mmap-group option is specified if opts.mmap_group and type( conn.target) == str and os.path.exists(conn.target): s = os.stat(conn.target) os.fchown(fd, -1, s.st_gid) os.fchmod(fd, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP) self.mmap_size = max( 4096, mmap.PAGESIZE) * 32 * 1024 #generally 128MB log("using mmap file %s, fd=%s, size=%s", self.mmap_file, fd, self.mmap_size) os.lseek(fd, self.mmap_size - 1, os.SEEK_SET) assert os.write(fd, '\x00') os.lseek(fd, 0, os.SEEK_SET) self.mmap = mmap.mmap(fd, length=self.mmap_size) #write the 16 byte token one byte at a time - no endianness self.mmap_token = uuid.uuid4().int log.debug("mmap_token=%s", self.mmap_token) v = self.mmap_token for i in range(0, 16): poke = ctypes.c_ubyte.from_buffer(self.mmap, 512 + i) poke.value = v % 256 v = v >> 8 assert v == 0 except Exception, e: log.error("failed to setup mmap: %s", e) self.supports_mmap = False self.clean_mmap() self.mmap = None self.mmap_file = None self.mmap_size = 0
def _execute_child(self, args, executable, preexec_fn, close_fds, cwd, env, universal_newlines, startupinfo, creationflags, shell, to_close, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite): """Execute program (POSIX version)""" if isinstance(args, types.StringTypes): args = [args] else: args = list(args) if shell: args = ['/bin/sh', '-c'] + args if executable: args[0] = executable if executable is None: executable = args[0] def _close_in_parent(fd): os.close(fd) to_close.remove(fd) errpipe_read, errpipe_write = self.pipe_cloexec() try: try: gc_was_enabled = gc.isenabled() gc.disable() try: self.pid = os.fork() except: if gc_was_enabled: gc.enable() raise self._child_created = True if self.pid == 0: try: if p2cwrite is not None: os.close(p2cwrite) if c2pread is not None: os.close(c2pread) if errread is not None: os.close(errread) os.close(errpipe_read) if c2pwrite == 0: c2pwrite = os.dup(c2pwrite) if errwrite == 0 or errwrite == 1: errwrite = os.dup(errwrite) def _dup2(a, b): if a == b: self._set_cloexec_flag(a, False) elif a is not None: os.dup2(a, b) return _dup2(p2cread, 0) _dup2(c2pwrite, 1) _dup2(errwrite, 2) closed = {None} for fd in [p2cread, c2pwrite, errwrite]: if fd not in closed and fd > 2: os.close(fd) closed.add(fd) if cwd is not None: os.chdir(cwd) if preexec_fn: preexec_fn() if close_fds: self._close_fds(but=errpipe_write) if env is None: os.execvp(executable, args) else: os.execvpe(executable, args, env) except: exc_type, exc_value, tb = sys.exc_info() exc_lines = traceback.format_exception(exc_type, exc_value, tb) exc_value.child_traceback = ''.join(exc_lines) os.write(errpipe_write, pickle.dumps(exc_value)) os._exit(255) if gc_was_enabled: gc.enable() finally: os.close(errpipe_write) data = _eintr_retry_call(os.read, errpipe_read, 1048576) finally: if p2cread is not None and p2cwrite is not None: _close_in_parent(p2cread) if c2pwrite is not None and c2pread is not None: _close_in_parent(c2pwrite) if errwrite is not None and errread is not None: _close_in_parent(errwrite) os.close(errpipe_read) if data != '': try: _eintr_retry_call(os.waitpid, self.pid, 0) except OSError as e: if e.errno != errno.ECHILD: raise child_exception = pickle.loads(data) raise child_exception return
def write(self, data): return os.write(self.fd, data)
def set(self): # It doesn't really matter what we are writing to the pipe, # as long, something gets written os.write(self.__write_fd, 'x')
def main(): """Handle command-line arguments.""" from docopt import docopt arguments = docopt(__doc__) if arguments['extract-qs-from-aux']: string = open(arguments['<file>']).read() print(" ".join(extract_qs_from_aux_string(string))) elif arguments['write-bbl-from-aux']: aux_filename = arguments['<file>'] base_filename, _ = splitext(aux_filename) bbl_filename = base_filename + '.bbl' string = open(aux_filename).read() qs = extract_qs_from_aux_string(string) entities = wb_get_entities(qs) widest_label = max([len(q) for q in qs]) bbl = u'\\begin{thebibliography}{%d}\n\n' % widest_label for q in qs: entity = entities[q] bbl += '\\bibitem{%s}\n' % q bbl += u", ".join(entity_to_authors(entity)) + '.\n' bbl += entity_to_title(entity) + '.\n' bbl += '\n' bbl += '\\end{thebibliography}\n' with open(bbl_filename, 'w') as f: f.write(bbl.encode('utf-8')) with open(aux_filename, 'a') as f: for n, q in enumerate(qs, 1): f.write('\\bibcite{%s}{%d}\n' % (q, n)) elif arguments['write-bib-from-aux']: aux_filename = arguments['<file>'] base_filename, _ = splitext(aux_filename) bib_filename = base_filename + '.bib' string = open(aux_filename).read() qs = list(set(extract_qs_from_aux_string(string))) keys = qs[:] dois = list(set(extract_dois_from_aux_string(string))) for doi in dois: qs_doi = doi_to_qs(doi) if len(qs_doi) == 0: print('Could not find Wikidata item for {doi}'.format(doi=doi)) continue if len(qs_doi) > 1: print(('Multiple Wikidata items for {doi}: {qs}.' 'Using first.').format(doi=doi, qs=qs_doi)) q = qs_doi[0] qs.append(q) keys.append(doi) entities = wb_get_entities(qs) bib = u("") for q, key in zip(qs, keys): entity = entities[q] bib += entity_to_bibtex_entry(entity, key=key) bib += '\n' # Write BibTeX-formatted string to file output_file = os.open(bib_filename, os.O_RDWR | os.O_CREAT) output_encoding = "utf-8" write(output_file, bib.encode(output_encoding))
"-s", "--suppress-output", help= "Supresses output from so that stdout and stderr can be kept pure for the ssh transmission", action="store_true") # shows help if no arguments are provided if len(sys.argv) == 1: parser.print_help() sys.exit(1) args = parser.parse_args() # Return discover of plot if requested if args.discover: os.write(1, discover()) # Return a plot if iovs are provided if args.iovs: # Run plugin with arguments result = deserialize_iovs(args.db, args.plugin, args.plot, args.tag, args.time_type, json.loads(args.iovs)) # If test -> output the result as formatted json if args.test: os.write(1, json.dumps(json.loads(result), indent=4)) # If image plot -> get image file from result, open it and output bytes elif args.image_plot:
def isexit(str): if str == "exit": os.write(1, ("Goodbye...\n").encode()) sys.exit(0)
def pipedump(object, fd): """Pickle an object to the stream given by a numeric descriptor.""" os.write(fd, pickle.dumps(object))
def visualize(self): has_gv = False # ghostview has_open = False # Mac OS X utility if platform.system() in ('Windows', 'Microsoft'): # win32 psviewer = 'gsview32' has_gv = True else: # Assume it's UNIX (either proper or Mac OS X) psviewer = 'gv' if os.system("which dot 2> /dev/null > /dev/null") != 0: sys.stderr.write("Error: You must install graphviz before trying to visualize a graph.\n") sys.exit(1) if os.system("which gv 2> /dev/null > /dev/null") == 0: has_gv = True if os.system("which open 2> /dev/null > /dev/null") == 0: has_open = True if not (has_gv or has_open): sys.stderr.write("Error: You must install 'gv' or 'open' before trying to visualize a graph.\n") sys.exit(1) o = [] for s in self.states: if s.is_start: shape = ", shape=doublecircle" else: shape = "" o.append("""%s [label="%s"%s];""" % (id(s), s.name, shape)) for t in self.transitions: if not t.is_marked: style = ", style=dashed" else: style = "" o.append("""%s -> %s [label="%s", arrowhead=vee%s];""" % (id(t.source), id(t.target), t.name, style)) handle, dot_path = tempfile.mkstemp() os.write(handle, "digraph { %s\n}" % "\n ".join(o)) os.close(handle) if has_open: handle, ps_path = tempfile.mkstemp(".pdf") os.close(handle) os.system("dot -T pdf -o %s %s" % (ps_path, dot_path)) os.system("open %s&" % ps_path) elif has_gv: handle, ps_path = tempfile.mkstemp(".ps") os.close(handle) os.system("dot -T ps -o %s %s" % (ps_path, dot_path)) os.system("%s %s&" % (psviewer, ps_path)) os.remove(dot_path)
def stop(self): _logger.info("graceful stop requested") self._stop = True # wakeup the select() in wait_notification os.write(self._stop_pipe[1], b".")
def translate( self, src, tgt=None, batch_size=None, batch_type="sents", attn_debug=False, align_debug=False, phrase_table="", ): """Translate content of ``src`` and get gold scores from ``tgt``. Args: src: See :func:`self.src_reader.read()`. tgt: See :func:`self.tgt_reader.read()`. batch_size (int): size of examples per mini-batch attn_debug (bool): enables the attention logging align_debug (bool): enables the word alignment logging Returns: (`list`, `list`) * all_scores is a list of `batch_size` lists of `n_best` scores * all_predictions is a list of `batch_size` lists of `n_best` predictions """ if batch_size is None: raise ValueError("batch_size must be set") if self.tgt_prefix and tgt is None: raise ValueError("Prefix should be feed to tgt if -tgt_prefix.") src_data = {"reader": self.src_reader, "data": src} tgt_data = {"reader": self.tgt_reader, "data": tgt} _readers, _data = inputters.Dataset.config([("src", src_data), ("tgt", tgt_data)]) data = inputters.Dataset( self.fields, readers=_readers, data=_data, sort_key=inputters.str2sortkey[self.data_type], filter_pred=self._filter_pred, ) data_iter = inputters.OrderedIterator( dataset=data, device=self._dev, batch_size=batch_size, batch_size_fn=max_tok_len if batch_type == "tokens" else None, train=False, sort=False, sort_within_batch=True, shuffle=False, ) xlation_builder = onmt.translate.TranslationBuilder( data, self.fields, self.n_best, self.replace_unk, tgt, self.phrase_table, ) # Statistics counter = count(1) pred_score_total, pred_words_total = 0, 0 gold_score_total, gold_words_total = 0, 0 all_scores = [] all_predictions = [] start_time = time.time() for batch in data_iter: batch_data = self.translate_batch(batch, data.src_vocabs, attn_debug) translations = xlation_builder.from_batch(batch_data) for trans in translations: all_scores += [trans.pred_scores[:self.n_best]] pred_score_total += trans.pred_scores[0] pred_words_total += len(trans.pred_sents[0]) if tgt is not None: gold_score_total += trans.gold_score gold_words_total += len(trans.gold_sent) + 1 n_best_preds = [ " ".join(pred) for pred in trans.pred_sents[:self.n_best] ] if self.report_align: align_pharaohs = [ build_align_pharaoh(align) for align in trans.word_aligns[:self.n_best] ] n_best_preds_align = [ " ".join(align) for align in align_pharaohs ] n_best_preds = [ pred + DefaultTokens.ALIGNMENT_SEPARATOR + align for pred, align in zip(n_best_preds, n_best_preds_align) ] all_predictions += [n_best_preds] self.out_file.write("\n".join(n_best_preds) + "\n") self.out_file.flush() if self.verbose: sent_number = next(counter) output = trans.log(sent_number) if self.logger: self.logger.info(output) else: os.write(1, output.encode("utf-8")) if attn_debug: preds = trans.pred_sents[0] preds.append(DefaultTokens.EOS) attns = trans.attns[0].tolist() if self.data_type == "text": srcs = trans.src_raw else: srcs = [str(item) for item in range(len(attns[0]))] output = report_matrix(srcs, preds, attns) if self.logger: self.logger.info(output) else: os.write(1, output.encode("utf-8")) if align_debug: tgts = trans.pred_sents[0] align = trans.word_aligns[0].tolist() if self.data_type == "text": srcs = trans.src_raw else: srcs = [str(item) for item in range(len(align[0]))] output = report_matrix(srcs, tgts, align) if self.logger: self.logger.info(output) else: os.write(1, output.encode("utf-8")) end_time = time.time() if self.report_score: msg = self._report_score("PRED", pred_score_total, pred_words_total) self._log(msg) if tgt is not None: msg = self._report_score("GOLD", gold_score_total, gold_words_total) self._log(msg) if self.report_time: total_time = end_time - start_time self._log("Total translation time (s): %f" % total_time) self._log("Average translation time (s): %f" % (total_time / len(all_predictions))) self._log("Tokens per second: %f" % (pred_words_total / total_time)) if self.dump_beam: import json json.dump( self.translator.beam_accum, codecs.open(self.dump_beam, "w", "utf-8"), ) return all_scores, all_predictions
def __call_nohup__(self, commander): (pid, self.r_path, self.w_path, self.stdin_path, self.stdout_path, self.stderr_path) = daemonize(self.basecmd.cmd_hash) if pid == 1: # Child process make commands commander._close_cmds_stdios(self) (self.pid, r_pipe, w_pipe, stdin_pipe, stdout_pipe, stderr_pipe) = create_process_cmd() if self.pid == 0: # Child process make commands self.msg = ms.Messenger(ms.StdIOWrapperIn(r_pipe), ms.StdIOWrapperOut(w_pipe)) try: self.basecmd.results = self.obj(*self.basecmd.args, **self.basecmd.kargs) except Exception: err_msg = traceback.format_exc() self.msg.write_msg(remote_interface.CmdTraceBack(err_msg)) sys.exit(-1) finally: self.msg.write_msg(self.basecmd.results) sys.exit(0) else: # helper child process open communication pipes. # This process is able to manage problem with connection width # main parent process. It allows start unchanged child process. self.r_pipe = os.open(self.r_path, os.O_RDONLY) self.w_pipe = os.open(self.w_path, os.O_WRONLY) sys.stdout = os.fdopen(os.open(self.stdout_path, os.O_WRONLY), "w", 0) sys.stderr = os.fdopen(os.open(self.stderr_path, os.O_WRONLY), "w", 0) sys.stdin = os.fdopen(os.open(self.stdin_path, os.O_RDONLY), "r", 0) w_fds = [r_pipe, w_pipe, stdin_pipe, stdout_pipe, stderr_pipe] m_fds = [self.r_pipe, self.w_pipe, sys.stdin.fileno(), sys.stdout.fileno(), sys.stderr.fileno()] p = select.poll() p.register(r_pipe) p.register(w_pipe) # p.register(stdin_pipe) p.register(stdout_pipe) p.register(stderr_pipe) p.register(self.r_pipe) # p.register(self.w_pipe) p.register(sys.stdin.fileno()) # p.register(sys.stdout.fileno()) # p.register(sys.stderr.fileno()) io_map = {r_pipe: self.w_pipe, self.r_pipe: w_pipe, sys.stdin.fileno(): stdin_pipe, stdout_pipe: sys.stdout.fileno(), stderr_pipe: sys.stderr.fileno()} while 1: d = p.poll() w_ev = [x for x in d if x[0] in w_fds] m_ev = [x for x in d if x[0] in m_fds] w_hup, w_read, _ = sort_fds_event(w_ev) m_hup, m_read, _ = sort_fds_event(m_ev) if m_hup: time.sleep(0.1) if w_hup: # child process finished for r in w_read: data = os.read(r, 16384) os.write(io_map[r], data) break for r in w_read: data = os.read(r, 16384) os.write(io_map[r], data) for r in m_read: data = os.read(r, 16384) os.write(io_map[r], data) self.msg = ms.Messenger(ms.StdIOWrapperIn(self.r_pipe), ms.StdIOWrapperOut(self.w_pipe)) self.msg.write_msg(CmdFinish()) exit(0) else: # main process open communication named pipes. self.w_pipe = os.open(self.w_path, os.O_WRONLY) self.r_pipe = os.open(self.r_path, os.O_RDONLY) self.stdout_pipe = os.open(self.stdout_path, os.O_RDONLY) self.stderr_pipe = os.open(self.stderr_path, os.O_RDONLY) self.stdin_pipe = os.open(self.stdin_path, os.O_WRONLY) self.msg = ms.Messenger(ms.StdIOWrapperIn(self.r_pipe), ms.StdIOWrapperOut(self.w_pipe))