Example #1
0
    def _run_post_scripts(self):
        for s in kickstart.get_post_scripts(self.ks):
            (fd, path) = tempfile.mkstemp(prefix = "ks-script-",
                                          dir = self._instroot + "/tmp")

            os.write(fd, s.script.encode("utf-8"))
            os.close(fd)
            os.chmod(path, 0o700)

            env = self._get_post_scripts_env(s.inChroot)

            if not s.inChroot:
                env["INSTALL_ROOT"] = self._instroot
                preexec = None
                script = path
            else:
                preexec = self._chroot
                script = "/tmp/" + os.path.basename(path)

            try:
                subprocess.check_call([s.interp, script],
                                      preexec_fn = preexec, env = env)
            except OSError as e:
                raise CreatorError("Failed to execute %%post script "
                                   "with '%s' : %s" % (s.interp, e.strerror))
            except subprocess.CalledProcessError as err:
                if s.errorOnFail:
                    raise CreatorError("%%post script failed with code %d "
                                       % err.returncode)
                logging.warning("ignoring %%post failure (code %d)"
                                % err.returncode)
            finally:
                os.unlink(path)
Example #2
0
    def create(obj, dummy_eng):
        #FIXME change share tmp directory
        from invenio.config import CFG_TMPSHAREDDIR
        from invenio.legacy.bibsched.bibtask import task_low_level_submission, \
            bibtask_allocate_sequenceid
        d = Deposition(obj)

        sip = d.get_latest_sip(sealed=False)
        sip.seal()

        tmp_file_fd, tmp_file_path = mkstemp(
            prefix="webdeposit-%s-%s" % (d.id, sip.uuid),
            suffix='.xml',
            dir=CFG_TMPSHAREDDIR,
        )

        os.write(tmp_file_fd, sip.package)
        os.close(tmp_file_fd)

        # Trick to have access to task_sequence_id in subsequent tasks.
        d.workflow_object.task_sequence_id = bibtask_allocate_sequenceid()

        task_id = task_low_level_submission(
            'bibupload', 'webdeposit',
            '-r' if 'recid' in sip.metadata else '-i', tmp_file_path,
            '-I', str(d.workflow_object.task_sequence_id)
        )

        sip.task_ids.append(task_id)

        d.update()
def install_nvm(options):
  print '---------- Installing NVM ---------'
  check_run_quick('sudo chmod 775 /usr/local')
  check_run_quick('sudo mkdir -m 777 -p /usr/local/node /usr/local/nvm')

  result = check_fetch(
    'https://raw.githubusercontent.com/creationix/nvm/{nvm_version}/install.sh'
    .format(nvm_version=NVM_VERSION))

  fd, temp = tempfile.mkstemp()
  os.write(fd, result.content)
  os.close(fd)

  try:
    run_and_monitor(
        'bash -c "NVM_DIR=/usr/local/nvm source {temp}"'.format(temp=temp))
  finally:
    os.remove(temp)

#  curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.26.0/install.sh | NVM_DIR=/usr/local/nvm bash


  check_run_and_monitor('sudo bash -c "cat > /etc/profile.d/nvm.sh"',
                        input=__NVM_SCRIPT)

  print '---------- Installing Node {version} ---------'.format(
    version=NODE_VERSION)

  run_and_monitor('bash -c "source /etc/profile.d/nvm.sh'
                  '; nvm install {version}'
                  '; nvm alias default {version}"'
                  .format(version=NODE_VERSION))
Example #4
0
 def test_runcmd_redirects_stdin_from_file(self):
     fd, filename = tempfile.mkstemp()
     os.write(fd, 'foobar')
     os.lseek(fd, 0, os.SEEK_SET)
     self.assertEqual(cliapp.runcmd_unchecked(['cat'], stdin=fd),
                      (0, 'foobar', ''))
     os.close(fd)
def save_data(data, output_filename):
    """
    Save data to file.

    If the file already exists, the function will not save the data and return
    1

    Parameters
    ----------
    data : str
         String containing the data you wish to write out to a file

    output_filename : str
         Path (full or relative) to the file you will save the data into.

    Returns
    -------
    out : int
        Return 0 if the data was saved successfully. Return 1 if the file 
        already exists.

    Hint
    ----
    Check out the os module for determining whether a file exists already.

    """
    if not os.path.exists(output_filename): # output_file doesn't currently exist
        
        fd = os.open(output_filename,os.O_WRONLY|os.O_CREAT)
        os.write(fd,data)
        os.close(fd)
        return 0
    else:
        return 1
Example #6
0
    def _load_master_key(self):
        """Load the master key from file, or create one if not available."""

        # TODO(jamielennox): This is but one way that a key file could be
        # stored. This can be pluggable later for storing/fetching keys from
        # better locations.

        mkey = None

        try:
            with open(CONF.crypto.master_key_file, 'r') as f:
                mkey = base64.b64decode(f.read())
        except IOError as e:
            if e.errno == errno.ENOENT:
                flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL
                mkey = self.crypto.new_key(self.KEY_SIZE)
                f = None
                try:
                    f = os.open(CONF.crypto.master_key_file, flags, 0o600)
                    os.write(f, base64.b64encode(mkey))
                except Exception as x:
                    _logger.warn('Failed to read master key initially: %s', e)
                    _logger.warn('Failed to create new master key: %s', x)
                    raise x
                finally:
                    if f:
                        os.close(f)
            else:
                # the file could be unreadable due to bad permissions
                # so just pop up whatever error comes
                raise e

        return mkey
Example #7
0
def child(pipeout):
	zzz = 0
	while True:
		time.sleep(1 if zzz == 0 else zzz)
		msg = ('Spam %03d' % zzz).encode()
		os.write(pipeout, msg)
		zzz = (zzz+1) % 5
Example #8
0
        def run_tests(disk_file):
            auditor_worker = auditor.AuditorWorker(self.conf, self.logger,
                                                   self.rcache, self.devices)
            data = '0' * 1024
            etag = md5()
            with disk_file.create() as writer:
                writer.write(data)
                etag.update(data)
                etag = etag.hexdigest()
                timestamp = str(normalize_timestamp(time.time()))
                metadata = {
                    'ETag': etag,
                    'X-Timestamp': timestamp,
                    'Content-Length': str(os.fstat(writer._fd).st_size),
                }
                writer.put(metadata)
                pre_quarantines = auditor_worker.quarantines

                auditor_worker.object_audit(
                    AuditLocation(disk_file._datadir, 'sda', '0'))
                self.assertEquals(auditor_worker.quarantines, pre_quarantines)

                os.write(writer._fd, 'extra_data')

                auditor_worker.object_audit(
                    AuditLocation(disk_file._datadir, 'sda', '0'))
                self.assertEquals(auditor_worker.quarantines,
                                  pre_quarantines + 1)
Example #9
0
def _write(root, path, buf, offset, fh):
    f_path = full_path(root, path)
    vnfs_ops = VNFSOperations(root)
    file_name = vnfs_ops.vnfs_get_file_name(f_path)
    #if file_name == "action":
    if file_name in special_files and special_files[file_name]+'_write' in globals():
        try:
            nf_config = get_nf_config(vnfs_ops, f_path)
            # call the custom write function
            logger.info('Writing to ' + file_name  + ' in ' + 
                nf_config['nf_instance_name'] + '@' + nf_config['host'])
            ret_str = globals()[special_files[file_name]+'_write'](vnfs_ops._hypervisor, 
                nf_config, buf.rstrip("\n"))
        except errors.HypervisorError, ex:
            logger.debug('raised OSErro ' + str(ex.errno))
            raise OSError(ex.errno, os.strerror(ex.errno))
        logger.info('Successfully wrote ' + file_name + 
            ' in ' + nf_config['nf_instance_name'] + '@' + nf_config['host'])

        #if buf.rstrip("\n") == "activate":
        #    try:
        #        vnfs_ops.vnfs_deploy_nf(nf_path)
        #    except errors.VNFCreateError:
        #        #raise OSError(errno.EBUSY, os.strerror(errno.EBUSY))
        #        raise OSError(747, 'Cannot create VNF')
        #elif buf.rstrip("\n") == "stop":
        #    vnfs_ops.vnfs_stop_vnf(nf_path)
        #elif buf.rstrip("\n") == "start":
        #    vnfs_ops.vnfs_start_vnf(nf_path)
        #elif buf.rstrip("\n") == "destroy":
        #    vnfs_ops.vnfs_destroy_vnf(nf_path)
        os.lseek(fh, offset, os.SEEK_SET)
        os.write(fh, buf.rstrip("\n"))
        return len(buf)
Example #10
0
def main(argv):
    if len(argv) > 1 and argv[1] == "-n": # no passphrase
        passphrase = None
    else:
        # first read a line containing the passphrase
        passphrase = string.strip(sys.stdin.readline())
    # fork with pty
    pid,master = pty.fork()
    assert pid != -1
    if pid == 0:
        # child. run ssh
        os.execvp("ssh", ssh)
    else:
        # parent. talk to child.
        s = parent(master, passphrase)
        # ensure child is gone 
        cleanup(pid)
        # write whatever we get from child
        os.write(1, s)
        # wait for child to disappear
        qid,status = os.wait()
        assert pid == qid
        if os.WIFEXITED(status):
            # child normally exited. forward its status
            os._exit(os.WEXITSTATUS(status))
        else:
            # child was killed. return 255
            os._exit(255)
Example #11
0
 def create_signed_cert(self, ou, san="IP:127.0.0.1,IP:::1,DNS:localhost"):
     print_ok("generating {0}.key, {0}.crt, {0}.p12".format(ou))
     fd, openssl_config = mkstemp(dir='.')
     os.write(fd, "extendedKeyUsage=clientAuth,serverAuth\n".encode('utf-8'))
     os.write(fd, "subjectAltName = {0}".format(san).encode('utf-8'))
     call("openssl genrsa -out {0}.key 1024".format(ou),
          shell=True, stderr=FNULL)
     call(
         "openssl req -new -key {0}.key -out {0}.csr -subj /C=US/ST=CA/O=ghostunnel/OU={0}".format(ou),
         shell=True,
         stderr=FNULL)
     call("chmod 600 {0}.key".format(ou), shell=True)
     call(
         "openssl x509 -req -in {0}.csr -CA {1}.crt -CAkey {1}.key -CAcreateserial -out {0}_temp.crt -days 5 -extfile {2}".format(
             ou,
             self.name,
             openssl_config),
         shell=True,
         stderr=FNULL)
     call(
         "openssl pkcs12 -export -out {0}_temp.p12 -in {0}_temp.crt -inkey {0}.key -password pass:"******"{0}_temp.crt".format(ou), "{0}.crt".format(ou))
     os.rename("{0}_temp.p12".format(ou), "{0}.p12".format(ou))
     os.close(fd)
     os.remove(openssl_config)
     self.leaf_certs.append(ou)
Example #12
0
    def _create_cookie(self, timefunc=time.time):
        
        lockfd = self._get_lock()

        cookies = self._get_cookies(timefunc)

        cookie_id = 1
        for tpl in cookies:
            if int(tpl[0]) >= cookie_id:
                cookie_id = int(tpl[0]) + 1

        cookie = hexlify(os.urandom(24))
        
        cookies.append( (str(cookie_id), str(int(timefunc())), cookie) )

        for c in cookies:
            os.write(lockfd, ' '.join(c).encode('ascii') + b'\n')

        os.close(lockfd)
        if os.geteuid() == 0:
            os.chown(self.lock_file, self.uid, self.gid)

        os.rename(self.lock_file, self.cookie_file)

        self.cookieId = cookie_id
        self.cookie   = cookie
Example #13
0
  def Run(self, args):
    """Initializes the driver."""
    self.SyncTransactionLog()

    # This will raise if the signature is bad.
    args.driver.Verify(config_lib.CONFIG["Client.driver_signing_public_key"])

    if args.force_reload:
      try:
        self.UninstallDriver(None, args.driver_name, delete_file=False)
      except Exception as e:  # pylint: disable=broad-except
        logging.debug("Error uninstalling driver: %s", e)

    path_handle, path_name = tempfile.mkstemp(suffix=".sys")
    try:
      # TODO(user): Ensure we have lock here, no races
      logging.info("Writing driver to %s", path_name)

      # Note permissions default to global read, user only write.
      try:
        os.write(path_handle, args.driver.data)
      finally:
        os.close(path_handle)

      self.InstallDriver(path_name, args.driver_name, args.driver_display_name)

    finally:
      os.unlink(path_name)
Example #14
0
def write_to_tempfile(content, path=None, suffix="", prefix="tmp"):
    """Create temporary file or use existing file.

    This util is needed for creating temporary file with
    specified content, suffix and prefix. If path is not None,
    it will be used for writing content. If the path doesn't
    exist it'll be created.

    :param content: content for temporary file.
    :param path: same as parameter 'dir' for mkstemp
    :param suffix: same as parameter 'suffix' for mkstemp
    :param prefix: same as parameter 'prefix' for mkstemp

    For example: it can be used in database tests for creating
    configuration files.
    """
    if path:
        ensure_tree(path)

    (fd, path) = tempfile.mkstemp(suffix=suffix, dir=path, prefix=prefix)
    try:
        os.write(fd, content)
    finally:
        os.close(fd)
    return path
Example #15
0
    def run_once(self):
        super(simple_base, self).run_once()
        self.loginfo("Starting background docker command, timeout %s seconds", self.config["docker_timeout"])

        attach_in_pipe_r, attach_in_pipe_w = os.pipe()
        self.sub_stuff["file_desc"].append(attach_in_pipe_r)
        self.sub_stuff["file_desc"].append(attach_in_pipe_w)

        self.sub_stuff["subargs_a"].append(self.sub_stuff["rand_name"])

        dkrcmd = AsyncDockerCmd(self, "attach", self.sub_stuff["subargs_a"], verbose=True)
        # Runs in background
        self.sub_stuff["cmd_attach"] = dkrcmd
        self.sub_stuff["cmdresult_attach"] = dkrcmd.execute(attach_in_pipe_r)
        self.wait_interactive_cmd()
        self.logdebug("Before input should be ignored: %s", dkrcmd.cmdresult)

        # This input should be ignored.
        os.write(self.sub_stuff["run_in_pipe_w"], self.config["interactive_cmd_run"] + "\n")

        self.logdebug("Before input should be passed: %s", dkrcmd.cmdresult)
        # This input should be passed to container.
        os.write(attach_in_pipe_w, self.config["interactive_cmd_attach"] + "\n")

        self.wait_interactive_cmd()
        self.logdebug("After input was passsed: %s", dkrcmd.cmdresult)
Example #16
0
def execute(tty, command, timeout=TIMEOUT, prompts=PROMPT, rm_esc_seq=True):
    """
    Execute command in the remote shell and return it's output.
    Raise an Exception on timeout and on recognized error messages.
    
    tty -- file descriptor of the conrolling pseudoterminal of the
           remote shell
    command -- command to be executed
    timeout -- timeout applied when trying to read one(!) character
    prompts -- list of prompts expected at the end ot the output,
               shouldn't be set if connetcion was established with
               rlogin
    rm_esc_seq -- if True, remove graphical escape sequences from output
    """

    #Send command:
    os.write(tty, command + os.linesep);
    #Read the echo of the command: (When command is long, echo has additional
    #" \r" or similar in it)
    repr(expect(tty, [command.replace(" ", "")], " \r\n", timeout));
    #Read the output of the command:
    output = expect(tty, prompts, timeout=timeout);

    #Strip the prompt from the output:
    for prompt in prompts:
        output = output.replace(prompt, "");

    #Remove escape sequences if necessary:
    #Escape secuences are: \033[NN;NN;NNm
    if rm_esc_seq:
        output = re.sub("(\033)\\[.*?m", "", output);

    return output.strip();
Example #17
0
def rlogout(tty, timeout=TIMEOUT):
    """
    End remote session.
    """

    os.write(tty, "exit" + os.linesep);
    expect(tty, ["closed."], timeout=timeout);
Example #18
0
	def setDelay(self, device, value): #REP_DELAY
		if self.getDeviceAttribute(device, 'enabled') == True:
			print "[InputDevice] setDelay for device '%s' to %d ms" % (device,value)
			event = struct.pack('iihhi', 0, 0, 0x14, 0x00, int(value))
			fd = os.open("/dev/input/" + device, os.O_RDWR)
			os.write(fd, event)
			os.close(fd)
Example #19
0
    def _sendstr(self, str):
        r"""
        Send a string to the pexpect interface, autorestarting the expect
        interface if anything goes wrong.

        INPUT:


        -  ``str`` - a string


        EXAMPLES: We illustrate this function using the R interface::

            sage: r._sendstr('a <- 10;\n')
            sage: r.eval('a')
            '[1] 10'

        We illustrate using the singular interface::

            sage: singular._sendstr('int i = 5;')
            sage: singular('i')
            5
        """
        if self._expect is None:
            self._start()
        try:
            os.write(self._expect.child_fd, str)
        except OSError:
            self._crash_msg()
            self.quit()
            self._sendstr(str)
Example #20
0
def start_gdb(gdb_path, gdb_commands, gdb_flags=None):
    """Start gdb in the background and block until it finishes.

    Args:
        gdb_path: Path of the gdb binary.
        gdb_commands: Contents of GDB script to run.
        gdb_flags: List of flags to append to gdb command.
    """

    # Windows disallows opening the file while it's open for writing.
    gdb_script_fd, gdb_script_path = tempfile.mkstemp()
    os.write(gdb_script_fd, gdb_commands)
    os.close(gdb_script_fd)
    gdb_args = [gdb_path, "-x", gdb_script_path] + (gdb_flags or [])

    kwargs = {}
    if sys.platform.startswith("win"):
        kwargs["creationflags"] = subprocess.CREATE_NEW_CONSOLE

    gdb_process = subprocess.Popen(gdb_args, **kwargs)
    while gdb_process.returncode is None:
        try:
            gdb_process.communicate()
        except KeyboardInterrupt:
            pass

    os.unlink(gdb_script_path)
Example #21
0
    def process_IAC(self, sock, cmd, option):
        """
            Read in and parse IAC commands as passed by telnetlib.

            SB/SE commands are stored in sbdataq, and passed in w/ a command
            of SE.  
        """
        if cmd == DO:
            if option == TM: # timing mark - send WILL into outgoing stream
                os.write(self.remote, IAC + WILL + TM)
            else:
                pass
        elif cmd == IP:
            # interrupt process
            os.write(self.local, chr(ord('C') & 0x1F))
        elif cmd == SB:
            pass
        elif cmd == SE:
            option = self.sbdataq[0]
            if option == NAWS: # negotiate window size.
                cols = ord(self.sbdataq[1])
                rows = ord(self.sbdataq[2])
                s = struct.pack('HHHH', rows, cols, 0, 0)
                fcntl.ioctl(self.local, termios.TIOCSWINSZ, s)
        elif cmd == DONT:
            pass
        else:
            pass
Example #22
0
 def __init__(self, data):
     fd, fname = tempfile.mkstemp()
     gzd = GzipFile(mode='r', fileobj=StringIO(b64decode(data)))
     os.write(fd, gzd.read())
     os.close(fd)
     gzd.close()
     self.name = fname
Example #23
0
 def cosimTimeZero():
     wt = int(os.environ['MYHDL_TO_PIPE'])
     rf = int(os.environ['MYHDL_FROM_PIPE'])
     buf = "TO 01 "
     for s, w in zip(fromSignames, fromSizes):
         buf += "%s %s " % (s, w)
     os.write(wt, to_bytes(buf))
Example #24
0
 def publish_from_path(self, path, content=None):
     """
     Gets filename and content for a path, attempts to create directory if 
     necessary, writes to file.
     
     """
     fn, directory, fngz = self.get_filename_from_path(path)
     
     if not content:
         content = self.get_content_from_path(path)
     
     if not os.path.exists(directory):
         try:
             os.makedirs(directory)
         except:
             raise StaticGeneratorException('Could not create the directory: %s' % directory)
     
     try:
         f, tmpname = tempfile.mkstemp(dir=directory)
         os.write(f, content)
         os.close(f)
         os.chmod(tmpname, stat.S_IREAD | stat.S_IWRITE | stat.S_IWUSR | stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH)
         os.rename(tmpname, fn)
         try:
             tmpnamegz = '%s.gz' % tmpname
             f = gzip.open(tmpnamegz, 'wb')
             f.write(content)
             f.close()
             os.chmod(tmpnamegz, stat.S_IREAD | stat.S_IWRITE | stat.S_IWUSR | stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH)
             os.rename(tmpnamegz, fngz)
         except:
             raise StaticGeneratorException('Could not create the file: %s' % fngz)
     except:
         raise StaticGeneratorException('Could not create the file: %s' % fn)
    def testContiunePollingOnSubscriberError(self):

        def cleanUp(_):
            w.stop()

        def fileDeleted(_):
            d = defer.Deferred()
            d.addCallback(cleanUp)
            reactor.callLater(0, d.callback, _)
            return d

        def fileChanged(_):
            os.remove(tempFile)
            raise Exception("This exception shouldn't be raisen")

        fd, tempFile = tempfile.mkstemp()
        w = watcher.FilesWatcher([tempFile], 0.001)
        w.subscribe(fileChanged = fileChanged)
        d = defer.Deferred()
        d.addCallback(fileDeleted)
        w.subscribe(fileDeleted = d.callback)
        w.start()
        os.write(fd, "test")
        os.close(fd)
        return d
Example #26
0
def set_(package, question, type, value, *extra):
    '''
    Set answers to debconf questions for a package.

    CLI Example:

    .. code-block:: bash

        salt '*' debconf.set <package> <question> <type> <value> [<value> ...]
    '''

    if extra:
        value = ' '.join((value,) + tuple(extra))

    fd_, fname = salt.utils.mkstemp(prefix="salt-", close_fd=False)

    line = "{0} {1} {2} {3}".format(package, question, type, value)
    os.write(fd_, line)
    os.close(fd_)

    _set_file(fname)

    os.unlink(fname)

    return True
Example #27
0
  def _exec(self, data):
    """Starts the registered function as a second process

    This will fork and start the registered function on a second process.
    This shouldn't block on anything. It merely starts then returns.

    Args:
      data: The data that should be written to stdin on the sub process.

    Returns:
      Nothing.
    """
    logging.warning('Executing process: %s' % self._description)
    try:
      p = MinimalSubprocess(self._description, data, timeout=self._timeout)
      p.fork_exec(self._run_func, self._uid, self._gid)
      self._lock.acquire()
      self._processes.append(p)
      self._lock.release()
      if default_ping_fd is not None:
        os.write(default_ping_fd, '\0')
    except UnknownUserError:
      logging.error('%s: Unable to find user %s', self._description,
                    self._uid)
    except UnknownGroupError:
      logging.error('%s: Unable to find group %s', self._description,
                    self._gid)
Example #28
0
 def cosimToSignalVals():
     wt = int(os.environ['MYHDL_TO_PIPE'])
     rf = int(os.environ['MYHDL_FROM_PIPE'])
     buf = "FROM 00 "
     for s, w in zip(fromSignames, fromSizes):
         buf += "%s %s " % (s, w)
     os.write(wt, to_bytes(buf))
     os.read(rf, MAXLINE)
     buf = "TO 00 "
     for s, w in zip(toSignames, toSizes):
         buf += "%s %s " % (s, w)
     os.write(wt, to_bytes(buf))
     os.read(rf, MAXLINE)
     os.write(wt, b"START")
     os.read(rf, MAXLINE)
     buf = "0 "
     for s, v in zip(toSignames, toVals):
         buf += s
         buf += " "
         buf += hex(v)[2:]
         buf += " "
     os.write(wt, to_bytes(buf))
     os.read(rf, MAXLINE)
     buf = "0 "
     for s, v in zip(toSignames, toXVals):
         buf += s
         buf += " "
         buf += v
         buf += " "
     os.write(wt, to_bytes(buf))
Example #29
0
def do_edit(given_cl, current_cl, cl_file_path):
  if given_cl.is_unspecified():
    # Show an editor if CL not specified on the command-line
    tmp_fd, tmp_path = tempfile.mkstemp(prefix='appspot-', suffix='.txt')
    os.write(tmp_fd, editable_change(current_cl))
    os.close(tmp_fd)

    retcode = subprocess.call(
        '%s %s' % (os.getenv('VISUAL', os.getenv('EDITOR', 'vi')),
                   commands.mkarg(tmp_path)),
        shell=True)
    try:
      if retcode < 0:
        raise Exception('editor closed with signal %s' % -retcode)
      elif retcode > 0:
        raise Exception('editor exited with error value %s' % retcode)
      edited_cl = parse_change(open(tmp_path).read())
    finally:
      os.remove(tmp_path)
    if edited_cl.is_unspecified():
      print >>sys.stderr, 'cancelled edit'
      return
    edited_cl.merge_into(current_cl)
  else:
    given_cl.merge_into(current_cl)
  out = open(cl_file_path, 'w')
  out.write(editable_change(current_cl))
  out.close()
Example #30
0
    def _call(self, request, async_object):
        """Ensure there's an active connection and put the request in
        the queue if there is.

        Returns False if the call short circuits due to AUTH_FAILED,
        CLOSED, EXPIRED_SESSION or CONNECTING state.

        """

        if self._state == KeeperState.AUTH_FAILED:
            async_object.set_exception(AuthFailedError())
            return False
        elif self._state == KeeperState.CLOSED:
            async_object.set_exception(ConnectionClosedError(
                "Connection has been closed"))
            return False
        elif self._state in (KeeperState.EXPIRED_SESSION,
                             KeeperState.CONNECTING):
            async_object.set_exception(SessionExpiredError())
            return False

        self._queue.append((request, async_object))

        # wake the connection, guarding against a race with close()
        write_pipe = self._connection._write_pipe
        if write_pipe is None:
            async_object.set_exception(ConnectionClosedError(
                "Connection has been closed"))
        try:
            os.write(write_pipe, b'\0')
        except:
            async_object.set_exception(ConnectionClosedError(
                "Connection has been closed"))
Example #31
0
 def write(self, string):
     f = self.obtain()
     os.write(f.fileno(), string)
Example #32
0
def oswritebytes(fd, obj):
    os.write(fd, tobytes(obj))
Example #33
0
    def spawn(
        cls: Type[PtyProcessType],
        spawn_command: List[str],
        echo: bool = True,
        rows: int = 80,
        cols: int = 256,
    ) -> PtyProcessType:
        """
        Start the given command in a child process in a pseudo terminal.

        This does all the fork/exec type of stuff for a pty, and returns an instance of PtyProcess.
        For some devices setting terminal width strictly in the operating system (the actual network
        operating system) does not seem to be sufficient by itself for setting terminal length or
        width -- so we have optional values for rows/cols that can be passed here as well.

        Args:
            spawn_command: command to execute with arguments (if applicable), as a list
            echo: enable/disable echo -- defaults to True, should be left as True for "normal"
                scrapli operations, optionally disable for scrapli_netconf operations.
            rows: integer number of rows for ptyprocess "window"
            cols: integer number of cols for ptyprocess "window"

        Returns:
            PtyProcessType: instantiated PtyProcess object

        Raises:
            ScrapliValueError: if no ssh binary found on PATH
            Exception: IOError - if unable to set window size of child process
            Exception: OSError - if unable to spawn command in child process
            IOError: failing to reset window size
            exception: if we get an exception decoding output

        """
        # Note that it is difficult for this method to fail.
        # You cannot detect if the child process cannot start.
        # So the only way you can tell if the child process started
        # or not is to try to read from the file descriptor. If you get
        # EOF immediately then it means that the child is already dead.
        # That may not necessarily be bad because you may have spawned a child
        # that performs some task; creates no stdout output; and then dies.

        import fcntl
        import pty
        import resource
        import termios
        from pty import CHILD, STDIN_FILENO

        spawn_executable = which(spawn_command[0])
        if spawn_executable is None:
            raise ScrapliValueError("ssh executable not found!")
        spawn_command[0] = spawn_executable

        # [issue #119] To prevent the case where exec fails and the user is
        # stuck interacting with a python child process instead of whatever
        # was expected, we implement the solution from
        # http://stackoverflow.com/a/3703179 to pass the exception to the
        # parent process
        # [issue #119] 1. Before forking, open a pipe in the parent process.
        exec_err_pipe_read, exec_err_pipe_write = os.pipe()

        pid, fd = pty.fork()

        # Some platforms must call setwinsize() and setecho() from the
        # child process, and others from the master process. We do both,
        # allowing IOError for either.
        if pid == CHILD:
            try:
                _setwinsize(fd=STDIN_FILENO, rows=rows, cols=cols)
            except IOError as err:
                if err.args[0] not in (errno.EINVAL, errno.ENOTTY):
                    raise

            # disable echo if requested
            if echo is False:
                try:
                    _setecho(STDIN_FILENO, False)
                except (IOError, termios.error) as err:
                    if err.args[0] not in (errno.EINVAL, errno.ENOTTY):
                        raise

            # [issue #119] 3. The child closes the reading end and sets the
            # close-on-exec flag for the writing end.
            os.close(exec_err_pipe_read)
            fcntl.fcntl(exec_err_pipe_write, fcntl.F_SETFD, fcntl.FD_CLOEXEC)

            # Do not allow child to inherit open file descriptors from parent,
            # with the exception of the exec_err_pipe_write of the pipe.
            # Impose ceiling on max_fd: AIX bugfix for users with unlimited
            # nofiles where resource.RLIMIT_NOFILE is 2^63-1 and os.closerange()
            # occasionally raises out of range error
            max_fd = min(1048576,
                         resource.getrlimit(resource.RLIMIT_NOFILE)[0])
            pass_fds = sorted({exec_err_pipe_write})
            for pair in zip([2] + pass_fds, pass_fds + [max_fd]):
                os.closerange(pair[0] + 1, pair[1])

            try:
                os.execv(spawn_executable, spawn_command)
            except OSError as err:
                # [issue #119] 5. If exec fails, the child writes the error
                # code back to the parent using the pipe, then exits.
                tosend = f"OSError:{err.errno}:{str(err)}".encode()
                os.write(exec_err_pipe_write, tosend)
                os.close(exec_err_pipe_write)
                os._exit(os.EX_OSERR)

        # Parent
        inst = cls(pid, fd)

        # [issue #119] 2. After forking, the parent closes the writing end
        # of the pipe and reads from the reading end.
        os.close(exec_err_pipe_write)
        exec_err_data = os.read(exec_err_pipe_read, 4096)
        os.close(exec_err_pipe_read)

        # [issue #119] 6. The parent reads eof (a zero-length read) if the
        # child successfully performed exec, since close-on-exec made
        # successful exec close the writing end of the pipe. Or, if exec
        # failed, the parent reads the error code and can proceed
        # accordingly. Either way, the parent blocks until the child calls
        # exec.
        if len(exec_err_data) != 0:
            try:
                errclass, errno_s, errmsg = exec_err_data.split(b":", 2)
                exctype = getattr(builtins, errclass.decode("ascii"),
                                  Exception)

                exception = exctype(errmsg.decode("utf-8", "replace"))
                if exctype is OSError:
                    exception.errno = int(errno_s)
            except Exception:
                raise Exception("Subprocess failed, got bad error data: %r" %
                                exec_err_data)
            else:
                raise exception

        try:
            inst.setwinsize(rows=rows, cols=cols)
        except IOError as err:
            if err.args[0] not in (errno.EINVAL, errno.ENOTTY, errno.ENXIO):
                raise

        return inst
Example #34
0
def makelock(info: str,
             pathname: str,
             checkdeadlock: bool = True) -> "Optional[int]":
    """Try to make a lock at given path. Write info inside it.

    Stale non-symlink or symlink locks are removed automatically. Symlink locks
    are only used by legacy code, or by the new code temporarily to prevent
    issues running together with the old code.

    Return file descriptor on success. The file descriptor must be kept
    for the lock to be effective.

    Raise EAGAIN, likely caused by another process holding the lock.
    Raise EEXIST or ELOOP, likely caused by another legacy hg process
    holding the lock.

    Can also raise other errors or those errors for other causes.
    Callers should convert errors to error.LockHeld or error.LockUnavailable.
    """

    # This is a bit complex, since it aims to support old lock code where the
    # lock file is removed when the lock is released.  The simpler version
    # where the lock file does not get unlinked when releasing the lock is:
    #
    #     # Open the file. Create on demand. Fail if it's a symlink.
    #     fd = os.open(pathname, os.O_CREAT | os.O_RDWR | os.O_NOFOLLOW | O_CLOEXEC)
    #     try:
    #         fcntl.flock(fd, fcntl.LOCK_NB | fcntl.LOCK_EX)
    #         os.write(fd, info)
    #     except (OSError, IOError):
    #         os.close(fd)
    #         raise
    #     else:
    #         return fd
    #
    # With "unlink" on release, the above simple logic can break in this way:
    #
    #     [process 1] got fd.
    #     [process 2] got fd pointing to a same file.
    #     [process 1] .... release lock. file unlinked.
    #     [process 2] flock on fd. (broken lock - file was gone)
    #
    # A direct fix is to use O_EXCL to make sure the file is created by the
    # current process, then use "flock". That means there needs to be a way to
    # remove stale lock, and that is not easy. A naive check and delete can
    # break subtly:
    #
    #     [process 1] to check stale lock - got fd.
    #     [process 2] ... release lock. file unlinked.
    #     [process 1] flock taken, decided to remove file.
    #     [process 3] create a new lock.
    #     [process 1] unlink lock file. (wrong - removed the wrong lock)
    #
    # Instead of figuring out how to handle all corner cases carefully, we just
    # always lock the parent directory when doing "racy" write operations
    # (creating a lock, or removing a stale lock). So they become "atomic" and
    # safe. There are 2 kinds of write operations that can happen without
    # taking the directory lock:
    #
    #   - Legacy symlink lock creation or deletion. The new code errors out
    #     when it saw a symlink lock (os.open(..., O_NOFOLLOW) and os.rename).
    #     So they play well with each other.
    #   - Unlinking lock file when when releasing. The release logic is holding
    #     the flock. So it knows nobody else has the lock. Therefore it can do
    #     the unlink without extra locking.
    dirname = os.path.dirname(pathname)
    if checkdeadlock and pathname in _processlocks:
        raise error.ProgrammingError(
            "deadlock: %s was locked in the same process" % pathname)
    with _locked(dirname or "."):
        # Check and remove stale lock
        try:
            fd = os.open(pathname, os.O_RDONLY | os.O_NOFOLLOW | O_CLOEXEC)
        except (OSError, IOError) as ex:
            # ELOOP: symlink lock. Check if it's stale.
            if ex.errno == errno.ELOOP:
                oldinfo = os.readlink(pathname)
                if _issymlinklockstale(oldinfo, info):
                    os.unlink(pathname)
            elif ex.errno != errno.ENOENT:
                raise
        else:
            try:
                # Use fcntl to test stale lock
                fcntl.flock(fd, fcntl.LOCK_NB | fcntl.LOCK_EX)
                os.unlink(pathname)
            except (OSError, IOError) as ex:
                # EAGAIN: lock taken - return directly
                # ENOENT: lock removed already - continue
                if ex.errno != errno.ENOENT:
                    raise
            finally:
                os.close(fd)

        # Create symlink placeholder. Make sure the file replaced by
        # "os.rename" can only be this symlink. This avoids race condition
        # when legacy code creates the symlink lock without locking the
        # parent directory.
        #
        # This is basically the legacy lock logic.
        placeholdercreated = False
        try:
            os.symlink(info, pathname)
            placeholdercreated = True
        except (IOError, OSError) as ex:
            if ex.errno == errno.EEXIST:
                raise
        except AttributeError:
            pass

        if not placeholdercreated:
            # No symlink support. Suboptimal. Create a placeholder by using an
            # empty file.  Other legacy process might see a "malformed lock"
            # temporarily. New processes won't see this because both "readlock"
            # and "islocked" take the directory lock.
            fd = os.open(pathname,
                         os.O_CREAT | os.O_WRONLY | os.O_EXCL | O_CLOEXEC)
            os.close(fd)

        infobytes = encodeutf8(info)
        try:
            # Create new lock.
            #
            # mkstemp sets FD_CLOEXEC automatically. For thread-safety. Threads
            # used here (progress, profiling, Winodws update worker) do not fork.
            # So it's fine to not patch `os.open` here.
            fd, tmppath = tempfile.mkstemp(prefix="makelock", dir=dirname)
            try:
                os.fchmod(fd, 0o664)
                fcntl.flock(fd, fcntl.LOCK_NB | fcntl.LOCK_EX)
                os.write(fd, infobytes)
                os.rename(tmppath, pathname)
                _processlocks[pathname] = fd
                return fd
            except Exception:
                unlink(tmppath)
                os.close(fd)
                raise
        except Exception:
            # Remove the placeholder
            unlink(pathname)
            raise
Example #35
0
    elif thisarg == 'write_flash':
        write_addr = sys.argv.pop(0)
        binary = sys.argv.pop(0)
    elif thisarg:
        cmdline = cmdline + [thisarg]

cmdline = cmdline + ['write_flash']
if write_option:
    cmdline = cmdline + [write_option]
cmdline = cmdline + ['--flash_size', 'detect']
cmdline = cmdline + [write_addr, binary]

erase_file = ''
if erase_addr:
    # Generate temporary empty (0xff) file
    eraser = tempfile.mkstemp()
    erase_file = eraser[1]
    os.write(eraser[0], bytearray([0xff] * int(erase_len, 0)))
    os.close(eraser[0])
    cmdline = cmdline + [erase_addr, erase_file]

try:
    esptool.main(cmdline)
except esptool.FatalError as e:
    sys.stderr.write('\nA fatal esptool.py error occurred: %s' % e)
finally:
    if erase_file:
        os.remove(erase_file)
    if any(sys.exc_info()):
        sys.exit(2)
Example #36
0
 def prout(s):
     os.write(WiredTigerTestCase._dupout, s + '\n')
Example #37
0
 def flush(self):
     if self.writable:
         os.write(self._fd, self.buffer.getvalue())
         self.buffer.truncate(0), self.buffer.seek(0)
Example #38
0
def rpython_print_newline():
    import os
    os.write(1, "\n")
Example #39
0
    def process(self, file_struct, directory=None, strict_ownership=1):
        # Older servers will not return directories; if filetype is missing,
        # assume file

        if file_struct.get('filetype') == 'directory':
            if directory is None:
                directory = ""
            return None, utils.mkdir_p(directory + file_struct['path'])

        if directory:
            directory += os.path.split(file_struct['path'])[0]
        if file_struct.get('filetype') == 'symlink':
            if 'symlink' not in file_struct:
                raise Exception("Missing key symlink")

            (fullpath, dirs_created, fd) = maketemp(prefix=".rhn-cfg-tmp",
                                                    directory=directory)
            os.close(fd)
            os.unlink(fullpath)
            os.symlink(file_struct['symlink'], fullpath)
            return fullpath, dirs_created

        for k in self.file_struct_fields.keys():
            if k not in file_struct:
                # XXX
                raise Exception("Missing key %s" % k)

        encoding = ''

        if 'encoding' in file_struct:
            encoding = file_struct['encoding']

        contents = file_struct['file_contents']

        if contents and (encoding == 'base64'):
            contents = decodestring(bstr(contents))

        delim_start = file_struct['delim_start']
        delim_end = file_struct['delim_end']

        if ('checksum' in file_struct and 'checksum_type' in file_struct
                and 'verify_contents' in file_struct
                and file_struct['verify_contents']):
            if file_struct['checksum'] != utils.getContentChecksum(
                    file_struct['checksum_type'], contents):
                raise Exception(
                    "Corrupt file received: Content checksums do not match!")
        elif ('md5sum' in file_struct and 'verify_contents' in file_struct
              and file_struct['verify_contents']):
            if file_struct['md5sum'] != utils.getContentChecksum(
                    'md5', contents):
                raise Exception(
                    "Corrupt file received: Content checksums do not match!")
        elif ('verify_contents' in file_struct
              and file_struct['verify_contents']):
            raise Exception(
                "Corrupt file received: missing checksum information!")

        (fullpath, dirs_created, fd) = maketemp(prefix=".rhn-cfg-tmp",
                                                directory=directory)

        try:
            os.write(fd, bstr(contents))
        except Exception:
            raise
        finally:
            os.close(fd)

        # try to set mtime and ctime of the file to
        # the last modified time on the server
        if 'modified' in file_struct:
            try:
                modified = xmlrpc_time(file_struct['modified'].value)
                epoch_time = time.mktime(modified)
                os.utime(fullpath, (epoch_time, epoch_time))
            except (ValueError, AttributeError):
                # we can't parse modified time
                pass

        return fullpath, dirs_created
Example #40
0
def preprocess(path):
    with open(path, "rb") as fh:
        lines = []
        it = iter(fh)

        for l in it:
            # zstd.h includes <stddef.h>, which is also included by cffi's
            # boilerplate. This can lead to duplicate declarations. So we strip
            # this include from the preprocessor invocation.
            #
            # The same things happens for including zstd.h, so give it the same
            # treatment.
            #
            # We define ZSTD_STATIC_LINKING_ONLY, which is redundant with the inline
            # #define in zstdmt_compress.h and results in a compiler warning. So drop
            # the inline #define.
            if l.startswith((
                    b"#include <stddef.h>",
                    b'#include "zstd.h"',
                    b"#define ZSTD_STATIC_LINKING_ONLY",
            )):
                continue

            # The preprocessor environment on Windows doesn't define include
            # paths, so the #include of limits.h fails. We work around this
            # by removing that import and defining INT_MAX ourselves. This is
            # a bit hacky. But it gets the job done.
            # TODO make limits.h work on Windows so we ensure INT_MAX is
            # correct.
            if l.startswith(b"#include <limits.h>"):
                l = b"#define INT_MAX 2147483647\n"

            # ZSTDLIB_API may not be defined if we dropped zstd.h. It isn't
            # important so just filter it out.
            if l.startswith(b"ZSTDLIB_API"):
                l = l[len(b"ZSTDLIB_API "):]

            lines.append(l)

    fd, input_file = tempfile.mkstemp(suffix=".h")
    os.write(fd, b"".join(lines))
    os.close(fd)

    try:
        env = dict(os.environ)
        # cffi attempts to decode source as ascii. And the preprocessor
        # may insert non-ascii for some annotations. So try to force
        # ascii output via LC_ALL.
        env["LC_ALL"] = "C"

        if getattr(compiler, "_paths", None):
            env["PATH"] = compiler._paths
        process = subprocess.Popen(args + [input_file],
                                   stdout=subprocess.PIPE,
                                   env=env)
        output = process.communicate()[0]
        ret = process.poll()
        if ret:
            raise Exception("preprocessor exited with error")

        return output
    finally:
        os.unlink(input_file)
Example #41
0
 def write(self, str):
     os.write(self.fd, str)
Example #42
0
def rpython_print_item(s):
    import os
    os.write(1, s)
Example #43
0
 def write(self, s):
     os.write(self.fd, s)
Example #44
0
    def daemonize(pidfile):
        try:
            pid = os.fork()
            if pid > 0:
                # exit first parent
                os._exit(0)

        except OSError as exc:
            raise error.ControlPlaneError('ERROR: fork #1 failed: %s' % exc)

        # decouple from parent environment
        try:
            os.chdir('/')
            os.setsid()

        except OSError:
            pass

        os.umask(0)

        # do second fork
        try:
            pid = os.fork()
            if pid > 0:
                # exit from second parent
                os._exit(0)

        except OSError as exc:
            raise error.ControlPlaneError('ERROR: fork #2 failed: %s' % exc)

        def signal_cb(s, f):
            raise KeyboardInterrupt

        for s in (signal.SIGTERM, signal.SIGINT, signal.SIGHUP,
                  signal.SIGQUIT):
            signal.signal(s, signal_cb)

        # write pidfile
        def atexit_cb():
            try:
                if pidfile:
                    os.remove(pidfile)

            except OSError:
                pass

        atexit.register(atexit_cb)

        try:
            if pidfile:
                fd, nm = tempfile.mkstemp(dir=os.path.dirname(pidfile))
                os.write(fd, ('%d\n' % os.getpid()).encode('utf-8'))
                os.close(fd)
                os.rename(nm, pidfile)

        except Exception as exc:
            raise error.ControlPlaneError('Failed to create PID file %s: %s' %
                                          (pidfile, exc))

        # redirect standard file descriptors
        sys.stdout.flush()
        sys.stderr.flush()
        si = open(os.devnull, 'r')
        so = open(os.devnull, 'a+')
        se = open(os.devnull, 'a+')

        os.dup2(si.fileno(), sys.stdin.fileno())
        os.dup2(so.fileno(), sys.stdout.fileno())
        os.dup2(se.fileno(), sys.stderr.fileno())
Example #45
0
def pdf_from_html(html: str,
                  header_html: str = None,
                  footer_html: str = None,
                  wkhtmltopdf_filename: str = None,
                  wkhtmltopdf_options: Dict[str, Any] = None,
                  output_path: str = None,
                  debug: bool = False) -> Union[bytes, bool]:
    """
    Takes HTML and either:
        - returns a PDF (as a binary object in memory), if output_path is None
        - creates a PDF in the file specified by output_path
    Uses wkhtmltopdf (with pdfkit)
        - faster than xhtml2pdf
        - tables not buggy like Weasyprint
        - however, doesn't support CSS Paged Media, so we have the
          header_html and footer_html options to allow you to pass appropriate
          HTML content to serve as the header/footer (rather than passing it
          within the main HTML).
    """
    # Customized for this Django site
    wkhtmltopdf_filename = wkhtmltopdf_filename or settings.WKHTMLTOPDF_FILENAME  # noqa
    if wkhtmltopdf_options is None:
        wkhtmltopdf_options = settings.WKHTMLTOPDF_OPTIONS
    else:
        wkhtmltopdf_options = merge_two_dicts(settings.WKHTMLTOPDF_OPTIONS,
                                              wkhtmltopdf_options)

    # Generic
    if not wkhtmltopdf_filename:
        config = None
    else:
        if FIX_PDFKIT_ENCODING_BUG:
            config = pdfkit.configuration(
                wkhtmltopdf=wkhtmltopdf_filename.encode('utf-8'))
            # the bug is that pdfkit.pdfkit.PDFKit.__init__ will attempt to
            # decode the string in its configuration object;
            # https://github.com/JazzCore/python-pdfkit/issues/32
        else:
            config = pdfkit.configuration(wkhtmltopdf=wkhtmltopdf_filename)
    # Temporary files that a subprocess can read:
    #   http://stackoverflow.com/questions/15169101
    # wkhtmltopdf requires its HTML files to have ".html" extensions:
    #   http://stackoverflow.com/questions/5776125
    h_filename = None
    f_filename = None
    try:
        if header_html:
            h_fd, h_filename = tempfile.mkstemp(suffix='.html')
            os.write(h_fd, header_html.encode('utf8'))
            os.close(h_fd)
            wkhtmltopdf_options["header-html"] = h_filename
        if footer_html:
            f_fd, f_filename = tempfile.mkstemp(suffix='.html')
            os.write(f_fd, footer_html.encode('utf8'))
            os.close(f_fd)
            wkhtmltopdf_options["footer-html"] = f_filename
        if debug:
            log.critical("wkhtmltopdf_options: " + repr(wkhtmltopdf_options))
        kit = pdfkit.pdfkit.PDFKit(html,
                                   'string',
                                   configuration=config,
                                   options=wkhtmltopdf_options)
        return kit.to_pdf(path=output_path)
        # ... if path is None, will return the PDF
        # ... if path is specified, will return True
        # https://github.com/JazzCore/python-pdfkit/blob/master/pdfkit/pdfkit.py  # noqa
    finally:
        if h_filename:
            os.remove(h_filename)
        if f_filename:
            os.remove(f_filename)
Example #46
0
 else:
     if len(argv) > 2:
         mpd_print(
             1,
             "when using gdb, pass cmd-line args to user pgms via the 'run' cmd"
         )
         exit(-1)
     gdb_args = argv[1]
 gdb_info = Popen4('gdb -q %s' % (gdb_args), 0)
 gdbPid = gdb_info.pid
 # print "PID=%d GDBPID=%d" % (getpid(),gdbPid) ; stdout.flush()
 gdb_sin = gdb_info.tochild
 gdb_sin_fileno = gdb_sin.fileno()
 gdb_sout_serr = gdb_info.fromchild
 gdb_sout_serr_fileno = gdb_sout_serr.fileno()
 write(gdb_sin_fileno, 'set prompt (gdb)\\n\n')
 gdb_line = gdb_sout_serr.readline()
 # check if gdb reports any errors
 if findall(r'.*: No such file or directory.', gdb_line) != []:
     print gdb_line,
     stdout.flush()
     exit(-1)
 mpd_print(0000, "LINE1=|%s|" % (gdb_line.rstrip()))
 write(gdb_sin_fileno, 'set confirm off\n')
 gdb_line = gdb_sout_serr.readline()
 mpd_print(0000, "LINE2=|%s|" % (gdb_line.rstrip()))
 write(gdb_sin_fileno, 'handle SIGUSR1 nostop noprint\n')
 gdb_line = gdb_sout_serr.readline()
 mpd_print(0000, "LINE3=|%s|" % (gdb_line.rstrip()))
 write(gdb_sin_fileno, 'handle SIGPIPE nostop noprint\n')
 gdb_line = gdb_sout_serr.readline()
    def call_from_executor(self, callback):
        self._calls_from_executor.append(callback)

        if self._schedule_pipe:
            os.write(self._schedule_pipe[1], b'x')
Example #48
0
    def _load_urllib(self, filename, kwargs):
        '''(internal) Loading a network file. First download it, save it to a
        temporary file, and pass it to _load_local().'''
        if PY2:
            import urllib2 as urllib_request

            def gettype(info):
                return info.gettype()
        else:
            import urllib.request as urllib_request

            def gettype(info):
                return info.get_content_type()

        proto = filename.split(':', 1)[0]
        if proto == 'smb':
            try:
                # note: it's important to load SMBHandler every time
                # otherwise the data is occasionally not loaded
                from smb.SMBHandler import SMBHandler
            except ImportError:
                Logger.warning(
                    'Loader: can not load PySMB: make sure it is installed')
                return
        import tempfile
        data = fd = _out_osfd = None
        try:
            _out_filename = ''

            if proto == 'smb':
                # read from samba shares
                fd = urllib_request.build_opener(SMBHandler).open(filename)
            else:
                # read from internet
                request = urllib_request.Request(filename)
                if Config.has_option('network', 'useragent'):
                    useragent = Config.get('network', 'useragent')
                    if useragent:
                        request.add_header('User-Agent', useragent)
                opener = urllib_request.build_opener()
                fd = opener.open(request)

            if '#.' in filename:
                # allow extension override from URL fragment
                suffix = '.' + filename.split('#.')[-1]
            else:
                ctype = gettype(fd.info())
                suffix = mimetypes.guess_extension(ctype)
                suffix = LoaderBase.EXT_ALIAS.get(suffix, suffix)
                if not suffix:
                    # strip query string and split on path
                    parts = filename.split('?')[0].split('/')[1:]
                    while len(parts) > 1 and not parts[0]:
                        # strip out blanks from '//'
                        parts = parts[1:]
                    if len(parts) > 1 and '.' in parts[-1]:
                        # we don't want '.com', '.net', etc. as the extension
                        suffix = '.' + parts[-1].split('.')[-1]
            _out_osfd, _out_filename = tempfile.mkstemp(prefix='kivyloader',
                                                        suffix=suffix)

            idata = fd.read()
            fd.close()
            fd = None

            # write to local filename
            write(_out_osfd, idata)
            close(_out_osfd)
            _out_osfd = None

            # load data
            data = self._load_local(_out_filename, kwargs)

            # FIXME create a clean API for that
            for imdata in data._data:
                imdata.source = filename
        except Exception as ex:
            Logger.exception('Loader: Failed to load image <%s>' % filename)
            # close file when remote file not found or download error
            try:
                if _out_osfd:
                    close(_out_osfd)
            except OSError:
                pass

            # update client
            for c_filename, client in self._client[:]:
                if filename != c_filename:
                    continue
                # got one client to update
                client.image = self.error_image
                client.dispatch('on_error', error=ex)
                self._client.remove((c_filename, client))

            return self.error_image
        finally:
            if fd:
                fd.close()
            if _out_osfd:
                close(_out_osfd)
            if _out_filename != '':
                unlink(_out_filename)

        return data
Example #49
0
    def __init__(self, conn, opts):
        XpraClientBase.__init__(self, opts)
        self.start_time = time.time()
        self._window_to_id = {}
        self._id_to_window = {}
        title = opts.title
        if opts.title_suffix is not None:
            title = "@title@ %s" % opts.title_suffix
        self.title = title
        self.readonly = opts.readonly
        self.session_name = opts.session_name
        self.compression_level = opts.compression_level
        self.auto_refresh_delay = opts.auto_refresh_delay
        self.max_bandwidth = opts.max_bandwidth
        if self.max_bandwidth > 0.0 and self.jpegquality == 0:
            """ jpegquality was not set, use a better start value """
            self.jpegquality = 50

        self.server_capabilities = {}

        self.mmap_enabled = False
        self.server_start_time = -1
        self.server_platform = ""
        self.server_actual_desktop_size = None
        self.server_desktop_size = None
        self.server_randr = False
        self.pixel_counter = maxdeque(maxlen=100)
        self.server_latency = maxdeque(maxlen=100)
        self.server_load = None
        self.client_latency = maxdeque(maxlen=100)
        self.toggle_cursors_bell_notify = False
        self.bell_enabled = True
        self.cursors_enabled = True
        self.notifications_enabled = True
        self.clipboard_enabled = False
        self.mmap = None
        self.mmap_token = None
        self.mmap_file = None
        self.mmap_size = 0

        self._client_extras = ClientExtras(self, opts)
        self.clipboard_enabled = not self.readonly and opts.clipboard and self._client_extras.supports_clipboard(
        )
        self.supports_mmap = opts.mmap and (
            "rgb24" in ENCODINGS) and self._client_extras.supports_mmap()
        if self.supports_mmap:
            try:
                import mmap
                import tempfile
                import uuid
                from stat import S_IRUSR, S_IWUSR, S_IRGRP, S_IWGRP
                mmap_dir = os.getenv("TMPDIR", "/tmp")
                if not os.path.exists(mmap_dir):
                    raise Exception("TMPDIR %s does not exist!" % mmap_dir)
                #create the mmap file, the mkstemp that is called via NamedTemporaryFile ensures
                #that the file is readable and writable only by the creating user ID
                temp = tempfile.NamedTemporaryFile(prefix="xpra.",
                                                   suffix=".mmap",
                                                   dir=mmap_dir)
                #keep a reference to it so it does not disappear!
                self._mmap_temp_file = temp
                self.mmap_file = temp.name
                fd = temp.file.fileno()
                #set the group permissions and gid if the mmap-group option is specified
                if opts.mmap_group and type(
                        conn.target) == str and os.path.exists(conn.target):
                    s = os.stat(conn.target)
                    os.fchown(fd, -1, s.st_gid)
                    os.fchmod(fd, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP)
                self.mmap_size = max(
                    4096, mmap.PAGESIZE) * 32 * 1024  #generally 128MB
                log("using mmap file %s, fd=%s, size=%s", self.mmap_file, fd,
                    self.mmap_size)
                os.lseek(fd, self.mmap_size - 1, os.SEEK_SET)
                assert os.write(fd, '\x00')
                os.lseek(fd, 0, os.SEEK_SET)
                self.mmap = mmap.mmap(fd, length=self.mmap_size)
                #write the 16 byte token one byte at a time - no endianness
                self.mmap_token = uuid.uuid4().int
                log.debug("mmap_token=%s", self.mmap_token)
                v = self.mmap_token
                for i in range(0, 16):
                    poke = ctypes.c_ubyte.from_buffer(self.mmap, 512 + i)
                    poke.value = v % 256
                    v = v >> 8
                assert v == 0
            except Exception, e:
                log.error("failed to setup mmap: %s", e)
                self.supports_mmap = False
                self.clean_mmap()
                self.mmap = None
                self.mmap_file = None
                self.mmap_size = 0
Example #50
0
        def _execute_child(self, args, executable, preexec_fn, close_fds, cwd, env, universal_newlines, startupinfo, creationflags, shell, to_close, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite):
            """Execute program (POSIX version)"""
            if isinstance(args, types.StringTypes):
                args = [args]
            else:
                args = list(args)
            if shell:
                args = ['/bin/sh', '-c'] + args
                if executable:
                    args[0] = executable
            if executable is None:
                executable = args[0]

            def _close_in_parent(fd):
                os.close(fd)
                to_close.remove(fd)

            errpipe_read, errpipe_write = self.pipe_cloexec()
            try:
                try:
                    gc_was_enabled = gc.isenabled()
                    gc.disable()
                    try:
                        self.pid = os.fork()
                    except:
                        if gc_was_enabled:
                            gc.enable()
                        raise

                    self._child_created = True
                    if self.pid == 0:
                        try:
                            if p2cwrite is not None:
                                os.close(p2cwrite)
                            if c2pread is not None:
                                os.close(c2pread)
                            if errread is not None:
                                os.close(errread)
                            os.close(errpipe_read)
                            if c2pwrite == 0:
                                c2pwrite = os.dup(c2pwrite)
                            if errwrite == 0 or errwrite == 1:
                                errwrite = os.dup(errwrite)

                            def _dup2(a, b):
                                if a == b:
                                    self._set_cloexec_flag(a, False)
                                elif a is not None:
                                    os.dup2(a, b)
                                return

                            _dup2(p2cread, 0)
                            _dup2(c2pwrite, 1)
                            _dup2(errwrite, 2)
                            closed = {None}
                            for fd in [p2cread, c2pwrite, errwrite]:
                                if fd not in closed and fd > 2:
                                    os.close(fd)
                                    closed.add(fd)

                            if cwd is not None:
                                os.chdir(cwd)
                            if preexec_fn:
                                preexec_fn()
                            if close_fds:
                                self._close_fds(but=errpipe_write)
                            if env is None:
                                os.execvp(executable, args)
                            else:
                                os.execvpe(executable, args, env)
                        except:
                            exc_type, exc_value, tb = sys.exc_info()
                            exc_lines = traceback.format_exception(exc_type, exc_value, tb)
                            exc_value.child_traceback = ''.join(exc_lines)
                            os.write(errpipe_write, pickle.dumps(exc_value))

                        os._exit(255)
                    if gc_was_enabled:
                        gc.enable()
                finally:
                    os.close(errpipe_write)

                data = _eintr_retry_call(os.read, errpipe_read, 1048576)
            finally:
                if p2cread is not None and p2cwrite is not None:
                    _close_in_parent(p2cread)
                if c2pwrite is not None and c2pread is not None:
                    _close_in_parent(c2pwrite)
                if errwrite is not None and errread is not None:
                    _close_in_parent(errwrite)
                os.close(errpipe_read)

            if data != '':
                try:
                    _eintr_retry_call(os.waitpid, self.pid, 0)
                except OSError as e:
                    if e.errno != errno.ECHILD:
                        raise

                child_exception = pickle.loads(data)
                raise child_exception
            return
Example #51
0
 def write(self, data):
     return os.write(self.fd, data)
Example #52
0
 def set(self):
     # It doesn't really matter what we are writing to the pipe,
     # as long, something gets written
     os.write(self.__write_fd, 'x')
Example #53
0
def main():
    """Handle command-line arguments."""
    from docopt import docopt

    arguments = docopt(__doc__)

    if arguments['extract-qs-from-aux']:
        string = open(arguments['<file>']).read()
        print(" ".join(extract_qs_from_aux_string(string)))

    elif arguments['write-bbl-from-aux']:
        aux_filename = arguments['<file>']
        base_filename, _ = splitext(aux_filename)
        bbl_filename = base_filename + '.bbl'

        string = open(aux_filename).read()
        qs = extract_qs_from_aux_string(string)
        entities = wb_get_entities(qs)

        widest_label = max([len(q) for q in qs])
        bbl = u'\\begin{thebibliography}{%d}\n\n' % widest_label

        for q in qs:
            entity = entities[q]
            bbl += '\\bibitem{%s}\n' % q
            bbl += u", ".join(entity_to_authors(entity)) + '.\n'
            bbl += entity_to_title(entity) + '.\n'

            bbl += '\n'

        bbl += '\\end{thebibliography}\n'

        with open(bbl_filename, 'w') as f:
            f.write(bbl.encode('utf-8'))

        with open(aux_filename, 'a') as f:
            for n, q in enumerate(qs, 1):
                f.write('\\bibcite{%s}{%d}\n' % (q, n))

    elif arguments['write-bib-from-aux']:
        aux_filename = arguments['<file>']
        base_filename, _ = splitext(aux_filename)
        bib_filename = base_filename + '.bib'

        string = open(aux_filename).read()
        qs = list(set(extract_qs_from_aux_string(string)))
        keys = qs[:]
        dois = list(set(extract_dois_from_aux_string(string)))
        for doi in dois:
            qs_doi = doi_to_qs(doi)
            if len(qs_doi) == 0:
                print('Could not find Wikidata item for {doi}'.format(doi=doi))
                continue
            if len(qs_doi) > 1:
                print(('Multiple Wikidata items for {doi}: {qs}.'
                       'Using first.').format(doi=doi, qs=qs_doi))
            q = qs_doi[0]
            qs.append(q)
            keys.append(doi)

        entities = wb_get_entities(qs)

        bib = u("")
        for q, key in zip(qs, keys):
            entity = entities[q]
            bib += entity_to_bibtex_entry(entity, key=key)
            bib += '\n'

        # Write BibTeX-formatted string to file
        output_file = os.open(bib_filename, os.O_RDWR | os.O_CREAT)
        output_encoding = "utf-8"
        write(output_file, bib.encode(output_encoding))
Example #54
0
        "-s",
        "--suppress-output",
        help=
        "Supresses output from so that stdout and stderr can be kept pure for the ssh transmission",
        action="store_true")

    # shows help if no arguments are provided
    if len(sys.argv) == 1:
        parser.print_help()
        sys.exit(1)

    args = parser.parse_args()

    # Return discover of plot if requested
    if args.discover:
        os.write(1, discover())

    # Return a plot if iovs are provided
    if args.iovs:

        # Run plugin with arguments
        result = deserialize_iovs(args.db, args.plugin, args.plot, args.tag,
                                  args.time_type, json.loads(args.iovs))

        # If test -> output the result as formatted json
        if args.test:
            os.write(1, json.dumps(json.loads(result), indent=4))

        # If image plot -> get image file from result, open it and output bytes
        elif args.image_plot:
Example #55
0
def isexit(str):
	if str == "exit":
		os.write(1, ("Goodbye...\n").encode())
		sys.exit(0)
Example #56
0
def pipedump(object, fd):
    """Pickle an object to the stream given by a numeric descriptor."""
    os.write(fd, pickle.dumps(object))
Example #57
0
    def visualize(self):

        has_gv = False # ghostview
        has_open = False # Mac OS X utility
        if platform.system() in ('Windows', 'Microsoft'):
            # win32
            psviewer = 'gsview32'
            has_gv = True
        else:
            # Assume it's UNIX (either proper or Mac OS X)
            psviewer = 'gv'
    
            if os.system("which dot 2> /dev/null > /dev/null") != 0:
                sys.stderr.write("Error: You must install graphviz before trying to visualize a graph.\n")
                sys.exit(1)

            if os.system("which gv 2> /dev/null > /dev/null") == 0:
                has_gv = True

            if os.system("which open 2> /dev/null > /dev/null") == 0:
                has_open = True

            if not (has_gv or has_open):
                sys.stderr.write("Error: You must install 'gv' or 'open' before trying to visualize a graph.\n")
                sys.exit(1)
    
        o = []
        for s in self.states:
            if s.is_start:
                shape = ", shape=doublecircle"
            else:
                shape = ""
            
            o.append("""%s [label="%s"%s];""" % (id(s), s.name, shape))

        for t in self.transitions:
            if not t.is_marked:
                style = ", style=dashed"
            else:
                style = ""
            
            o.append("""%s -> %s [label="%s", arrowhead=vee%s];""" % (id(t.source), id(t.target), t.name, style))
            
        handle, dot_path = tempfile.mkstemp()
        os.write(handle, "digraph {  %s\n}" % "\n  ".join(o))
        os.close(handle)
        
        if has_open:
            handle, ps_path = tempfile.mkstemp(".pdf")
            os.close(handle)
        
            os.system("dot -T pdf -o %s %s" % (ps_path, dot_path))
            os.system("open %s&" % ps_path)
        elif has_gv:
            handle, ps_path = tempfile.mkstemp(".ps")
            os.close(handle)
        
            os.system("dot -T ps -o %s %s" % (ps_path, dot_path))
            os.system("%s %s&" % (psviewer, ps_path))

        os.remove(dot_path)
Example #58
0
 def stop(self):
     _logger.info("graceful stop requested")
     self._stop = True
     # wakeup the select() in wait_notification
     os.write(self._stop_pipe[1], b".")
Example #59
0
    def translate(
        self,
        src,
        tgt=None,
        batch_size=None,
        batch_type="sents",
        attn_debug=False,
        align_debug=False,
        phrase_table="",
    ):
        """Translate content of ``src`` and get gold scores from ``tgt``.

        Args:
            src: See :func:`self.src_reader.read()`.
            tgt: See :func:`self.tgt_reader.read()`.
            batch_size (int): size of examples per mini-batch
            attn_debug (bool): enables the attention logging
            align_debug (bool): enables the word alignment logging

        Returns:
            (`list`, `list`)

            * all_scores is a list of `batch_size` lists of `n_best` scores
            * all_predictions is a list of `batch_size` lists
                of `n_best` predictions
        """

        if batch_size is None:
            raise ValueError("batch_size must be set")

        if self.tgt_prefix and tgt is None:
            raise ValueError("Prefix should be feed to tgt if -tgt_prefix.")

        src_data = {"reader": self.src_reader, "data": src}
        tgt_data = {"reader": self.tgt_reader, "data": tgt}
        _readers, _data = inputters.Dataset.config([("src", src_data),
                                                    ("tgt", tgt_data)])

        data = inputters.Dataset(
            self.fields,
            readers=_readers,
            data=_data,
            sort_key=inputters.str2sortkey[self.data_type],
            filter_pred=self._filter_pred,
        )

        data_iter = inputters.OrderedIterator(
            dataset=data,
            device=self._dev,
            batch_size=batch_size,
            batch_size_fn=max_tok_len if batch_type == "tokens" else None,
            train=False,
            sort=False,
            sort_within_batch=True,
            shuffle=False,
        )

        xlation_builder = onmt.translate.TranslationBuilder(
            data,
            self.fields,
            self.n_best,
            self.replace_unk,
            tgt,
            self.phrase_table,
        )

        # Statistics
        counter = count(1)
        pred_score_total, pred_words_total = 0, 0
        gold_score_total, gold_words_total = 0, 0

        all_scores = []
        all_predictions = []

        start_time = time.time()

        for batch in data_iter:
            batch_data = self.translate_batch(batch, data.src_vocabs,
                                              attn_debug)
            translations = xlation_builder.from_batch(batch_data)

            for trans in translations:
                all_scores += [trans.pred_scores[:self.n_best]]
                pred_score_total += trans.pred_scores[0]
                pred_words_total += len(trans.pred_sents[0])
                if tgt is not None:
                    gold_score_total += trans.gold_score
                    gold_words_total += len(trans.gold_sent) + 1

                n_best_preds = [
                    " ".join(pred) for pred in trans.pred_sents[:self.n_best]
                ]
                if self.report_align:
                    align_pharaohs = [
                        build_align_pharaoh(align)
                        for align in trans.word_aligns[:self.n_best]
                    ]
                    n_best_preds_align = [
                        " ".join(align) for align in align_pharaohs
                    ]
                    n_best_preds = [
                        pred + DefaultTokens.ALIGNMENT_SEPARATOR + align for
                        pred, align in zip(n_best_preds, n_best_preds_align)
                    ]
                all_predictions += [n_best_preds]
                self.out_file.write("\n".join(n_best_preds) + "\n")
                self.out_file.flush()

                if self.verbose:
                    sent_number = next(counter)
                    output = trans.log(sent_number)
                    if self.logger:
                        self.logger.info(output)
                    else:
                        os.write(1, output.encode("utf-8"))

                if attn_debug:
                    preds = trans.pred_sents[0]
                    preds.append(DefaultTokens.EOS)
                    attns = trans.attns[0].tolist()
                    if self.data_type == "text":
                        srcs = trans.src_raw
                    else:
                        srcs = [str(item) for item in range(len(attns[0]))]
                    output = report_matrix(srcs, preds, attns)
                    if self.logger:
                        self.logger.info(output)
                    else:
                        os.write(1, output.encode("utf-8"))

                if align_debug:
                    tgts = trans.pred_sents[0]
                    align = trans.word_aligns[0].tolist()
                    if self.data_type == "text":
                        srcs = trans.src_raw
                    else:
                        srcs = [str(item) for item in range(len(align[0]))]
                    output = report_matrix(srcs, tgts, align)
                    if self.logger:
                        self.logger.info(output)
                    else:
                        os.write(1, output.encode("utf-8"))

        end_time = time.time()

        if self.report_score:
            msg = self._report_score("PRED", pred_score_total,
                                     pred_words_total)
            self._log(msg)
            if tgt is not None:
                msg = self._report_score("GOLD", gold_score_total,
                                         gold_words_total)
                self._log(msg)

        if self.report_time:
            total_time = end_time - start_time
            self._log("Total translation time (s): %f" % total_time)
            self._log("Average translation time (s): %f" %
                      (total_time / len(all_predictions)))
            self._log("Tokens per second: %f" %
                      (pred_words_total / total_time))

        if self.dump_beam:
            import json

            json.dump(
                self.translator.beam_accum,
                codecs.open(self.dump_beam, "w", "utf-8"),
            )
        return all_scores, all_predictions
Example #60
0
    def __call_nohup__(self, commander):
        (pid, self.r_path, self.w_path, self.stdin_path, self.stdout_path,
         self.stderr_path) = daemonize(self.basecmd.cmd_hash)
        if pid == 1:  # Child process make commands
            commander._close_cmds_stdios(self)
            (self.pid, r_pipe, w_pipe, stdin_pipe,
             stdout_pipe, stderr_pipe) = create_process_cmd()
            if self.pid == 0:  # Child process make commands
                self.msg = ms.Messenger(ms.StdIOWrapperIn(r_pipe),
                                        ms.StdIOWrapperOut(w_pipe))
                try:
                    self.basecmd.results = self.obj(*self.basecmd.args,
                                                    **self.basecmd.kargs)
                except Exception:
                    err_msg = traceback.format_exc()
                    self.msg.write_msg(remote_interface.CmdTraceBack(err_msg))
                    sys.exit(-1)
                finally:
                    self.msg.write_msg(self.basecmd.results)
                sys.exit(0)
            else:
                # helper child process open communication pipes.
                # This process is able to manage problem with connection width
                # main parent process. It allows start unchanged child process.
                self.r_pipe = os.open(self.r_path, os.O_RDONLY)
                self.w_pipe = os.open(self.w_path, os.O_WRONLY)
                sys.stdout = os.fdopen(os.open(self.stdout_path, os.O_WRONLY),
                                       "w",
                                       0)
                sys.stderr = os.fdopen(os.open(self.stderr_path, os.O_WRONLY),
                                       "w",
                                       0)
                sys.stdin = os.fdopen(os.open(self.stdin_path, os.O_RDONLY),
                                      "r",
                                      0)

                w_fds = [r_pipe, w_pipe, stdin_pipe, stdout_pipe, stderr_pipe]
                m_fds = [self.r_pipe,
                         self.w_pipe,
                         sys.stdin.fileno(),
                         sys.stdout.fileno(),
                         sys.stderr.fileno()]
                p = select.poll()
                p.register(r_pipe)
                p.register(w_pipe)
                # p.register(stdin_pipe)
                p.register(stdout_pipe)
                p.register(stderr_pipe)
                p.register(self.r_pipe)
                # p.register(self.w_pipe)
                p.register(sys.stdin.fileno())
                # p.register(sys.stdout.fileno())
                # p.register(sys.stderr.fileno())
                io_map = {r_pipe: self.w_pipe,
                          self.r_pipe: w_pipe,
                          sys.stdin.fileno(): stdin_pipe,
                          stdout_pipe: sys.stdout.fileno(),
                          stderr_pipe: sys.stderr.fileno()}
                while 1:
                    d = p.poll()
                    w_ev = [x for x in d if x[0] in w_fds]
                    m_ev = [x for x in d if x[0] in m_fds]
                    w_hup, w_read, _ = sort_fds_event(w_ev)
                    m_hup, m_read, _ = sort_fds_event(m_ev)
                    if m_hup:
                        time.sleep(0.1)
                    if w_hup:  # child process finished
                        for r in w_read:
                            data = os.read(r, 16384)
                            os.write(io_map[r], data)
                        break
                    for r in w_read:
                        data = os.read(r, 16384)
                        os.write(io_map[r], data)
                    for r in m_read:
                        data = os.read(r, 16384)
                        os.write(io_map[r], data)
                self.msg = ms.Messenger(ms.StdIOWrapperIn(self.r_pipe),
                                        ms.StdIOWrapperOut(self.w_pipe))
                self.msg.write_msg(CmdFinish())
                exit(0)
        else:  # main process open communication named pipes.
            self.w_pipe = os.open(self.w_path, os.O_WRONLY)
            self.r_pipe = os.open(self.r_path, os.O_RDONLY)
            self.stdout_pipe = os.open(self.stdout_path, os.O_RDONLY)
            self.stderr_pipe = os.open(self.stderr_path, os.O_RDONLY)
            self.stdin_pipe = os.open(self.stdin_path, os.O_WRONLY)
            self.msg = ms.Messenger(ms.StdIOWrapperIn(self.r_pipe),
                                    ms.StdIOWrapperOut(self.w_pipe))