Exemplo n.º 1
0
def ReadFiles(FileSize, location):
    if FileSize == '2k':
        NumFiles = No2kFiles
        ddBlockSize = 512
    elif FileSize == '4k':
        NumFiles = No4kFiles
        ddBlockSize = '1k'
    elif FileSize == '8k':
        NumFiles = No8kFiles
        ddBlockSize = '2k'
    elif FileSize == '16k':
        NumFiles = No16kFiles
        ddBlockSize = '4k'
    elif FileSize == '32k':
        NumFiles = No32kFiles
        ddBlockSize = '8k'
    else:
        print "Nonsense"
    i = 1
    while (i <= NumFiles):

        #FSTAT
        path = '%s/file%s-%s' % (location, FileSize, i)
        print path
        fd = os.open(path, os.O_RDWR)
        print "FD is %s" % (fd)
        os.fstatvfs(fd)
        os.close(fd)
        sleep(1)

        #GETATTR, ACCESS, READ
        call("dd if=%s/file%s-%s of=/dev/null bs=%s count=4" %
             (location, FileSize, i, ddBlockSize),
             shell=True)
        i = i + 1
Exemplo n.º 2
0
def test_fstatvfs():
    if not hasattr(os, 'fstatvfs'):
        py.test.skip('posix specific function')
    try:
        os.fstatvfs(0)
    except OSError, e:
        py.test.skip("the underlying os.fstatvfs() failed: %s" % e)
Exemplo n.º 3
0
def fstatvfs(space, fd):
    try:
        st = os.fstatvfs(fd)
    except OSError as e:
        raise wrap_oserror(space, e)
    else:
        return build_statvfs_result(space, st)
Exemplo n.º 4
0
def fstatvfs(space, fd):
    try:
        st = os.fstatvfs(fd)
    except OSError as e:
        raise wrap_oserror(space, e)
    else:
        return build_statvfs_result(space, st)
Exemplo n.º 5
0
Arquivo: lsf.py Projeto: PeteW/luigi
    def _init_local(self):

        base_tmp_dir = self.shared_tmp_dir

        random_id = '%016x' % random.getrandbits(64)
        task_name = random_id + self.task_id
        # If any parameters are directories, if we don't
        # replace the separators on *nix, it'll create a weird nested directory
        task_name = task_name.replace("/", "::")

        # Max filename length
        max_filename_length = os.fstatvfs(0).f_namemax
        self.tmp_dir = os.path.join(base_tmp_dir, task_name[:max_filename_length])

        LOGGER.info("Tmp dir: %s", self.tmp_dir)
        os.makedirs(self.tmp_dir)

        # Dump the code to be run into a pickle file
        LOGGER.debug("Dumping pickled class")
        self._dump(self.tmp_dir)

        # Make sure that all the class's dependencies are tarred and available
        LOGGER.debug("Tarballing dependencies")
        # Grab luigi and the module containing the code to be run
        packages = [luigi, __import__(self.__module__, None, None, 'dummy')]
        create_packages_archive(packages, os.path.join(self.tmp_dir, "packages.tar"))

        # Now, pass onto the class's specified init_local() method.
        self.init_local()
Exemplo n.º 6
0
    def __call__(self, fd, mode, offset, length):
        """
        生成大文件
        :param fd:
        :param mode:
        :param offset:
        :param length:
        :return:

         1, fallocate = Fallocate(False)
            f = open("/tmp/f_test.data", "wb")
            print(fallocate(f, 0, 0, 2 * 1024 * 1024 * 1024))

         2, fallocate = Fallocate.instance()
            f = open("/tmp/f_test.data", "wb")
            print(fallocate(f, 0, 0, 2 * 1024 * 1024 * 1024))
        """

        fd = getattr(fd, "fileno", lambda: fd)()
        if self.check_free:
            st = os.fstatvfs(fd)
            free = st.f_frsize * st.f_bavail - length
            if float(free) <= 0:
                raise OSError(errno.ENOSPC, "fallocate fail free <= 0")
        c_length = ctypes.c_uint64(length)
        args = {
            'fallocate': (fd, mode, offset, c_length),
            'posix_fallocate': (fd, offset, c_length)
        }
        return self.fallocate(*args[self.func_name])
Exemplo n.º 7
0
 def _get_plasma_size():
     fd = os.open(options.worker.plasma_dir, os.O_RDONLY)
     stats = os.fstatvfs(fd)
     os.close(fd)
     size = stats.f_bsize * stats.f_bavail
     # keep some safety margin for allocator fragmentation
     return 8 * size // 10
Exemplo n.º 8
0
    def _init_local(self):

        # Set up temp folder in shared directory (trim to max filename length)
        base_tmp_dir = self.shared_tmp_dir
        random_id = '%016x' % random.getrandbits(64)
        folder_name = self.task_id + '-' + random_id
        self.tmp_dir = os.path.join(base_tmp_dir, folder_name)
        max_filename_length = os.fstatvfs(0).f_namemax
        self.tmp_dir = self.tmp_dir[:max_filename_length]
        logger.info("Tmp dir: %s", self.tmp_dir)
        os.makedirs(self.tmp_dir)

        # Dump the code to be run into a pickle file
        logging.debug("Dumping pickled class")
        self._dump(self.tmp_dir)

        if not self.no_tarball:
            # Make sure that all the class's dependencies are tarred and available
            # This is not necessary if luigi is importable from the cluster node
            logging.debug("Tarballing dependencies")
            # Grab luigi and the module containing the code to be run
            packages = [luigi
                        ] + [__import__(self.__module__, None, None, 'dummy')]
            luigi.hadoop.create_packages_archive(
                packages, os.path.join(self.tmp_dir, "packages.tar"))
def calc_disk(tag_info):
    '''
    Calculate disk related statistics

    :param tag_info: The dictionary used to hold the tag metadata and
        values.
    :type tag_info: dict(str, dict)
    '''
    if salt.utils.platform.is_windows():
        root_disk_path = 'C:\\'
    else:
        root_disk_path = '/'

    if six.PY3:
        disk_usage = shutil.disk_usage(root_disk_path)  # pylint: disable=no-member
        disk_total = float(disk_usage.total)
        disk_used = float(disk_usage.used)
        disk_free = float(disk_usage.free)
    else:
        fd_ = os.open(root_disk_path, os.O_RDONLY)
        try:
            disk_info = os.fstatvfs(fd_)  # pylint: disable=no-member
            disk_total = float(disk_info.f_frsize) * float(disk_info.f_blocks)
            disk_free = float(disk_info.f_frsize) * float(disk_info.f_bfree)
            disk_used = disk_total - disk_free
        finally:
            os.close(fd_)

    tag_info['disk_free']['value'] = disk_free
    tag_info['disk_used']['value'] = disk_used
    tag_info['disk_total']['value'] = disk_total
    tag_info['disk_perc']['value'] = (disk_used / disk_total) * 100
Exemplo n.º 10
0
def user_freespace_gb():
    """
    Returns the amount of free space user space in GB
    """
    fd = os.open('.', os.O_RDONLY | os.O_DIRECTORY)
    info = os.fstatvfs(fd)
    os.close(fd)
    return (info.f_bsize * info.f_bavail) / (1024 * 1024 * 1024)
Exemplo n.º 11
0
def statfs(path):
    fd = os.open(path, os.O_RDONLY)
    stv = os.fstatvfs(fd)
    os.close(fd)
    return dict(
        (key, getattr(stv, key))
        for key in ('f_bavail', 'f_bfree', 'f_blocks', 'f_bsize', 'f_favail',
                    'f_ffree', 'f_files', 'f_flag', 'f_frsize', 'f_namemax'))
Exemplo n.º 12
0
 def __init__(self, name):
     self.name = name
     fd = os.open(self.name, os.O_RDONLY)
     disk = os.fstatvfs(fd)
     self.totalBytes = float(disk.f_bsize * disk.f_blocks)
     self.totalUsed = float(disk.f_bsize * (disk.f_blocks - disk.f_bfree))
     self.totalAvail = float(disk.f_bsize * disk.f_bfree)
     os.close(fd)
Exemplo n.º 13
0
    def upload(self, batch_id, file_path, filename=None, file_index=0):
        """Upload a file through an Automation batch

        Uses poster.httpstreaming to stream the upload
        and not load the whole file in memory.
        """
        # Request URL
        url = self.automation_url.encode('ascii') + self.batch_upload_url

        # HTTP headers
        if filename is None:
            filename = os.path.basename(file_path)
        file_size = os.path.getsize(file_path)
        ctype, _ = mimetypes.guess_type(filename)
        if ctype:
            mime_type = ctype
        else:
            mime_type = "application/octet-stream"
        # Quote UTF-8 filenames even though JAX-RS does not seem to be able
        # to retrieve them as per: https://tools.ietf.org/html/rfc5987
        filename = safe_filename(filename)
        quoted_filename = urllib2.quote(filename.encode('utf-8'))
        headers = {
            "X-Batch-Id": batch_id,
            "X-File-Idx": file_index,
            "X-File-Name": quoted_filename,
            "X-File-Size": file_size,
            "X-File-Type": mime_type,
            "Content-Type": "application/octet-stream",
            "Content-Length": file_size,
        }
        headers.update(self._get_common_headers())

        # Request data
        input_file = open(file_path, 'rb')
        # Use file system block size if available for streaming buffer
        if sys.platform != 'win32':
            fs_block_size = os.fstatvfs(input_file.fileno()).f_bsize
        else:
            fs_block_size = DEFAULT_STREAMING_BUFFER_SIZE
        log.trace("Using file system block size"
                  " for the streaming upload buffer: %u bytes", fs_block_size)
        data = self._read_data(input_file, fs_block_size)

        # Execute request
        cookies = self._get_cookies()
        log.trace("Calling %s with headers %r and cookies %r for file %s",
            url, headers, cookies, file_path)
        req = urllib2.Request(url, data, headers)
        try:
            resp = self.streaming_opener.open(req, timeout=self.blob_timeout)
        except Exception as e:
            self._log_details(e)
            raise
        finally:
            input_file.close()

        return self._read_response(resp, url)
Exemplo n.º 14
0
def start_objstore(node_ip_address, redis_address, object_manager_port=None,
                   cleanup=True, redirect_output=False, objstore_memory=None):
  """This method starts an object store process.

  Args:
    node_ip_address (str): The IP address of the node running the object store.
    redis_address (str): The address of the Redis instance to connect to.
    object_manager_port (int): The port to use for the object manager. If this
      is not provided, one will be generated randomly.
    cleanup (bool): True if using Ray in local mode. If cleanup is true, then
      this process will be killed by serices.cleanup() when the Python process
      that imported services exits.
    redirect_output (bool): True if stdout and stderr should be redirected to
      /dev/null.

  Return:
    A tuple of the Plasma store socket name, the Plasma manager socket name, and
      the plasma manager port.
  """
  if objstore_memory is None:
    # Compute a fraction of the system memory for the Plasma store to use.
    system_memory = psutil.virtual_memory().total
    if sys.platform == "linux" or sys.platform == "linux2":
      # On linux we use /dev/shm, its size is half the size of the physical
      # memory. To not overflow it, we set the plasma memory limit to 0.4 times
      # the size of the physical memory.
      objstore_memory = int(system_memory * 0.4)
      # Compare the requested memory size to the memory available in /dev/shm.
      shm_fd = os.open("/dev/shm", os.O_RDONLY)
      try:
        shm_fs_stats = os.fstatvfs(shm_fd)
        # The value shm_fs_stats.f_bsize is the block size and the value
        # shm_fs_stats.f_bavail is the number of available blocks.
        shm_avail = shm_fs_stats.f_bsize * shm_fs_stats.f_bavail
        if objstore_memory > shm_avail:
          print("Warning: Reducing object store memory because /dev/shm has only {} bytes available. You may be able to free up space by deleting files in /dev/shm. If you are inside a Docker container, you may need to pass an argument with the flag '--shm-size' to 'docker run'.".format(shm_avail))
          objstore_memory = int(shm_avail * 0.8)
      finally:
        os.close(shm_fd)
    else:
      objstore_memory = int(system_memory * 0.8)
  # Start the Plasma store.
  plasma_store_name, p1 = plasma.start_plasma_store(plasma_store_memory=objstore_memory, use_profiler=RUN_PLASMA_STORE_PROFILER, redirect_output=redirect_output)
  # Start the plasma manager.
  if object_manager_port is not None:
    plasma_manager_name, p2, plasma_manager_port = plasma.start_plasma_manager(plasma_store_name, redis_address, plasma_manager_port=object_manager_port, node_ip_address=node_ip_address, num_retries=1, run_profiler=RUN_PLASMA_MANAGER_PROFILER, redirect_output=redirect_output)
    assert plasma_manager_port == object_manager_port
  else:
    plasma_manager_name, p2, plasma_manager_port = plasma.start_plasma_manager(plasma_store_name, redis_address, node_ip_address=node_ip_address, run_profiler=RUN_PLASMA_MANAGER_PROFILER, redirect_output=redirect_output)
  if cleanup:
    all_processes[PROCESS_TYPE_PLASMA_STORE].append(p1)
    all_processes[PROCESS_TYPE_PLASMA_MANAGER].append(p2)

  return ObjectStoreAddress(plasma_store_name, plasma_manager_name,
                            plasma_manager_port)
Exemplo n.º 15
0
    def _init_local(self):
        # Set up temp folder in shared directory (trim to max filename length)
        base_tmp_dir = self.shared_tmp_dir
        random_id = '%016x' % random.getrandbits(64)
        folder_name = self.task_id + '-' + random_id
        self.tmp_dir = os.path.join(base_tmp_dir, folder_name)
        max_filename_length = os.fstatvfs(0).f_namemax
        self.tmp_dir = self.tmp_dir[:max_filename_length]
        logger.info("Tmp dir: %s", self.tmp_dir)
        os.makedirs(self.tmp_dir)

        # Dump the code to be run into a pickle file
        self._dump(self.tmp_dir)
Exemplo n.º 16
0
    def _get_plasma_limit(self):
        if sys.platform == 'win32':  # pragma: no cover
            return
        elif sys.platform == 'darwin':
            default_plasma_dir = '/tmp'
        else:
            default_plasma_dir = '/dev/shm'

        fd = os.open(self._plasma_dir or default_plasma_dir, os.O_RDONLY)
        stats = os.fstatvfs(fd)
        os.close(fd)
        size = stats.f_bsize * stats.f_bavail
        # keep some safety margin for allocator fragmentation
        return 8 * size // 10
Exemplo n.º 17
0
def _get_io_blocksize_MB_(pathname):
    """
    Return the I/O blocksize for a given path (used to estimate IOPS)
    """
    if isfile(pathname):
        fname = pathname
        fflag = O_RDONLY
    elif isdir(pathname):
        fname = join(pathname, '.bsize_test_file')
        fflag = O_CREAT
    else:
        return None
    fd = fdopen(fname, fflag)
    bsize = fstatvfs(fd).f_bsize
    fdclose(fd)
    return bsize
Exemplo n.º 18
0
def _get_io_blocksize_MB_(pathname):
    """
    Return the I/O blocksize for a given path (used to estimate IOPS)
    """
    if isfile(pathname):
        fname = pathname
        fflag = O_RDONLY
    elif isdir(pathname):
        fname = join(pathname, '.bsize_test_file')
        fflag = O_CREAT
    else:
        return None
    fd = fdopen(fname, fflag)
    bsize = fstatvfs(fd).f_bsize
    fdclose(fd)
    return bsize
Exemplo n.º 19
0
    def func_diskspace(self, *args):
        """
        diskspace("/var") -> 125 (12.5% used)
        """
        if len(args) != 1:
            log.warning("Warning: function diskspace takes exactly 1 argument.")
            return
        mountpoint = args[0]

        fd = os.open(mountpoint, os.O_RDONLY)
        stats = os.fstatvfs(fd)
        os.close(fd)
        size = stats[1] * stats[2]
        free = stats[1] * stats[4]
        diskspace = 1000 * (size - free) / size
        return diskspace
Exemplo n.º 20
0
    def check_free(self):
        if self.check_free_blocks == -1:
            return False

        fst = os.fstatvfs(self.out_fd.fileno())

        if not self.check_free_blocks and config.has_section("free"):
            best = None
            for ent in config.options("free"):
                if self.output_real[0:len(ent)] != ent:
                    continue
                if best and len(ent) > len(best):
                    best = ent
                else:
                    best = ent
            # no match at all ?
            if not best:
                return True
            # calculate the free block counter
            should = config.get("free", best)
            # percentage
            if should in ('off', 'disabled', 'disable', 'none'):
                self.check_free_blocks = -1
                return False

            elif should[-1] == "%":
                self.check_free_blocks = (fst.f_blocks/100)*int(should[:-1] or 10)
            else:
                factors = {
                        "k": 1000,
                        "m": 1000 ** 2,
                        "g": 1000 ** 3,
                        "t": 1000 ** 4,
                        "p": 1000 ** 5 }
                if factors.has_key(should[-1].lower()):
                    should = int(should[:-1]) * factors[should[-1].lower()]
                else:
                    # default to megabyte
                    should = int(should) * factors["m"]
                
                self.check_free_blocks = should / fst.f_frsize  # should be multiples of f_frsize, i guess
        # actual check
        if self.check_free_blocks is not None:
            if fst.f_bfree < self.check_free_blocks:
                return False

        return True
Exemplo n.º 21
0
def fibmap2(fd, start=0, end=None, flags=0):
    xstat = os.fstat(fd)
    if not stat.S_ISREG(xstat.st_mode):
        return
    if flags & FIEMAP_FLAG_XATTR:
        return

    if end is None:
        end = xstat.st_size

    statvfs = os.fstatvfs(fd)
    block_size = statvfs.f_bsize
    start_block = start // block_size
    end_block = (end + block_size - 1) // block_size

    block = start_block
    fe_pblk = None
    fe_lblk = None
    fe_len = None
    max_extent = MAX_EXTENT_LENGTH / block_size
    while block <= end_block:
        indata = struct.pack('i', block)
        res = fcntl.ioctl(fd, _FIBMAP, indata)
        pblock = struct.unpack('i', res)[0]
        if fe_pblk is not None:
            if pblock > 0 and pblock == fe_pblk + fe_len and \
               fe_len <= max_extent:
                fe_len += 1
            else:
                yield fiemap_rec(fe_lblk * block_size, \
                  fe_pblk * block_size, \
                  fe_len * block_size, 0, \
                  FIEMAP_EXTENT_MERGED)
                fe_pblk = fe_lblk = fe_len = None
        else:
            if pblock > 0:
                fe_pblk = pblock
                fe_lblk = block
                fe_len = 1
        block += 1

    if fe_pblk is not None:
        yield fiemap_rec(fe_lblk * block_size, \
          fe_pblk * block_size, fe_len * block_size, 0, \
          FIEMAP_EXTENT_MERGED)
Exemplo n.º 22
0
def get_free_disk_space():
	''' Returns disk space currently free in MB'''
	docker_path = '/var/lib/docker'
	try:
		fd = os.open(docker_path, os.O_RDONLY)
		stats = os.fstatvfs(fd)
		bytes_free = stats.f_frsize * stats.f_bavail  # block size * number of free blocks
		mb_free = bytes_free * 1e-6
		return mb_free
	except Exception as exc:
		utils_logger.debug('Free disk space could not be determined.')
	finally:
		try:
			os.close(fd)
		except (NameError, OSError):
			# file has not been opened
			pass
	return
Exemplo n.º 23
0
    def __call__(
        self,
        fd,
        mode,
        offset,
        length,
        ):
        """ The length parameter must be a ctypes.c_uint64 """

        if FALLOCATE_RESERVE > 0:
            st = os.fstatvfs(fd)
            free = st.f_frsize * st.f_bavail - length.value
            if free <= FALLOCATE_RESERVE:
                raise OSError('FALLOCATE_RESERVE fail %s <= %s'
                              % (free, FALLOCATE_RESERVE))
        args = {'fallocate': (fd, mode, offset, length),
                'posix_fallocate': (fd, offset, length)}
        return self.fallocate(*args[self.func_name])
Exemplo n.º 24
0
    def check_free(self):
        if self.check_free_blocks == -1:
            return False

        fst = os.fstatvfs(self.out_fd.fileno())

        if not self.check_free_blocks and config.has_section("free"):
            best = None
            for ent in config.options("free"):
                if self.output_real[0 : len(ent)] != ent:
                    continue
                if best and len(ent) > len(best):
                    best = ent
                else:
                    best = ent
            # no match at all ?
            if not best:
                return True
            # calculate the free block counter
            should = config.get("free", best)
            # percentage
            if should in ("off", "disabled", "disable", "none"):
                self.check_free_blocks = -1
                return False

            elif should[-1] == "%":
                self.check_free_blocks = (fst.f_blocks / 100) * int(should[:-1] or 10)
            else:
                factors = {"k": 1000, "m": 1000 ** 2, "g": 1000 ** 3, "t": 1000 ** 4, "p": 1000 ** 5}
                if factors.has_key(should[-1].lower()):
                    should = int(should[:-1]) * factors[should[-1].lower()]
                else:
                    # default to megabyte
                    should = int(should) * factors["m"]

                self.check_free_blocks = should / fst.f_frsize  # should be multiples of f_frsize, i guess
        # actual check
        if self.check_free_blocks is not None:
            if fst.f_bfree < self.check_free_blocks:
                return False

        return True
Exemplo n.º 25
0
def get_shared_memory_bytes():
    """Get the size of the shared memory file system.

    Returns:
        The size of the shared memory file system in bytes.
    """
    # Make sure this is only called on Linux.
    assert sys.platform == "linux" or sys.platform == "linux2"

    shm_fd = os.open("/dev/shm", os.O_RDONLY)
    try:
        shm_fs_stats = os.fstatvfs(shm_fd)
        # The value shm_fs_stats.f_bsize is the block size and the
        # value shm_fs_stats.f_bavail is the number of available
        # blocks.
        shm_avail = shm_fs_stats.f_bsize * shm_fs_stats.f_bavail
    finally:
        os.close(shm_fd)

    return shm_avail
Exemplo n.º 26
0
def get_shared_memory_bytes():
    """Get the size of the shared memory file system.

    Returns:
        The size of the shared memory file system in bytes.
    """
    # Make sure this is only called on Linux.
    assert sys.platform == "linux" or sys.platform == "linux2"

    shm_fd = os.open("/dev/shm", os.O_RDONLY)
    try:
        shm_fs_stats = os.fstatvfs(shm_fd)
        # The value shm_fs_stats.f_bsize is the block size and the
        # value shm_fs_stats.f_bavail is the number of available
        # blocks.
        shm_avail = shm_fs_stats.f_bsize * shm_fs_stats.f_bavail
    finally:
        os.close(shm_fd)

    return shm_avail
Exemplo n.º 27
0
    def __call__(
        self,
        fd,
        mode,
        offset,
        length,
    ):
        """ The length parameter must be a ctypes.c_uint64 """

        if FALLOCATE_RESERVE > 0:
            st = os.fstatvfs(fd)
            free = st.f_frsize * st.f_bavail - length.value
            if free <= FALLOCATE_RESERVE:
                raise OSError('FALLOCATE_RESERVE fail %s <= %s' %
                              (free, FALLOCATE_RESERVE))
        args = {
            'fallocate': (fd, mode, offset, length),
            'posix_fallocate': (fd, offset, length)
        }
        return self.fallocate(*args[self.func_name])
Exemplo n.º 28
0
    def _init_local(self):

        base_tmp_dir = self.shared_tmp_dir

        random_id = "%016x" % random.getrandbits(64)
        task_name = random_id + self.task_id
        # If any parameters are directories, if we don't
        # replace the separators on *nix, it'll create a weird nested directory
        task_name = task_name.replace("/", "::")

        # Max filename length
        max_filename_length = os.fstatvfs(0).f_namemax
        self.tmp_dir = os.path.join(base_tmp_dir,
                                    task_name[:max_filename_length])

        LOGGER.info("Tmp dir: %s", self.tmp_dir)
        os.makedirs(self.tmp_dir)

        # Now, pass onto the class's specified init_local() method.
        self.init_local()
Exemplo n.º 29
0
    def _init_local(self):

        # Set up temp folder in shared directory (trim to max filename length)
        base_tmp_dir = self.shared_tmp_dir
        random_id = '%016x' % random.getrandbits(64)
        folder_name = _clean_task_id(self.task_id) + '-' + random_id
        self.tmp_dir = os.path.join(base_tmp_dir, folder_name)
        max_filename_length = os.fstatvfs(0).f_namemax
        self.tmp_dir = self.tmp_dir[:max_filename_length]
        logger.info("Tmp dir: %s", self.tmp_dir)
        os.makedirs(self.tmp_dir)

        # Dump the code to be run into a pickle file
        logging.debug("Dumping pickled class")
        self._dump(self.tmp_dir)

        # Make sure that all the class's dependencies are tarred and available
        logging.debug("Tarballing dependencies")
        # Grab luigi and the module containing the code to be run
        packages = [luigi] + [__import__(self.__module__, None, None, 'dummy')]
        luigi.hadoop.create_packages_archive(packages, os.path.join(self.tmp_dir, "packages.tar"))
Exemplo n.º 30
0
USER=os.getlogin()
print "User="******"GID:UID", GID, ":", UID

print "Creating a temp file and reading/writing it"
tfile=os.tmpfile()
tfile.write("Hello 1\n")
tfile.write("Hello 2\n")
tfile.seek(0)
for x in tfile:
    x=x.rstrip()
    print x

tfd=tfile.fileno()
tstat=os.fstat(tfd)
print "Stat info on temp file=", tstat
vfsinfo=os.fstatvfs(tfd)
print "VFS info on temp file=", vfsinfo

print "truncating file, then writing new data"
os.ftruncate(tfd,0)
tfile.write("Hello a\n")
tfile.write("Hello b\n")
tfile.seek(0)
for x in tfile:
    x=x.rstrip()
    print x

tfile.close();

print "Getting list of files in this dir"
# note, list is not sorted so we sort it here
Exemplo n.º 31
0


#open a file1
fd = os.open("foo.txt", os.O_RDWR | os.O_CREAT)

#Now get the touple
info = os.fstat(fd)
print "File Info : ", info
print "File UUID : ", info.st_uid
print "File GIS  : ", info.st_gid
print "File ctime : ", info.st_ctime
print "File Block Size :", info.st_blksize
print "File protection :", info.st_mode

sys_info = os.fstatvfs(fd)

print "File system info : ", sys_info
print "Maximum filename length : ", sys_info.f_namemax

print "create hard link to file foo.txt"
try:
    os.link("foo.txt", "foo1.txt")
except Exception as ex:
    print "Link already exist"

print "List current directory's contents : \n",
for file in os.listdir(newdir):
    print file

ls_info = os.lstat("foo1.txt")
Exemplo n.º 32
0
    def get_file(self, user, suite, path, path_in_tar=None, mode=None):
        """Returns file information / content or a cherrypy response."""
        f_name = self._get_user_suite_dir(user, suite, path)
        self._check_file_path(path)
        view_size_max = self.VIEW_SIZE_MAX
        if path_in_tar:
            tar_f = tarfile.open(f_name, "r:gz")
            try:
                tar_info = tar_f.getmember(path_in_tar)
            except KeyError:
                raise cherrypy.HTTPError(404)
            f_size = tar_info.size
            handle = tar_f.extractfile(path_in_tar)
            if handle.read(2) == "#!":
                mime = self.MIME_TEXT_PLAIN
            else:
                mime = mimetypes.guess_type(
                    urllib.pathname2url(path_in_tar))[0]
            handle.seek(0)
            if (mode == "download" or f_size > view_size_max or mime and
                (not mime.startswith("text/") or mime.endswith("html"))):
                temp_f = NamedTemporaryFile()
                f_bsize = os.fstatvfs(temp_f.fileno()).f_bsize
                while True:
                    bytes_ = handle.read(f_bsize)
                    if not bytes_:
                        break
                    temp_f.write(bytes_)
                cherrypy.response.headers["Content-Type"] = mime
                try:
                    return cherrypy.lib.static.serve_file(temp_f.name, mime)
                finally:
                    temp_f.close()
            text = handle.read()
        else:
            f_size = os.stat(f_name).st_size
            if open(f_name).read(2) == "#!":
                mime = self.MIME_TEXT_PLAIN
            else:
                mime = mimetypes.guess_type(urllib.pathname2url(f_name))[0]
            if not mime:
                mime = self.MIME_TEXT_PLAIN
            if (mode == "download" or f_size > view_size_max or mime and
                (not mime.startswith("text/") or mime.endswith("html"))):
                cherrypy.response.headers["Content-Type"] = mime
                return cherrypy.lib.static.serve_file(f_name, mime)
            text = open(f_name).read()
        try:
            if mode in [None, "text"]:
                text = jinja2.escape(text)
            lines = [unicode(line) for line in text.splitlines()]
        except UnicodeDecodeError:
            if path_in_tar:
                handle.seek(0)
                # file closed by cherrypy
                return cherrypy.lib.static.serve_fileobj(
                    handle, self.MIME_TEXT_PLAIN)
            else:
                return cherrypy.lib.static.serve_file(f_name,
                                                      self.MIME_TEXT_PLAIN)
        else:
            if path_in_tar:
                handle.close()
        name = path
        if path_in_tar:
            name = "log/" + path_in_tar
        job_entry = None
        if name.startswith("log/job"):
            names = name.replace("log/job/", "").split("/", 3)
            if len(names) == 4:
                cycle, task, submit_num, _ = names
                entries = self.suite_dao.get_suite_job_entries(
                    user, suite, [cycle], [task], None, None, None, None,
                    None)[0]
                for entry in entries:
                    if entry["submit_num"] == int(submit_num):
                        job_entry = entry
                        break
        if fnmatch(os.path.basename(path), "suite*.rc*"):
            file_content = "cylc-suite-rc"
        elif fnmatch(os.path.basename(path), "rose*.conf"):
            file_content = "rose-conf"
        else:
            file_content = None

        return lines, job_entry, file_content, f_name
Exemplo n.º 33
0
Arquivo: bush.py Projeto: m214089/rose
 def view(self, user, suite, path, path_in_tar=None, mode=None):
     """View a text log file."""
     f_name = self._get_user_suite_dir(user, suite, path)
     if not os.access(f_name, os.F_OK | os.R_OK):
         raise cherrypy.HTTPError(404)
     if path_in_tar:
         tar_f = tarfile.open(f_name, 'r:gz')
         try:
             tar_info = tar_f.getmember(path_in_tar)
         except KeyError:
             raise cherrypy.HTTPError(404)
         f_size = tar_info.size
         f = tar_f.extractfile(path_in_tar)
         if f.read(2) == "#!":
             mime = MIME_TEXT_PLAIN
         else:
             mime = mimetypes.guess_type(
                         urllib.pathname2url(path_in_tar))[0]
         f.seek(0)
         if (mode == "download" or
             f_size > self.VIEW_SIZE_MAX or
             mime and (not mime.startswith("text/") or
             mime.endswith("html"))):
             t = NamedTemporaryFile()
             f_bsize = os.fstatvfs(t.fileno()).f_bsize
             while True:
                 bytes = f.read(f_bsize)
                 if not bytes:
                     break
                 t.write(bytes)
             cherrypy.response.headers["Content-Type"] = mime
             try:
                 return cherrypy.lib.static.serve_file(t.name, mime)
             finally:
                 t.close()
         s = f.read()
         f.close()
     else:
         f_size = os.stat(f_name).st_size
         if open(f_name).read(2) == "#!":
             mime = MIME_TEXT_PLAIN
         else:
             mime = mimetypes.guess_type(urllib.pathname2url(f_name))[0]
         if (mode == "download" or
             f_size > self.VIEW_SIZE_MAX or
             mime and (not mime.startswith("text/") or
             mime.endswith("html"))):
             cherrypy.response.headers["Content-Type"] = mime
             return cherrypy.lib.static.serve_file(f_name, mime)
         s = open(f_name).read()
     if mode == "text":
         s = jinja2.escape(s)
     try:
         lines = [unicode(line) for line in s.splitlines()]
     except UnicodeDecodeError:
         return cherrypy.lib.static.serve_file(f_name, MIME_TEXT_PLAIN)
     name = path
     if path_in_tar:
         name = path_in_tar
     if fnmatch(os.path.basename(path), "rose*.conf"):
         file_content = "rose-conf"
     else:
         file_content = self.suite_engine_proc.is_conf(path)
     template = self.template_env.get_template("view.html")
     return template.render(
             rose_version=self.rose_version,
             script=cherrypy.request.script_name,
             time=strftime("%Y-%m-%dT%H:%M:%S+0000", gmtime()),
             host=self.host_name,
             user=user,
             suite=suite,
             path=path,
             path_in_tar=path_in_tar,
             f_name=f_name,
             mode=mode,
             file_content=file_content,
             lines=lines)
Exemplo n.º 34
0
 def test_fstatvfs(self):
     try:
         os.fstatvfs(0)
     except OSError as e:
         py.test.skip("the underlying os.fstatvfs() failed: %s" % e)
     rposix_stat.fstatvfs(0)
Exemplo n.º 35
0
    def get_file(self, user, suite, path, path_in_tar=None, mode=None):
        """Returns file information / content or a cherrypy response."""
        f_name = self._get_user_suite_dir(user, suite, path)
        conf = ResourceLocator.default().get_conf()
        view_size_max = int(conf.get_value(
            ["rose-bush", "view-size-max"], self.VIEW_SIZE_MAX))
        if path_in_tar:
            tar_f = tarfile.open(f_name, "r:gz")
            try:
                tar_info = tar_f.getmember(path_in_tar)
            except KeyError:
                raise cherrypy.HTTPError(404)
            f_size = tar_info.size
            handle = tar_f.extractfile(path_in_tar)
            if handle.read(2) == "#!":
                mime = self.MIME_TEXT_PLAIN
            else:
                mime = mimetypes.guess_type(
                    urllib.pathname2url(path_in_tar))[0]
            handle.seek(0)
            if (mode == "download" or
                    f_size > view_size_max or
                    mime and
                    (not mime.startswith("text/") or mime.endswith("html"))):
                temp_f = NamedTemporaryFile()
                f_bsize = os.fstatvfs(temp_f.fileno()).f_bsize
                while True:
                    bytes_ = handle.read(f_bsize)
                    if not bytes_:
                        break
                    temp_f.write(bytes_)
                cherrypy.response.headers["Content-Type"] = mime
                try:
                    return cherrypy.lib.static.serve_file(temp_f.name, mime)
                finally:
                    temp_f.close()
            text = handle.read()
        else:
            f_size = os.stat(f_name).st_size
            if open(f_name).read(2) == "#!":
                mime = self.MIME_TEXT_PLAIN
            else:
                mime = mimetypes.guess_type(urllib.pathname2url(f_name))[0]
            if not mime:
                mime = self.MIME_TEXT_PLAIN
            if (mode == "download" or
                    f_size > view_size_max or
                    mime and
                    (not mime.startswith("text/") or mime.endswith("html"))):
                cherrypy.response.headers["Content-Type"] = mime
                return cherrypy.lib.static.serve_file(f_name, mime)
            text = open(f_name).read()
        try:
            if mode in [None, "text"]:
                text = jinja2.escape(text)
            lines = [unicode(line) for line in text.splitlines()]
        except UnicodeDecodeError:
            if path_in_tar:
                handle.seek(0)
                # file closed by cherrypy
                return cherrypy.lib.static.serve_fileobj(
                    handle, self.MIME_TEXT_PLAIN)
            else:
                return cherrypy.lib.static.serve_file(
                    f_name, self.MIME_TEXT_PLAIN)
        else:
            if path_in_tar:
                handle.close()
        name = path
        if path_in_tar:
            name = "log/" + path_in_tar
        job_entry = None
        if name.startswith("log/job"):
            names = self.bush_dao.parse_job_log_rel_path(name)
            if len(names) == 4:
                cycle, task, submit_num, _ = names
                entries = self.bush_dao.get_suite_job_entries(
                    user, suite, [cycle], [task],
                    None, None, None, None, None)[0]
                for entry in entries:
                    if entry["submit_num"] == int(submit_num):
                        job_entry = entry
                        break
        if fnmatch(os.path.basename(path), "rose*.conf"):
            file_content = "rose-conf"
        else:
            file_content = self.bush_dao.is_conf(path)

        return lines, job_entry, file_content, f_name
Exemplo n.º 36
0
os.fchown(fd,uid,gid)
# 修改一个文件的所有权,这个函数修改一个文件的用户ID和用户组ID,该文件由文件描述符fd指定

os.fdatasync(fd)
# 强制将文件谢谢如磁盘,该文件由文件描述符fd指定,但是不强制更新文件的状态信息

os.fdopen(fd[,mode[,bufsize]])
# 通过文件描述符fd创建一个文件独享,并返回这个文件对象

os.fpathconf(fd,name)
# 返回一个打开的文件的系统配置信息。name为检索的系统配置的值,它也许是一个定义系统值的字符串,这些名字在很多标准中指定(POSIX.1, Unix 95, Unix 98, 和其它)。

os.fstat(fd)
# 返回文件描述符fd的状态,像stat()

os.fstatvfs(fd)
# 返回包含文件描述符fd的文件的文件系统的信息,想stavfs()

os.fsync(fd)
# 强制将文件描述符为fd的文件写入硬盘

os.ftruncate(fd,length)
# 裁剪文件描述符fd对应的文件,所以它最大不能超过文件大小

os.isatty(fd)
# 如果文件描述符fd是打开的,同时与tty(-like)设备相连,则返回True,否则False

os.tcgetpgrp(fd)
# 返回与终端fd(一个由os.open()返回的打开的文件描述符)关联的进程组
	
os.tcsetpgrp(fd, pg)
Exemplo n.º 37
0
 def fstatvfs(self, handle):
     fo = self._files.get(handle, None)
     if fo:
         return os.fstatvfs(fo.fileno())
Exemplo n.º 38
0
    def _swift_upload(self, swift, conn, infile):
        """Puts the contents of infile into a container on swift. If the file
        is larger than 100M, it will be divided into segments using
        the "dynamic large objects" convention.

        infile is expected to be a pipe, so each segment is stored in
        a temporary file before uploading. This allows the
        python-swiftclient library to retry failed uploads.
        """
        SEGMENT_SIZE = getattr(settings, "SWIFT_BACKEND_SEGMENT_SIZE",
                               100 * 1024 * 1024)

        segment_buf = tempfile.NamedTemporaryFile()

        st = os.fstatvfs(segment_buf.fileno())
        free_space = st.f_bavail * st.f_bsize
        if free_space < SEGMENT_SIZE:
            logger.error("There is only %dM space available for %s and %dM is required." % (free_space / 1024 / 1024, segment_buf.name, SEGMENT_SIZE / 1024 / 1024))

        def load_segment():
            """Copies enough data for a segment from the fifo into a temporary
            file. Returns how much data was actually written. """
            written = 0
            while written < SEGMENT_SIZE:
                bufsize = min(64 * 1024, SEGMENT_SIZE - written)
                buf = infile.read(bufsize)
                if not buf:
                    break
                segment_buf.write(buf)
                written += len(buf)
            segment_buf.seek(0)
            return written

        def upload_segment(bucket, prefix, length):
            try:
                conn.put_object(bucket, prefix, segment_buf,
                                content_length=length,
                                chunk_size=self.CHUNKSIZE)
            except swiftclient.exceptions.ClientException:
                logger.exception("Error when uploading segment")
                raise
            except IOError:
                logger.exception("Error when uploading segment")
                raise

        def upload_manifest(bucket, prefix):
            headers = {"X-Object-Manifest": "%s_segments/%s/" % (bucket, prefix)}
            conn.put_object(bucket, prefix, "", content_length=0,
                            headers=headers)

        seg_num = 0
        last_seg = False

        while not last_seg:
            seg_size = load_segment()

            last_seg = seg_size < SEGMENT_SIZE

            if seg_num == 0 and last_seg:
                bucket = swift.bucket
                prefix = swift.prefix
            else:
                bucket = swift.bucket + "_segments"
                prefix = "%s/%s/%08d" % (swift.prefix, SEGMENT_SIZE, seg_num)
                if seg_num == 0:
                    logger.debug("%s %s will be uploaded in %s byte segments." % (swift.bucket, swift.prefix, SEGMENT_SIZE))
                    # ensure that a segments container exists
                    conn.put_container(bucket)

            upload_segment(bucket, prefix, seg_size)

            segment_buf.seek(0)
            segment_buf.truncate(0)

            seg_num += 1

        if seg_num > 1:
            upload_manifest(swift.bucket, swift.prefix)

        return True
Exemplo n.º 39
0
def ino_check(dev, min_ino=MIN_INODES):
    return os.fstatvfs(dev).f_ffree >= int(min_ino)
Exemplo n.º 40
0
 def get_upload_buffer(self, input_file):
     if sys.platform != 'win32':
         return os.fstatvfs(input_file.fileno()).f_bsize
     else:
         return FILE_BUFFER_SIZE
Exemplo n.º 41
0
def space_check(dev, size):
    stdev = os.fstatvfs(dev)
    return size < stdev.f_bfree * stdev.f_bsize
 def get_upload_buffer(self, input_file):
     if sys.platform != 'win32':
         return os.fstatvfs(input_file.fileno()).f_bsize
     else:
         return FILE_BUFFER_SIZE
Exemplo n.º 43
0
def initialize_ray(
    override_is_cluster=False,
    override_redis_address: str = None,
    override_redis_password: str = None,
):
    """
    Initialize Ray based on parameters, ``modin.config`` variables and internal defaults.

    Parameters
    ----------
    override_is_cluster : bool, default: False
        Whether to override the detection of Modin being run in a cluster
        and always assume this runs on cluster head node.
        This also overrides Ray worker detection and always runs the initialization
        function (runs from main thread only by default).
        If not specified, ``modin.config.IsRayCluster`` variable is used.
    override_redis_address : str, optional
        What Redis address to connect to when running in Ray cluster.
        If not specified, ``modin.config.RayRedisAddress`` is used.
    override_redis_password : str, optional
        What password to use when connecting to Redis.
        If not specified, ``modin.config.RayRedisPassword`` is used.
    """
    import ray

    if not ray.is_initialized() or override_is_cluster:
        cluster = override_is_cluster or IsRayCluster.get()
        redis_address = override_redis_address or RayRedisAddress.get()
        redis_password = override_redis_password or RayRedisPassword.get()

        if cluster:
            # We only start ray in a cluster setting for the head node.
            ray.init(
                address=redis_address or "auto",
                include_dashboard=False,
                ignore_reinit_error=True,
                _redis_password=redis_password,
            )
        else:
            from modin.error_message import ErrorMessage

            # This string is intentionally formatted this way. We want it indented in
            # the warning message.
            ErrorMessage.not_initialized(
                "Ray",
                """
    import ray
    ray.init()
""",
            )
            object_store_memory = Memory.get()
            # In case anything failed above, we can still improve the memory for Modin.
            if object_store_memory is None:
                virtual_memory = psutil.virtual_memory().total
                if sys.platform.startswith("linux"):
                    shm_fd = os.open("/dev/shm", os.O_RDONLY)
                    try:
                        shm_stats = os.fstatvfs(shm_fd)
                        system_memory = shm_stats.f_bsize * shm_stats.f_bavail
                        if system_memory / (virtual_memory / 2) < 0.99:
                            warnings.warn(
                                f"The size of /dev/shm is too small ({system_memory} bytes). The required size "
                                f"at least half of RAM ({virtual_memory // 2} bytes). Please, delete files in /dev/shm or "
                                "increase size of /dev/shm with --shm-size in Docker. Also, you can set "
                                "the required memory size for each Ray worker in bytes to MODIN_MEMORY environment variable."
                            )
                    finally:
                        os.close(shm_fd)
                else:
                    system_memory = virtual_memory
                object_store_memory = int(0.6 * system_memory // 1e9 * 1e9)
                # If the memory pool is smaller than 2GB, just use the default in ray.
                if object_store_memory == 0:
                    object_store_memory = None
            else:
                object_store_memory = int(object_store_memory)

            ray_init_kwargs = {
                "num_cpus": CpuCount.get(),
                "num_gpus": GpuCount.get(),
                "include_dashboard": False,
                "ignore_reinit_error": True,
                "object_store_memory": object_store_memory,
                "address": redis_address,
                "_redis_password": redis_password,
                "_memory": object_store_memory,
            }
            ray.init(**ray_init_kwargs)

        if Backend.get() == "Cudf":
            from modin.engines.ray.cudf_on_ray.frame.gpu_manager import GPUManager
            from modin.engines.ray.cudf_on_ray.frame.partition_manager import (
                GPU_MANAGERS, )

            # Check that GPU_MANAGERS is empty because _update_engine can be called multiple times
            if not GPU_MANAGERS:
                for i in range(GpuCount.get()):
                    GPU_MANAGERS.append(GPUManager.remote(i))
    _move_stdlib_ahead_of_site_packages()
    ray.worker.global_worker.run_function_on_all_workers(
        _move_stdlib_ahead_of_site_packages)
    ray.worker.global_worker.run_function_on_all_workers(_import_pandas)
    num_cpus = int(ray.cluster_resources()["CPU"])
    num_gpus = int(ray.cluster_resources().get("GPU", 0))
    if Backend.get() == "Cudf":
        NPartitions._put(num_gpus)
    else:
        NPartitions._put(num_cpus)
Exemplo n.º 44
0
def get_max_filename_length():
    fd = os.open(".", os.O_RDONLY)
    info = os.fstatvfs(fd)
    return info.f_namemax
Exemplo n.º 45
0
# 获取元组
info = os.fstat(fd)
print "文件信息 :", info
# 获取文件 uid
print "文件 UID :%d" % info.st_uid
# 获取文件 gid
print "文件 GID  :%d" % info.st_gid
# 关闭文件
os.close(fd)

# 18 os.fstatvfs(fd)
# 返回包含文件描述符fd的文件的文件系统的信息,像 statvfs() Unix上可用。
# 打开文件
fd = os.open( "foo.txt", os.O_RDWR|os.O_CREAT )
# 获取元组
info = os.fstatvfs(fd)
print "文件信息 :", info
# 获取文件名最大长度
print "文件名最大长度 :%d" % info.f_namemax
# 获取可用块数
print "可用块数 :%d" % info.f_bfree
# 关闭文件
os.close(fd)

# 19 os.fsync(fd)
# 强制将文件描述符为fd的文件写入硬盘。
# 打开文件
fd = os.open( "foo.txt", os.O_RDWR|os.O_CREAT )
# 写入字符串
os.write(fd, "This is test")
# 使用 fsync() 方法.
Exemplo n.º 46
0
def get_max_filename_length():
    fd = os.open(".", os.O_RDONLY)
    info = os.fstatvfs(fd)
    return info.f_namemax
Exemplo n.º 47
0
    def run(self):
        if self.run_locally == 1:
            return self.work()
        else:

            # Set up temp folder in shared directory (trim to max filename length)
            base_tmp_dir = self.shared_tmp_dir
            random_id = '%016x' % random.getrandbits(64)
            folder_name = self.task_id + '-' + random_id
            self.tmp_dir = os.path.join(base_tmp_dir, folder_name)
            max_filename_length = os.fstatvfs(0).f_namemax
            self.tmp_dir = self.tmp_dir[:max_filename_length]
            logger.info("Tmp dir: %s", self.tmp_dir)

            to_copy = [d for d in os.listdir() if d != ".git"]
            if not os.path.exists(self.tmp_dir):
                os.makedirs(self.tmp_dir)

            for f in to_copy:
                if os.path.isfile(f):
                    copyfile(f, os.path.join(self.tmp_dir, f))
                else:
                    copytree(f, os.path.join(self.tmp_dir, f))

            # Dump the code to be run into a pickle file
            logging.debug("Dumping pickled class")
            self._dump(self.tmp_dir)

            if not self.no_tarball:
                # Make sure that all the class's dependencies are tarred and available
                # This is not necessary if luigi is importable from the cluster node
                logging.debug("Tarballing dependencies")
                # Grab luigi and the module containing the code to be run
                packages = [luigi] + [
                    __import__(self.__module__, None, None, 'dummy')
                ]
                create_packages_archive(
                    packages, os.path.join(self.tmp_dir, "packages.tar"))

            # make a stamp indicator in the folder
            # generate unique descriptive stamp for current commit
            get_commit_property = lambda s: subprocess.check_output(
                "git --no-pager log -1 --format=%{}".format(
                    s), shell=True).decode("utf-8").strip()

            commit_time   = get_commit_property("ci")[:16] \
                .replace(" ", "-") \
                .replace(":", "-")

            commit_author = get_commit_property("ce").split("@")[0]
            commit_hash = get_commit_property("h")
            stamp = "{}-{}-{}".format(commit_time, commit_author, commit_hash)

            # write out stamp to temp folder
            with open(os.path.join(self.tmp_dir, "stamp"), "w") as stamp_file:
                stamp_file.write(stamp)

            # Build a qsub argument that will run sge_runner.py on the directory
            # we've specified
            runner_path = os.path.join("utility", "sge_runner.py")
            if runner_path.endswith("pyc"):
                runner_path = runner_path[:-3] + "py"
            # enclose tmp_dir in quotes to protect from special escape chars
            job_str = 'python {0} "{1}" "{2}"'.format(runner_path,
                                                      self.tmp_dir,
                                                      os.getcwd())
            if self.no_tarball:
                job_str += ' "--no-tarball"'

            qsub_template = """echo {cmd} | {qsub_command} -V -r y -pe {pe} {n_cpu} -N {job_name} -l m_mem_free={mem_free} -sync y"""

            submit_cmd = qsub_template.format(cmd=job_str,
                                              job_name=self.job_name,
                                              pe=self.parallel_env,
                                              n_cpu=self.n_cpu,
                                              mem_free=self.mem_free,
                                              qsub_command=self.qsub_command)

            logger.debug('qsub command: \n' + submit_cmd)

            # Submit the job and grab job ID
            try:
                output = subprocess.check_output(submit_cmd,
                                                 shell=True,
                                                 stderr=subprocess.STDOUT)

                logger.debug("qsub job complete with output:\n" +
                             output.decode('utf-8'))
            except subprocess.CalledProcessError as e:
                logger.error("qsub submission failed with output:\n" +
                             e.output.decode('utf-8'))
                if os.path.exists(os.path.join(self.tmp_dir, "job.err")):
                    with open(os.path.join(self.tmp_dir, "job.err"),
                              "r") as err:
                        logger.error(err.read())

            # wait a beat, to give things a chance to settle
            time.sleep(2)

            # check whether the file exists
            if not self.output().exists():
                raise Exception("qsub failed to produce output")
            else:
                # delete the temporaries, if they're there.
                if self.tmp_dir and os.path.exists(self.tmp_dir):
                    logger.info('Removing temporary directory %s',
                                self.tmp_dir)
                    subprocess.call(["rm", "-rf", self.tmp_dir])
Exemplo n.º 48
0
 def test_fstatvfs(self):
     try:
         os.fstatvfs(0)
     except OSError, e:
         py.test.skip("the underlying os.fstatvfs() failed: %s" % e)
Exemplo n.º 49
0
    def view(self, user, suite, path, path_in_tar=None, mode=None):
        """View a text log file."""
        f_name = self._get_user_suite_dir(user, suite, path)
        conf = ResourceLocator.default().get_conf()
        view_size_max = int(conf.get_value(
            ["rose-bush", "view-size-max"], self.VIEW_SIZE_MAX))
        if path_in_tar:
            tar_f = tarfile.open(f_name, 'r:gz')
            try:
                tar_info = tar_f.getmember(path_in_tar)
            except KeyError:
                raise cherrypy.HTTPError(404)
            f_size = tar_info.size
            f = tar_f.extractfile(path_in_tar)
            if f.read(2) == "#!":
                mime = MIME_TEXT_PLAIN
            else:
                mime = mimetypes.guess_type(
                    urllib.pathname2url(path_in_tar))[0]
            f.seek(0)
            if (mode == "download" or
                    f_size > view_size_max or
                    mime and
                    (not mime.startswith("text/") or mime.endswith("html"))):
                t = NamedTemporaryFile()
                f_bsize = os.fstatvfs(t.fileno()).f_bsize
                while True:
                    bytes = f.read(f_bsize)
                    if not bytes:
                        break
                    t.write(bytes)
                cherrypy.response.headers["Content-Type"] = mime
                try:
                    return cherrypy.lib.static.serve_file(t.name, mime)
                finally:
                    t.close()
            s = f.read()
            f.close()
        else:
            f_size = os.stat(f_name).st_size
            if open(f_name).read(2) == "#!":
                mime = MIME_TEXT_PLAIN
            else:
                mime = mimetypes.guess_type(urllib.pathname2url(f_name))[0]
            if (mode == "download" or
                    f_size > view_size_max or
                    mime and
                    (not mime.startswith("text/") or mime.endswith("html"))):
                cherrypy.response.headers["Content-Type"] = mime
                return cherrypy.lib.static.serve_file(f_name, mime)
            s = open(f_name).read()
        try:
            if mode in [None, "text"]:
                s = jinja2.escape(s)
            lines = [unicode(line) for line in s.splitlines()]
        except UnicodeDecodeError:
            return cherrypy.lib.static.serve_file(f_name, MIME_TEXT_PLAIN)
        name = path
        if path_in_tar:
            name = path_in_tar
        job_entry = None
        if name.startswith("log/job/"):
            names = self.suite_engine_proc.parse_job_log_rel_path(name)
            if len(names) == 4:
                cycle, task, submit_num, ext = names
                entries = self.suite_engine_proc.get_suite_job_events(
                    user, suite, [cycle], [task], None, None, None, None)[0]
                for entry in entries:
                    if entry["submit_num"] == int(submit_num):
                        job_entry = entry
                        break
        if fnmatch(os.path.basename(path), "rose*.conf"):
            file_content = "rose-conf"
        else:
            file_content = self.suite_engine_proc.is_conf(path)
        template = self.template_env.get_template("view.html")

        data = {}
        data.update(self._get_suite_logs_info(user, suite))
        return template.render(
            rose_version=self.rose_version,
            script=cherrypy.request.script_name,
            method="view",
            time=strftime("%Y-%m-%dT%H:%M:%S+0000", gmtime()),
            logo=self.logo,
            title=self.title,
            host=self.host_name,
            user=user,
            suite=suite,
            path=path,
            path_in_tar=path_in_tar,
            f_name=f_name,
            mode=mode,
            file_content=file_content,
            lines=lines,
            entry=job_entry,
            **data)
Exemplo n.º 50
0
 def fstatvfs(self, handle):
     fo = self._files.get(handle, None)
     if fo:
         return os.fstatvfs(fo.fileno())
os.chown(path, uid, gid)        # 更改文件所有者
os.chroot(path)                 # 改变当前进程的根目录
os.close(fd)                    # 关闭文件描述符 fd
os.closerange(fd_low, fd_high)  # 关闭所有文件描述符,从 fd_low (包含) 到 fd_high (不包含), 错误会忽略
os.curdir                       # 返回当前目录:('.')
os.dup(fd)                      # 复制文件描述符 fd
os.dup2(fd, fd2)                # 将一个文件描述符 fd 复制到另一个 fd2
os.environ                      # 获取系统环境变量
os.fchdir(fd)                   # 通过文件描述符改变当前工作目录
os.fchmod(fd, mode)             # 改变一个文件的访问权限,该文件由参数fd指定,参数mode是Unix下的文件访问权限。
os.fchown(fd, uid, gid)         # 修改一个文件的所有权,这个函数修改一个文件的用户ID和用户组ID,该文件由文件描述符fd指定。
os.fdatasync(fd)                # 强制将文件写入磁盘,该文件由文件描述符fd指定,但是不强制更新文件的状态信息。
os.fdopen(fd[, mode[, bufsize]])  # 通过文件描述符 fd 创建一个文件对象,并返回这个文件对象
os.fpathconf(fd, name)          # 返回一个打开的文件的系统配置信息。name为检索的系统配置的值,它也许是一个定义系统值的字符串,这些名字在很多标准中指定(POSIX.1, Unix 95, Unix 98, 和其它)。
os.fstat(fd)                    # 返回文件描述符fd的状态,像stat()。
os.fstatvfs(fd)                 # 返回包含文件描述符fd的文件的文件系统的信息,像 statvfs()
os.fsync(fd)                    # 强制将文件描述符为fd的文件写入硬盘。
os.ftruncate(fd, length)        # 裁剪文件描述符fd对应的文件, 所以它最大不能超过文件大小。
os.getcwd()                     # 返回当前工作目录
os.getcwdu()                    # 返回一个当前工作目录的Unicode对象
os.isatty(fd)                   # 如果文件描述符fd是打开的,同时与tty(-like)设备相连,则返回true, 否则False。
os.lchflags(path, flags)        # 设置路径的标记为数字标记,类似 chflags(),但是没有软链接
os.lchmod(path, mode)           # 修改连接文件权限
os.lchown(path, uid, gid)       # 更改文件所有者,类似 chown,但是不追踪链接。
os.link(src, dst)               # 创建硬链接,名为参数 dst,指向参数 src
os.listdir(path)                # 返回path指定的文件夹包含的文件或文件夹的名字的列表。
os.lseek(fd, pos, how)          # 设置文件描述符 fd当前位置为pos, how方式修改: SEEK_SET 或者 0 设置从文件开始的计算的pos; SEEK_CUR或者 1 则从当前位置计算; os.SEEK_END或者2则从文件尾部开始. 在unix,Windows中有效
os.lstat(path)                  # 像stat(),但是没有软链接
os.linesep                      # 当前平台使用的行终止符,win下为"\t\n",Linux下为"\n"
os.major(device)                # 从原始的设备号中提取设备major号码 (使用stat中的st_dev或者st_rdev field)。
os.makedev(major, minor)        # 以major和minor设备号组成一个原始设备号