Пример #1
0
 def read_forever(self):
     """
     This function reads the log file one line at a time, reopening it if it
     changes (due to logrotate for example).
     This function will loop until the program ends or is explicitly told to
     stop reading the log file.
     """
     try:
         self.mc_log_fh = open(self.mc_log, 'r')
         
         self.curinode = os.fstat(self.mc_log_fh.fileno()).st_ino
         
         while self.log_read:
             while self.log_read:
                 self.buf = self.mc_log_fh.readline()
                 if self.buf == "":
                     break
                 
                 self.handle_message(self.buf, False)
             try:
                 if os.stat(self.mc_log).st_ino != self.curinode:
                     self.mc_new_fh = open(self.mc_log, "r")
                     self.mc_log_fh.close()
                     self.mc_log_fh = self.mc_new_fg
                     self.curinode = os.fstat(self.mc_log_fh.fileno()).st_ino
                     continue # dont bother sleeping since there is a new log file
             except IOError:
                 pass
             time.sleep(0.1)
     except:
         self.log.error('Craftoria: unable to open or read log file %s: %s',
             sys.exc_type, sys.exc_value)
    def _actually_load_xml(self, fobj, fn, this_id, load_backup):

        #print("ACTUALLY LOAD")

        tmpobj = None

        if (self._load_timestamp.get(this_id, 0) != os.fstat(fobj.fileno()).st_ctime):

            import time
            b4=time.time()
            tmpobj, errs = self.from_file(fobj)
            a4=time.time()
            logger.debug("Loading XML file for ID: %s took %s sec" % (this_id, str(a4-b4)))

            has_children = (self.sub_split is not None) and (self.sub_split in tmpobj.getNodeData()) and len(tmpobj.getNodeAttribute(self.sub_split)) == 0

            if this_id in self.objects:

                self._must_actually_load_xml(fobj, fn, this_id, load_backup, has_children, tmpobj, errs)

            else:
                #tmpobj.setNodeIndexCache(None)
                self._internal_setitem__(this_id, tmpobj)

            if hasattr(self.objects[this_id], self.sub_split):
                sub_attr = getattr(self.objects[this_id], self.sub_split)
                if sub_attr is not None and hasattr(sub_attr, '_setParent'):
                    sub_attr._setParent(self.objects[this_id])

            self._load_timestamp[this_id] = os.fstat(fobj.fileno()).st_ctime

        else:
            logger.debug("Didn't Load Job ID: %s" % str(this_id))

        logger.debug("Finished Loading XML")
Пример #3
0
def test_contextmanager():
    fn = None
    with Image.open("Images/lena.gif") as im:
        fn = im.fp.fileno()
        assert_no_exception(lambda: os.fstat(fn))

    assert_exception(OSError, lambda: os.fstat(fn))    
Пример #4
0
def get_data_file(args):
    if args.file:
        return open(args.file, 'rb')
    else:
        # distinguish cases where:
        # (1) stdin is not valid (as in cron jobs):
        #     glance ... <&-
        # (2) image data is provided through standard input:
        #     glance ... < /tmp/file or cat /tmp/file | glance ...
        # (3) no image data provided:
        #     glance ...
        try:
            os.fstat(0)
        except OSError:
            # (1) stdin is not valid (closed...)
            return None
        if not sys.stdin.isatty():
            # (2) image data is provided through standard input
            image = sys.stdin
            if hasattr(sys.stdin, 'buffer'):
                image = sys.stdin.buffer
            if msvcrt:
                msvcrt.setmode(image.fileno(), os.O_BINARY)
            return image
        else:
            # (3) no image data provided
            return None
Пример #5
0
def fix_savedata(dir):
    if (not os.path.isdir(dir) or not os.path.isfile(dir+"/SYS.BIN") ): 
        ErrorMessageBox("Ŀ¼´íÎó")
        
    import mmap
    fd = os.open(dir+"/SYS.BIN", os.O_RDWR)
    buf = mmap.mmap(fd, os.fstat(fd).st_size, access=mmap.ACCESS_WRITE)
    if (buf[0:8] != "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF"): 
        print "Bad savedata or not decrypted. SYS.BIN"
        ErrorMessageBox("´æµµ´íÎó")
    for pos in range(0x269480, 0x269480 + 0x1258 * 100) :
        if buf[pos:pos+4] == "\0\0\0\2" :
            buf[pos+0x18:pos+0x58] = "\0\0\0\0" * 0x10
        pos+=0x1258
    os.close(fd)
    print 'Fix SYS.BIN.'
    
    import fnmatch
    zstr = "\0\0\0\0" * ((0x8A358 - 0x46358) / 4)
    for directory, subdirectories, files in os.walk(dir):
      for file in files:
        if fnmatch.fnmatch(file, 'SAVE???.BIN'):
            fd = os.open(os.path.join(directory, file), os.O_RDWR)
            buf = mmap.mmap(fd, os.fstat(fd).st_size, access=mmap.ACCESS_WRITE)
            if (buf[0:4] != "\0\0\0\2") :
                print "Bad savedata or not decrypted. %s" % file
                ErrorMessageBox("´æµµ´íÎó»òδ½âÃÜ")
            buf[0x18:0x58] = "\0\0\0\0" * 0x10
            buf[0x46358:0x8A358] = zstr
            os.close(fd)
            print 'Fix %s.' % (file)
    windll.user32.MessageBoxA(None, "´æµµÐÞÕýÍê³É!", EXE_TITLE, 0)
Пример #6
0
def read_module_files(module_data, limit=None, **addl_props):
  """ Read the files belonging to a module from disk and return
  their data as a dictionary. """

  files = {}
  
  if 'te_file' in module_data:
    with open(module_data['te_file']) as fin:
      info = os.fstat(fin.fileno())
      handle = itertools.islice(fin, limit)
      files['te'] = {'text': "".join(handle)}
      files['te'].update(**addl_props)
      files['te']['size'] = info.st_size

  if 'if_file' in module_data:
    with open(module_data['if_file']) as fin:
      info = os.fstat(fin.fileno())
      handle = itertools.islice(fin, limit)
      files['if'] = {'text': "".join(handle)}
      files['if'].update(**addl_props)
      files['if']['size'] = info.st_size

  if 'fc_file' in module_data:
    with open(module_data['fc_file']) as fin:
      info = os.fstat(fin.fileno())
      handle = itertools.islice(fin, limit)
      files['fc'] = {'text': "".join(handle)}
      files['fc'].update(**addl_props)
      files['fc']['size'] = info.st_size

  return files
Пример #7
0
def file_contents_ro(fd, stream=False, allow_mmap=True):
    """:return: read-only contents of the file represented by the file descriptor fd

    :param fd: file descriptor opened for reading
    :param stream: if False, random access is provided, otherwise the stream interface
        is provided.
    :param allow_mmap: if True, its allowed to map the contents into memory, which
        allows large files to be handled and accessed efficiently. The file-descriptor
        will change its position if this is False"""
    try:
        if allow_mmap:
            # supports stream and random access
            try:
                return mmap.mmap(fd, 0, access=mmap.ACCESS_READ)
            except EnvironmentError:
                # python 2.4 issue, 0 wants to be the actual size
                return mmap.mmap(fd, os.fstat(fd).st_size, access=mmap.ACCESS_READ)
            # END handle python 2.4
    except OSError:
        pass
    # END exception handling

    # read manully
    contents = os.read(fd, os.fstat(fd).st_size)
    if stream:
        return _RandomAccessBytesIO(contents)
    return contents
Пример #8
0
    def __init__ (self, fd, args=[], timeout=30, maxread=2000, searchwindowsize=None, logfile=None):

        """This takes a file descriptor (an int) or an object that support the
        fileno() method (returning an int). All Python file-like objects
        support fileno(). """

        ### TODO: Add better handling of trying to use fdspawn in place of spawn
        ### TODO: (overload to allow fdspawn to also handle commands as spawn does.

        if type(fd) != type(0) and hasattr(fd, 'fileno'):
            fd = fd.fileno()

        if type(fd) != type(0):
            raise ExceptionPexpect ('The fd argument is not an int. If this is a command string then maybe you want to use pexpect.spawn.')

        try: # make sure fd is a valid file descriptor
            os.fstat(fd)
        except OSError:
            raise ExceptionPexpect('The fd argument is not a valid file descriptor.')

        self.args = None
        self.command = None
        spawn.__init__(self, None, args, timeout, maxread, searchwindowsize, logfile)
        self.child_fd = fd
        self.own_fd = False
        self.closed = False
        self.name = '<file descriptor %d>' % fd
Пример #9
0
    def followTail(self, fileobj=None, fstat=None):
        if fileobj is None:
            fileobj = open(self.filename)
            if self.seekend:
                fileobj.seek(0, 2)

        line = fileobj.read()

        if line:
            self.dataReceived(line)

        if fstat is None:
            fstat = os.fstat(fileobj.fileno())

        try:
            stat = os.stat(self.filename)
        except:
            stat = fstat

        if self.fileIdentity(stat) != self.fileIdentity(fstat):
            fileobj = open(self.filename)
            fstat = os.fstat(fileobj.fileno())
            self.fileReset()

        if self.keeprunning:
            reactor.callLater(self.delay, lambda: self.followTail(fileobj, fstat))
Пример #10
0
 def test_object_run_once_multi_devices(self):
     self.auditor = auditor.AuditorWorker(self.conf, self.logger)
     timestamp = str(normalize_timestamp(time.time()))
     pre_quarantines = self.auditor.quarantines
     data = '0' * 10
     etag = md5()
     with self.disk_file.writer() as writer:
         writer.write(data)
         etag.update(data)
         etag = etag.hexdigest()
         metadata = {
             'ETag': etag,
             'X-Timestamp': timestamp,
             'Content-Length': str(os.fstat(writer.fd).st_size),
         }
         writer.put(metadata)
     self.auditor.audit_all_objects()
     self.disk_file = DiskFile(self.devices, 'sdb', '0', 'a', 'c',
                               'ob', self.logger)
     data = '1' * 10
     etag = md5()
     with self.disk_file.writer() as writer:
         writer.write(data)
         etag.update(data)
         etag = etag.hexdigest()
         metadata = {
             'ETag': etag,
             'X-Timestamp': timestamp,
             'Content-Length': str(os.fstat(writer.fd).st_size),
         }
         writer.put(metadata)
         os.write(writer.fd, 'extra_data')
     self.auditor.audit_all_objects()
     self.assertEquals(self.auditor.quarantines, pre_quarantines + 1)
 def test_handler_existing_file(self):
     handler = FakeHandler([])
     with tempfile.NamedTemporaryFile(dir="") as f_obj_a:
         with handler.parent_open(f_obj_a.name) as f_obj_b:
             self.assertEqual(os.fstat(f_obj_a.fileno()).st_ino,
                              os.fstat(f_obj_b.fileno()).st_ino)
             self.assertEqual(f_obj_a.name, f_obj_b.name)
Пример #12
0
    def __init__ (self, fd, args=None, timeout=30, maxread=2000, searchwindowsize=None,
                  logfile=None, encoding=None, codec_errors='strict'):
        '''This takes a file descriptor (an int) or an object that support the
        fileno() method (returning an int). All Python file-like objects
        support fileno(). '''

        if type(fd) != type(0) and hasattr(fd, 'fileno'):
            fd = fd.fileno()

        if type(fd) != type(0):
            raise ExceptionPexpect('The fd argument is not an int. If this is a command string then maybe you want to use pexpect.spawn.')

        try: # make sure fd is a valid file descriptor
            os.fstat(fd)
        except OSError:
            raise ExceptionPexpect('The fd argument is not a valid file descriptor.')

        self.args = None
        self.command = None
        SpawnBase.__init__(self, timeout, maxread, searchwindowsize, logfile,
                           encoding=encoding, codec_errors=codec_errors)
        self.child_fd = fd
        self.own_fd = False
        self.closed = False
        self.name = '<file descriptor %d>' % fd
Пример #13
0
    def test_contextmanager(self):
        fn = None
        with Image.open("Tests/images/hopper.gif") as im:
            fn = im.fp.fileno()
            os.fstat(fn)

        self.assertRaises(OSError, os.fstat, fn)
Пример #14
0
    def _open_next(self, fd):
        current = os.path.join(self.dir, 'current')
        current_fd = _open_nonblock(current)

        pat = os.path.join(self.dir, '@*.[su]')
        files = glob.glob(pat)
        files.sort()
        files.append(current)

        inode = os.fstat(fd).st_ino
        os.close(fd)

        for idx, f in enumerate(files):
            if os.stat(f).st_ino == inode:
                break
        else:
            log.warn("multilog: can't find before file. (inode=%d)", inode)
            return os.fstat(current_fd).st_ino, current_fd

        idx += 1
        if len(files) == idx:
            log.warn("multilog: rotate current => current?")
            return os.fstat(current_fd).st_ino, current_fd

        next_file = files[idx]
        if next_file == current:
            return os.fstat(current_fd).st_ino, current_fd

        os.close(current_fd)
        return os.stat(next_file).st_ino, _open_nonblock(next_file)
Пример #15
0
    def test_object_run_fast_track_non_zero(self):
        self.auditor = auditor.ObjectAuditor(self.conf)
        self.auditor.log_time = 0
        data = '0' * 1024
        etag = md5()
        with self.disk_file.mkstemp() as (fd, tmppath):
            os.write(fd, data)
            etag.update(data)
            etag = etag.hexdigest()
            metadata = {
                'ETag': etag,
                'X-Timestamp': str(normalize_timestamp(time.time())),
                'Content-Length': str(os.fstat(fd).st_size),
                'Original-Content-Length': str(os.fstat(fd).st_size)
            }
            self.disk_file.put(fd, tmppath, metadata)
            etag = md5()
            etag.update('1' + '0' * 1023)
            etag = etag.hexdigest()
            metadata['ETag'] = etag
            write_metadata(fd, metadata)

        quarantine_path = os.path.join(self.devices,
                                       'sda', 'quarantined', 'objects')
        self.auditor.run_once(zero_byte_fps=50)
        self.assertFalse(os.path.isdir(quarantine_path))
        self.auditor.run_once()
        self.assertTrue(os.path.isdir(quarantine_path))
Пример #16
0
    def actionGetFile(self, params):
        site = self.sites.get(params["site"])
        if not site or not site.settings["serving"]:  # Site unknown or not serving
            self.response({"error": "Unknown site"})
            return False
        try:
            file_path = site.storage.getPath(params["inner_path"])
            if config.debug_socket:
                self.log.debug("Opening file: %s" % file_path)
            with StreamingMsgpack.FilePart(file_path, "rb") as file:
                file.seek(params["location"])
                file.read_bytes = FILE_BUFF
                back = {
                    "body": file,
                    "size": os.fstat(file.fileno()).st_size,
                    "location": min(file.tell() + FILE_BUFF, os.fstat(file.fileno()).st_size)
                }
                if config.debug_socket:
                    self.log.debug(
                        "Sending file %s from position %s to %s" %
                        (file_path, params["location"], back["location"])
                    )
                self.response(back, streaming=True)
            if config.debug_socket:
                self.log.debug("File %s sent" % file_path)

            # Add peer to site if not added before
            connected_peer = site.addPeer(self.connection.ip, self.connection.port)
            if connected_peer:  # Just added
                connected_peer.connect(self.connection)  # Assign current connection to peer

        except Exception, err:
            self.log.debug("GetFile read error: %s" % Debug.formatException(err))
            self.response({"error": "File read error: %s" % Debug.formatException(err)})
            return False
Пример #17
0
    def actionStreamFile(self, params):
        site = self.sites.get(params["site"])
        if not site or not site.settings["serving"]:  # Site unknown or not serving
            self.response({"error": "Unknown site"})
            return False
        try:
            if config.debug_socket:
                self.log.debug("Opening file: %s" % params["inner_path"])
            with site.storage.open(params["inner_path"]) as file:
                file.seek(params["location"])
                stream_bytes = min(FILE_BUFF, os.fstat(file.fileno()).st_size-params["location"])
                back = {
                    "size": os.fstat(file.fileno()).st_size,
                    "location": min(file.tell() + FILE_BUFF, os.fstat(file.fileno()).st_size),
                    "stream_bytes": stream_bytes
                }
                if config.debug_socket:
                    self.log.debug(
                        "Sending file %s from position %s to %s" %
                        (params["inner_path"], params["location"], back["location"])
                    )
                self.response(back)
                self.sendRawfile(file, read_bytes=FILE_BUFF)
            if config.debug_socket:
                self.log.debug("File %s sent" % params["inner_path"])

            # Add peer to site if not added before
            connected_peer = site.addPeer(self.connection.ip, self.connection.port)
            if connected_peer:  # Just added
                connected_peer.connect(self.connection)  # Assign current connection to peer

        except Exception, err:
            self.log.debug("GetFile read error: %s" % Debug.formatException(err))
            self.response({"error": "File read error: %s" % Debug.formatException(err)})
            return False
Пример #18
0
    def test_object_audit_diff_data(self):
        self.auditor = auditor.AuditorWorker(self.conf)
        data = '0' * 1024
        etag = md5()
        timestamp = str(normalize_timestamp(time.time()))
        with self.disk_file.mkstemp() as (fd, tmppath):
            os.write(fd, data)
            etag.update(data)
            etag = etag.hexdigest()
            metadata = {
                'ETag': etag,
                'X-Timestamp': timestamp,
                'Content-Length': str(os.fstat(fd).st_size),
                'Original-Content-Length': str(os.fstat(fd).st_size)
            }
            self.disk_file.put(fd, tmppath, metadata)
            pre_quarantines = self.auditor.quarantines
            # remake so it will have metadata
            self.disk_file = DiskFile(self.devices, 'sda', '0', 'a', 'c', 'o',
                                      self.logger)

            self.auditor.object_audit(
                os.path.join(self.disk_file.datadir, timestamp + '.data'),
                'sda', '0')
            self.assertEquals(self.auditor.quarantines, pre_quarantines)
            etag = md5()
            etag.update('1' + '0' * 1023)
            etag = etag.hexdigest()
            metadata['ETag'] = etag
            write_metadata(fd, metadata)

            self.auditor.object_audit(
                os.path.join(self.disk_file.datadir, timestamp + '.data'),
                'sda', '0')
            self.assertEquals(self.auditor.quarantines, pre_quarantines + 1)
Пример #19
0
def main():
    parser = argparse.ArgumentParser(description='Check Memrise for data and possibly upload to Beeminder.')
    parser.add_argument('-c', '--config-file', metavar='FILENAME', type=argparse.FileType('r'),
                        help='The config file containing your settings.  A default one is included in the distribution.')
    parser.add_argument('-d', '--database-file', metavar='FILENAME', type=argparse.FileType('r+'),
                        help='The database file to store persistent data in (e.g. we find newly learned words by monitoring total word count)')
    args = parser.parse_args()
    if not args.config_file:
        if os.path.exists(CONFIG_DEFAULT):
            args.config_file = open(CONFIG_DEFAULT, 'r')
        else:
            exit('Error: No config file specified and no default config in {0}'.format(CONFIG_DEFAULT))
    config = load_config(args.config_file)

    if not args.database_file:
        if not os.path.exists(DATABASE_DEFAULT):
            print "Database file not specified, and default doesn't exist. Creating..."
            args.database_file = open(DATABASE_DEFAULT, 'w+')
            init_database(args.database_file, config)
        else:
            args.database_file = open(DATABASE_DEFAULT, 'r+')
    else:
        print os.fstat(args.database_file.fileno())
        if os.fstat(args.database_file.fileno()).st_size == 0:
            init_database(args.database_file, config)

    b = BeeminderSession(config.get('beeminder', 'username'), config.get('beeminder', 'auth_token'))
    # Here we hardcode total_count monitoring, since the API isn't that great, and what I hoped to do seems impossible (without HTML scraping...)
    do_monitor_totalcount(config.get('memrise', 'username'), args.database_file, b, config.get('beeminder', 'goal_slug'))
    cleanup(args.database_file, args.config_file)
Пример #20
0
def _set_data_field(fields, args):
    if 'location' not in fields and 'copy_from' not in fields:
        if args.file:
            fields['data'] = open(args.file, 'rb')
        else:
            # distinguish cases where:
            # (1) stdin is not valid (as in cron jobs):
            #     glance ... <&-
            # (2) image data is provided through standard input:
            #     glance ... < /tmp/file or cat /tmp/file | glance ...
            # (3) no image data provided:
            #     glance ...
            try:
                os.fstat(0)
            except OSError:
                # (1) stdin is not valid (closed...)
                fields['data'] = None
                return
            if not sys.stdin.isatty():
                # (2) image data is provided through standard input
                if msvcrt:
                    msvcrt.setmode(sys.stdin.fileno(), os.O_BINARY)
                fields['data'] = sys.stdin
            else:
                # (3) no image data provided
                fields['data'] = None
Пример #21
0
  def __init__(self, fh, mimetype, chunksize=DEFAULT_CHUNK_SIZE,
      resumable=False):
    """Constructor.

    Args:
      fh: io.Base or file object, The source of the bytes to upload. MUST be
        opened in blocking mode, do not use streams opened in non-blocking mode.
      mimetype: string, Mime-type of the file. If None then a mime-type will be
        guessed from the file extension.
      chunksize: int, File will be uploaded in chunks of this many bytes. Only
        used if resumable=True.
      resumable: bool, True if this is a resumable upload. False means upload
        in a single request.
    """
    self._fh = fh
    self._mimetype = mimetype
    self._chunksize = chunksize
    self._resumable = resumable
    self._size = None
    try:
      if hasattr(self._fh, 'fileno'):
        fileno = self._fh.fileno()

        # Pipes and such show up as 0 length files.
        size = os.fstat(fileno).st_size
        if size:
          self._size = os.fstat(fileno).st_size
    except IOError:
      pass
Пример #22
0
 def _fd_closed(fd):
     try:
         os.fstat(fd)
     except (OSError, IOError), exc:
         if exc.errno == errno.EBADF:
             return True
         raise
Пример #23
0
 def __iter__(self):
     if not self.file:
         self._reopen(True)
     try:
         need_reopen = not os.path.samestat(os.fstat(self.file.fileno()), os.stat(self.fname))
     except:
         need_reopen = True
     if self.file:
         newpos = os.fstat(self.file.fileno()).st_size
         if newpos > self.pos + self.max_size:
             self.pos = newpos
             self.buf = ''
             return
         self.file.seek(self.pos)
         if self.buf:
             self.buf += self.file.readline()
         if self.buf.endswith('\n') or len(self.buf) > self.max_size:
             yield self.buf
             self.buf = ''
         line = None
         for line in self.file:
             if line.endswith('\n'):
                 yield line
         if line and not line.endswith('\n'):
             self.buf = line
         self.pos = self.file.tell()
     if need_reopen:
         self._reopen(True)
Пример #24
0
    def pipe_read( self, pipe, minimum_to_read ):

        ##  Hackaround since Windows doesn't support select( ) except for sockets.

        dbg_print( 'pipe_read: minimum to read is ' + str( minimum_to_read ) )

        time.sleep( self.delay )

        count = 0
        count = os.fstat( pipe )[stat.ST_SIZE]
            
        data = ''

        while ( count > 0 ):
            data += os.read( pipe, 1 )
            count = os.fstat( pipe )[stat.ST_SIZE]

            ##  Be sure to break the read, if asked to do so,
            ##  after we've read in a line termination.

            if minimum_to_read != 0 and data[ len(data) -1 ] == '\n':
                if len( data ) >= minimum_to_read:
                    dbg_print( 'pipe_read: read at least the minimum asked for' )
                    break
Пример #25
0
 def close(self):
     self.file.close()
     hash = urlsafe_b64encode(self.hash.digest())
     filename = blob_filename(hash)
     dirname = os.path.dirname(filename)
     if os.path.exists(filename):
         origfile = open(filename, 'rb')
         newfile = open(self.file.name, 'rb')
         origlen = os.fstat(origfile.fileno()).st_size
         newlen = os.fstat(newfile.fileno()).st_size
         if origlen != newlen:
             raise Exception('HASH COLLISION! {0} {1}'.format(filename, self.file.name))
         while origlen > 0:
             origdata = origfile.read(min(origlen, 1024))
             newdata = newfile.read(min(origlen, 1024))
             if origdata != newdata:
                 raise Exception('HASH COLLISION! {0} {1}'.format(filename, self.file.name))
             origlen -= 1024
         origfile.close()
         newfile.close()
     ensuredirs(dirname, 0700)
     os.rename(self.file.name, filename)
     os.utime(filename, None)
     if self.on_close:
         self.on_close(hash)
     return hash
Пример #26
0
def split_multimol2(multimol2):
    """
    Splits a multi-mol2 file.

    Parameters
    ----------
    multimol2 : str
      Path to the multi-mol2 file.

    Returns
    ----------
    A generator object for lists for every extracted mol2-file. Lists contain
      the molecule ID and the mol2 file contents.
      e.g., ['ID1234', '@<TRIPOS>MOLECULE...'

    """
    with open(multimol2, 'r') as mol2file:
        line = mol2file.readline()

        while not mol2file.tell() == os.fstat(mol2file.fileno()).st_size:
            if line.startswith("@<TRIPOS>MOLECULE"):
                mol2cont = []
                mol2cont.append(line)
                line = mol2file.readline()
                molecule_id = line.strip()

                while not line.startswith("@<TRIPOS>MOLECULE"):
                    mol2cont.append(line)
                    line = mol2file.readline()
                    if mol2file.tell() == os.fstat(mol2file.fileno()).st_size:
                        mol2cont.append(line)
                        break
                mol2cont[-1] = mol2cont[-1].rstrip() # removes blank line at file end
                yield [molecule_id, "".join(mol2cont)]
Пример #27
0
def _putFile(inPath, outUrl, method='PUT', session=None, callback=None):
    if hasattr(inPath, 'read'):
        inFile = inPath
        if hasattr(inPath, 'size'):
            size = inFile.size
        elif hasattr(inPath, 'getvalue'):
            size = len(inPath.getvalue())
        elif hasattr(inPath, 'fileno'):
            size = os.fstat(inPath.fileno()).st_size
        else:
            raise TypeError("Can't determine size of file")
    else:
        inFile = open(inPath)
        size = os.fstat(inFile.fileno()).st_size

    headers = {}
    if session:
        headers['Cookie'] =  'vmware_soap_session=%s; $Path=/' % session
    response = _makeConnection(outUrl, method, headers, inFile, bodyLength=size,
        callback=callback)

    if response and response.status not in (200, 201):
        raise RuntimeError('%s failed: %d - %s' % (
            method, response.status, response.reason))
    elif not response:
        raise RuntimeError('%s failed' % method)
    response.close()
Пример #28
0
def truncate_and_unlink(path, increment=256*1024**2, delay=0.2):
    """Truncate a file to zero bytes before unlinking

    Truncation is done in ``increment`` bytes with a sleep delay
    of ``delay`` seconds between each truncation step.

    Once the file is zero bytes in size it will be removed/unlinked
    from the filesystem.

    :raises: OSError on error
    """
    LOG.debug("truncate_and_unlink(path=%r, increment=%r, delay=%.2fs)",
              path, format_bytes(increment), delay)
    fd = os.open(path, os.O_RDWR)

    size = os.fstat(fd).st_size
    while size > 0:
        start = time.time()
        length = size - increment
        if length < 0:
            length = 0
        os.ftruncate(fd, length)
        LOG.debug("truncate(%s, length=%s) in %.2fs",
                  path, format_bytes(length), time.time() - start)
        time.sleep(delay)
        size = os.fstat(fd).st_size
    os.unlink(path)
    LOG.debug("unlink(%s)", path)
Пример #29
0
def execute(target, directory):
    page_size = resource.getpagesize()
    rows = []

    if directory:
        for (path, dirs, files) in os.walk(target):
            for myfile in files:
                f = os.path.join(path, myfile)
                fd = file(f,'r')
                file_size = os.fstat(fd.fileno())[stat.ST_SIZE]
                if file_size == 0:
                    fd.close()
                    continue
                pages_cached, pages_total = ftools.fincore_ratio(fd.fileno())
                fd.close()
                rows.append([f, file_size, pages_total, pages_cached, (pages_cached * page_size), (float(pages_cached) / float(pages_total)) * 100.0])
    else:
        fd = file(target, 'r')
        file_size = os.fstat(fd.fileno())[stat.ST_SIZE]
        if file_size == 0:
            fd.close()
            return None
        else:
            pages_cached, pages_total = ftools.fincore_ratio(fd.fileno())
            fd.close()
            rows.append([target, file_size, pages_total, pages_cached, (pages_cached * page_size), (float(pages_cached) / float(pages_total)) * 100.0])

    rows = sorted(rows, key=lambda t: t[5], reverse=True)
    return rows
Пример #30
0
    def test_contextmanager(self):
        fn = None
        with Image.open("Images/lena.gif") as im:
            fn = im.fp.fileno()
            os.fstat(fn)

        self.assertRaises(OSError, lambda: os.fstat(fn))
Пример #31
0
def send_data(v_vars, v_files, boundary, sock=None):
    l = 0
    for (k, v) in v_vars:
        buffer = ''
        buffer += '--%s\r\n' % boundary
        buffer += 'Content-Disposition: form-data; name="%s"\r\n' % k
        buffer += '\r\n'
        buffer += v + '\r\n'
        if sock:
            sock.send(buffer)
        l += len(buffer)
    for (k, v) in v_files:
        fd = v

        if not hasattr(fd, 'read'):
            raise TypeError("file descriptor MUST have read attribute")

        if hasattr(fd, 'fileno'):
            # a File
            try:
                name = fd.name.split(os.path.sep)[-1]
                file_size = os.fstat(fd.fileno())[stat.ST_SIZE]
            except io.UnsupportedOperation:
                name = fd.name
                file_size = len(fd.getvalue())
            fd.seek(0)
        elif hasattr(fd, 'len'):
            # StringIO
            name = k
            file_size = fd.len
            fd.seek(0)  # START
        else:
            raise TypeError(
                "file descriptor might be File of StringIO but MUST have fileno or len attribute"
            )

        if isinstance(name, unicode):
            name = name.encode('UTF-8')
        buffer = ''
        buffer += '--%s\r\n' % boundary
        buffer += 'Content-Disposition: form-data; name="%s"; filename="%s"\r\n' \
                  % (k, name)
        buffer += 'Content-Type: %s\r\n' % get_content_type(name)
        buffer += 'Content-Length: %s\r\n' % file_size
        buffer += '\r\n'

        l += len(buffer)
        if sock:
            sock.send(buffer)
            if hasattr(fd, 'seek'):
                fd.seek(0)
        # read file only of sock is defined
        if sock:
            while True:
                chunk = fd.read(CHUNK_SIZE)
                if not chunk:
                    break
                if sock:
                    sock.send(chunk)
        l += file_size
    buffer = '\r\n'
    buffer += '--%s--\r\n' % boundary
    buffer += '\r\n'
    if sock:
        sock.send(buffer)
    l += len(buffer)
    return l
Пример #32
0
def fd_id(fd):
    # Returns a tuple which uniquely identifies a file descriptor. In Mac OS,
    # this doesn't work with shared memory handles, which is why we don't
    # support the "file_descriptor" sharing method on that platform.
    stat = os.fstat(fd)
    return (stat.st_ino, stat.st_dev)
Пример #33
0
    # freshen permissions, if possible.
    try:
        os.chmod(MSULOGDIR, 01777)
    except OSError:
        pass

    # find a safe log file to write to for this user
    filename = os.path.join(MSULOGDIR, MSULOGFILE % username)
    attempt = 0
    ours = False

    while attempt < 10:
        try:
            fref = os.open(filename, os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW,
                           0600)
            st = os.fstat(fref)
            ours = stat.S_ISREG(st.st_mode) and st.st_uid == os.getuid()
            os.close(fref)
            if ours:
                break
        except (OSError, IOError):
            pass  # permission denied, symlink, ...

        # avoid creating many separate log files by using one static suffix
        # as the first alternative.  if unsuccessful, switch to totally
        # randomly suffixed files.
        if attempt == 0:
            random.seed(hash(username))
        elif attempt == 1:
            random.seed()
Пример #34
0
## All windows closed, clean up
    Editor.deinit_tty()
    Editor.yank_buffer = []
    ## close
    return slot[0].content if (slot[0].fname == "") else slot[0].fname


#ifdef LINUX
if __name__ == "__main__":
    if is_linux:
        import stat
        fd_tty = 0
        if len(sys.argv) > 1:
            name = sys.argv[1:]
            pye(*name, undo=500, device=fd_tty)
        else:
            name = ""
            if not is_micropython:
                mode = os.fstat(0).st_mode
                if stat.S_ISFIFO(mode) or stat.S_ISREG(mode):
                    name = sys.stdin.readlines()
                    os.close(0)  ## close and repopen /dev/tty
                    fd_tty = os.open("/dev/tty",
                                     os.O_RDONLY)  ## memorized, if new fd
                    for i, l in enumerate(name):  ## strip and convert
                        name[i], tc = expandtabs(l.rstrip('\r\n\t '))
            pye(name, undo=500, device=fd_tty)
    else:
        print("\nSorry, this OS is not supported (yet)")
#endif
Пример #35
0
def flen(f):
    return os.fstat(f.fileno()).st_size
Пример #36
0
def padToBlockSize(path):
    with open(path, 'a') as f:
        size = os.fstat(f.fileno()).st_size
        newSize = utils.round(size, sc.BLOCK_SIZE)
        log.info("Truncating file %s to %d bytes", path, newSize)
        os.ftruncate(f.fileno(), newSize)
Пример #37
0
import  os
os.dup2()  #复制
os.fdatasync()#强制将文件写入磁盘
os.fstat()#返回文件描述fd的状态
os.open()#打开一个文件,并设置需要的打开选项mode 参数是可选的
os.remove()#删除
os.removedirs()#递归删除目录
os.rename()#重命名
os.renames()#递归重命名
#parser()方法  parseString()


Пример #38
0
def main(opts):
    logfile = open_logfile()

    if not logfile:
        say('No logfile found. Is vserver_launcher running?')
        return 1

    if (opts.last_crash > 0) and not is_alive():
        say('Program is dead. Last logs:')
        print_last_lines(logfile, opts.last_crash)
    elif opts.last > 0:
        say('Last logs:')
        print_last_lines(logfile, opts.last)

    if opts.restart and logfile:
        say('Requesting vserver.py restart')
        os.unlink(LOG_NAME)
        logfile.close()
        for _ in xrange(20):
            logfile = open_logfile()
            if logfile:
                break
            time.sleep(0.25)
        else:
            say('Logfile did not appear. Is vserver_launcher running?')
            return 2

    stop_at = None
    if (opts.wait_exit is not None) and (opts.wait_exit >= 0):
        stop_at = time.time() + opts.wait_exit
    last_msg = time.time()
    say("Live logs")

    log_inode = None
    # Do the tail
    while True:
        if logfile is not None:
            block = logfile.read(1024)
        else:
            # wait for file to appear
            block = ''
            logfile = open_logfile()
            if logfile:
                say('File re-opened')
        if block != '':
            sys.stdout.write(block)
            sys.stdout.flush()
            last_msg = time.time()
            continue
        if not is_alive():
            say('Program has exited. Terminating')
            return 1
        if (stop_at is not None) and (stop_at < time.time()):
            break
        if opts.idle_marker > 0 and (
            (last_msg + opts.idle_marker) < time.time()):
            say('Program still running, no messages')
            last_msg = time.time()

        # Sleep
        time.sleep(opts.poll_time)
        if logfile is not None:
            # Reset EOF marker if we have the file
            logfile.seek(0, os.SEEK_CUR)
            # Check if file was replaced
            if log_inode is None:
                log_inode = os.fstat(logfile.fileno()).st_ino
            try:
                disk_inode = os.stat(LOG_NAME).st_ino
            except OSError:
                disk_inode = -1
            if log_inode != disk_inode:
                say('Logfile disappeared')
                logfile.close()
                logfile = None
                log_inode = None

    say('Time expired, program still running')
    return 0
Пример #39
0
def isapipe(fd):
    fd = getattr(fd, 'fileno', lambda: fd)()
    return stat.S_ISFIFO(os.fstat(fd).st_mode)
Пример #40
0
    def send_head(self):
        """Common code for GET and HEAD commands.

        This sends the response code and MIME headers.

        Return value is either a file object (which has to be copied
        to the outputfile by the caller unless the command was HEAD,
        and must be closed by the caller under all circumstances), or
        None, in which case the caller has nothing further to do.

        """
        path = self.translate_path(self.path)
        f = None
        if os.path.isdir(path):
            parts = urllib.parse.urlsplit(self.path)
            if not parts.path.endswith('/'):
                # redirect browser - doing basically what apache does
                self.send_response(HTTPStatus.MOVED_PERMANENTLY)
                new_parts = (parts[0], parts[1], parts[2] + '/', parts[3],
                             parts[4])
                new_url = urllib.parse.urlunsplit(new_parts)
                self.send_header("Location", new_url)
                self.send_header("Content-Length", "0")
                self.end_headers()
                return None
            for index in "index.html", "index.htm":
                index = os.path.join(path, index)
                if os.path.exists(index):
                    path = index
                    break
            else:
                return self.list_directory(path)
        ctype = self.guess_type(path)
        # check for trailing "/" which should return 404. See Issue17324
        # The test for this was added in test_httpserver.py
        # However, some OS platforms accept a trailingSlash as a filename
        # See discussion on python-dev and Issue34711 regarding
        # parsing and rejection of filenames with a trailing slash
        if path.endswith("/"):
            self.send_error(HTTPStatus.NOT_FOUND, "File not found")
            return None
        try:
            f = open(path, 'rb')
        except OSError:
            self.send_error(HTTPStatus.NOT_FOUND, "File not found")
            return None

        try:
            fs = os.fstat(f.fileno())
            # Use browser cache if possible
            if ("If-Modified-Since" in self.headers
                    and "If-None-Match" not in self.headers):
                # compare If-Modified-Since and time of last file modification
                try:
                    ims = email.utils.parsedate_to_datetime(
                        self.headers["If-Modified-Since"])
                except (TypeError, IndexError, OverflowError, ValueError):
                    # ignore ill-formed values
                    pass
                else:
                    if ims.tzinfo is None:
                        # obsolete format with no timezone, cf.
                        # https://tools.ietf.org/html/rfc7231#section-7.1.1.1
                        ims = ims.replace(tzinfo=datetime.timezone.utc)
                    if ims.tzinfo is datetime.timezone.utc:
                        # compare to UTC datetime of last modification
                        last_modif = datetime.datetime.fromtimestamp(
                            fs.st_mtime, datetime.timezone.utc)
                        # remove microseconds, like in If-Modified-Since
                        last_modif = last_modif.replace(microsecond=0)

                        if last_modif <= ims:
                            self.send_response(HTTPStatus.NOT_MODIFIED)
                            self.end_headers()
                            f.close()
                            return None

            self.send_response(HTTPStatus.OK)
            self.send_header("Content-type", ctype)
            self.send_header("Content-Length", str(fs[6]))
            self.send_header("Last-Modified",
                             self.date_time_string(fs.st_mtime))
            self.end_headers()
            return f
        except:
            f.close()
            raise
Пример #41
0
 def stat(self):
     try:
         return SFTPAttributes.from_stat(os.fstat(self.readfile.fileno()))
     except OSError as e:
         return SFTPServer.convert_errno(e.errno)
Пример #42
0
os.unsetenv(varname)            # Unset (delete) the environment variable named varname

################################ File Object Creation ################################
os.fdopen(fd[, mode[, bufsize]]) 
os.popen(command[, mode[, bufsize]])    # Deprecated since version 2.6: This function is obsolete. Use the subprocess module
os.tmpfile()                            # Deprecated since version 2.6: All of the popen*() functions are obsolete. Use the subprocess module
os.popen2(cmd[, mode[, bufsize]])       # Deprecated since version 2.6: This function is obsolete. Use the subprocess module
os.popen3(cmd[, mode[, bufsize]])       # Deprecated since version 2.6: This function is obsolete. Use the subprocess module
os.popen4(cmd[, mode[, bufsize]])       # Deprecated since version 2.6: This function is obsolete. Use the subprocess module

################################ File Descriptor Operations ################################
os.close(fd)        # Close file descriptor fd.
os.closerange(fd_low, fd_high)      # Close all file descriptors from fd_low (inclusive) to fd_high (exclusive), ignoring errors
os.dup(fd)          # Return a duplicate of file descriptor fd.
os.dup2(fd, fd2)    # Duplicate file descriptor fd to fd2
os.fstat(fd)        # Return status for file descriptor fd, like stat().
os.fsync(fd)        # Force write of file with filedescriptor fd to disk
os.isatty(fd)       # Return True if the file descriptor fd is open and connected to a tty(-like) device, else False.
os.lseek(fd, pos, how)  # Set the current position of file descriptor fd to position pos, modified by how: SEEK_SET or 0
os.open(file, flags[, mode])    # Open the file file and set various flags according to flags
os.pipe()           # Create a pipe. Return a pair of file descriptors (r, w)
os.read(fd, n)      # Read at most n bytes from file descriptor fd
os.write(fd, str)   # Write the string str to file descriptor fd

################################ Files and Directories ################################

os.access(path, mode)       # Use the real uid/gid to test for access to path
os.chdir(path)              # Change the current working directory to path.
os.getcwd()                 # Return a string representing the current working directory.
os.getcwdu()                # Return a Unicode object representing the current working directory.
os.chmod(path, mode)        # Change the mode of path to the numeric mode
Пример #43
0
    def maybeParseConfig(self):
        if self.resolv is None:
            # Don't try to parse it, don't set up a call loop
            return

        try:
            resolvConf = file(self.resolv)
        except IOError, e:
            if e.errno == errno.ENOENT:
                # Missing resolv.conf is treated the same as an empty resolv.conf
                self.parseConfig(())
            else:
                raise
        else:
            mtime = os.fstat(resolvConf.fileno()).st_mtime
            if mtime != self._lastResolvTime:
                log.msg('%s changed, reparsing' % (self.resolv, ))
                self._lastResolvTime = mtime
                self.parseConfig(resolvConf)

        # Check again in a little while
        from twisted.internet import reactor
        self._parseCall = reactor.callLater(self._resolvReadInterval,
                                            self.maybeParseConfig)

    def parseConfig(self, resolvConf):
        servers = []
        for L in resolvConf:
            L = L.strip()
            if L.startswith('nameserver'):
Пример #44
0
    # imagesos = os.listdir("/home/neo/Documents/Python/Images/all/")
    images = [
        "/home/neo/Documents/Python/Images/all/" + item for item in imagesos
    ]
    images.append("/home/neo/Documents/Python/Images/Yann.jpg")

    known_face_encodings = []
    known_face_names = []

    csvwriter = csv.writer(
        filewrite, delimiter=",")  # , quotechar='', quoting=csv.QUOTE_MINIMAL)
    if load:
        # npdata = np.load('face_recognitiontest.npy', allow_pickle=True):
        try:
            with open("face_recognitiontestdb.csv", "r") as db:
                if os.fstat(db.fileno()).st_size:
                    known_face_encodings = np.loadtxt(db,
                                                      delimiter=",",
                                                      skiprows=1,
                                                      unpack=True).tolist()
                else:
                    print("Error db loading : File is empty")
                    load = False
        except (StopIteration, UserWarning):
            load = False
        csvreader = csv.reader(fileread, delimiter=",")  # , quotechar='')
        csvlist = []
        for idlist, item in enumerate(csvreader):
            # known_face_encodings.append(npdata[idlist])
            # print(npdata[idlist])
            print(item[0])
Пример #45
0
 def remapfile(self):
     import mmap
     size = os.fstat(self.fd).st_size
     self.mm = mmap.mmap(self.fd, size, access=self.access)
Пример #46
0
def copyupdown_internal(wh, sd, upp):
    '''Copy up/down a file or dir.

    wh: 'copyup' or 'copydown'
    sd: (source, destination) paths
    upp: True for copyup, False for copydown
    '''
    if not downtmp:
        bomb("%s when not open" % wh)
    if not sd[0] or not sd[1]:
        bomb("%s paths must be nonempty" % wh)
    dirsp = sd[0][-1] == '/'
    if dirsp != (sd[1][-1] == '/'):
        bomb("%s paths must agree about directoryness"
             " (presence or absence of trailing /)" % wh)

    # if we have a shared directory, we just need to copy it from/to there; in
    # most cases, it's testbed end is already in the downtmp dir
    downtmp_host = get_downtmp_host()
    if downtmp_host:
        try:
            if upp:
                copyup_shareddir(sd[0], sd[1], dirsp, downtmp_host)
            else:
                copydown_shareddir(sd[0], sd[1], dirsp, downtmp_host)
            return
        except Timeout:
            raise FailedCmd(['timeout'])
        except (shutil.Error, subprocess.CalledProcessError) as e:
            adtlog.debug(
                'Cannot copy %s to %s through shared dir: %s, falling back to tar'
                % (sd[0], sd[1], str(e)))

    isrc = 0
    idst = 1
    ilocal = 0 + upp
    iremote = 1 - upp

    deststdout = devnull_read
    srcstdin = devnull_read
    remfileq = pipes.quote(sd[iremote])
    if not dirsp:
        rune = 'cat %s%s' % ('><'[upp], remfileq)
        if upp:
            deststdout = open(sd[idst], 'w')
        else:
            srcstdin = open(sd[isrc], 'r')
            status = os.fstat(srcstdin.fileno())
            if status.st_mode & 0o111:
                rune += '; chmod +x -- %s' % (remfileq)
        localcmdl = ['cat']
    else:
        taropts = [None, None]
        taropts[isrc] = '--warning=none -c .'
        taropts[idst] = '--warning=none --preserve-permissions --extract ' \
                        '--no-same-owner'

        rune = 'cd %s; tar %s -f -' % (remfileq, taropts[iremote])
        if upp:
            try:
                os.mkdir(sd[ilocal])
            except (IOError, OSError) as oe:
                if oe.errno != errno.EEXIST:
                    raise
        else:
            rune = ('if ! test -d %s; then mkdir -- %s; fi; ' %
                    (remfileq, remfileq)) + rune

        localcmdl = ['tar', '--directory', sd[ilocal]] + (
            ('%s -f -' % taropts[ilocal]).split())
    downcmdl = auxverb + ['sh', '-ec', rune]

    if upp:
        cmdls = (downcmdl, localcmdl)
    else:
        cmdls = (localcmdl, downcmdl)

    adtlog.debug(str(["cmdls", str(cmdls)]))
    adtlog.debug(
        str([
            "srcstdin",
            str(srcstdin), "deststdout",
            str(deststdout), "devnull_read", devnull_read
        ]))

    subprocs = [None, None]
    adtlog.debug(" +< %s" % ' '.join(cmdls[0]))
    subprocs[0] = subprocess.Popen(cmdls[0],
                                   stdin=srcstdin,
                                   stdout=subprocess.PIPE)
    adtlog.debug(" +> %s" % ' '.join(cmdls[1]))
    subprocs[1] = subprocess.Popen(cmdls[1],
                                   stdin=subprocs[0].stdout,
                                   stdout=deststdout)
    subprocs[0].stdout.close()
    try:
        timeout_start(copy_timeout)
        for sdn in [1, 0]:
            adtlog.debug(" +" + "<>"[sdn] + "?")
            status = subprocs[sdn].wait()
            if not (status == 0 or (sdn == 0 and status == -13)):
                timeout_stop()
                bomb("%s %s failed, status %d" %
                     (wh, ['source', 'destination'][sdn], status))
        timeout_stop()
    except Timeout:
        for sdn in [1, 0]:
            subprocs[sdn].kill()
            subprocs[sdn].wait()
        raise FailedCmd(['timeout'])
Пример #47
0
import itertools
import os
import re
import stat
import sys
import traceback

from voussoirkit import clipext
from voussoirkit import expressionmatch
from voussoirkit import pathclass
from voussoirkit import safeprint
from voussoirkit import spinal

# Thanks georg
# http://stackoverflow.com/a/13443424
STDIN_MODE = os.fstat(sys.stdin.fileno()).st_mode
if stat.S_ISFIFO(STDIN_MODE):
    STDIN_MODE = 'pipe'
else:
    STDIN_MODE = 'terminal'

def all_terms_match(search_text, terms, match_function):
    matches = (
        (not terms['yes_all'] or all(match_function(search_text, term) for term in terms['yes_all'])) and
        (not terms['yes_any'] or any(match_function(search_text, term) for term in terms['yes_any'])) and
        (not terms['not_all'] or not all(match_function(search_text, term) for term in terms['not_all'])) and
        (not terms['not_any'] or not any(match_function(search_text, term) for term in terms['not_any']))
    )
    return matches

def search(
Пример #48
0
    def _handle_get_static(self, path_match, data):
        """ Returns a static file. """
        req_file = util.sanitize_path(path_match.group('file'))

        # Strip md5 hash out of frontend filename
        if re.match(r'^frontend-[A-Za-z0-9]{32}\.html$', req_file):
            req_file = "frontend.html"

        path = os.path.join(os.path.dirname(__file__), 'www_static', req_file)

        inp = None

        try:
            inp = open(path, 'rb')

            do_gzip = 'gzip' in self.headers.get('accept-encoding', '')

            self.send_response(HTTP_OK)

            ctype = self.guess_type(path)
            self.send_header("Content-Type", ctype)

            # Add cache if not development
            if not self.server.development:
                # 1 year in seconds
                cache_time = 365 * 86400

                self.send_header("Cache-Control",
                                 "public, max-age={}".format(cache_time))
                self.send_header(
                    "Expires", self.date_time_string(time.time() + cache_time))

            if do_gzip:
                gzip_data = gzip.compress(inp.read())

                self.send_header("Content-Encoding", "gzip")
                self.send_header("Vary", "Accept-Encoding")
                self.send_header("Content-Length", str(len(gzip_data)))

            else:
                fs = os.fstat(inp.fileno())
                self.send_header("Content-Length", str(fs[6]))

            self.end_headers()

            if self.command == 'HEAD':
                return

            elif do_gzip:
                self.wfile.write(gzip_data)

            else:
                self.copyfile(inp, self.wfile)

        except IOError:
            self.send_response(HTTP_NOT_FOUND)
            self.end_headers()

        finally:
            if inp:
                inp.close()
Пример #49
0
def sizesha1(f):
    size = os.fstat(f.fileno())[6]
    f.seek(0)
    sha1sum = apt_pkg.sha1sum(f)
    return (sha1sum, size)
Пример #50
0
def isEndOfFile(fic):
    return (fic.tell() == os.fstat(fic.fileno()).st_size)
Пример #51
0
 def last_update(self):
     return os.fstat(self._tmp.fileno()).st_ctime
Пример #52
0
    def in_place(
        self,
        mode='r',
        buffering=-1,
        encoding=None,
        errors=None,
        newline=None,
        backup_extension=None,
    ):
        """
        A context in which a file may be re-written in-place with
        new content.

        Yields a tuple of :samp:`({readable}, {writable})` file
        objects, where `writable` replaces `readable`.

        If an exception occurs, the old file is restored, removing the
        written data.

        Mode *must not* use ``'w'``, ``'a'``, or ``'+'``; only
        read-only-modes are allowed. A :exc:`ValueError` is raised
        on invalid modes.

        For example, to add line numbers to a file::

            p = Path(filename)
            assert p.isfile()
            with p.in_place() as (reader, writer):
                for number, line in enumerate(reader, 1):
                    writer.write('{0:3}: '.format(number)))
                    writer.write(line)

        Thereafter, the file at `filename` will have line numbers in it.
        """
        if set(mode).intersection('wa+'):
            raise ValueError('Only read-only file modes can be used')

        # move existing file to backup, create new file with same permissions
        # borrowed extensively from the fileinput module
        backup_fn = self + (backup_extension or os.extsep + 'bak')
        backup_fn.remove_p()
        self.rename(backup_fn)
        readable = io.open(
            backup_fn,
            mode,
            buffering=buffering,
            encoding=encoding,
            errors=errors,
            newline=newline,
        )
        try:
            perm = os.fstat(readable.fileno()).st_mode
        except OSError:
            writable = self.open(
                'w' + mode.replace('r', ''),
                buffering=buffering,
                encoding=encoding,
                errors=errors,
                newline=newline,
            )
        else:
            os_mode = os.O_CREAT | os.O_WRONLY | os.O_TRUNC
            os_mode |= getattr(os, 'O_BINARY', 0)
            fd = os.open(self, os_mode, perm)
            writable = io.open(
                fd,
                "w" + mode.replace('r', ''),
                buffering=buffering,
                encoding=encoding,
                errors=errors,
                newline=newline,
            )
            with contextlib.suppress(OSError, AttributeError):
                self.chmod(perm)
        try:
            yield readable, writable
        except Exception:
            # move backup back
            readable.close()
            writable.close()
            self.remove_p()
            backup_fn.rename(self)
            raise
        else:
            readable.close()
            writable.close()
        finally:
            backup_fn.remove_p()
Пример #53
0
 def fsize(self):
     return os.fstat(self.f.fileno())[stat.ST_SIZE]
Пример #54
0
def upload_to_s3(aws_access_key_id,
                 aws_secret_access_key,
                 fname,
                 bucket,
                 key,
                 callback=None,
                 md5=None,
                 reduced_redundancy=False,
                 content_type=None,
                 host='s3.eu-central-1.amazonaws.com'):
    """
    XXX copied from somewher on stackoverflow. Hope to find it again.

    Uploads the given file to the AWS S3
    bucket and key specified.

    callback is a function of the form:

    def callback(complete, total)

    The callback should accept two integer parameters,
    the first representing the number of bytes that
    have been successfully transmitted to S3 and the
    second representing the size of the to be transmitted
    object.

    Returns boolean indicating success/failure of upload.
    """
    switch_validation = False
    if host is not None:
        if 'eu-central' in host:
            switch_validation = True
            os.environ['S3_USE_SIGV4'] = 'True'
    com = boto.connect_s3(aws_access_key_id, aws_secret_access_key, host=host)
    bucket = com.get_bucket(bucket, validate=True)
    s3_key = Key(bucket)
    s3_key.key = key
    if content_type:
        s3_key.set_metadata('Content-Type', content_type)

    with open(fname) as fid:
        try:
            size = os.fstat(fname.fileno()).st_size
        except:
            # Not all file objects implement fileno(),
            # so we fall back on this
            fid.seek(0, os.SEEK_END)
            size = fid.tell()
        sent = s3_key.set_contents_from_file(
            fid,
            cb=callback,
            md5=md5,
            reduced_redundancy=reduced_redundancy,
            rewind=True)
        # Rewind for later use
        fid.seek(0)

    if switch_validation:
        del os.environ['S3_USE_SIGV4']

    if sent == size:
        return True
    return False
Пример #55
0
        if check(arr):
            return (i+1)
    return n

def check(arr):
    for i,x in enumerate(arr):
        if (i+1) !=x:
            return False
    return True

def f(arr,i):
    if arr[i] > arr[i+1]:
        tmp = arr[i]
        arr[i] = arr[i+1]
        arr[i+1] = tmp
import os
import io
# import time
# a=time.time()
if __name__ == "__main__":
    input = io.BytesIO(os.read(0, os.fstat(0).st_size)).readline


    T = int(input().decode().strip())
    for t in range(T):
        n = int(input().decode().strip())
        arr = [int(x) for x in input().decode().strip().split(" ")]
        res = solve_dumb(arr, n)
        # check(res)
        print(res)
Пример #56
0
    def update_body_from_data(self, data, skip_auto_headers):
        if not data:
            return

        if isinstance(data, str):
            data = data.encode(self.encoding)

        if isinstance(data, (bytes, bytearray)):
            self.body = data
            if (hdrs.CONTENT_TYPE not in self.headers
                    and hdrs.CONTENT_TYPE not in skip_auto_headers):
                self.headers[hdrs.CONTENT_TYPE] = 'application/octet-stream'
            if hdrs.CONTENT_LENGTH not in self.headers and not self.chunked:
                self.headers[hdrs.CONTENT_LENGTH] = str(len(self.body))

        elif isinstance(data, (asyncio.StreamReader, streams.DataQueue)):
            self.body = data

        elif asyncio.iscoroutine(data):
            self.body = data
            if (hdrs.CONTENT_LENGTH not in self.headers
                    and self.chunked is None):
                self.chunked = True

        elif isinstance(data, io.IOBase):
            assert not isinstance(data, io.StringIO), \
                'attempt to send text data instead of binary'
            self.body = data
            if not self.chunked and isinstance(data, io.BytesIO):
                # Not chunking if content-length can be determined
                size = len(data.getbuffer())
                self.headers[hdrs.CONTENT_LENGTH] = str(size)
                self.chunked = False
            elif not self.chunked and isinstance(data, io.BufferedReader):
                # Not chunking if content-length can be determined
                try:
                    size = os.fstat(data.fileno()).st_size - data.tell()
                    self.headers[hdrs.CONTENT_LENGTH] = str(size)
                    self.chunked = False
                except OSError:
                    # data.fileno() is not supported, e.g.
                    # io.BufferedReader(io.BytesIO(b'data'))
                    self.chunked = True
            else:
                self.chunked = True

            if hasattr(data, 'mode'):
                if data.mode == 'r':
                    raise ValueError('file {!r} should be open in binary mode'
                                     ''.format(data))
            if (hdrs.CONTENT_TYPE not in self.headers
                    and hdrs.CONTENT_TYPE not in skip_auto_headers
                    and hasattr(data, 'name')):
                mime = mimetypes.guess_type(data.name)[0]
                mime = 'application/octet-stream' if mime is None else mime
                self.headers[hdrs.CONTENT_TYPE] = mime

        elif isinstance(data, MultipartWriter):
            self.body = data.serialize()
            self.headers.update(data.headers)
            self.chunked = self.chunked or 8192

        else:
            if not isinstance(data, helpers.FormData):
                data = helpers.FormData(data)

            self.body = data(self.encoding)

            if (hdrs.CONTENT_TYPE not in self.headers
                    and hdrs.CONTENT_TYPE not in skip_auto_headers):
                self.headers[hdrs.CONTENT_TYPE] = data.content_type

            if data.is_multipart:
                self.chunked = self.chunked or 8192
            else:
                if (hdrs.CONTENT_LENGTH not in self.headers
                        and not self.chunked):
                    self.headers[hdrs.CONTENT_LENGTH] = str(len(self.body))
Пример #57
0
Файл: s3c.py Проект: iphydf/s3ql
    def _send_request(self,
                      method,
                      path,
                      headers,
                      subres=None,
                      query_string=None,
                      body=None):
        '''Add authentication and send request

        Returns the response object.
        '''

        if not isinstance(headers, CaseInsensitiveDict):
            headers = CaseInsensitiveDict(headers)

        if not self.hostname.startswith(self.bucket_name):
            path = '/%s%s' % (self.bucket_name, path)
        headers['host'] = self.hostname

        self._authorize_request(method, path, headers, subres, query_string)

        path = urllib.parse.quote(path)
        if query_string:
            s = urllib.parse.urlencode(query_string, doseq=True)
            if subres:
                path += '?%s&%s' % (subres, s)
            else:
                path += '?%s' % s
        elif subres:
            path += '?%s' % subres

        # We can probably remove the assertions at some point and
        # call self.conn.read_response() directly
        def read_response():
            resp = self.conn.read_response()
            assert resp.method == method
            assert resp.path == path
            return resp

        use_expect_100c = not self.options.get('disable-expect100', False)
        try:
            log.debug('sending %s %s', method, path)
            if body is None or isinstance(body,
                                          (bytes, bytearray, memoryview)):
                self.conn.send_request(method,
                                       path,
                                       body=body,
                                       headers=headers)
            else:
                body_len = os.fstat(body.fileno()).st_size
                self.conn.send_request(method,
                                       path,
                                       expect100=use_expect_100c,
                                       headers=headers,
                                       body=BodyFollowing(body_len))

                if use_expect_100c:
                    resp = read_response()
                    if resp.status != 100:  # Error
                        return resp

                try:
                    copyfileobj(body, self.conn, BUFSIZE)
                except ConnectionClosed:
                    # Server closed connection while we were writing body data -
                    # but we may still be able to read an error response
                    try:
                        resp = read_response()
                    except ConnectionClosed:  # No server response available
                        pass
                    else:
                        if resp.status >= 400:  # Got error response
                            return resp
                        log.warning(
                            'Server broke connection during upload, but signaled '
                            '%d %s', resp.status, resp.reason)

                    # Re-raise first ConnectionClosed exception
                    raise

            return read_response()

        except Exception as exc:
            if is_temp_network_error(exc) or isinstance(exc, ssl.SSLError):
                # We probably can't use the connection anymore
                self.conn.disconnect()
            raise
Пример #58
0
    def _do_request(self,
                    connection,
                    method,
                    path,
                    headers=None,
                    body=None,
                    download_body=True):
        '''Send request, read and return response object'''

        log.debug('started with %s %s', method, path)

        if headers is None:
            headers = CaseInsensitiveDict()

        if self.authorization_token is None:
            self._authorize_account()

        if 'Authorization' not in headers:
            headers['Authorization'] = self.authorization_token

        if self.test_mode_expire_some_tokens:
            headers[
                'X-Bz-Test-Mode'] = 'expire_some_account_authorization_tokens'

        if self.test_mode_force_cap_exceeded:
            headers['X-Bz-Test-Mode'] = 'force_cap_exceeded'

        log.debug('REQUEST: %s %s %s', connection.hostname, method, path)

        if body is None or isinstance(body, (bytes, bytearray, memoryview)):
            connection.send_request(method, path, headers=headers, body=body)
        else:
            body_length = os.fstat(body.fileno()).st_size
            connection.send_request(method,
                                    path,
                                    headers=headers,
                                    body=BodyFollowing(body_length))

            copyfileobj(body, connection, BUFSIZE)

        response = connection.read_response()

        if download_body is True or response.status != 200:  # Backblaze always returns a json with error information in body
            response_body = connection.readall()
        else:
            response_body = None

        content_length = response.headers.get('Content-Length', '0')
        log.debug('RESPONSE: %s %s %s %s', response.method, response.status,
                  response.reason, content_length)

        if (
                response.status == 404 or  # File not found
            (response.status != 200 and method == 'HEAD')
        ):  # HEAD responses do not have a body -> we have to raise a HTTPError with the code
            raise HTTPError(response.status, response.reason, response.headers)

        if response.status != 200:
            json_error_response = json.loads(
                response_body.decode('utf-8')) if response_body else None
            code = json_error_response['code'] if json_error_response else None
            message = json_error_response[
                'message'] if json_error_response else response.reason
            b2_error = B2Error(json_error_response['status'], code, message,
                               response.headers)
            raise b2_error

        return response, response_body
Пример #59
0
 def working_tree_time_ns(self) -> int:
     self._mtime_probe.seek(0)
     self._mtime_probe.write(b'0')  # updates mtime
     return os.fstat(self._mtime_probe.fileno()).st_mtime_ns
Пример #60
0
 def _fill(self):
     s = os.read(self._fd, max(os.fstat(self._fd).st_size, BUFSIZE))
     self.seek((self.tell(), self.seek(0,2), super(FastIO, self).write(s))[0])
     return s