Beispiel #1
0
    def __download_range(self, k, dst):
        try:
            _, ext = os.path.splitext(dst)
            ds = []
            parts = []

            logging.info("Download %s start", k.name)

            for startByte in range(0, k.size, self.splitMB):
                output_part = self.new_temp_file(suffix=ext)
                parts.append(output_part)

                endByte = min(startByte + self.splitMB - 1, k.size)

                logging.debug(
                    "deferToThreadPool %s start=%d end=%d size=%d cnt=%d",
                    k.name,
                    startByte,
                    endByte,
                    endByte - startByte,
                    len(ds),
                )

                d = twisted.internet.threads.deferToThreadPool(
                    reactor,
                    reactor.getThreadPool(),  # @UndefinedVariable
                    self.__downloadOne,
                    k,
                    startByte,
                    endByte,
                    output_part,
                    len(ds),
                )
                ds.append(d)

            if os.path.exists(dst):
                os.remove(dst)
            fout = file(dst, "wb")
            start = timeit.default_timer()
            for cnt, p in enumerate(parts):
                yield ds[cnt]
                shutil.copyfileobj(file(p, "rb"), fout)

                size = min(k.size, (cnt + 1) * self.splitMB)
                elapsed = timeit.default_timer() - start
                speedstr = formatFileSize(size / elapsed)
                sizestr = formatFileSize(size)
                percent = (float(cnt) / len(parts)) * 100.0
                logging.info(
                    "%03d/%03d (%.2f%%) speed=%s/s, elapsed=%.2f, size=%s",
                    cnt,
                    len(parts),
                    percent,
                    speedstr,
                    elapsed,
                    sizestr,
                )
        except Exception:
            logging.error("download error", exc_info=True)
            raise
Beispiel #2
0
 def getFilesAndDirectories(self, directory):
     files = []
     dirs = []
     for path in directory:
         url = urllib.quote(path.encode('utf8'), "/")
         escapedPath = cgi.escape(path)
         if os.path.isdir(os.path.join(self.path, path)):
             url = url + '/'
             dirs.append({
                 'text': escapedPath + "/",
                 'href': url,
                 'size': '',
                 'type': '[Directory]',
                 'encoding': ''
             })
         else:
             mimetype, encoding = static.getTypeAndEncoding(
                 path, self.contentTypes, self.contentEncodings,
                 self.defaultType)
             try:
                 size = os.stat(os.path.join(self.path, path)).st_size
             except OSError:
                 continue
             files.append({
                 'text': escapedPath,
                 "href": url,
                 'type': '[%s]' % mimetype,
                 'encoding': (encoding and '[%s]' % encoding or ''),
                 'size': static.formatFileSize(size)
             })
     return dirs, files
Beispiel #3
0
 def getFilesAndDirectories(self, directory):
     files = []
     dirs = []
     for path in directory:
         url = urllib.quote(path.encode('utf8'), "/")
         escapedPath = cgi.escape(path)
         if os.path.isdir(os.path.join(self.path, path)):
             url = url + '/'
             dirs.append({'text': escapedPath + "/", 'href': url,
                          'size': '', 'type': '[Directory]',
                          'encoding': ''})
         else:
             mimetype, encoding = static.getTypeAndEncoding(path, self.contentTypes,
                                                     self.contentEncodings,
                                                     self.defaultType)
             try:
                 size = os.stat(os.path.join(self.path, path)).st_size
             except OSError:
                 continue
             files.append({
                 'text': escapedPath, "href": url,
                 'type': '[%s]' % mimetype,
                 'encoding': (encoding and '[%s]' % encoding or ''),
                 'size': static.formatFileSize(size)})
     return dirs, files
Beispiel #4
0
    def download(self, key, dst):
        start = timeit.default_timer()

        if key.size < self.splitMB:
            key.get_contents_to_filename(dst)
        else:
            try:
                yield self.__download_range(key, dst)
            finally:
                for f in self.tempfiles:
                    try:
                        os.unlink(f)
                    except Exception:
                        pass

        elapsed = timeit.default_timer() - start
        speedstr = formatFileSize(key.size / elapsed)
        sizestr = formatFileSize(key.size)
        logging.info("Done=%s dst=%s speed=%s/s, elapsed=%.2f, size=%s", key.name, dst, speedstr, elapsed, sizestr)
Beispiel #5
0
    def __uploadOne(self, mp, src, dst, seek, cnt):
        size = os.path.getsize(src)
        sizestr = formatFileSize(size)

        logging.debug("upload start src=%s, dst=%s, size=%s, seek=%d, cnt=%d", src, dst, sizestr, seek, cnt)
        start = timeit.default_timer()

        fd = file(src, "rb")

        sd = cStringIO.StringIO()
        fd.seek(seek)
        content = fd.read(self.splitMB)
        size = len(content)
        sd.write(content)
        sd.seek(0)

        mp.upload_part_from_file(sd, cnt + 1, policy=self.grant)

        elapsed = timeit.default_timer() - start
        speedstr = formatFileSize(size / elapsed)
        logging.info(
            "upload end src=%s, dst=%s, size=%s, speed=%s/s, seek=%d, cnt=%d", src, dst, sizestr, speedstr, seek, cnt
        )
Beispiel #6
0
    def _getFilesAndDirectories(self, directory):
        """
        Helper returning files and directories in given directory listing, with
        attributes to be used to build a table content with
        C{self.linePattern}.

        @return: tuple of (directories, files)
        @rtype: C{tuple} of C{list}
        """
        files = []
        dirs = []

        for path in directory:
            if _PY3:
                if isinstance(path, bytes):
                    path = path.decode("utf8")

            url = quote(path, "/")
            escapedPath = escape(path)
            childPath = filepath.FilePath(self.path).child(path)

            if childPath.isdir():
                dirs.append({
                    'text': escapedPath + "/",
                    'href': url + "/",
                    'size_int': -1,
                    'size': '',
                    'type': '[Directory]',
                    'encoding': '',
                    'ctime': ''
                })
            else:
                mimetype, encoding = getTypeAndEncoding(
                    path, self.contentTypes, self.contentEncodings,
                    self.defaultType)
                try:
                    size = childPath.getsize()
                except OSError:
                    continue
                files.append({
                    'text': escapedPath,
                    "href": url,
                    'type': '[%s]' % mimetype,
                    'encoding': (encoding and '[%s]' % encoding or ''),
                    'size_int': size,
                    'size': formatFileSize(size),
                    'ctime': self.getCreateTime(childPath.path)
                })
        return dirs, files
Beispiel #7
0
 def test_formatFileSize(self):
     """
     L{static.formatFileSize} format an amount of bytes into a more readable
     format.
     """
     self.assertEquals(static.formatFileSize(0), "0B")
     self.assertEquals(static.formatFileSize(123), "123B")
     self.assertEquals(static.formatFileSize(4567), "4K")
     self.assertEquals(static.formatFileSize(8900000), "8M")
     self.assertEquals(static.formatFileSize(1234000000), "1G")
     self.assertEquals(static.formatFileSize(1234567890000), "1149G")
Beispiel #8
0
 def test_formatFileSize(self):
     """
     L{static.formatFileSize} format an amount of bytes into a more readable
     format.
     """
     self.assertEquals(static.formatFileSize(0), "0B")
     self.assertEquals(static.formatFileSize(123), "123B")
     self.assertEquals(static.formatFileSize(4567), "4K")
     self.assertEquals(static.formatFileSize(8900000), "8M")
     self.assertEquals(static.formatFileSize(1234000000), "1G")
     self.assertEquals(static.formatFileSize(1234567890000), "1149G")
Beispiel #9
0
def targzip_package(tarpath, basedir, src_paths, src_excludes, checksum=False):
    from twisted.web.static import formatFileSize

    start = timeit.default_timer()

    files = []
    for src_path in src_paths:
        try:
            src_path.index("/")
        except ValueError:
            files.append(src_path)
            continue

        exp = re.compile(src_path)

        # look for the deeper possible sub_path to search (e.g. faster)
        exp_subpath = re.compile(r'^([\w\/]+)')
        sub_path = exp_subpath.match(src_path).group(1)
        sub_path = sub_path[:sub_path.rfind('/')]
        logging.info("searching %s for '%s'", sub_path, src_path)

        for dirpath, dirnames, filenames in os.walk(
            os.path.join(basedir, sub_path)):
            for filename in filenames:
                path = os.path.join(dirpath, filename)
                path = path[len(basedir) + 1:]
                path = path.replace('\\', "/")
                if exp.match(path):
                    files.append(path)
            for dirname in dirnames:
                path = os.path.join(dirpath, dirname)
                if not os.path.islink(path):
                    continue
                path = path[len(basedir) + 1:]
                path = path.replace('\\', "/")
                if exp.match(path):
                    files.append(path)

    newfiles = []
    src_excludes_exp = map(lambda x: re.compile(x), src_excludes)
    for f in files:
        add = True
        for cnt, e in enumerate(src_excludes_exp):
            if e.match(f):
                add = False
                logging.debug("excluding %s (based on %s)", f,
                              src_excludes[cnt])
                break
        if add:
            newfiles.append(f)

    files = newfiles
    files.sort()

    if not files:
        logging.warning("skipping targzip_package, no files")
        return None

    logging.info("targzip_package: search elapsed=%.2f",
                 timeit.default_timer() - start)

    # checksum
    image_size = 0
    if checksum:
        start = timeit.default_timer()
        sha_image = sha()

        for f in files:
            logging.debug("checksum %s", f)
            path = os.path.join(basedir, f)
            if os.path.islink(path):
                sha_image.update(os.readlink(path))  # @UndefinedVariable
                continue
            fd = open(path, "rb")
            while 1:
                buf = fd.read(IMAGE_IO_CHUNK)
                if not buf:
                    break
                sha_image.update(buf)

            image_size += os.path.getsize(path)

        image_digest = hexlify(sha_image.digest())
        logging.info("targzip_package: sha=%s, size=%d, elapsed=%.2f",
                     image_digest, image_size, timeit.default_timer() - start)
    else:
        image_digest = None
        for f in files:
            path = os.path.join(basedir, f)
            if not os.path.islink(path):
                image_size += os.path.getsize(path)

    # tar/gzip
    start = timeit.default_timer()
    tar_file = targzip_files(tarpath, files, basedir)
    tar_size = os.path.getsize(tar_file)

    logging.info(
        "targzip_package: size=%s, compressed=%s, ratio=%.2f%%, elapsed=%.2f",
        formatFileSize(image_size), formatFileSize(tar_size),
        (float(tar_size) / image_size) * 100,
        timeit.default_timer() - start)

    return tar_file, image_digest
Beispiel #10
0
    def _getFilesAndDirectories(self, directory):
        """
        Helper returning files and directories in given directory listing, with
        attributes to be used to build a table content with
        C{self.linePattern}.

        @return: tuple of (directories, files)
        @rtype: C{tuple} of C{list}
        """

        files = []
        dirs = []
        dirs.append({'text':"<i>Parent Directory</i>", 'href': "..",
                    'size': '', 'type': '',
#                    'encoding': '',
                    'mtime': "",
                    'commstr': ""
        })
        for path in directory:
            if path[0] == ".":      # ignore filenames that begin with "."
                continue
            mtime = time.asctime(time.localtime(os.path.getmtime(os.path.join(self.path, path))))

            url = urllib.quote(path, "/")
            escapedPath = cgi.escape(path)
            # print "path %s url %s escapedPath %s" %(path, url, escapedPath)
            if os.path.isdir(os.path.join(self.path, path)):
                url = url + '/'
                dirs.append({'text': escapedPath + "/", 'href': url,
                             'size': '', 'type': '[Directory]',
#                             'encoding': '',
                             'mtime': mtime,
                             'commstr': ''
                })
            else:
                mimetype, encoding = getTypeAndEncoding(path, self.contentTypes,
                    self.contentEncodings,
                    self.defaultType)
                try:
                    size = os.stat(os.path.join(self.path, path)).st_size
                except OSError:
                    continue

                extension = os.path.splitext(path)[1]
                if extension == ".dbm":					# ignore .dbm files
                    continue
                if extension == ".db":                  # if it's ".db"
                    str = path[:-3]
                    extension = os.path.splitext(str)[1]
                    if extension == ".dbm":             # and if there's also a ".dbm"
                        continue                        # just ignore it - MacOSX (esp. Mountain Lion) likes to
                                                        # add .dbm.db (see comment in snmpsimd.py) - so don't show it


                theCommStr = ""
                if (extension == ".snmprec") or (extension == ".sapwalk") or (extension == ".snmpwalk"):
                    bdLen = len(myBaseDir) + len("/data/")
                    theCommStr = self.path[bdLen:]      # get the path up to .../data
                    if theCommStr != "":                # if it's non-empty
                        theCommStr += "/"               # append the "/"
                    theCommStr += path[:-len(extension)]   # append the file name, less the extension

                ## ** Comment **
                ##   Add attributes the the "elements" displayed in each line

                files.append({
                    'text': escapedPath, "href": url,
                    'type': '[%s]' % mimetype,
#                    'encoding': (encoding and '[%s]' % encoding or ''),
                    'size': formatFileSize(size),
                    'mtime': mtime,
                    'commstr': theCommStr,
                    }
                )
        return dirs, files