コード例 #1
0
ファイル: fileUtils.py プロジェクト: benimang/easyjob
def cutFile(file, size, isDelFile=True, outputFolder=None):
    '''根据尺寸切文件'''
    result = []
    num = 1
    fileName = os.path.basename(file)
    fileSize = getsize(file)
    if outputFolder == None:
        outputFolder = os.path.dirname(file)
    if fileSize <= size:
        targetFile = os.sep.join((outputFolder, fileName))
        if targetFile != file:
            copyFile(file, targetFile)
        result.append(targetFile)
        trace('无需生成碎片文件 ' + targetFile)
    else:
        with open(file, 'rb') as f:
            while True:
                content = f.read(size)
                if len(content) == 0:
                    break
                targetFile = os.sep.join((outputFolder, fileName+'.'+'%03d'%num))
                trace('生成碎片文件 ' + targetFile)
                with open(targetFile, 'wb') as subFile:
                    subFile.write(content)
                result.append(targetFile)
                num += 1
        if isDelFile:
            delFile(file)
    return tuple(sorted(result))
コード例 #2
0
ファイル: shell.py プロジェクト: pkit/python-zwiftclient
 def _exec_job(job, conn):
     path = job['path']
     content_type = job['content_type']
     try:
         exec_headers = {}
         # Merge the command line header options to the exec_headers
         exec_headers.update(split_headers(options.header, '',
                                           thread_manager))
         container = None
         obj = None
         if options.obj_path:
             container, obj = options.obj_path.split('/', 1)
         resp = {}
         body, headers = conn.exec_account(
             container, obj, open(path, 'rb'), content_type=content_type,
             content_length=getsize(path), headers=exec_headers,
             response_dict=resp)
         thread_manager.print_headers(headers,
                                      meta_prefix='x-object-meta-',
                                      exclude_headers=['content-encoding',
                                                       'vary', 'etag'],
                                      offset=18)
         thread_manager.print_msg(body)
     except OSError as err:
         if err.errno != ENOENT:
             raise
         thread_manager.error('Local file %r not found', path)
コード例 #3
0
 def test_getsize(self):
     f = open(support.TESTFN, "wb")
     try:
         f.write(b"foo")
         f.close()
         self.assertEqual(genericpath.getsize(support.TESTFN), 3)
     finally:
         if not f.closed:
             f.close()
         os.remove(support.TESTFN)
コード例 #4
0
 def test_getsize(self):
     f = open(support.TESTFN, "wb")
     try:
         f.write(b"foo")
         f.close()
         self.assertEqual(genericpath.getsize(support.TESTFN), 3)
     finally:
         if not f.closed:
             f.close()
         os.remove(support.TESTFN)
コード例 #5
0
ファイル: adgenerator.py プロジェクト: liyinhgqw/Coordinator
def check_success():
  lfs = coord.common.LFS()
  # Decide run or not according to the buffer size
  totalsz = lfs.get_unfinished_dir_size(ADS_PATH, 'RankSelect')
  if lfs.exists(REST_PATH):
    totalsz += getsize(REST_PATH)
  if totalsz > 1500000:
    print 'block'
    return False
  return True
コード例 #6
0
def hash_file(path):
    """
	Take a hash of a file. Contains filesize to apparently prevent length extension attacks (idea taken from git).
	File is read in 1MB blocks to work on low-memory devices.
	"""
    #todo: not here, but I should check that all filenames in the package are boring
    sha2 = sha256()
    size = pack('l', getsize(path))
    sha2.update(size + b'notex')
    with open(path, 'rb') as fh:
        while True:
            block = fh.read(1024)
            if not block:
                break
            sha2.update(block)
    return urlsafe_b64encode(sha2.digest()).decode('ascii')[:-1]
コード例 #7
0
def hash_for_shooter(path):
    with open(path, "rb") as file_obj:
        file_length = getsize(path)

        if file_length < 8192:
            return ""
        else:
            block_size = 4096
            offset_collection = [block_size, file_length / 3 * 2, file_length / 3, file_length - 8192]
            hash_result = ""
            for offset_el in offset_collection:
                file_obj.seek(int(offset_el))
                data_block = file_obj.read(block_size)
                if len(hash_result) > 0:
                    hash_result += ";"
                hash_result += hashlib.md5(data_block).hexdigest().lower()
            return hash_result
コード例 #8
0
ファイル: utils.py プロジェクト: vladsaveliev/AZ_Orthofinder
def verify_file(fpath, description='', silent=False):
    if not fpath:
        if not silent:
            log.error((description + ': f' if description else 'F') + 'ile name is empty.')
        return False
    fpath = expanduser(fpath)
    if not exists(fpath):
        if not silent:
            log.error((description + ': ' if description else '') + fpath + ' does not exist.')
        return False
    if not isfile(fpath):
        if not silent:
            log.error((description + ': ' if description else '') + fpath + ' is not a file.')
        return False
    if getsize(fpath) <= 0:
        if not silent:
            log.error((description + ': ' if description else '') + fpath + ' is empty.')
        return False
    return True
コード例 #9
0
    def build_download_requests(self,
                                movie_dir,
                                release_name=None,
                                force=False):
        reqs = []
        for root_dir, child_folders, file_names in os.walk(movie_dir):
            for file_name in file_names:
                if release_name and not file_name.startswith(release_name):
                    # Not match with the release we need
                    continue

                for ext in self.movie_extensions:
                    if file_name.endswith('.%s' % ext):
                        if self.min_movie_size and getsize(
                                join(root_dir,
                                     file_name)) < self.min_movie_size:
                            # Ignore small movie file
                            continue

                        save_dir = root_dir
                        m = self.movie_file_pattern.search(file_name)
                        if not m:
                            continue

                        movie_release_name = m.group(1)

                        # Detect if the sub exists
                        if not force:
                            missed_langs = self._find_missed_langs(
                                movie_release_name, file_names)

                            if missed_langs:
                                reqs.append((movie_release_name, save_dir,
                                             missed_langs))
                        else:
                            reqs.append(
                                (movie_release_name, save_dir, self.languages))

        return reqs
コード例 #10
0
def get_bedgraph_coverage(cnf,
                          bam_fpath,
                          chr_len_fpath=None,
                          output_fpath=None,
                          bed_fpath=None,
                          exit_on_error=True):
    chr_len_fpath = chr_len_fpath or get_chr_len_fpath(cnf)
    dedup_bam = intermediate_fname(cnf, bam_fpath, source.dedup_bam)
    if not verify_bam(dedup_bam, silent=True):
        info('Deduplicating bam file ' + bam_fpath)
        remove_dups(cnf, bam_fpath, dedup_bam)
    else:
        info(dedup_bam + ' exists')
    index_bam(cnf, dedup_bam)
    bam_bed_fpath = bam_to_bed(cnf, dedup_bam, to_gzip=False)
    if getsize(bam_bed_fpath) <= 0:
        info('No coverage for ' + bam_fpath + ', skipping.')
        return None

    sorted_bed_fpath = sort_bed_by_alphabet(cnf,
                                            bam_bed_fpath,
                                            chr_len_fpath=chr_len_fpath)
    if bed_fpath:
        in_bed_fpath = intersect_bed(cnf, sorted_bed_fpath, bed_fpath)
    else:
        in_bed_fpath = sorted_bed_fpath

    if not verify_file(in_bed_fpath, silent=True):
        info('No coverage in ' + in_bed_fpath)
        return None

    bedgraph_fpath = output_fpath or '%s.bedgraph' % splitext(bam_fpath)[0]
    with file_transaction(cnf.work_dir, bedgraph_fpath) as tx_fpath:
        bedtools = get_system_path(cnf, 'bedtools')
        cmdl = '{bedtools} genomecov -bg -split -g {chr_len_fpath} -i {in_bed_fpath}'.format(
            **locals())
        call(cnf, cmdl, exit_on_error=exit_on_error, output_fpath=tx_fpath)
    return bedgraph_fpath
コード例 #11
0
def verify_file(fpath, description='', silent=False):
    if not fpath:
        if not silent:
            log.error((description + ': f' if description else 'F') +
                      'ile name is empty.')
        return False
    fpath = expanduser(fpath)
    if not exists(fpath):
        if not silent:
            log.error((description + ': ' if description else '') + fpath +
                      ' does not exist.')
        return False
    if not isfile(fpath):
        if not silent:
            log.error((description + ': ' if description else '') + fpath +
                      ' is not a file.')
        return False
    if getsize(fpath) <= 0:
        if not silent:
            log.error((description + ': ' if description else '') + fpath +
                      ' is empty.')
        return False
    return True
コード例 #12
0
ファイル: models.py プロジェクト: Cruel/Anondex
    def save_file(self, file):
        if isinstance(file, UploadedFile):
            base, ext = os.path.splitext(file.name)
            filename = settings.MEDIA_ROOT + 'tmp/' + slugify(base[:40]) + ext
            content_type = file.content_type
            destination = open(filename, 'wb+')
            print "Opened %s for writing as %s..." % (filename, content_type)
            for chunk in file.chunks():
                destination.write(chunk)
            destination.close()
            file = filename
        else:
            content_type = mimetypes.guess_type(file)[0]

        self.md5 = md5_file(file)
        try:
            # Get class values from existing file with same md5 (if it exists)
            f = LibraryFile.objects.get(md5=self.md5)
            self.__dict__ = f.__dict__
        except LibraryFile.DoesNotExist:
            self.name = basename(file)
            self.filesize = getsize(file)
            if self.filesize > LIBRARYFILE_MAX_FILESIZE:
                raise Exception, 'File "%s" (%d) is greater than maximum file size of %d' % (self.name, self.filesize, LIBRARYFILE_MAX_FILESIZE)

            if content_type in self.IMAGE_EXTENSIONS:
                self.save_image(file)
            elif content_type in self.VIDEO_EXTENSIONS:
                self.save_video(file)
            elif content_type in self.AUDIO_EXTENSIONS:
                self.save_audio(file)
            elif content_type in self.FLASH_EXTENSIONS:
                self.save_flash(file)
            else:
                raise Exception, 'Not a valid LibraryFile MIME type "%s" for file "%s"' % (content_type, self.name)
            super(LibraryFile, self).save()
コード例 #13
0
 def getFileSize(self):
     return getsize(self.fileName)
コード例 #14
0
 def time_save(self, arr, pth):
     t0 = time()
     self.save(arr, pth)
     self.save_time = time() - t0
     self.storage_space = getsize(pth)
コード例 #15
0
 def update_event(self, inp=-1):
     self.set_output_val(0, genericpath.getsize(self.input(0)))
コード例 #16
0
 def getFileSizeH(self):
     return self.sizeofHuman(getsize(self.fileName))
コード例 #17
0
def filesize(path):
    """ returns the file size from Os """
    if not genericpath.isfile(path):
        return -1
    return genericpath.getsize(path)
コード例 #18
0
 def getFileSize(self):
     return getsize(self.fileName)
コード例 #19
0
 def getFileSizeH(self):
     return self.sizeofHuman(getsize(self.fileName))