def cutFile(file, size, isDelFile=True, outputFolder=None):
'''根据尺寸切文件'''
result = []
num = 1
fileName = os.path.basename(file)
fileSize = getsize(file)
if outputFolder == None:
outputFolder = os.path.dirname(file)
if fileSize <= size:
targetFile = os.sep.join((outputFolder, fileName))
if targetFile != file:
copyFile(file, targetFile)
result.append(targetFile)
trace('无需生成碎片文件 ' + targetFile)
else:
with open(file, 'rb') as f:
while True:
content = f.read(size)
if len(content) == 0:
break
targetFile = os.sep.join((outputFolder, fileName+'.'+'%03d'%num))
trace('生成碎片文件 ' + targetFile)
with open(targetFile, 'wb') as subFile:
subFile.write(content)
result.append(targetFile)
num += 1
if isDelFile:
delFile(file)
return tuple(sorted(result))
def _exec_job(job, conn):
path = job['path']
content_type = job['content_type']
try:
exec_headers = {}
# Merge the command line header options to the exec_headers
exec_headers.update(split_headers(options.header, '',
thread_manager))
container = None
obj = None
if options.obj_path:
container, obj = options.obj_path.split('/', 1)
resp = {}
body, headers = conn.exec_account(
container, obj, open(path, 'rb'), content_type=content_type,
content_length=getsize(path), headers=exec_headers,
response_dict=resp)
thread_manager.print_headers(headers,
meta_prefix='x-object-meta-',
exclude_headers=['content-encoding',
'vary', 'etag'],
offset=18)
thread_manager.print_msg(body)
except OSError as err:
if err.errno != ENOENT:
raise
thread_manager.error('Local file %r not found', path)
def test_getsize(self):
f = open(support.TESTFN, "wb")
try:
f.write(b"foo")
f.close()
self.assertEqual(genericpath.getsize(support.TESTFN), 3)
finally:
if not f.closed:
f.close()
os.remove(support.TESTFN)
def test_getsize(self):
f = open(support.TESTFN, "wb")
try:
f.write(b"foo")
f.close()
self.assertEqual(genericpath.getsize(support.TESTFN), 3)
finally:
if not f.closed:
f.close()
os.remove(support.TESTFN)
def check_success():
lfs = coord.common.LFS()
# Decide run or not according to the buffer size
totalsz = lfs.get_unfinished_dir_size(ADS_PATH, 'RankSelect')
if lfs.exists(REST_PATH):
totalsz += getsize(REST_PATH)
if totalsz > 1500000:
print 'block'
return False
return True
def hash_file(path):
"""
Take a hash of a file. Contains filesize to apparently prevent length extension attacks (idea taken from git).
File is read in 1MB blocks to work on low-memory devices.
"""
#todo: not here, but I should check that all filenames in the package are boring
sha2 = sha256()
size = pack('l', getsize(path))
sha2.update(size + b'notex')
with open(path, 'rb') as fh:
while True:
block = fh.read(1024)
if not block:
break
sha2.update(block)
return urlsafe_b64encode(sha2.digest()).decode('ascii')[:-1]
def hash_for_shooter(path):
with open(path, "rb") as file_obj:
file_length = getsize(path)
if file_length < 8192:
return ""
else:
block_size = 4096
offset_collection = [block_size, file_length / 3 * 2, file_length / 3, file_length - 8192]
hash_result = ""
for offset_el in offset_collection:
file_obj.seek(int(offset_el))
data_block = file_obj.read(block_size)
if len(hash_result) > 0:
hash_result += ";"
hash_result += hashlib.md5(data_block).hexdigest().lower()
return hash_result
def verify_file(fpath, description='', silent=False):
if not fpath:
if not silent:
log.error((description + ': f' if description else 'F') + 'ile name is empty.')
return False
fpath = expanduser(fpath)
if not exists(fpath):
if not silent:
log.error((description + ': ' if description else '') + fpath + ' does not exist.')
return False
if not isfile(fpath):
if not silent:
log.error((description + ': ' if description else '') + fpath + ' is not a file.')
return False
if getsize(fpath) <= 0:
if not silent:
log.error((description + ': ' if description else '') + fpath + ' is empty.')
return False
return True
def build_download_requests(self,
movie_dir,
release_name=None,
force=False):
reqs = []
for root_dir, child_folders, file_names in os.walk(movie_dir):
for file_name in file_names:
if release_name and not file_name.startswith(release_name):
# Not match with the release we need
continue
for ext in self.movie_extensions:
if file_name.endswith('.%s' % ext):
if self.min_movie_size and getsize(
join(root_dir,
file_name)) < self.min_movie_size:
# Ignore small movie file
continue
save_dir = root_dir
m = self.movie_file_pattern.search(file_name)
if not m:
continue
movie_release_name = m.group(1)
# Detect if the sub exists
if not force:
missed_langs = self._find_missed_langs(
movie_release_name, file_names)
if missed_langs:
reqs.append((movie_release_name, save_dir,
missed_langs))
else:
reqs.append(
(movie_release_name, save_dir, self.languages))
return reqs
def get_bedgraph_coverage(cnf,
bam_fpath,
chr_len_fpath=None,
output_fpath=None,
bed_fpath=None,
exit_on_error=True):
chr_len_fpath = chr_len_fpath or get_chr_len_fpath(cnf)
dedup_bam = intermediate_fname(cnf, bam_fpath, source.dedup_bam)
if not verify_bam(dedup_bam, silent=True):
info('Deduplicating bam file ' + bam_fpath)
remove_dups(cnf, bam_fpath, dedup_bam)
else:
info(dedup_bam + ' exists')
index_bam(cnf, dedup_bam)
bam_bed_fpath = bam_to_bed(cnf, dedup_bam, to_gzip=False)
if getsize(bam_bed_fpath) <= 0:
info('No coverage for ' + bam_fpath + ', skipping.')
return None
sorted_bed_fpath = sort_bed_by_alphabet(cnf,
bam_bed_fpath,
chr_len_fpath=chr_len_fpath)
if bed_fpath:
in_bed_fpath = intersect_bed(cnf, sorted_bed_fpath, bed_fpath)
else:
in_bed_fpath = sorted_bed_fpath
if not verify_file(in_bed_fpath, silent=True):
info('No coverage in ' + in_bed_fpath)
return None
bedgraph_fpath = output_fpath or '%s.bedgraph' % splitext(bam_fpath)[0]
with file_transaction(cnf.work_dir, bedgraph_fpath) as tx_fpath:
bedtools = get_system_path(cnf, 'bedtools')
cmdl = '{bedtools} genomecov -bg -split -g {chr_len_fpath} -i {in_bed_fpath}'.format(
**locals())
call(cnf, cmdl, exit_on_error=exit_on_error, output_fpath=tx_fpath)
return bedgraph_fpath
def verify_file(fpath, description='', silent=False):
if not fpath:
if not silent:
log.error((description + ': f' if description else 'F') +
'ile name is empty.')
return False
fpath = expanduser(fpath)
if not exists(fpath):
if not silent:
log.error((description + ': ' if description else '') + fpath +
' does not exist.')
return False
if not isfile(fpath):
if not silent:
log.error((description + ': ' if description else '') + fpath +
' is not a file.')
return False
if getsize(fpath) <= 0:
if not silent:
log.error((description + ': ' if description else '') + fpath +
' is empty.')
return False
return True
def save_file(self, file):
if isinstance(file, UploadedFile):
base, ext = os.path.splitext(file.name)
filename = settings.MEDIA_ROOT + 'tmp/' + slugify(base[:40]) + ext
content_type = file.content_type
destination = open(filename, 'wb+')
print "Opened %s for writing as %s..." % (filename, content_type)
for chunk in file.chunks():
destination.write(chunk)
destination.close()
file = filename
else:
content_type = mimetypes.guess_type(file)[0]
self.md5 = md5_file(file)
try:
# Get class values from existing file with same md5 (if it exists)
f = LibraryFile.objects.get(md5=self.md5)
self.__dict__ = f.__dict__
except LibraryFile.DoesNotExist:
self.name = basename(file)
self.filesize = getsize(file)
if self.filesize > LIBRARYFILE_MAX_FILESIZE:
raise Exception, 'File "%s" (%d) is greater than maximum file size of %d' % (self.name, self.filesize, LIBRARYFILE_MAX_FILESIZE)
if content_type in self.IMAGE_EXTENSIONS:
self.save_image(file)
elif content_type in self.VIDEO_EXTENSIONS:
self.save_video(file)
elif content_type in self.AUDIO_EXTENSIONS:
self.save_audio(file)
elif content_type in self.FLASH_EXTENSIONS:
self.save_flash(file)
else:
raise Exception, 'Not a valid LibraryFile MIME type "%s" for file "%s"' % (content_type, self.name)
super(LibraryFile, self).save()
def getFileSize(self):
return getsize(self.fileName)
def time_save(self, arr, pth):
t0 = time()
self.save(arr, pth)
self.save_time = time() - t0
self.storage_space = getsize(pth)
def update_event(self, inp=-1):
self.set_output_val(0, genericpath.getsize(self.input(0)))
def getFileSizeH(self):
return self.sizeofHuman(getsize(self.fileName))
def filesize(path):
""" returns the file size from Os """
if not genericpath.isfile(path):
return -1
return genericpath.getsize(path)
def getFileSize(self):
return getsize(self.fileName)
def getFileSizeH(self):
return self.sizeofHuman(getsize(self.fileName))
