def split(fName, num=2, dest=None):
"""
Splits a file into parts
"""
if not osw.fileExists(fName):
raise OSError('file \'%s\' does not exists' %(fName))
base = osw.basename(fName)
if not dest:
dest = osw.dirname(fName)
dirname = base + '.fsplit'
location = osw.cdir(dirname, dest)
size = osw.getsize(fName)
chunk_size = size / num
inputf = open(fName)
for cou in range(num):
filename = str(cou + 1) + '.split'
path = osw.getpath(location, filename)
with open(path, 'w') as outputf:
osw.fwrite(inputf, outputf, chunk_size)
filename = str(num) + '.split'
path = osw.getpath(location, filename)
with open(path, 'a') as outputf:
left = size - (chunk_size * num)
osw.fwrite(inputf, outputf, left)
inputf.close()
def split(fName, lines=2000, dest=None):
"""
Splits a file into parts
"""
if not osw.fileExists(fName):
raise OSError('file \'%s\' does not exists' %(fName))
base = osw.basename(fName)
if not dest:
dest = osw.dirname(fName)
#dirname = base + '.t2k'
dirname = osw.pDir()+'/t2k_tmp'
path = osw.getpath(dest, dirname)
if(os.path.exists(path)):
shutil.rmtree(path)
location = osw.cdir(dirname, dest)
#to avoid chinese wrong code
# we need to change the read to read lines!
"""
# OLD CODE
size = osw.getsize(fName)
chunk_size = size / num
inputf = open(fName)
for cou in range(num):
filename = str(cou + 1) + '.split'
path = osw.getpath(location, filename)
with open(path, 'w') as outputf:
osw.fwrite(inputf, outputf, chunk_size)
filename = str(num) + '.split'
path = osw.getpath(location, filename)
with open(path, 'a') as outputf:
left = size - (chunk_size * num)
osw.fwrite(inputf, outputf, left)
"""
# NEW CODE
size = osw.getlines(fName)
print "... Book has totally %d lines."%(size)
chunk_size = lines
num = size/lines
if(size%lines!=0):
num = num + 1
print "... And will be cut into %d pieces in processing."%(num)
inputf = open(fName)
for cou in range(num):
filename = str(cou + 1) + '.split'
path = osw.getpath(location, filename)
with open(path, 'w') as outputf:
osw.flwrite(inputf, outputf, chunk_size)
filename = str(num) + '.split'
path = osw.getpath(location, filename)
with open(path, 'a') as outputf:
left = size - (chunk_size * num)
osw.flwrite(inputf, outputf, left)
inputf.close()
return num
def split(fName, lines=2000, dest=None):
"""
Splits a file into parts
"""
if not osw.fileExists(fName):
raise OSError('file \'%s\' does not exists' % (fName))
base = osw.basename(fName)
if not dest:
dest = osw.dirname(fName)
#dirname = base + '.t2k'
dirname = osw.pDir() + '/t2k_tmp'
path = osw.getpath(dest, dirname)
if (os.path.exists(path)):
shutil.rmtree(path)
location = osw.cdir(dirname, dest)
#to avoid chinese wrong code
# we need to change the read to read lines!
"""
# OLD CODE
size = osw.getsize(fName)
chunk_size = size / num
inputf = open(fName)
for cou in range(num):
filename = str(cou + 1) + '.split'
path = osw.getpath(location, filename)
with open(path, 'w') as outputf:
osw.fwrite(inputf, outputf, chunk_size)
filename = str(num) + '.split'
path = osw.getpath(location, filename)
with open(path, 'a') as outputf:
left = size - (chunk_size * num)
osw.fwrite(inputf, outputf, left)
"""
# NEW CODE
size = osw.getlines(fName)
print "... Book has totally %d lines." % (size)
chunk_size = lines
num = size / lines
if (size % lines != 0):
num = num + 1
print "... And will be cut into %d pieces in processing." % (num)
inputf = open(fName)
for cou in range(num):
filename = str(cou + 1) + '.split'
path = osw.getpath(location, filename)
with open(path, 'w') as outputf:
osw.flwrite(inputf, outputf, chunk_size)
filename = str(num) + '.split'
path = osw.getpath(location, filename)
with open(path, 'a') as outputf:
left = size - (chunk_size * num)
osw.flwrite(inputf, outputf, left)
inputf.close()
return num
def createTar(fName, dest=None):
"""
Converts a file into tar
"""
basename, dirname = osw.basename(fName), osw.dirname(fName)
if not osw.fileExists(basename, dirname):
raise OSError('file \'%s\' does not exist' %(fName))
if not dest:
dest = osw.getpath(dirname, '')
dest = osw.getpath(dest, basename + '.tar.gz')
with tarfile.open(dest, 'w:gz') as tar:
tar.add(fName, arcname=basename)
def join(dire=osw.pDir(), dest=''):
"""
Joins all the .split files together systematically
"""
if not osw.dirExists(dire):
raise OSError('directory \'%s\' does not exist' %(dire))
filename = osw.getFname(dire)
if not filename:
raise exce.InvalidDirError("Not a valid fsplit directory \'%s\'" %(dire))
if not dest:
dest = osw.getpath(dire, '../')
dest = osw.getpath(dest, filename)
outputf = open(dest, 'w')
dire = osw.getpath(dire)
for part in osw.olistdir(dire):
partpath = osw.getpath(dire, part)
with open(partpath) as inputf:
osw.fwrite(inputf, outputf)
outputf.close()
def createTarAll(dire=osw.pDir(), dest=None):
"""
Creates tar of all the files in a directory
removes the files
"""
if not osw.dirExists(dire):
raise OSError('directory \'%s\' does not exist' %(dire))
for file in os.listdir(dire):
path = osw.getpath(dire, file)
createTar(path, dest)
os.remove(path)
def extractTar(tarname, dest=None):
"""
Extracts a tarfile to the destination
"""
basename, dirname = osw.basename(tarname), osw.dirname(tarname)
if not osw.fileExists(basename, dirname):
raise OSError('file \'%s\' does not exist' %(tarname))
if not isTar(tarname):
raise exce.NotTarError("Not a valid tarfile \'%s\'" %(tarname))
if not dest:
dest = osw.getpath(dirname, '')
with tarfile.open(tarname, 'r|gz') as tar:
tar.extractall(dest)
def extractTarAll(dire=osw.pDir(), dest=None):
"""
Extracts all tarfiles to the destination
removes the tarfiles
"""
status = False
if not osw.dirExists(dire):
raise OSError('directory \'%s\' does not exist' %(dire))
for file in os.listdir(dire):
path = osw.getpath(dire, file)
if isTar(path):
status = True
extractTar(path, dest)
os.remove(path)
return status
sys.exit()
dest = args.dest
if dest:
if not os.path.exists(dest):
print 'directory \'%s\' does not exist.' %(dest)
sys.exit()
elif not os.path.isdir(dest):
print 'ouptut destination \'%s\' should be a directory' %(dest)
sys.exit()
verbose = args.verbose
if target.endswith('/'):
target = target[:-1]
base = osw.basename(target)
path = osw.getpath(dest, base[:-7])
try:
if verbose:
print 'Joining \'%s\' into \'%s\'' %(target, path)
print 'Looking for tarfiles'
status = tar.extractTarAll(target)
if verbose:
if status:
print 'Tarfiles extracted'
else:
print 'No tarfile found'
print 'Starting join'
main.join(target, dest)
if verbose:
print 'Joined into \'%s\'' %(path)
print 'Joining Complete'
sys.exit()
dest = args.dest
if dest:
if not os.path.exists(dest):
print 'directory \'%s\' does not exist.' % (dest)
sys.exit()
elif not os.path.isdir(dest):
print 'ouptut destination \'%s\' should be a directory' % (dest)
sys.exit()
verbose = args.verbose
if target.endswith('/'):
target = target[:-1]
base = osw.basename(target)
path = osw.getpath(dest, base[:-7])
try:
if verbose:
print 'Joining \'%s\' into \'%s\'' % (target, path)
print 'Looking for tarfiles'
status = tar.extractTarAll(target)
if verbose:
if status:
print 'Tarfiles extracted'
else:
print 'No tarfile found'
print 'Starting join'
main.join(target, dest)
if verbose:
print 'Joined into \'%s\'' % (path)
print 'Joining Complete'
print 'target \'%s\' should be a file' % (target)
sys.exit()
dest = args.dest
if not os.path.exists(dest):
print 'directory \'%s\' does not exist.' % (dest)
sys.exit()
elif not os.path.isdir(dest):
print 'ouptut destination \'%s\' should be a directory' % (dest)
sys.exit()
parts, istar = args.num, args.tar
verbose = args.verbose
base = osw.basename(target)
path = osw.getpath(dest, base + '.fsplit')
try:
if verbose:
print 'Splitting \'%s\' into \'%d\' parts' % (target, parts)
main.split(target, num=parts, dest=dest) # now split it
if istar:
if verbose:
print 'Creating Tarfiles'
tar.createTarAll(path)
if verbose:
print 'Tarfiles created'
if verbose:
print 'splits saved in \'%s\'' % (path)
print 'Splitting Complete'
except Exception, e:
if os.path.isdir(path):