def open_possibly_compressed_file(filename):
""" Read a file. It might be compressed. """
#if isinstance(filename, file):
# return filename #Whatever
#peek, f = peek_and_pipe(open_possibly_remote_file(filename), 4)
from nesoni import sam
if sam.is_bam(filename):
return sam.open_bam(filename)
f = open(filename, 'rb')
peek = f.read(4)
f.close()
#f = open(filename,'rb')
if peek.startswith('\x1f\x8b'):
#command = 'gunzip'
return gzip.open(filename, 'rb')
elif peek.startswith('BZh'):
#command = 'bunzip2'
return bz2.BZFile(filename, 'rb')
else:
#command = None
return open(filename, 'rb')
def archive_shares(self, dbi):
log.debug("DBInterface.archive_shares called")
found_time = dbi.archive_check()
if found_time == 0:
return False
log.info("Archiving shares newer than timestamp %f " % found_time)
dbi.archive_found(found_time)
if settings.ARCHIVE_MODE == 'db':
dbi.archive_to_db(found_time)
dbi.archive_cleanup(found_time)
elif settings.ARCHIVE_MODE == 'file':
shares = dbi.archive_get_shares(found_time)
filename = settings.ARCHIVE_FILE
if settings.ARCHIVE_FILE_APPEND_TIME:
filename = filename + "-" + datetime.fromtimestamp(
found_time).strftime("%Y-%m-%d-%H-%M-%S")
filename = filename + ".csv"
if settings.ARCHIVE_FILE_COMPRESS == 'gzip':
import gzip
filename = filename + ".gz"
filehandle = gzip.open(filename, 'a')
elif settings.ARCHIVE_FILE_COMPRESS == 'bzip2' and settings.ARCHIVE_FILE_APPEND_TIME:
import bz2
filename = filename + ".bz2"
filehandle = bz2.BZFile(filename, mode='wb', buffering=4096)
else:
filehandle = open(filename, "a")
while True:
row = shares.fetchone()
if row == None:
break
str1 = '","'.join([str(x) for x in row])
filehandle.write('"%s"\n' % str1)
filehandle.close()
clean = False
while not clean:
try:
dbi.archive_cleanup(found_time)
clean = True
except Exception as e:
clean = False
log.error(
"Archive Cleanup Failed... will retry to cleanup in 30 seconds"
)
sleep(30)
return True
def load_file(filename):
""" LOADING FILES """
if filename in ['-', 'stdin']:
filehandle = sys.stdin
elif filename.split('.')[-1] == 'gz':
filehandle = gzip.open(filename)
elif filename.split('.')[-1] == 'bz2':
filehandle = bz2.BZFile(filename)
elif filename.split('.')[-1] == 'zip':
filehandle = zipfile.Zipfile(filename)
else:
filehandle = open(filename)
return filehandle