def main(argv):
parser = argparse.ArgumentParser()
parser.add_argument('-m', required=True, dest='manifest', help='list of files to map')
args = parser.parse_args()
manifest = open(args.manifest)
now = datetime.datetime.now()
if not os.path.exists("logs"): os.mkdir("logs")
errorlog = open("logs/bowtie_log" + "_" + str(now.year) + str(now.month) +
str(now.day), 'wa')
for line in manifest:
if not re.search("#", line):
line = line.split()
date = line[0] ## date of sequencing
lane = line[1] ## lane of flow cell
single_end = line[2] ## single or paired end
indices_names = line[3:] ## list of indices to process
indices = []
for index_name in indices_names:
index_name = index_name.split("-")
indices.append((index_name[0], index_name[1]))
for index in indices:
print line
if single_end == "PE": single_end = ""
try:
bowtie.bowtie(date, lane, bool(single_end), index)
except:
errorlog.write(str(sys.exc_info()[0]) + " ".join([str(index[0]), index[1]]) + "\n")
continue
errorlog.close()
def main(argv):
parser = argparse.ArgumentParser()
parser.add_argument('-m', required=True, dest='manifest', help='list of files to map')
args = parser.parse_args()
manifest = open(args.manifest)
for line in manifest:
line = line.split()
date = line[0]
sample = line[1]
index = line[2]
line_args = ['-d', date, '-s', sample, '-i', index]
bowtie.bowtie(date, sample, index)
def preprocess(self, preargs):
self.base = preargs[0]
self.fwd = preargs[1]
self.rev = preargs[2]
self.datadir = preargs[3]
#Run quality control ananlysis and quality based trimming
qcdir = os.path.abspath(self.datadir) + '/qc'
if not os.path.exists(qcdir):
os.mkdir(qcdir)
self.fwd, self.rev = trimgalore.run_qc(self.fwd, self.rev, qcdir,
self.base)
#Run alignment
self.bamfile = bowtie.bowtie(self.fwd, self.rev, self.datadir,
self.base)
#Run picard tools
self.bamfile = picard.addreadgroup(self.bamfile, self.base)
self.bamfile, self.metrics = picard.markdup(self.bamfile)
#Run GATK realignment and quality recalibration
self.interval = gatk.target(self.bamfile)
self.bamfile = gatk.realigner(self.bamfile, self.interval)
self.table = gatk.baserecal(self.bamfile)
self.bamfile = gatk.printreads(self.bamfile, self.table)
self.gvcf = gatk.haplocaller(self.bamfile)
return(self.gvcf)
def align(self, alargs):
# Run alignment
self.base = alargs[0]
self.fwd = alargs[1]
self.rev = alargs[2]
self.datadir = alargs[3]
self.bamfile = bowtie.bowtie(self.fwd, self.rev, self.datadir, self.base)
# Run picard tools
self.bamfile = picard.addreadgroup(self.bamfile, self.base)
self.bamfile, self.metrics = picard.markdup(self.bamfile)
# Run GATK realignment and quality recalibration
self.interval = gatk.target(self.bamfile)
self.bamfile = gatk.realigner(self.bamfile, self.interval)
self.table = gatk.baserecal(self.bamfile)
self.bamfile = gatk.printreads(self.bamfile, self.table)
self.vcf = gatk.haplocaller(self.bamfile)
print(self.vcf)
def align(self, alargs):
#Run alignment
self.base = alargs[0]
self.fwd = alargs[1]
self.rev = alargs[2]
self.datadir = alargs[3]
self.bamfile = bowtie.bowtie(self.fwd, self.rev, self.datadir,
self.base)
#Run picard tools
self.bamfile = picard.addreadgroup(self.bamfile, self.base)
#self.bamfile, self.metrics = picard.markdup(self.bamfile)
#Run GATK realignment and quality recalibration
#self.interval = gatk.target(self.bamfile)
#self.bamfile = gatk.realigner(self.bamfile, self.interval)
#self.table = gatk.baserecal(self.bamfile)
#self.bamfile = gatk.printreads(self.bamfile, self.table)
#self.vcf = gatk.haplocaller(self.bamfile)
print(self.bamfile)
def modisread(folder, resolution='1km', bandlist=[0], calc_overlap='True'):
"""reads modis Level1B image(s), geolocations and removes the bow-tie effect.
folder: location of the image files
resolution: '1km', '500m' or '250m'
bandlist: list of the bands to read,
where the bands are the i_th band of the given resolution starting from 0
calc_overlap: set this to 'True', if the bowtie-overlap pattern
shall be calculated for the specific image
returns lon,lat,image_list
lon: array of Longitude coordinates
lat: array of Latitude coordinates
image_list: List of 2d Arrays which are the images corresponding to bandlist
written by Johannes Roehrs and Lars Kaleschke, IfM, University of Hamburg
"""
print ' '
print '____________Read Modis Data_____________'
print ' '
#os.system('rm bowtie.so') # Comment out for rebuilding Fortran module
if not (os.path.exists('bowtie.so')):
os.system('f2py --fcompiler=gnu95 -m bowtie -c ./bowtie.f95 ')
import bowtie #reload(bowtie)
d = {}
cs_dict = {'1km': 1000, '500m': 500, '250m': 250}
cs = cs_dict[resolution]
dataset = {
'1km': 'EV_1KM_RefSB',
'500m': 'EV_500_RefSB',
'250m': 'EV_250_RefSB'
}
#Find Modis Files and recognize available Data
granules = modis_granules(folder)
#Chose one File
k = granules.keys()[0]
#
# Read SolarZenith, Latitude and Longitude
#
filename_1km = granules[k]['1km']
print 'read ' + folder + filename_1km
f_1km = SD(folder + filename_1km)
print ' '
for key in [
'SensorAzimuth', 'SensorZenith', 'SolarAzimuth', 'SolarZenith',
'Latitude', 'Longitude'
]:
d[key] = (sp.array(f_1km.select(key).get(), sp.float32))
if key == 'SolarZenith':
#d[key]=d[key]/100.0
d[key] = d[key] / 100.0 / 360 * 2 * sp.pi
if key == 'SolarAzimuth':
d[key] = d[key] / 100.0 #scalefactor 0.01
if key == 'SensorZenith':
d[key] = d[key] / 100.0 #scalefactor 0.01
if key == 'SensorAzimuth':
d[key] = d[key] / 100.0 #scalefactor 0.01
print 'interpolating ' + key + ' onto a grid...'
# remove bowtie effect:
# this is quick and dirty but works quite well in the test region
# Averaging should not be done with lon/lat values!
#avfilter=sp.array([[0,1,0],[0,2,0],[0,1,0]])/4.
#d[key]=nd.filters.convolve(d[key],avfilter) #weighted average between neighbored location
# resample to image resolution:
d[key] = nd.interpolation.zoom(d[key],
5000 / cs,
order=1,
mode='constant')[:, :-1000 / cs]
print ' done.'
#Read Chanel
filename = granules[k][resolution]
print 'load ' + folder + filename + '... '
f = SD(folder + filename)
print ' done'
d['img'] = []
for band in bandlist:
IMG = read_band(f, dataset[resolution], band, d) #d.kd.
# get the bowtie overlap pattern
if calc_overlap == 'True':
p = sp.array(bowtie_polynom(IMG, cs, folder),
dtype=float,
order='Fortran')
else:
p = sp.array(ovp[resolution], dtype=float, order='Fortran')
# remove bowtie-effect from image
modis_img = sp.array(IMG.transpose(), dtype=float, order='Fortran')
bowtie.bowtie(modis_img, p, int(10000 / cs), modis_img.shape[0],
modis_img.shape[1])
IMG = sp.array(modis_img.transpose(), order='C')
d['img'].append(
IMG[:, 1000 / cs:]
) #remove the left-hand-side edge pixel, so that geolocations fit
#Make the image fit to the geolocations:
d['Latitude'] = d['Latitude'][:, :-1000 /
cs] #remove the right-hand-side edge pixels
d['Longitude'] = d['Longitude'][:, :-1000 / cs]
d['SolarZenith'] = d['SolarZenith'][:, :-1000 / cs]
d['SolarAzimuth'] = d['SolarAzimuth'][:, :-1000 / cs]
d['SensorAzimuth'] = d['SensorAzimuth'][:, :-1000 / cs]
d['SensorZenith'] = d['SensorZenith'][:, :-1000 / cs]
print 'image shape: ' + str(d['img'][0].shape)
print ' '
return d['Longitude'], d['Latitude'], d['img'], d['SolarZenith'], d[
'SolarAzimuth'], d['SensorAzimuth'], d['SensorZenith']