def _build_resources(self):
for source, dest, required in self.resources:
dest = self._get_abs_path(dest)
logging.debug('Copying %s to %s' % (source, dest))
try:
tools.copy(source, dest, required)
except IOError, err:
msg = 'Error: The file "%s" could not be copied to "%s": %s'
raise SystemExit(msg % (source, dest, err))
def build_from_source():
try:
cmdstr ="mvn -DskipTests clean install"
shell.ShellCmd(cmdstr, workdir=self.zstack_java.source, pipe=False)()
warstr = "mvn war:war"
shell.ShellCmd(warstr, workdir=os.path.join(self.zstack_java.source, 'build'), pipe=False)()
war = os.path.join(self.zstack_java.source, "build/target/zstack.war")
tools.copy([(war, self.zstack_java.dist_war)])
self._info("zstack.war is created at %s" % self.zstack_java.dist_war)
except Exception as e:
raise BuildError(str(e))
def init():
global goal, states, init_state
init_state = agent.init_state
goal = agent.food_location
states = tools.copy(data.states)
init_north_south()
init_east_west()
init_map()
def build_from_source():
try:
cmdstr = "mvn -DskipTests clean install"
shell.ShellCmd(cmdstr,
workdir=self.zstack_java.source,
pipe=False)()
warstr = "mvn war:war"
shell.ShellCmd(warstr,
workdir=os.path.join(self.zstack_java.source,
'build'),
pipe=False)()
war = os.path.join(self.zstack_java.source,
"build/target/zstack.war")
tools.copy([(war, self.zstack_java.dist_war)])
self._info("zstack.war is created at %s" %
self.zstack_java.dist_war)
except Exception as e:
raise BuildError(str(e))
def remove_uneeded_actions(state_name, needed_actions):
global actions
uneeded_keys = []
uneeded_actions = tools.copy(actions)
for action in needed_actions:
uneeded_actions.remove(action)
for action in uneeded_actions:
uneeded_keys.append((state_name, action))
for key in uneeded_keys:
if key in q_table:
del q_table[key]
def update_prices():
global map_dict
for state in states:
map_list = list(map_dict.get(state))
working_map_list = tools.copy(map_list)
for entry in working_map_list:
new_price = data.q_table.get((str(state), str(entry[0])))
map_list.remove(entry)
entry = (entry[0], entry[1], new_price)
map_list.append(entry)
del map_dict[state]
map_dict[state] = map_list
def _build_resources(self):
for name, content in self.new_files:
filename = self._get_abs_path(name)
with open(filename, 'w') as file:
logging.debug('Writing file "%s"' % filename)
file.write(content)
if name == 'run':
# Make run script executable
os.chmod(filename, 0755)
for source, dest, required, symlink in self.resources:
if required and not os.path.exists(source):
logging.error('The required resource can not be found: %s' %
source)
sys.exit(1)
dest = self._get_abs_path(dest)
if symlink:
source = self._get_rel_path(source)
os.symlink(source, dest)
logging.debug('Linking from %s to %s' % (source, dest))
continue
logging.debug('Copying %s to %s' % (source, dest))
tools.copy(source, dest, required)
def main():
options, args = cmdparameter(sys.argv)
#-----------------------------------
labelL = options.filein.split(' ')
makefile_am = options.makefile_am
sampleFile = options.sampleFile
main_dir = options.dir
sampD, condD = readSample(sampleFile)
curation_label = os.path.split(sys.argv[0])[-1].replace('.', '_')
seqTypeD = {}
for line in open(makefile_am):
if line.startswith("prefix="):
prefix = line.strip().split('=')[1]
elif line.find('seq_type=') != -1:
sample, seq_type = \
line.strip().replace('_seq_type','').split('=')
seqTypeD[sample] = seq_type
newfileL = []
read_base_countL = []
new_labelL = []
len_labelL = len(labelL)
labelMapD = {}
for i in range(len_labelL):
label = labelL[i]
seq_type = seqTypeD[label]
if seq_type == "PE":
left = label + '_1'
right = label + '_2'
new_labelL.extend([left, right])
newfileL.extend([
left + '_fastqc/fastqc_data.txt',
right + '_fastqc/fastqc_data.txt'
])
read_base_countL.extend([
left + ".statistics_fastq_reads_bases",
right + ".statistics_fastq_reads_bases"
])
labelMapD[left] = label
labelMapD[right] = label
elif seq_type == "SE":
new_labelL.append(label)
newfileL.append(label + '_fastqc/fastqc_data.txt')
read_base_countL.extend([label + ".statistics_fastq_reads_bases"])
labelMapD[label] = label
fileL = newfileL
labelL = new_labelL
#print read_base_countL
baseCntD_T0_1, baseCntD_T0_1_1 = readBases(read_base_countL, labelMapD)
#print >>sys.stderr, baseCntD_T0_1
#print >>sys.stderr, baseCntD_T0_1_1
len_labelL = len(labelL)
#print fileL
#print labelL
verbose = options.verbose
global dbug
debug = options.debug
#-----------------------------------
print "## 测序质量总结 {#sub-sequence-summary}\n"
curation_label = "Count_sequenced_reads_bases"
knitr_read_txt(main_dir, curation_label)
tableL = []
headerL = [
"Sample", "Total reads", "Total bases", "Sequence length (nt)",
"GC content (%)", "Encoding"
]
tableL.append(headerL)
#Do not change keyL
keyL = [
"Total Sequences", "Total bases", "Sequence length", "%GC", "Encoding"
]
aDict = {}
readCntD = {}
baseCntD = {}
T0_L = []
T0_1_L = []
sampleRepD = {}
for i in range(len_labelL):
file = fileL[i]
label = labelL[i]
aDict[label] = {}
parseFile(file, aDict[label])
aDict[label]["Total bases"] = baseCntD_T0_1_1[label]
sampleL = [label]
sampleL.extend([aDict[label].get(key) for key in keyL])
tableL.append(sampleL)
T0_1_1 = label
T0_1 = labelMapD[T0_1_1]
#print >>sys.stderr, T0_1_1, T0_1
T0 = sampD[T0_1]
#print >>sys.stderr, T0_1_1, T0_1, T0
if T0 not in T0_L:
T0_L.append(T0)
if T0 not in readCntD:
readCntD[T0] = {}
baseCntD[T0] = {}
T0_1 = T0_1.replace(T0, "")
if not T0_1:
T0_1 = '1'
else:
T0_1 = T0_1[1:]
#sampleRepD[T0] = sampleRepD.get(T0, 0)+1
#T0_1 = str(sampleRepD[T0])
#print >>sys.stderr, T0_1_1, T0_1
if T0_1 not in T0_1_L:
T0_1_L.append(T0_1)
readCntD[T0][T0_1] = sampleL[1]
baseCntD[T0][T0_1] = str(baseCntD_T0_1[labelMapD[T0_1_1]])
#--------------------------------------
#print >>sys.stderr, readCntD
#print >>sys.stderr, baseCntD
#sys.exit(1)
#print tableL
print "Table: (\#tab:seq-sta-sum) Summary of sequencing reads 测序量总结 \
(对于双端测序, *\_1* 表示左端reads, *\_2* 表示右端reads) \n"
print '\n'.join(transferListToMultiLTable(tableL))
print
tardir = "1_sequencing_quality_check"
targetdir = main_dir + '/' + tardir + '/'
os.system("mkdir -p " + targetdir)
read_cnt_file = prefix + ".read_cnt.xls"
read_cnt_fh = open(read_cnt_file, 'w')
read_cnt_pdf = read_cnt_file + ".barplot.pdf"
print >> read_cnt_fh, "Sample\t%s" % '\t'.join(T0_1_L)
for T0 in T0_L:
print >> read_cnt_fh, "%s\t%s" % (T0, '\t'.join(
[readCntD[T0].get(i, '0') for i in T0_1_L]))
read_cnt_fh.close()
if len_labelL <= 60:
cmd = "s-plot barPlot -f %s -d dodge -y 'Number of sequenced reads' -R 45 -P none -w 30" % read_cnt_file
else:
height = len_labelL // 5
if height < 10:
height = 10
cmdL = [
"s-plot barPlot -f", read_cnt_file,
"-d dodge -y 'Number of sequenced reads' -P none -w 20",
"-F TRUE -u",
str(height)
]
cmd = ' '.join(cmdL)
os.system(cmd)
base_cnt_file = prefix + ".base_cnt.xls"
base_cnt_fh = open(base_cnt_file, 'w')
base_cnt_pdf = base_cnt_file + ".barplot.pdf"
print >> base_cnt_fh, "Sample\t%s" % '\t'.join(T0_1_L)
for T0 in T0_L:
print >> base_cnt_fh, "%s\t%s" % (T0, '\t'.join(
[baseCntD[T0].get(i, '0') for i in T0_1_L]))
base_cnt_fh.close()
if len_labelL <= 60:
cmd = "s-plot barPlot -f %s -d dodge -y 'Number of sequenced bases' -R 45 -P none -w 30" % base_cnt_file
else:
#height = len_labelL // 3
#if height < 10:
# height = 10
cmdL = [
"s-plot barPlot -f", base_cnt_file,
"-d dodge -y 'Number of sequenced bases' -P none -w 20",
"-F TRUE -u",
str(height)
]
cmd = ' '.join(cmdL)
os.system(cmd)
read_cnt_melt_file = prefix + ".read_cnt_melt.xls"
read_cnt_melt_fh = open(read_cnt_melt_file, 'w')
read_cnt_melt_pdf = read_cnt_melt_file + ".densityHist.pdf"
print >> read_cnt_melt_fh, "variable\tvalue"
for T0 in T0_L:
for i in T0_1_L:
cnt = int(readCntD[T0].get(i, '0'))
if cnt:
print >>read_cnt_melt_fh, \
"Number of sequenced reads (million)\t%s" % (\
str(cnt/(10**6)))
read_cnt_melt_fh.close()
cmdL = [
"s-plot densityHistPlot -f", read_cnt_melt_file,
"-d hist -g ..count..",
"-v TRUE -P none -R 45 -x 'Number of sequenced reads (million)'",
"-y 'Sample count' -a 1"
]
cmd = ' '.join(cmdL)
os.system(cmd)
base_cnt_melt_file = prefix + ".base_cnt_melt.xls"
base_cnt_melt_fh = open(base_cnt_melt_file, 'w')
base_cnt_melt_pdf = base_cnt_melt_file + ".densityHist.pdf"
print >> base_cnt_melt_fh, "variable\tvalue"
for T0 in T0_L:
for i in T0_1_L:
cnt = int(baseCntD[T0].get(i, '0'))
if cnt:
print >>base_cnt_melt_fh, \
"Number of sequenced bases (G)\t%s" % (\
str(cnt/(10**9)))
base_cnt_melt_fh.close()
cmdL = [
"s-plot densityHistPlot -f", base_cnt_melt_file,
"-d hist -g ..count..",
"-v TRUE -P none -R 45 -x 'Number of sequenced bases (G)'",
"-y 'Sample count' -a 1"
]
cmd = ' '.join(cmdL)
os.system(cmd)
copy(targetdir, read_cnt_file, base_cnt_file)
copypdf(targetdir, read_cnt_pdf, base_cnt_pdf, read_cnt_melt_pdf,
base_cnt_melt_pdf)
#print "## Visualize statistics of sequenced reads\n"
#knitr_read_txt(main_dir, curation_label)
print "所有样品测序reads数目和碱基数目分布，用于查看是否存在测序量异常的样品 (Figure \@ref(fig:read-num-all-sample) and \@ref(fig:base-num-all-sample))。\n"
print "\n(ref:read-num-all-sample) Distribution of sequenced reads in all samples. Vertical line represents average of sequenced reads of all samples. 1 Million = 10^6^. \
[PDF](%s/%s)\n" % (tardir, read_cnt_melt_pdf)
print '''
```{r read-num-all-sample, fig.cap="(ref:read-num-all-sample)"}
knitr::include_graphics("%s/%s")
```
''' % (tardir, read_cnt_melt_pdf.replace('pdf', 'png'))
print "(ref:base-num-all-sample) Distribution of sequenced bases in all samples. Vertical line represents average of sequenced bases of all samples. 1G = 10^9^. \
[PDF](%s/%s)\n" % (tardir, base_cnt_melt_pdf)
print '''```{r base-num-all-sample, fig.cap="(ref:base-num-all-sample)"}
knitr::include_graphics("%s/%s")
```
''' % (tardir, base_cnt_melt_pdf.replace('pdf', 'png'))
print "单个样品测序Reads数目和碱基分布，如果一个样品有多个重复，会以不同颜色的柱子展示 (Figure \@ref(fig:read-num-per-sample))。\n"
print "(ref:read-num-per-sample) Number of sequenced reads. Color bars represent \
each replicate. 每个样品各个重复测序获得的reads数目。1 Million = 10^6^.\
[PDF](%s/%s) [SOURCE](%s/%s)\n" % (tardir, read_cnt_pdf, tardir,
read_cnt_file)
print '''```{r read-num-per-sample, fig.cap="(ref:read-num-per-sample)"}
knitr::include_graphics("%s/%s")
```
''' % (tardir, read_cnt_pdf.replace('pdf', 'png'))
print "(ref:base-num-per-sample) Number of sequenced bases. Color bars represent \
each replicate. 每个样品各个重复测序获得的碱基数。1 G = 10^9^.\
[PDF](%s/%s) [SOURCE](%s/%s)\n" % (tardir, base_cnt_pdf, tardir,
base_cnt_file)
print '''```{r base-num-per-sample, fig.cap="(ref:base-num-per-sample)"}
knitr::include_graphics("%s/%s")
```
''' % (tardir, base_cnt_pdf.replace('pdf', 'png'))
###--------multi-process------------------
#pool = ThreadPool(5) # 5 represents thread_num
#result = pool.map(func, iterable_object)
#pool.close()
#pool.join()
###--------multi-process------------------
if verbose:
print >>sys.stderr,\
"--Successful %s" % strftime(timeformat, localtime())
def test_copy_dir_to_dir():
copy(src_dir, dest_dir3)
assert os.path.isdir(os.path.join(base, 'dest_dir_also_not_existing'))
assert os.path.isfile(os.path.join(base, 'dest_dir_also_not_existing',
'nested_src_file'))
def test_copy_file_to_not_ex_dir():
copy(src_file, dest_dir2)
assert os.path.isfile(os.path.join(base, 'dest_dir_not_existing'))
def test_copy_file_to_ex_dir():
copy(src_file, dest_dir1)
assert os.path.isfile(os.path.join(base, 'dest_dir_existing', 'src_file'))
def test_copy_file_to_file():
copy(src_file, dest_file)
assert os.path.isfile(os.path.join(base, 'dest1', 'dest_file'))