def create():
self = Run()
# parse command line args
args = Args.parse()
# setup logging
if args.verbose:
level = logging.DEBUG
else:
level = logging.INFO
logging.basicConfig(level=level,
format="%(asctime)s pid=%(process)d "
"%(levelname)s - %(message)s")
if args.resume:
self.results = Results(args.output_dir)
self.status = Status.load(self.results.status_file)
self.samples = Sample.parse_tsv(self.results.sample_file,
header=True)
self.args = Args.load(self.results.args_file)
else:
self.args = args
self.results = Results(args.output_dir)
self.status = Status()
self.samples = Sample.parse_tsv(self.args.sample_file)
# create output directories
results = self.results
if not os.path.exists(results.output_dir):
logging.debug("Creating output directory '%s'" %
(results.output_dir))
os.makedirs(results.output_dir)
if not os.path.exists(results.tmp_dir):
logging.debug("Creating tmp directory '%s'" % (results.tmp_dir))
os.makedirs(results.tmp_dir)
if not args.resume:
# write command line args
Args.dump(self.args, self.results.args_file)
# write samples
Sample.write_tsv(self.samples, self.results.sample_file)
# bypass aggregate step
if args.assemble is not None:
# create a symbolic link to BED file
os.symlink(args.assemble, self.results.transfrags_bed_file)
self.status.aggregate = True
# update status and write to file
self.status.create = True
self.status.write(self.results.status_file)
return self
def parse():
parser = Args.create()
args = parser.parse_args()
# check if we are trying to resume a previous run
if args.resume:
if not os.path.exists(args.output_dir):
parser.error("Output directory '%s' does not exist" %
args.output_dir)
# check that basic files exist in directory
results = Results(args.output_dir)
can_resume = os.path.exists(results.args_file)
can_resume = can_resume and os.path.exists(results.status_file)
can_resume = can_resume and os.path.exists(results.sample_file)
if not can_resume:
parser.error("Output directory '%s' not valid" %
args.output_dir)
else:
if os.path.exists(args.output_dir):
parser.error("Output directory '%s' already exists" %
args.output_dir)
if args.sample_file is None:
parser.error("sample file not specified")
if not os.path.exists(args.sample_file):
parser.error("sample file %s not found" % (args.sample_file))
args.sample_file = os.path.abspath(args.sample_file)
if args.min_transfrag_length < 0:
parser.error("min_transfrag_length < 0")
if args.min_expr < 0:
parser.error("min_expr < 0")
if (args.isoform_frac < 0) or (args.isoform_frac > 1):
parser.error("isoform_frac out of range (0.0-1.0)")
if (args.max_isoforms < 0):
parser.error("max_isoforms < 0")
if not (0 <= args.path_graph_loss_threshold <= 1):
parser.error("loss_threshold not in range (0.0-1.0)")
if args.path_graph_kmax < 0:
parser.error("kmax must be >= 0")
if args.max_paths < 0:
parser.error("max_paths must be >= 0")
if not (0 <= args.path_frac <= 1):
parser.error("path_frac not in range (0.0-1.0)")
if (args.change_point):
if not (0.0 <= args.change_point_pvalue <= 1.0):
parser.error('change point pvalue invalid')
if not (0.0 <= args.change_point_fold_change <= 1.0):
parser.error('change point fold change invalid')
if args.ref_gtf_file is not None:
if not os.path.exists(args.ref_gtf_file):
parser.error("reference GTF file %s not found" %
(args.ref_gtf_file))
args.ref_gtf_file = os.path.abspath(args.ref_gtf_file)
elif (args.guided_assembly or args.guided_ends
or args.guided_strand):
parser.error('Guided assembly modes require a '
'reference GTF (--ref-gtf)')
return args
def parse():
parser = Args.create()
args = parser.parse_args()
if args.resume:
# check if we are trying to resume a previous run
if not os.path.exists(args.output_dir):
parser.error("Output directory '%s' does not exist" %
args.output_dir)
# check that basic files exist in directory
results = Results(args.output_dir)
can_resume = os.path.exists(results.args_file)
can_resume = can_resume and os.path.exists(results.status_file)
can_resume = can_resume and os.path.exists(results.sample_file)
if not can_resume:
parser.error("Output directory '%s' not valid" %
args.output_dir)
else:
if os.path.exists(args.output_dir):
parser.error("Output directory '%s' already exists" %
args.output_dir)
# check if we are trying bypass aggregate and assemble from a
# preexisting BED file
if args.assemble is not None:
if not os.path.exists(args.assemble):
parser.error('(--assemble) BED file %s not found')
args.assemble = os.path.abspath(args.assemble)
if args.sample_file is None:
parser.error(
'Sample file not specified. Run with --help for more info.'
)
if not os.path.exists(args.sample_file):
parser.error('Sample file %s not found' % (args.sample_file))
args.sample_file = os.path.abspath(args.sample_file)
if args.filter_min_length < 0:
parser.error("filter_min_length < 0")
if args.filter_min_expr < 0:
parser.error("filter_min_expr < 0")
if (args.isoform_frac < 0) or (args.isoform_frac > 1):
parser.error("isoform_frac out of range (0.0-1.0)")
if (args.max_isoforms < 0):
parser.error("max_isoforms < 0")
if args.path_graph_kmax < 0:
parser.error("kmax must be >= 0")
if args.max_paths < 0:
parser.error("max_paths must be >= 0")
if not (0 <= args.path_frac <= 1):
parser.error("path_frac not in range (0.0-1.0)")
if args.change_point:
if not (0.0 <= args.change_point_pvalue <= 1.0):
parser.error('change point pvalue invalid')
if not (0.0 <= args.change_point_fold_change <= 1.0):
parser.error('change point fold change invalid')
if args.filter_splice_juncs:
if not args.ref_genome_fasta_file:
parser.error('Filtering of splice junctions enabled '
'(--filter-splice-juncs) but '
'reference genome FASTA file not specified')
if not os.path.exists(args.ref_genome_fasta_file):
parser.error('Reference genome FASTA file not found')
args.ref_genome_fasta_file = os.path.abspath(
args.ref_genome_fasta_file)
if args.ref_gtf_file is not None:
if not os.path.exists(args.ref_gtf_file):
parser.error("reference GTF file %s not found" %
(args.ref_gtf_file))
args.ref_gtf_file = os.path.abspath(args.ref_gtf_file)
elif (args.guided_assembly or args.guided_ends
or args.guided_strand):
parser.error('Guided assembly modes require a '
'reference GTF (--ref-gtf)')
return args
def aggregate_parallel(samples, args, results):
'''
Process and aggregate GTF input files
samples: list of Sample objects
args: from Argparse module. command-line arguments to configure the
assembly process
results: Results object containing input and output filenames
'''
logging.info('Aggregating in parallel using %d processes' %
(args.num_processes))
if args.filter_splice_juncs and args.ref_genome_fasta_file:
# test opening FastaFile
logging.info('Indexing reference genome fasta file (if necessary)')
fasta_fh = FastaFile(args.ref_genome_fasta_file)
fasta_fh.close()
# create queue
input_queue = JoinableQueue(maxsize=args.num_processes * 2)
# start worker processes
procs = []
worker_results = []
for i in xrange(args.num_processes):
worker_id = 'aggregate_worker%03d' % i
worker_dir = os.path.join(results.tmp_dir, worker_id)
if not os.path.exists(worker_dir):
os.makedirs(worker_dir)
worker_results.append(Results(worker_dir))
p = Process(target=aggregate_worker,
args=(input_queue, args, worker_dir))
p.start()
procs.append(p)
# reference gtf
if args.ref_gtf_file is not None:
logging.debug('Reference: %s' % args.ref_gtf_file)
input_queue.put(Sample(args.ref_gtf_file, Sample.REF_ID))
# parse samples
for sample in samples:
input_queue.put(sample)
for p in procs:
input_queue.put(None)
# close input queue
input_queue.join()
input_queue.close()
# join worker processes
for p in procs:
p.join()
# merge output files
logging.info('Merging aggregated files')
logging.debug('\tmerging bed files')
retcode = merge_bed(
input_files=[r.transfrags_bed_file for r in worker_results],
output_file=results.transfrags_bed_file,
num_processes=args.num_processes,
tmp_dir=results.tmp_dir)
if retcode != 0:
raise TacoError('Error running linux merge')
logging.debug('\tmerging filtered bed files')
retcode = merge_bed(
input_files=[r.transfrags_filtered_bed_file for r in worker_results],
output_file=results.transfrags_filtered_bed_file,
num_processes=args.num_processes,
tmp_dir=results.tmp_dir)
if retcode != 0:
raise TacoError('Error running linux merge')
logging.debug('\tmerging sample stats')
def sort_key_field0(line):
fields = line.split('\t', 1)
return fields[0]
stats_header = [
'sample_id', 'num_transfrags', 'filtered_length', 'filtered_expr',
'filtered_splice\n'
]
stats_header = '\t'.join(stats_header)
merge_files(input_files=[r.sample_stats_file for r in worker_results],
output_file=results.sample_stats_file,
key=sort_key_field0,
header=stats_header)
# cleanup worker data
logging.info('Removing temporary files')
def shutil_error_callback(func, path, excinfo):
logging.error('Error removing tmp files path=%s message=%s' %
(path, excinfo))
for r in worker_results:
shutil.rmtree(r.output_dir, onerror=shutil_error_callback)
logging.info('Aggregate done')
return 0
def aggregate_worker(input_queue, args, output_dir):
results = Results(output_dir)
# create temp directories
tmp_dir = os.path.join(results.output_dir, 'tmp')
if not os.path.exists(tmp_dir):
logging.debug('\tcreating tmp dir %s' % (tmp_dir))
os.makedirs(tmp_dir)
# create set of unsorted results
tmp_results = Results(tmp_dir)
# setup genome fasta file
genome_fasta_fh = None
if args.filter_splice_juncs and args.ref_genome_fasta_file:
genome_fasta_fh = FastaFile(args.ref_genome_fasta_file)
# setup output files
bed_fh = open(tmp_results.transfrags_bed_file, 'w')
filtered_bed_fh = open(tmp_results.transfrags_filtered_bed_file, 'w')
stats_fh = open(results.sample_stats_file, 'w')
# process samples via input queue
while True:
sample = input_queue.get()
if sample is None:
break
aggregate_sample(sample,
gtf_expr_attr=args.gtf_expr_attr,
is_ref=(sample._id == Sample.REF_ID),
min_length=args.filter_min_length,
min_expr=args.filter_min_expr,
filter_splice_juncs=args.filter_splice_juncs,
add_splice_motif=args.add_splice_motif,
genome_fasta_fh=genome_fasta_fh,
bed_fh=bed_fh,
filtered_bed_fh=filtered_bed_fh,
stats_fh=stats_fh)
input_queue.task_done()
input_queue.task_done()
# cleanup and close files
bed_fh.close()
filtered_bed_fh.close()
stats_fh.close()
if genome_fasta_fh:
genome_fasta_fh.close()
# sort output files
logging.debug('Sorting aggregated files: "%s"' % (output_dir))
# sort bed file
logging.debug('\ttransfrags bed file: %s' % (results.output_dir))
retcode = sort_bed(tmp_results.transfrags_bed_file,
results.transfrags_bed_file,
num_processes=1,
tmp_dir=tmp_dir)
if retcode != 0:
raise TacoError('Error running linux sort')
os.remove(tmp_results.transfrags_bed_file)
# sort filtered bed file
logging.debug('\tfiltered transfrags bed file: %s' % (results.output_dir))
retcode = sort_bed(tmp_results.transfrags_filtered_bed_file,
results.transfrags_filtered_bed_file,
num_processes=1,
tmp_dir=tmp_dir)
os.remove(tmp_results.transfrags_filtered_bed_file)
if retcode != 0:
raise TacoError('Error running linux sort')
# remove temporary directories
logging.debug('\t%s cleaning up' % (results.output_dir))
os.rmdir(tmp_dir)
from hardware import WORK_ON_RPI
from .app_conf import ActionBase, logger, States, Task
from hardware import gl
ActionNameType = NewType('ActionNameType', str)
TaskNameType = NewType('TaskNameType', str)
# 树莓派跑 or 自机测试
if WORK_ON_RPI:
from hardware import MotorAction, WeightSensor, Light, Fan, Printer
# TODO: 完成后删除测试库
import random
# 类实例化
hardware_info = Hardware()
results_info = Results()
if WORK_ON_RPI:
# drawer_hard = MotorAction('托盘', [31, 33, 35, 37], [12, 16, 18, 22], 8000)
# lifting_hard = MotorAction('抬升', [32, 36, 38, 40], [13, 15, 7, 11], 3200)
drawer_hard = MotorAction('托盘', [31, 33, 35, 37], [12, 16, 18, 22], 6000)
time.sleep(0.01)
lifting_hard = MotorAction('抬升', [32, 36, 38, 40], [13, 15, 7, 11], 4000)
# drawer_hard = MotorAction('托盘', [32, 36, 38, 40], [7, 11, 7, 11], 400)
# lifting_hard = MotorAction('抬升', [31, 33, 35, 37], [7, 11, 7, 11], 400)
weight_hard = WeightSensor('/dev/ttyAMA0')
time.sleep(0.01)
light_hard = Light(21)
time.sleep(0.01)
light_plate_hard = Light(23)
time.sleep(0.01)
fan_hard = Fan(29)
#!/usr/bin/env python
# encoding: utf-8
from flask import Flask, render_template, Response, request, send_file, redirect
from flask.json import jsonify
from base import Hardware
from base import Results
from base import utility
from base import Motor, TravelSwitch, MotorAction
app = Flask(__name__)
app.config['hardware'] = Hardware()
app.config['results'] = Results()
drawer = MotorAction('托盘', [31, 33, 35, 37], [12, 16, 18, 22], 8000)
lifting = MotorAction('抬升', [32, 36, 38, 40], [13, 15, 7, 11], 1600)
# 验证文件大小，通过设置Flask内置的配置变量MAX_CONTENT_LENGTH，可以显示请求报文的最大长度，单位是字节
# 当上传文件的大小超过这个限制后，flask内置的开服务器会中断连接，在生产环境的服务器上会返回413错误响应
# app.config['MAX_CONTENT_LENGTH'] = 1 * 1024 * 1024
@app.errorhandler(404)
def error404(args):
return args
@app.route('/')
def hello_world():
return render_template('index.html')