def split(args):
if args.fragments is None:
return {"chunks": [], "join": {}}
ctg_mgr = ReferenceManager(args.reference_path)
all_contigs = ctg_mgr.primary_contigs(allow_sex_chromosomes=True)
chunks = []
for contig in all_contigs:
chunks.append({"contig": contig, "__mem_gb": 5})
return {"chunks": chunks, "join": {"__mem_gb": 5}}
def split(args):
if args.fragments is None:
return {'chunks': [], 'join': {}}
ctg_mgr = ReferenceManager(args.reference_path)
all_contigs = ctg_mgr.primary_contigs(allow_sex_chromosomes=True)
chunks = []
for contig in all_contigs:
chunks.append({'contig': contig, '__mem_gb': 5})
return {'chunks': chunks, 'join': {'__mem_gb': 5}}
def split(args):
if args.fragments is None:
return {'chunks': [], 'join': {}}
ctg_mgr = ReferenceManager(args.reference_path)
all_contigs = ctg_mgr.primary_contigs(allow_sex_chromosomes=True)
contig_len = ctg_mgr.get_contig_lengths()
BYTES_PER_INT32_WITH_SAFETY = 5
chunks = []
for contig in all_contigs:
chunks.append({'contig': contig,
'__mem_gb': int(np.ceil(BYTES_PER_INT32_WITH_SAFETY * contig_len[contig] / 1024 / 1024 / 1024))})
return {'chunks': chunks, 'join': {'__mem_gb': 5}}
def split(args):
if args.fragments is None:
return {"chunks": [], "join": {}}
ctg_mgr = ReferenceManager(args.reference_path)
all_contigs = ctg_mgr.primary_contigs(allow_sex_chromosomes=True)
with open(args.barcode_counts, "r") as infile:
barcode_counts = Counter(json.load(infile))
barcode_array = np.array([bc for bc in barcode_counts])
gem_group_array = np.array(
[get_barcode_gem_group(bc) for bc in barcode_counts])
gem_groups = set(gem_group_array)
frag_count_array = np.array([barcode_counts[bc] for bc in barcode_array])
valid_barcodes = list()
for gem_group in gem_groups:
count_mask = (frag_count_array > MINIMUM_COUNTS) & (gem_group_array
== gem_group)
# find at most top N barcodes
topN_indices = barcode_array[count_mask].argsort(
)[-min(MAXIMUM_BARCODES, len(count_mask)):]
valid_barcodes.extend(list(barcode_array[count_mask][topN_indices]))
# mem allocs
JOIN_LOAD_FACTOR = 2
BUFFER_GB = 2
BYTES_PER_ENTRY = 4 # this depends on the dtype
chunk_mem_gb = BUFFER_GB + np.ceil(
BYTES_PER_ENTRY * len(gem_groups) * MAXIMUM_BARCODES**2 /
1024**3).astype('int32')
join_mem_gb = BUFFER_GB + np.ceil(
JOIN_LOAD_FACTOR * BYTES_PER_ENTRY * len(gem_groups) *
MAXIMUM_BARCODES**2 / 1024**3).astype('int32')
valid_barcodes_path = martian.make_path("valid_barcodes.txt")
with open(valid_barcodes_path, 'w') as f:
f.write(",".join(valid_barcodes))
chunks = []
for contig in all_contigs:
chunks.append({
"contig": contig,
"valid_barcodes": valid_barcodes_path,
"__mem_gb": chunk_mem_gb,
})
return {"chunks": chunks, "join": {"__mem_gb": join_mem_gb}}
def split(args):
if args.fragments is None:
return {'chunks': [], 'join': {}}
if args.peaks is None:
martian.throw("peaks BED file expected")
if args.cell_barcodes is None:
martian.throw("cell barcodes CSV file expected")
ctg_mgr = ReferenceManager(args.reference_path)
all_contigs = ctg_mgr.primary_contigs(allow_sex_chromosomes=True)
chunks = []
for contig in all_contigs:
chunks.append({'contig': contig, '__mem_gb': 4})
return {'chunks': chunks, 'join': {'__mem_gb': 8}}
def main(args, outs):
args.coerce_strings()
outs.coerce_strings()
with open(args.barcodes, 'r') as barcode_file:
barcodes_dict = OrderedDict(
(bc.strip('\n'), num) for num, bc in enumerate(barcode_file))
outs.insert_summary = None
if args.fragments is None or len(barcodes_dict) == 0:
outs.insert_sizes = None
outs.total = None
return
ref_contig_manager = ReferenceManager(args.reference_path)
# iterate over fragments and count fragment sizes for each barcode
insert_sizes = {bc: Counter() for bc in barcodes_dict.iterkeys()}
primary_contigs = set(
ref_contig_manager.primary_contigs(allow_sex_chromosomes=True))
for contig, start, stop, barcode, _ in open_fragment_file(args.fragments):
if barcode not in barcodes_dict:
continue
if args.exclude_non_nuclear and contig not in primary_contigs:
continue
size = stop - start
insert_sizes[barcode][
str(size) if size <= MAX_INSERT_SIZE else GT_MAX_INSERT_SIZE] += 1
# compute total and write out csv
total = np.zeros(MAX_INSERT_SIZE)
with open(outs.insert_sizes, 'w') as outfile:
outfile.write(','.join(['Barcode'] +
[str(n)
for n in range(1, MAX_INSERT_SIZE + 1)] +
['>{}'.format(MAX_INSERT_SIZE)]) + '\n')
for barcode in insert_sizes:
outfile.write(','.join([barcode] + [
str(insert_sizes[barcode][str(n)])
for n in range(1, MAX_INSERT_SIZE + 1)
] + [str(insert_sizes[barcode][GT_MAX_INSERT_SIZE])]) + '\n')
for n in range(1, 1001):
total[n - 1] += insert_sizes[barcode][str(n)]
# write out totals for reduce in join
np.savetxt(outs.total, total, delimiter=',')
def split(args):
if args.fragments is None:
return {"chunks": [], "join": {}}
with open(args.barcode_counts, "r") as infile:
barcode_counts = Counter(json.load(infile))
valid_barcodes = barcode_counts.keys()
part_a_seqs, part_c_seqs, part_b_seqs, gem_group_seqs = query_barcode_subsequences(
valid_barcodes)
ctg_mgr = ReferenceManager(args.reference_path)
all_contigs = ctg_mgr.primary_contigs(allow_sex_chromosomes=True)
chunks = []
for gem_group in gem_group_seqs:
for contig in all_contigs:
chunks.append({
"contig": contig,
"gem_group": gem_group,
"__mem_gb": 4,
})
return {"chunks": chunks, "join": {"__mem_gb": 16}}
def join(args, outs, chunk_defs, chunk_outs):
# Sample ID / pipestance name
check_sample_id(args.sample_id)
# force_cells
check_force_cells(args.force_cells, ulimit=10000000) # allow arbitrarily large limit for reanalyzer
# # Reference
# ref directory structure and timestamps
ok, msg = check_refdata(args.reference_path, max_contigs=None)
if ok:
martian.log_info(msg)
else:
martian.exit(msg)
# formatting
check_reference_format(args.reference_path)
contig_manager = ReferenceManager(args.reference_path)
# peaks format check and nonoverlapping
if args.peaks is None:
martian.exit("peaks file not provided")
exists_and_readable(args.peaks, "peaks")
bed_format_checker(args.peaks, contig_manager.fasta_index)
contain_three_columns(args.peaks)
if is_overlapping(args.peaks):
martian.exit("{} contains overlapping peak regions".format(args.peaks))
# check parameters files
if args.parameters is not None:
if not os.path.exists(args.parameters):
martian.exit("{} does not exist".format(args.parameters))
# fragments checks
whitelist_barcodes = load_barcode_whitelist(args.barcode_whitelist)
species_list = contig_manager.list_species()
observed_gem_groups = set()
observed_species = set()
if args.fragments is None:
martian.exit("fragments file not provided")
exists_and_readable(args.fragments, "fragments")
contig_lens = contig_manager.get_contig_lengths()
# check bounds and matching contigs in reference and species
for chrom, start, stop, bc, _ in open_fragment_file(args.fragments):
spec = chrom.split("_")
observed_species.add(spec[0] if spec[0] != chrom else "")
barcode, gem_group = bc.split("-")
observed_gem_groups.add(gem_group)
if args.check_executables: # run this only non-locally
if barcode not in whitelist_barcodes:
martian.exit("{} is not a valid whitelist barcode".format(barcode))
if chrom not in contig_lens:
martian.exit("contig {} not present in reference".format(chrom))
if stop > contig_lens[chrom]:
martian.exit("fragment {}:{}-{} boundaries exceed contig size ({} bp)".format(chrom, start, stop, contig_lens[chrom]))
# ensure fragments are on the correct reference
for species in observed_species:
if species not in species_list:
martian.exit("{} contains fragments mapped to species not recognized in the reference".format(args.fragments))
if len(observed_gem_groups) > 1:
martian.log_info("multiple gem groups present in {}, likely generated in a previous aggregation run".format(args.fragments))
# fragments index is synced with fragments
if args.fragments_index is None:
martian.exit("fragments index file not provided")
if not os.path.exists(args.fragments_index):
martian.exit("{} does not exist".format(args.fragments_index))
try:
all_contigs = contig_manager.primary_contigs(allow_sex_chromosomes=True)
for contig in all_contigs:
en = 0
for chrom, start, end, bc, dups in parsed_fragments_from_contig(contig, args.fragments, index=args.fragments_index):
if en >= FRAGMENTS_SCAN_SIZE:
break
en += 1
except:
martian.exit("fragments index is not in sync with the fragments file")
# aggr csv checks
if args.aggregation_csv is not None:
check_aggr_csv(args.aggregation_csv, args.reference_path, cursory=True)
# cell barcode checks
if args.cell_barcodes is not None:
if not os.path.exists(args.cell_barcodes):
martian.exit("{} does not exist".format(args.cell_barcodes))
check_singlecell_format(args.cell_barcodes, species_list, whitelist_barcodes)
# Open file handles limit
if args.check_executables:
check_filehandle_limit()
martian.log_info(tk_preflight.record_package_versions())