def check_reference_format(reference_path): """Check file formats for files present in the reference""" try: contig_manager = ReferenceManager(reference_path) except Exception as e: martian.exit("Contig manager could not be initialized, Error:\n%s" % str(e)) # formatting error_msg = contig_manager.verify_contig_defs() if error_msg is not None: martian.exit(error_msg) # filecheck contig_manager.genes # check if motif file is in right format (naming convention) if len(contig_manager.list_species()) == 1: motif_format_checker(contig_manager.motifs) # checks for valid bed file formats in regions/ faidx_file = os.path.join(reference_path, 'fasta', 'genome.fa.fai') bed_format_checker(contig_manager.tss_track, faidx_file) bed_format_checker(contig_manager.transcripts_track, faidx_file) bed_format_checker(contig_manager.ctcf_track, faidx_file) bed_format_checker(contig_manager.blacklist_track, faidx_file) bed_format_checker(contig_manager.dnase_track, faidx_file) bed_format_checker(contig_manager.enhancer_track, faidx_file) bed_format_checker(contig_manager.promoter_track, faidx_file)
def split(args): # validate if args.kit_type not in ["5'", "3'"]: martian.exit("Kit type is not one of 5' or 3'.") # group by gene tx_dict = get_gene_pred_dict(args.transcripts) tx_by_name2 = defaultdict(list) valid_chroms = set(args.valid_chroms) for tx in tx_dict.itervalues(): if tx.chromosome in valid_chroms: tx_by_name2[tx.name2].append(tx) singletons = [ x[0] for x in tx_by_name2.itervalues() if len(x) == 1 and args.lower_size_cutoff <= len(x[0]) <= args.upper_size_cutoff ] avg = np.mean([len(x) for x in singletons]) med = np.median([len(x) for x in singletons]) martian.log_info( '{} singleton genes under consideration (avg size = {} median size = {}' .format(len(singletons), avg, med)) def tx_to_str(singletons): for x in singletons: yield x.get_gene_pred() chunks = [{ 'tx_subset': list(x), '__mem_gb': 4 } for x in grouper(tx_to_str(singletons), 20)] return {'chunks': chunks, 'join': {'__mem_gb': 32}}
def split(args): chunk_dict = defaultdict(list) # TENKIT-88 If we don't want to run qc, just have a blank set of chunks if not args.run_qc: return {'chunks': []} # find projects inside run dir dirnames = next(os.walk(args.fastq_path))[1] # this should be fine even if --reports-dir and --stats-dir are elsewhere # theoretically, you could name your sample 'Reports' or 'Stats' and # move the --reports-dir and --stats-dir somewhere else but at that # point you're just being daft project_dirs = [dn for dn in dirnames if dn not in ('Reports', 'Stats')] if not project_dirs: # add root folder project_dirs = ['.'] for project in sorted(project_dirs): project_path = os.path.join(args.fastq_path, project) # split by detected sample (all types) r1_files = tk_fasta.find_input_fastq_files_bcl2fastq_demult( project_path, 'R1', None, None) r2_files = tk_fasta.find_input_fastq_files_bcl2fastq_demult( project_path, 'R2', None, None) r3_files = tk_fasta.find_input_fastq_files_bcl2fastq_demult( project_path, 'R3', None, None) r4_files = tk_fasta.find_input_fastq_files_bcl2fastq_demult( project_path, 'R4', None, None) i1_files = tk_fasta.find_input_fastq_files_bcl2fastq_demult( project_path, 'I1', None, None) i2_files = tk_fasta.find_input_fastq_files_bcl2fastq_demult( project_path, 'I2', None, None) # group files together by like sample all_files = r1_files + r2_files + r3_files + r4_files + i1_files + i2_files for path in sorted(all_files): file_spec = tk_fasta.IlmnFastqFile(path) # do not add chunk if Undetermined if file_spec.prefix != 'Undetermined': chunk_dict[(project, file_spec.lane, file_spec.prefix, file_spec.s)].append(path) # if all reads were undetermined or bcl2fastq didn't generate anything if len(chunk_dict) == 0: martian.exit( "No FASTQs matched your sample sheet's lane, sample, and indexes. Please recheck your sample sheet." ) chunk_defs = [{ 'input_files': file_list, 'project': tup[0], 'lane': tup[1], 'sample': tup[2], 'subfolder': tup[3] } for tup, file_list in sorted(chunk_dict.items())] return {'chunks': chunk_defs}
def main(args, outs): try: run_preflight_checks(args) except cr_preflight.PreflightException as e: martian.exit(e.msg) cr_preflight.record_package_versions()
def check_factorization(factorization): """checks if factorization is valid""" if factorization is not None: if len(factorization) == 0: martian.exit("factorization must be a non-empty list.") if not all(elem in ALLOWED_FACTORIZATIONS for elem in factorization): martian.exit("Unsupported factorization provided. Options are {}.". format(", ".join(ALLOWED_FACTORIZATIONS)))
def check_key(n, dict_in, name, tys): if not name in dict_in: martian.exit("Entry %d in sample_def missing required field: %s" % (n, name)) if not (type(dict_in[name]) in tys): martian.exit( "Entry %d in sample_def for '%s' has incorrect type -- expecting %s, got %s" % (n, name, str(tys), type(dict_in[name])))
def check_spec(spec): # rule: if spec doesn't contain csv, it must contain lane/sample/index, with optional project. if not spec.get('csv'): # allow None for lanes, downstream will default to all if not spec.get('sample') or not spec.get('indices'): martian.exit( "Samplesheet spec without CSV must include lanes, sample and indices keys." )
def check_chunk_chemistries(chunks): """ Ensure all samples were generated with the same chemistry. """ unique_chemistries = set([chunk['chemistry']['name'] for chunk in chunks]) descriptions = map(cr_chem.get_chemistry_description_from_name, list(unique_chemistries)) if len(unique_chemistries) > 1: martian.exit( "Found multiple chemistries: %s. Combined analysis of libraries generated with different chemistries is not supported." % ', '.join(descriptions))
def check_force_cells(force_cells, ulimit=20000): """check if force cells is correctly specified""" if force_cells is not None: if len(force_cells) == 0: martian.exit("force_cells must be a non-empty dictionary.") for force_cells_k in force_cells.keys(): if (force_cells[force_cells_k] < 1 or force_cells[force_cells_k] > ulimit): martian.exit("MRO parameter force-cells[{}] must be a positive integer <= 20000: {}". format(force_cells_k, force_cells[force_cells_k]))
def check_gatk_ref(reference_path): if not os.path.exists(os.path.join(reference_path, "fasta", "genome.dict")): msg = "GATK requires that you create a .dict reference index file.\n" msg += "Note: to use this reference with GATK, also run this Picard command:\n" msg += "java -jar /path/to/picard.jar CreateSequenceDictionary R=%s O=%s\n" % (os.path.join(reference_path, "fasta", "genome.fa"), os.path.join(reference_path, "fasta", "genome.dict")) msg += "If you have GATK4, use this command:\n" msg += "java -jar /path/to/gatk4.jar CreateSequenceDictionary -R=%s" % os.path.join(reference_path, "fasta", "genome.fa") martian.exit(msg)
def check_vmem_for_reference(ref): """Finds out vmem required to load a genome reference fully""" refpath = os.path.join(ref, "fasta", "genome.fa") if not os.path.exists(refpath): hostname = socket.gethostname() martian.exit("Your reference does not contain the expected files, or they are not readable. Please check your reference folder on {}.".format(hostname)) refsize = os.path.getsize(refpath) / 1e9 vmem_gb = int(np.ceil(refsize)) + 4 return vmem_gb
def check_rta_complete(folder_path): """ :return: path to valid RTAComplete.txt in folder_path :rtype: string """ hostname = socket.gethostname() check_folder("sequencing run", folder_path, hostname) rta_complete = os.path.join(folder_path, "RTAComplete.txt") if not os.path.exists(rta_complete): martian.exit("On machine: %s, run does not appear to be complete yet. RTAComplete.txt not found." % hostname) return rta_complete
def contain_three_columns(in_file, check_top_n = 100): with open(in_file) as bediter: for i, row in enumerate(bediter): fields = row.strip().split('\t') if len(fields) != 3: martian.exit("Peak input file must contain only 3 columns") if i > check_top_n: break return
def check_bcl2fastq_v1(hostname): try: subprocess.check_call(["which", "configureBclToFastq.pl"]) except subprocess.CalledProcessError: martian.exit( "On machine: %s, bcl2fastq or configureBclToFastq.pl not found on PATH." % hostname) print "configureBclToFastq.pl version on %s: < 2.0" % hostname try: subprocess.check_call(["which", "perl"]) except subprocess.CalledProcessError: martian.exit("On machine: %s, perl not found on PATH." % hostname)
def join(args, outs, chunk_defs, chunk_outs): if args.raw_matrix is None: outs.filtered_matrix = None return # consume cell barcodes across all species and raise errors if not found if args.cell_barcodes is None: martian.exit("cell barcodes not provided") cell_barcodes = utils.load_cell_barcodes(args.cell_barcodes, with_species=True) # Read the peak matrix file and keep only cell barcodes # remove cell barcodes that were specified externally, such in reanalyzer, # which may not be present in raw matrix because they're missing from the fragments file present_cell_barcodes = {} peak_matrix = cr_matrix.CountMatrix.load_h5_file(args.raw_matrix) peak_matrix_bcs = set(peak_matrix.bcs) for species in cell_barcodes: present_cell_barcodes[species] = set() for bc in cell_barcodes[species]: if bc not in peak_matrix_bcs: martian.log_info("{} not found in the raw peak - bc matrix".format(bc)) else: present_cell_barcodes[species].add(bc) peak_matrix = peak_matrix.filter_barcodes(present_cell_barcodes) if peak_matrix.features_dim == 0: martian.log_info("data has no peaks, skipping the clustering analysis") outs.filtered_matrix = None outs.filtered_matrix_mex = None return peak_matrix = prune(peak_matrix, num_analysis_bcs=args.num_analysis_bcs, random_state=args.random_seed) if peak_matrix.bcs_dim <= analysis_constants.MAX_N_CLUSTERS_DEFAULT: martian.log_info("Insufficient number of cell barcodes present after processing") outs.filtered_matrix = None outs.filtered_matrix_mex = None return if peak_matrix.features_dim < analysis_constants.MAX_N_CLUSTERS_DEFAULT: martian.log_info("Insufficient number of peaks present after processing") outs.filtered_matrix = None outs.filtered_matrix_mex = None return # save processed matrix peak_matrix.save_h5_file(outs.filtered_matrix, sw_version=martian.get_pipelines_version()) if not os.path.exists(outs.filtered_matrix_mex): os.mkdir(outs.filtered_matrix_mex) atac_matrix.save_mex(peak_matrix, outs.filtered_matrix_mex, cr_lib_constants.ATACSEQ_LIBRARY_TYPE, sw_version=martian.get_pipelines_version())
def main(args, outs): if args.reanalyze and not args.aggregation_csv: return # CSV not required for reanalyze outs.sample_defs = parse_sample_sheet(args.pipestance_root, args.aggregation_csv) if args.reanalyze and args.matrix_h5: library_map = cr_matrix.get_gem_group_index(args.matrix_h5) matrix_library_ids = set([library_id for library_id, gem_group in library_map.values()]) csv_library_ids = set([row[cr_constants.AGG_ID_FIELD] for row in outs.sample_defs]) if matrix_library_ids != csv_library_ids: martian.exit("Library IDs specified in CSV (%s) do not match those contained in the input matrix (%s)" % (csv_library_ids, matrix_library_ids)) copy_csv(args.aggregation_csv, outs.aggregation_csv)
def check_runinfo_xml(folder_path): """ :return: path to valid RunInfo.xml in folder_path :rtype: string """ hostname = socket.gethostname() check_folder("sequencing run", folder_path, hostname) runinfo = os.path.join(folder_path, "RunInfo.xml") if not os.path.exists(runinfo): martian.exit("On machine: %s, RunInfo.xml not found. Cannot verify run was 10X-prepped." % hostname) if not os.access(runinfo, os.R_OK): martian.exit("On machine: %s, insufficient permission to open RunInfo.xml." % hostname) return runinfo
def main(args, outs): if args.pipestance_type != "count" and args.pipestance_type != "aggr": martian.exit("The type argument must be one of: count, aggr") if args.pipestance_type == "count": pname = "SC_RNA_COUNTER_CS" if args.pipestance_type == "aggr": pname = "SC_RNA_AGGREGATOR_CS" pipestance_exists = os.path.exists(args.pipestance_path) if not pipestance_exists: martian.exit("Invalid pipestance path: %s" % args.pipestance_path) # check to see if an analysis file exists. If it doesn't, then # this is likely a barnyard sample, and we cannot generate a # .loupe file (CELLRANGER-773); analysis_h5_path = os.path.join(args.pipestance_path, "outs/analysis/analysis.h5") # 1.2.0 location only internal_count_h5_path = os.path.join( args.pipestance_path, "SC_RNA_COUNTER_CS/SC_RNA_COUNTER/SC_RNA_ANALYZER/SUMMARIZE_ANALYSIS/fork0/files/analysis/analysis.h5" ) internal_aggr_h5_path = os.path.join( args.pipestance_path, "SC_RNA_AGGREGATOR_CS/SC_RNA_AGGREGATOR/SC_RNA_ANALYZER/SUMMARIZE_ANALYSIS/fork0/files/analysis/analysis.h5" ) if not os.path.exists(analysis_h5_path) \ and not os.path.exists(internal_count_h5_path) \ and not os.path.exists(internal_aggr_h5_path): martian.exit( "Could not find single-species analysis HDF5 file. " + "Loupe Cell Browser files are not generated for multi-species experiments." ) # has to be 1.2 or higher cellranger_pd_before_1_2_path = os.path.join(args.pipestance_path, "CELLRANGER_PD") cellranger_cs_before_1_2_path = os.path.join(args.pipestance_path, "CELLRANGER_CS") if os.path.exists(cellranger_pd_before_1_2_path) or os.path.exists( cellranger_cs_before_1_2_path): martian.exit( "mkloupe is only supported for Cell Ranger 1.2 and later.") call = [ "crconverter", args.sample_id, pname, "--pipestance", args.pipestance_path, "--output", outs.output_for_cloupe ] martian.log_info("Running crconverter: %s" % " ".join(call)) try: results = subprocess.check_output(call) martian.log_info("crconverter output: %s" % results) except subprocess.CalledProcessError, e: outs.output_for_cloupe = None martian.throw("Could not generate .cloupe file: \n%s" % e.output)
def main(args, outs): hostname = socket.gethostname() print "Checking run folder..." tk_preflight.check_rta_complete(args.run_path) print "Checking RunInfo.xml..." runinfo = tk_preflight.check_runinfo_xml(args.run_path) if not args.allow_no_barcodes: ok, msg = check_reads(runinfo) if not ok: martian.exit(msg) print "Checking system environment..." ok, msg = tk_preflight.check_ld_library_path() if not ok: martian.exit(msg) # Presence of SampleSheet.csv interferes with demux. # Ask customer to move it. Under older RTA, bcl2fastq looks for it # in Data/Intensities/BaseCalls while under newer RTA, it looks for it # at the top of the run folder. bc_dir = os.path.join(args.run_path, "Data", "Intensities", "BaseCalls") for ss_dir in [args.run_path, bc_dir]: ilmn_sample_sheet = os.path.join(ss_dir, "SampleSheet.csv") external = True try: import kitten external = False except ImportError: pass if external and os.path.exists(ilmn_sample_sheet): martian.exit( "On machine: %s, SampleSheet.csv found in run folder that would interfere with demux:\n%s\nPlease move, rename, or delete the file and run demux again." % (hostname, ilmn_sample_sheet)) if args.check_executables: print "Checking bcl2fastq..." # Determine the RTA version of the run and whether this instrument # requires i2 to RC'd (rta_version, rc_i2_read, bcl_params) = tenkit.bcl.get_rta_version(args.run_path) martian.log_info("RTA Version: %s" % rta_version) martian.log_info("BCL Params: %s" % str(bcl_params)) # Determine the best available bcl2fastq version to use # Will call martian.exit() with an error message if there isn't # a compatible version available (major_ver, full_ver) = tenkit.bcl.check_bcl2fastq(hostname, rta_version) martian.log_info("Running bcl2fastq mode: %s. Version: %s" % (major_ver, full_ver)) ok, msg = tk_preflight.check_open_fh() if not ok: martian.exit(msg)
def issue(self, metric, value, format_string=""): for alert in self.alerts: ## find the right metric if alert["metric"] == metric: ## should we trigger? if (alert["compare"] == ">") ^ (value < alert["threshold"]): ## optional formatting of alert message with format_string or value if len(format_string) == 0: format_string = str(value) message = alert["message"].replace("{}", format_string) ## issue an alert if alert["action"] == "alarm": martian.alarm(message) elif alert["action"] == "exit": martian.exit(message)
def check_file(file_type, file_path, hostname): if not file_path.startswith('/'): martian.exit("Specified %s file must be an absolute path: %s" % (file_type, file_path)) if not os.path.exists(file_path): martian.exit("On machine: %s, specified %s file does not exist: %s" % (hostname, file_type, file_path)) if os.path.isdir(file_path): martian.exit("On machine: %s, specified %s file is a folder: %s" % (hostname, file_type, file_path)) if not os.access(file_path, os.R_OK): martian.exit("On machine: %s, specified %s file is not readable: %s" % (hostname, file_type, file_path))
def check_folder(folder_type, folder_path, hostname, permission=os.X_OK): if not folder_path.startswith('/'): martian.exit("Specified %s folder must be an absolute path: %s" % (folder_type, folder_path)) if not os.path.exists(folder_path): martian.exit("On machine: %s, specified %s folder does not exist: %s" % (hostname, folder_type, folder_path)) if not os.path.isdir(folder_path): martian.exit("On machine: %s, specified %s path is not a folder: %s" % (hostname, folder_type, folder_path)) if not os.access(folder_path, permission): martian.exit("On machine: %s, insufficient permissions on %s folder: %s" % (hostname, folder_type, folder_path))
def main(args, outs): in_h5_list = [args.map_track, args.genome_tracks] out = pd.HDFStore(outs.tracks, "w") ## first add in mappability and GC tracks msizes = {} for in_h5 in in_h5_list: if in_h5 is None: continue if not os.path.exists(in_h5): martian.exit("Could not find " + in_h5) continue indata = pd.HDFStore(in_h5, "r") for key in indata.keys(): X = indata[key] # # column-wise concatenation for the constants section # if key in out: out[key] = pd.concat([out[key], X], axis=0) else: out[key] = X chrom = key.split("/")[-1] msizes[chrom] = X.shape[0] indata.close() ## convert confident windows into numpy and store if args.confident_windows is None or not os.path.exists( args.confident_windows): for chrom, length in msizes.iteritems(): out["/CONF/" + chrom] = pd.Series(np.ones(length, dtype=float)) else: conf = defaultdict(list) for line in open(args.confident_windows): fields = line.strip().split() chrom, perc = fields[0], float(fields[3]) conf[chrom].append(perc) for chrom, length in msizes.iteritems(): X = np.array(conf[chrom]) cbins = (X > crdna.constants.CONFIDENT_BIN_THRESHOLD).sum() if X.shape[0] == 0 or cbins == 0: out["/CONF/" + chrom] = pd.Series(np.ones(length, dtype=float)) else: assert X.shape[0] == length out["/CONF/" + chrom] = pd.Series(X) out.close()
def main(args, outs): ''' Convert sample_def = { "libraries_csv": "/path/to/libraries.csv" } into a standard sample_def map used by the rest of the pipeline. Only used by the CS pipeline to handle the --libraries cmd-line argument.''' if len(args.raw_sample_def) == 1: if args.raw_sample_def[0].keys() == ["libraries"]: # We've got a 'libraries mode' argument coming in -- load & check the CSV, and expand it into a normal sample def try: outs.sample_def = cr_preflight.expand_libraries_csv( args.raw_sample_def[0]["libraries"]) return except cr_preflight.PreflightException as e: martian.exit(e.msg) # Default case -- just copy over the sample_def outs.sample_def = args.raw_sample_def
def sequencer_detection_message(fastq_files): seqrs = set() # accumulate (sequencer, status) set for fastq in fastq_files: with gzip.open(fastq) as f: head = "" line = f.readline() if len(line) > 0: if line[0] == "@": head = line else: martian.exit( "Incorrectly formatted first read in FASTQ file: %s" % fastq) iid, fcid = parse_readhead(head) seqr, msg = infer_sequencer_with_message(iid, fcid) for sr in seqr: signal = (sr, msg) seqrs.add(signal) # get a list of sequencing platforms platforms = set() for platform, _ in seqrs: platforms.add(platform) sequencers = list(platforms) # if no sequencer detected at all message = "" fails = 0 for platform, status in seqrs: if status == fail_msg: fails += 1 if fails == len(seqrs): message = "could not detect the sequencing platform(s) used to generate the input FASTQ files" return message, sequencers # if partial or no detection failures if fails > 0: message = "could not detect the sequencing platform used to generate some of the input FASTQ files, " message += "detected the following sequencing platforms- " for platform, status in seqrs: if status != fail_msg: message += platform + " " + status + ", " message = message.strip(", ") return message, sequencers
def join(args, outs, chunk_defs, chunk_outs): outs.updated_sample_defs = [ chunk_out.updated_sample_def for chunk_out in chunk_outs ] if any("batch" in sample_def for sample_def in outs.updated_sample_defs): ncells = 0 for sample_def in outs.updated_sample_defs: with cr_mol_counter.MoleculeCounter.open( sample_def[cr_constants.AGG_H5_FIELD], 'r') as mc: library_info = mc.get_library_info() ncells += sum( mc.get_num_filtered_barcodes_for_library(i) for i in xrange(len(library_info))) if ncells > CBC_MAX_NCELLS: martian.exit( "You provided {:,} cells in total, but chemistry batch correction only supports up to {:,} cells." .format(ncells, CBC_MAX_NCELLS))
def get_run_data(fn): """ Parse flowcell + lane from the first FASTQ record. NOTE: we don't check whether there are multiple FC / lanes in this file. """ if fn[-2:] == 'gz': reader = gzip.open(fn) else: reader = open(fn, 'r') gen = tk_fasta.read_generator_fastq(reader) try: (name, seq, qual) = gen.next() (flowcell, lane) = re.split(':', name)[2:4] return (flowcell, lane) except StopIteration: # empty fastq martian.exit("FASTQ is empty: %s" % fn)
def main_bcl_processor(sample_id, sample_def, chemistry_arg, custom_chemistry_def): chunks = [] sample_index_strings, msg = tk_preflight.check_sample_indices(sample_def) if sample_index_strings is None: martian.exit(msg) path = sample_def['read_path'] lanes = sample_def['lanes'] for sample_index in sample_index_strings: # Determine the read-type => fastq filename mapping try: chemistry_name = cr_chem.infer_sc3p_chemistry_bcl_processor( chemistry_arg, path, sample_index, lanes) except cr_chem.NoInputFastqsException: continue if chemistry_name == cr_chem.CUSTOM_CHEMISTRY_NAME: chemistry = custom_chemistry_def else: chemistry = cr_chem.get_chemistry(chemistry_name) read_type_map = cr_chem.get_read_type_map( chemistry, tk_constants.BCL_PROCESSOR_FASTQ_MODE) # Collect the fastq files for each read type filename_lists = {} for dest_read_type in cr_constants.FASTQ_READ_TYPES: src_read_type = read_type_map[dest_read_type] filename_lists[ dest_read_type] = tk_fasta.find_input_fastq_files_10x_preprocess( path, src_read_type, sample_index, lanes) fill_in_missing_reads(filename_lists) if validate_fastq_lists(filename_lists): chunks += construct_chunks(filename_lists, sample_id, sample_def, reads_interleaved=True, chemistry=chemistry) return chunks
def validate_csv(csv_file, entry_type, entry_colname): if not os.path.exists(csv_file): martian.exit("Specified %s file does not exist: %s" % (entry_type, csv_file)) elif not os.access(csv_file, os.R_OK): martian.exit("Specified %s file is not readable, please check file permissions: %s" % (entry_type, csv_file)) with open(csv_file) as f: header = f.readline().strip().split(',') if header[0] != entry_colname: martian.exit("First line of %s file must be a header line, with '%s' as the first column." % (entry_type, entry_colname)) counts = sum(1 for line in f) # count remaining lines if counts == 0: martian.exit("Specified %s file must contain at least one entry." % entry_type) return counts
def check_folder_or_create(folder_type, folder_path, hostname, permission=os.X_OK): if not folder_path.startswith('/'): martian.exit("Specified %s folder must be an absolute path: %s" % (folder_type, folder_path)) if os.path.exists(folder_path): if not os.path.isdir(folder_path): martian.exit("On machine: %s, specified %s path is not a folder: %s" % (hostname, folder_type, folder_path)) if not os.access(folder_path, permission): martian.exit("On machine: %s, insufficient permissions on %s folder: %s" % (hostname, folder_type, folder_path)) else: try: os.makedirs(folder_path) except: martian.exit("On machine: %s, could not create %s folder: %s" % (hostname, folder_type, folder_path))