def process(self, information): # Setup django for templates from django.conf import settings settings.configure(TEMPLATE_DIRS=self.templateDirs, INSTALLED_APPS=('APIgen.tags', )) # Reset temp dir if os.path.exists(self.tempDir): shutil.rmtree(self.tempDir) # Make sure we have out and temp directories mkdir(self.outDir) mkdir(self.tempDir) # Copy assets to output for directory in self.assetDirs: shutil.copytree(directory, self.tempDir, ignore=shutil.ignore_patterns(IGNORE_PATTERNS)) log.info("\n---------------------GENERATING------------------------\n") for module in information[MODULES].values(): self.gen_module(information, module) log.info("\n---------------------DONE------------------------\n")
def process(self, information): # Setup django for templates from django.conf import settings settings.configure( TEMPLATE_DIRS=self.templateDirs, INSTALLED_APPS = ('APIgen.tags', ) ) # Reset temp dir if os.path.exists(self.tempDir): shutil.rmtree(self.tempDir) # Make sure we have out and temp directories mkdir(self.outDir) mkdir(self.tempDir) # Copy assets to output for directory in self.assetDirs: shutil.copytree(directory, self.tempDir, ignore=shutil.ignore_patterns(IGNORE_PATTERNS)) log.info("\n---------------------GENERATING------------------------\n") for module in information[MODULES].values(): self.gen_module(information, module) log.info("\n---------------------DONE------------------------\n")
def gen_module(self, information, module): moduleName = module[NAME] self.createPage( information , "%s.txt" % moduleName , [ os.sep.join(['modules', moduleName, 'module.rst']) , 'module.rst' ] , module = module , current = module , fullname = moduleName ) moduleDir = os.path.join(self.outDir, moduleName) mkdir(moduleDir) for kls in module[CLASS_LIST]: klsName = kls[NAME] fullName = "%s.%s" % (moduleName, klsName) if moduleName == klsName: fullName = klsName self.createPage( information , os.sep.join([moduleName, "%s.txt" % klsName]) , [ os.sep.join(["classes", "%s.rst" % klsName]) , os.sep.join(["classes", moduleName, "%s.rst" % klsName]) , os.sep.join(["modules", moduleName, "class.rst"]) , os.sep.join(["modules", moduleName, "classes", "%s.rst" % klsName]) , "class.rst" ] , module = module , current = kls , fullname = fullName )
def gen_module(self, information, module): moduleName = module[NAME] self.createPage( information, "%s.txt" % moduleName, [os.sep.join(['modules', moduleName, 'module.rst']), 'module.rst'], module=module, current=module, fullname=moduleName) moduleDir = os.path.join(self.outDir, moduleName) mkdir(moduleDir) for kls in module[CLASS_LIST]: klsName = kls[NAME] fullName = "%s.%s" % (moduleName, klsName) if moduleName == klsName: fullName = klsName self.createPage( information, os.sep.join([moduleName, "%s.txt" % klsName]), [ os.sep.join(["classes", "%s.rst" % klsName]), os.sep.join(["classes", moduleName, "%s.rst" % klsName]), os.sep.join(["modules", moduleName, "class.rst"]), os.sep.join( ["modules", moduleName, "classes", "%s.rst" % klsName]), "class.rst" ], module=module, current=kls, fullname=fullName)
def __logging_args(args): #make directories args.log_dir += args.dataset + '_noaug/' if args.aug_type == 0 else args.dataset + '/' logfile_name = '%s_d%dw%d' % (args.network, args.depth, args.widen_factor) #name of logging text file if args.model_prefix is None: args.model_prefix = args.log_dir + args.network + '/' logfile_name += '_exp' if args.exp_name is None else '_' + args.exp_name random_idx = 1 while os.path.isfile(args.model_prefix + logfile_name + str(random_idx) + '.txt'): random_idx += 1 logfile_name += str(random_idx) #logging log_file_full_name = args.model_prefix + logfile_name + '.txt' args.model_prefix += 'weights/' + logfile_name + '/' utility.mkdir(args.model_prefix) args.model_prefix += logfile_name head = '%(asctime)-15s %(message)s' logger = logging.getLogger() handler = logging.FileHandler(log_file_full_name) formatter = logging.Formatter(head) handler.setFormatter(formatter) logger.addHandler(handler) handler = logging.StreamHandler() handler.setFormatter(formatter) logger.addHandler(handler) logger.setLevel(logging.DEBUG) logger.info('%s', log_file_full_name) logger.info('start with arguments %s', args)
def bmp2jpg(path_image, pathGoal_image): utility.mkdir(pathGoal_image) for i_view in range(8): path_image_view = "%s%d/" % (path_image, i_view) pathGoal_image_view = "%s%d/" % (pathGoal_image, i_view) utility.mkdir(pathGoal_image_view) for root, dirs, files in os.walk(path_image_view): for i_file in range(files.__len__()): pass image = cv2.imread(path_image_view + files[i_file]) pathFileSave = pathGoal_image_view + files[i_file][ 0:3] + ".jpg" cv2.imwrite(pathFileSave, img=image) print("Finish %s%s" % (pathFileSave))
def run(self): apt_packages = [ "nginx", "nginx-extras", "apache2-utils", ] for package in apt_packages: utility.apt(package) utility.mkdir('/etc/nginx/sites-available', use_sudo=True) utility.mkdir('/etc/nginx/sites-enabled', use_sudo=True) upload_template('nginx.conf', '/etc/nginx/nginx.conf', context={}, template_dir=TEMPLATE_DIR, use_jinja=True, use_sudo=True) utility.rm('/etc/nginx/sites-enabled/default', use_sudo=True) self.restart()
def __init__(self, args, train_loader, train_sampler, valid_loader, my_model, my_loss, ckp): self.args = args self.scale = args.scale[0] self.ckp = ckp self.loader_train = train_loader self.loader_valid = valid_loader self.train_sampler = train_sampler self.model = my_model self.loss = my_loss self.optimizer = utility.make_optimizer(args, self.model) self.psnr_fn = PSNR(boundary_ignore=40) # Postprocessing function to obtain sRGB images self.postprocess_fn = SimplePostProcess(return_np=True) if 'L1' in args.loss: self.aligned_loss = L1(boundary_ignore=None).cuda(args.local_rank) elif 'MSE' in args.loss: self.aligned_loss = L2(boundary_ignore=None).cuda(args.local_rank) elif 'CB' in args.loss: self.aligned_loss = CharbonnierLoss(boundary_ignore=None).cuda( args.local_rank) elif 'MSSSIM' in args.loss: self.aligned_loss = MSSSIMLoss(boundary_ignore=None).cuda( args.local_rank) if self.args.fp16: self.scaler = GradScaler() self.best_psnr = 0. self.best_epoch = 0 if self.args.load != '': self.optimizer.load(ckp.dir, epoch=len(ckp.log)) self.error_last = 1e8 self.glob_iter = 0 self.log_dir = LOG_DIR + "/" + args.save self.img_save_dir = IMG_SAVE_DIR + "/" + args.save # Where to load model self.load_model_dir = LOAD_MODEL_DIR + "/" + args.save # Where to save new model self.save_model_dir = SAVE_MODEL_DIR + "/" + args.save # Where to save visualization images (for report) self.results_dir = RESULTS_DIR + "/" + args.save self.writer = SummaryWriter(log_dir=self.log_dir) utility.mkdir(self.save_model_dir) utility.mkdir(self.img_save_dir) utility.mkdir(self.log_dir) utility.mkdir('frames')
def __logging_args(args): #make directories args.log_dir += args.dataset + '_noaug/' if args.aug_type == 0 else args.dataset + '/' logfile_name = '%s_d%dL%dM%d' % (args.network, args.depth, args.primary_partition, args.secondary_partition) #name of logging text file if args.model_prefix is None: args.model_prefix = args.log_dir + args.network + '/' logfile_name += '_exp' if args.exp_name is None else '_' + args.exp_name random_idx = 1 while os.path.isfile(args.model_prefix + logfile_name + str(random_idx) + '.txt'): random_idx += 1 if args.load_epoch is not None: #deprecated. Use `--model-prefix` to manually set the name random_idx -= 1 logfile_name += str(random_idx) #model related if args.checkpoint_epochs is None: #if num_epochs=400, then we will save the model every 50 epochs args.checkpoint_epochs = args.num_epochs / 8 if args.rand_seed is None: import time args.rand_seed = int( time.time()) #different random init for serveral runs mx.random.seed(args.rand_seed) #cudnn conv backward is non-deterministic #logging log_file_full_name = args.model_prefix + logfile_name + '.txt' args.model_prefix += 'weights/' + logfile_name + '/' utility.mkdir(args.model_prefix) args.model_prefix += logfile_name head = '%(asctime)-15s %(message)s' logger = logging.getLogger() map(logger.removeHandler, logger.handlers[:]) #reset handler = logging.FileHandler(log_file_full_name) formatter = logging.Formatter(head) handler.setFormatter(formatter) logger.addHandler(handler) handler = logging.StreamHandler() handler.setFormatter(formatter) logger.addHandler(handler) logger.setLevel(logging.DEBUG) logger.info('%s', log_file_full_name) logger.info('start with arguments %s', args)
def handle(self): print('connected from:', self.client_address) while True: # 定义文件信息。128s表示文件名为128bytes长,l表示一个int或log文件类型,在此为文件大小 fileinfo_size = struct.calcsize('128sl') self.buf = self.request.recv(fileinfo_size) # 如果不加这个if,第一个文件传输完成后会自动走到下一句 if self.buf: # 根据128sl解包文件信息,与client端的打包规则相同 self.str_received, self.filesize = struct.unpack( '128sl', self.buf) # # 文件名长度为128,大于文件名实际长度 # print('filesize is: ',self.filesize,'filename size is: ',len(self.filename)) # # 使用strip()删除打包时附加的多余空字符 [self.filename, self.pathSave] = self.str_received.decode().split("##") pathSave = (self.pathSave).strip('\00') utility.mkdir(pathSave) for i_view in range(8): pathSave_view = pathSave + "%d_bmp/" % i_view utility.mkdir(pathSave_view) pathSave_view = pathSave + "v%d/" % i_view utility.mkdir(pathSave_view) self.filenewname = os.path.join(pathSave, (self.filename).strip('\00')) print(self.filenewname, type(self.filenewname)) # 定义接收了的文件大小 recvd_size = 0 file = open(self.filenewname, 'wb') # print('stat receiving...') while not recvd_size == self.filesize: if self.filesize - recvd_size > 1024: rdata = self.request.recv(1024) recvd_size += len(rdata) else: rdata = self.request.recv(self.filesize - recvd_size) recvd_size = self.filesize file.write(rdata) file.close()
if PC == "Desktop": path_data = "J:/0 SOG_201807/data/" elif PC in CameraSystem_L: path_data = "F:/0 SoG/data/" elif PC in CameraSystem_S: path_data = "C:/0_SoG/data/" # Path Save if 1: # level 1 str_timeDate_level_1 = "181128" str_dir_level_1 = 'dataResult_%s_image1811_1_K1/' % (str_timeDate_level_1) str_dir_level_1 = 'dataResult_%s_image1811_2_K1/' % (str_timeDate_level_1) pathSave = path_data + str_dir_level_1 if PC in CameraSystem_L: utility.mkdir(pathSave) str_level_2_list = [ ["181114", "0018", "D116"], ["181120", "1959", "D115"], ["181123", "2139", "D115_newSil"], ["181123", "2333", "D115_newSil_fixImageProcessed"], ["181124", "0021", "D114"], ["181125", "1927", "D114_root16"], ["181125", "2130", "D113_root16"], ["181126", "2152", "D113_root16_weight2"], ["181126", "2152", "D112_root16_weight2"], ["181128", "0310", "D111_root16_weight2"], ["181211", "1731", "D111_test"], ["181211", "2103", "D111_cluster"], ["181211", "2236", "D111_cluster25000"],
def join_runs(run_list, joined_fname_root): try: shutil.__name__ deque.__name__ except NameError: import shutil from collections import deque # Create the directory structure mkdir(joined_fname_root+'/') mkdir(joined_fname_root+'/chains/') mkdir(joined_fname_root+'/results/') mkdir(joined_fname_root+'/plots/') # Do some checks to ensure that the runs we are joining are compatible with # each other... for param in ['problem_name', 'n_avg', 'flag_autocor_on_file', 'flag_no_map_write', 'n_P', 'P_name', 'P_limit_min', 'P_limit_max', 'n_DS_arrays_total', 'n_DS']: check_param_compatibility(run_list, param) # Copy the run.dat file for the first run as our new run.dat file shutil.copy(run_list[0].filename_root+'/run.dat', joined_fname_root) # Read in the chain config files and work out the values for our new file n_iterations = 0 n_iterations_burn = None chain_list = deque() for run in run_list: for i_chain in xrange(run.n_chains): chain = Chain(run, i_chain) chain_list.append(chain) if n_iterations_burn==None: n_iterations_burn=chain.n_iterations_burn n_iterations += chain.n_iterations else: n_iterations += chain.n_iterations-chain.n_iterations_burn covariance_matrix = chain_list[0].covariance_matrix temp = chain_list[0].temp # Write the new chain config file fout = open(joined_fname_root+'/chains/chain_config_%06d.dat'%(0), 'wb') np.array([n_iterations], dtype=np.int32).tofile(fout) np.array([n_iterations_burn], dtype=np.int32).tofile(fout) np.array([temp], dtype=np.float64).tofile(fout) covariance_matrix.flatten().tofile(fout) fout.close() # Concatenate all of the chain stats files together fout = open(joined_fname_root+'/chains/chain_stats_%06d.dat'%(0), 'wb') first_burn = True for i_run, run in enumerate(run_list): for i_chain in xrange(run.n_chains): fin = open(run.filename_root+'/chains/chain_stats_%06d.dat'%(i_chain), 'rb') if first_burn==False: # Seek past the burn of this chain n_iterations_burn = chain_list[i_run+i_chain].n_iterations_burn byte_seek = (8*run.n_P*12)+(8*6) if run.flag_autocor_on_file==1: byte_seek+=(run.n_avg-1)*8 fin.seek(byte_seek*n_iterations_burn,1) else: first_burn = False shutil.copyfileobj(fin, fout) fin.close() fout.close() # Concatenate all of the chain trace files together fout = open(joined_fname_root+'/chains/chain_trace_%06d.dat'%(0), 'wb') first_burn = True for run in run_list: for i_chain in xrange(run.n_chains): fin = open(run.filename_root+'/chains/chain_trace_%06d.dat'%(i_chain), 'rb') if first_burn == False: # Seek past the burn of this chain chain = chain_list[i_run+i_chain] n_iterations_burn = chain.n_iterations_burn n_avg = run.n_avg byte_seek = 1+8+(8*run.n_P) if run.flag_no_map_write==0: for i_DS in xrange(run.n_DS): byte_seek += 8*run.n_M[i_DS] fin.seek(byte_seek*n_iterations_burn*n_avg,1) else: first_burn = False shutil.copyfileobj(fin, fout) fin.close() fout.close()
tfboard_name = exp_name + "_" exp_train_log_dir = os.path.join(train_log_dir, exp_name) LOG_DIR = os.path.join(exp_train_log_dir, 'logs') # save img path IMG_SAVE_DIR = os.path.join(exp_train_log_dir, 'img_log') # Where to load model LOAD_MODEL_DIR = os.path.join(exp_train_log_dir, 'models') # Where to save new model SAVE_MODEL_DIR = os.path.join(exp_train_log_dir, 'real_models') # Where to save visualization images (for report) RESULTS_DIR = os.path.join(exp_train_log_dir, 'report') utility.mkdir(SAVE_MODEL_DIR) utility.mkdir(IMG_SAVE_DIR) utility.mkdir(LOG_DIR) class Trainer(): def __init__(self, args, train_loader, train_sampler, valid_loader, my_model, my_loss, ckp): self.args = args self.scale = args.scale[0] self.ckp = ckp self.loader_train = train_loader self.loader_valid = valid_loader self.train_sampler = train_sampler self.model = my_model
def main(): args = get_args() sample_name = os.path.splitext(os.path.basename(args.fasta1))[0] if args.presets == "ont": preset = "map-ont" elif args.presets == "pacbio": preset = "map-pb" else: print("unrecognized preset provided, used pacbio mode") preset = "map-pb" overlap = 20 gap = 20 stat_summary = args.out + "/" + "stat.summary.txt" rm_file(stat_summary) # telr output files telr_bed = [] for file in glob.glob(args.telr_dir + "/*.telr.vcf"): telr_bed.append(file) prefix = os.path.basename(telr_bed[0]).replace('.telr.vcf', '') telr_bed = args.telr_dir + "/" + prefix + '.telr.vcf' telr_contigs = args.telr_dir + "/intermediate_files/" + prefix + '.contigs.fa' telr_meta = args.telr_dir + '/intermediate_files/' + prefix + ".final.meta.tsv" telr_contig_te = args.telr_dir +"/intermediate_files/" + prefix + ".te2contig_filter.bed" telr_family = args.telr_dir + "/intermediate_files/" + prefix + ".te2contig_rm.merge.bed" # filter genome 1 TE annotation by focus regions and families ## TODO: check annotation file format? if "bed" in args.annotation1: annotation_filter = args.out + "/" + sample_name + ".annotation.filter.bed" filter_annotation(args.annotation1, annotation_filter, args.out, sample_name, region = args.region1, discard_family = args.discard_family) else: annotation_filter = args.out + "/" + sample_name + ".annotation.filter.gff" filter_annotation(args.annotation1, annotation_filter, args.out, sample_name, region = args.region1, discard_family = args.discard_family) annotation_filter_bed = args.out + "/" + sample_name + ".annotation.filter.bed" liftover.gff_to_bed(annotation_filter, annotation_filter_bed) annotation_filter = annotation_filter_bed annotation_set = get_set(annotation_filter) out_line = "annotated TEs in genome 1: " + str(len(annotation_set)) write_report(stat_summary, out_line) # filter TELR predictions telr_filter = args.out + "/" + prefix + ".telr.filter.bed" filter_annotation(telr_meta, telr_filter, args.out, sample_name, region = args.region1, discard_family = args.discard_family) telr_contig_set = set() with open(telr_filter, "r") as input: for line in input: entry = line.replace('\n', '').split("\t") telr_contig_set.add(entry[5]) telr_contig_set = get_set(telr_filter, type = 2) out_line = "non-ref TEs predicted by TELR: " + str(len(telr_contig_set)) write_report(stat_summary, out_line) ################### liftover and overlap with TELR predictions ################### out_line = "\n### stats by liftover TEs from genome1 to genome2 and compare with TELR predictions ###" write_report(stat_summary, out_line) # liftover from one genome to another te_lift, te_lift_meta = liftover.lift_annotation(fasta1 = args.fasta1, fasta2 = args.fasta2, bed = annotation_filter, sample_name = sample_name, out_dir = args.out, preset = preset, overlap = overlap, gap = gap, flank_len = 500) # remove reference lifted annotations that overlap with genome2 TE annotations te_lift_filter = args.out + "/" + sample_name + ".lift.nonref.bed" with open(te_lift_filter, "w") as output: subprocess.call(["bedtools", "intersect", "-a", te_lift, "-b", args.annotation2, "-v"], stdout = output) lift_set = get_set(te_lift_filter) out_line = "non-ref TEs lifted from genome1 to genome2: " + str(len(lift_set)) write_report(stat_summary, out_line) # overlap overlap = args.out + "/" + sample_name + ".overlap.bed" window = 3 with open(overlap, "w") as output: subprocess.call(["bedtools", "window", "-w", str(window), "-a", te_lift_filter, "-b", telr_filter], stdout = output) share_lift_set = set() share_telr_contig_set = set() with open(overlap, "r") as input: for line in input: entry = line.replace('\n', '').split("\t") if entry[3].lower() in entry[9].lower(): share_lift_set.add('_'.join(entry[0:4])) share_telr_contig_set.add(entry[11]) out_line = "lifted non-ref TEs supported by TELR predictions: " + str(len(share_lift_set)) write_report(stat_summary, out_line) lift_only_set = lift_set.difference(share_lift_set) out_line = "non-ref TEs predicted by TELR supported by liftover set: " + str(len(share_telr_contig_set)) write_report(stat_summary, out_line) lift_only_set = lift_set.difference(share_lift_set) out_line = "non-ref TEs only in liftover set: " + str(len(lift_only_set)) write_report(stat_summary, out_line) telr_only_contig_set = telr_contig_set.difference(share_telr_contig_set) out_line = "non-ref TEs only in TELR set: " + str(len(telr_only_contig_set)) write_report(stat_summary, out_line) rm_file(overlap) ################## TELR flanks mapped to genome 1 ################### print("Lift flanking sequences from TELR contigs to genome 1...") out_line = "\n### stats by liftover TEs from TELR contigs to genome1 and compare with genome1 annotations ###" write_report(stat_summary, out_line) # generate bed file contig_te_strand_dict = dict() with open(telr_family, "r") as input: for line in input: entry = line.replace('\n', '').split("\t") contig_te_strand_dict[entry[0]] = entry[4] contig_te_bed = args.out + "/" + prefix + ".contig.te.filter.bed" with open(contig_te_bed, "w") as output, open(telr_meta, "r") as input: for line in input: entry = line.replace('\n', '').split("\t") if entry[5] in telr_contig_set: family = entry[3] contig_name = re.sub(":.*", "", entry[5]) coord = re.sub(".*:", "", entry[5]) start = coord.split('-')[0] end = coord.split('-')[1] strand = contig_te_strand_dict[contig_name] out_line = '\t'.join([contig_name, start, end, family, strand]) output.write(out_line + '\n') sample_name = prefix overlap = 20 gap = 30000 flank_lift, flank_lift_meta = liftover.lift_annotation(fasta1 = telr_contigs, fasta2 = args.fasta1, bed = contig_te_bed, sample_name = sample_name, out_dir = args.out, preset = preset, overlap = overlap, gap = gap, flank_len = 500) telr_lift_contig_set = set() with open(flank_lift_meta, "r") as input: for line in input: entry = line.replace('\n', '').split("\t") telr_lift_contig_set.add(entry[5]) telr_unlift_set = telr_contig_set.difference(telr_lift_contig_set) out_line = "TEs can not be lifted from TELR contigs to genome1: " + str(len(telr_unlift_set)) write_report(stat_summary, out_line) out_line = "TEs lifted from TELR contigs to genome1: " + str(len(telr_lift_contig_set)) write_report(stat_summary, out_line) # overlap overlap = args.out + "/" + sample_name + ".genome1.overlap.bed" window = 3 with open(overlap, "w") as output: subprocess.call(["bedtools", "window", "-w", str(window), "-a", flank_lift_meta, "-b", annotation_filter], stdout = output) share_telr_lift_contig_set = set() with open(overlap, "r") as input: for line in input: entry = line.replace('\n', '').split("\t") family_telr_lift_genome1 = entry[3].lower() family_annotation_genome1 = entry[10].lower() if family_annotation_genome1 in family_telr_lift_genome1: share_telr_lift_contig_set.add(entry[5]) out_line = "TEs lifted from TELR contigs supported by genome 1 annotation: " + str(len(share_telr_lift_contig_set)) write_report(stat_summary, out_line) telr_lift_only_set = telr_lift_contig_set.difference(share_telr_lift_contig_set) out_line = "TEs lifted from TELR contigs not supported by genome 1 annotation: " + str(len(telr_lift_only_set)) write_report(stat_summary, out_line) rm_file(overlap) #TODO compare sequence quality for all the overlapped? # for TELR only and unlifted flanks, map the contig to dm6 and do dnadiff comparison print("For TELR unlifted flanks and lifted flanks without genome1 annotation support, map contig to genome1") telr_check_set = telr_unlift_set.union(telr_lift_only_set) # out_line = "TELR TEs not lifted or not supported by genome1 annotation: " + str(len(telr_check_set)) # write_report(stat_summary, out_line) # then, extract contigs and map to dm6 telr_check_contig_list = args.out + "/" + sample_name + ".check.contig.txt" telr_check_contig_set = set() with open(telr_check_contig_list, "w") as output: for item in telr_check_set: contig = item.split(':')[0] telr_check_contig_set.add(contig) output.write(contig + '\n') telr_check_contigs = args.out + "/" + sample_name + ".check.contig.fa" with open(telr_check_contigs, "w") as output: subprocess.call(["seqtk", "subseq", telr_contigs, telr_check_contig_list], stdout = output) rm_file(telr_check_contig_list) telr_check_align = args.out + "/" + sample_name + ".check.contig.paf" with open(telr_check_align, "w") as output: subprocess.call(["minimap2", "-cx", preset, "-v", "0", "--secondary=no", args.fasta1, telr_check_contigs], stdout = output) rm_file(telr_check_contigs) # select full length mapped contig, compare using dnadiff and make mummer plot telr_check_align_contig_set = set() telr_check_dir = args.out + '/' + 'compare_telr_contig_genome1' mkdir(telr_check_dir) with open(telr_check_align, "r") as input: for line in input: entry = line.replace('\n', '').split("\t") contig_telr = entry[0].split('_')[0] if contig_telr == entry[5] and (int(entry[1]) * 0.8) < (int(entry[3]) - int(entry[2])): telr_check_align_contig_set.add(entry[0]) genome1_extract_fa = telr_check_dir + "/" + '_'.join([entry[5], entry[7], entry[8]]) + '.genome1.fa' extract_seq(args.fasta1, entry[5], entry[7], entry[8], genome1_extract_fa) contig_fa = telr_check_dir + '/' + entry[0] + '.contig.fa' with open(contig_fa, "w") as output: command = "samtools faidx " + telr_contigs + " " + entry[0] subprocess.call(command, shell = True, stdout = output) # dnadiff prefix_mummer = telr_check_dir + '/' + entry[0] subprocess.call(["dnadiff", contig_fa, genome1_extract_fa, "-p", prefix_mummer], stderr = subprocess.DEVNULL) mdelta = prefix_mummer + '.mdelta' subprocess.call(["mummerplot", "-p", prefix_mummer, "-s", "medium", "-f", "--png", mdelta, "--color"], stdout = subprocess.DEVNULL, stderr = subprocess.STDOUT) clean_mummer(prefix_mummer) rm_file(genome1_extract_fa) rm_file(contig_fa) telr_no_align_set = telr_check_contig_set.difference(telr_check_align_contig_set) rm_file(telr_check_align) # generate contig summary file contig_summary = args.out + "/" + "contig.summary.tsv" with open(contig_summary, "w") as output: out_line = '\t'.join(["contig_name", "genome2_support", "genome1_support"]) output.write(out_line + '\n') for ins in telr_contig_set: if ins in share_telr_contig_set: genome2_support = "yes" else: genome2_support = "no" if ins in telr_lift_contig_set: if ins in share_telr_lift_contig_set: genome1_support = "yes" else: genome1_support = "no" else: genome1_support = "TELR_flank_unlifted" contig_name = ins.split(':')[0] out_line = '\t'.join([contig_name, genome2_support, genome1_support]) output.write(out_line + '\n') # generate stats for contig summary file and write in final stats report out_line = "\n### stats by comparing two evaluation methods ###" write_report(stat_summary, out_line) out_line = "within " + str(len(telr_only_contig_set)) + " non-ref TEs in TELR set not in liftover TE set in genome2" write_report(stat_summary, out_line) num = len(telr_only_contig_set.intersection(share_telr_lift_contig_set)) out_line = str(num) + " TEs can be lifted to genome1 and supported by genome1 annotations (real support insertions)" write_report(stat_summary, out_line) num = len(telr_only_contig_set.intersection(telr_lift_only_set)) out_line = str(num) + " TEs can be lifted to genome1 but not supported by genome1 annotations (potential new insertions in dataset relative to genome1 or misannotation in genome1)" write_report(stat_summary, out_line) num = len(telr_only_contig_set.intersection(telr_unlift_set)) out_line = str(num) + " TEs can be not be lifted to genome1 (potential false positive or new insertions in dataset relative to genome1)" write_report(stat_summary, out_line) print("evaluation workflow finished!")
def join_runs(run_list, joined_fname_root): try: shutil.__name__ deque.__name__ except NameError: import shutil from collections import deque # Create the directory structure mkdir(joined_fname_root + '/') mkdir(joined_fname_root + '/chains/') mkdir(joined_fname_root + '/results/') mkdir(joined_fname_root + '/plots/') # Do some checks to ensure that the runs we are joining are compatible with # each other... for param in [ 'problem_name', 'n_avg', 'flag_autocor_on_file', 'flag_no_map_write', 'n_P', 'P_name', 'P_limit_min', 'P_limit_max', 'n_DS_arrays_total', 'n_DS' ]: check_param_compatibility(run_list, param) # Copy the run.dat file for the first run as our new run.dat file shutil.copy(run_list[0].filename_root + '/run.dat', joined_fname_root) # Read in the chain config files and work out the values for our new file n_iterations = 0 n_iterations_burn = None chain_list = deque() for run in run_list: for i_chain in xrange(run.n_chains): chain = Chain(run, i_chain) chain_list.append(chain) if n_iterations_burn == None: n_iterations_burn = chain.n_iterations_burn n_iterations += chain.n_iterations else: n_iterations += chain.n_iterations - chain.n_iterations_burn covariance_matrix = chain_list[0].covariance_matrix temp = chain_list[0].temp # Write the new chain config file fout = open(joined_fname_root + '/chains/chain_config_%06d.dat' % (0), 'wb') np.array([n_iterations], dtype=np.int32).tofile(fout) np.array([n_iterations_burn], dtype=np.int32).tofile(fout) np.array([temp], dtype=np.float64).tofile(fout) covariance_matrix.flatten().tofile(fout) fout.close() # Concatenate all of the chain stats files together fout = open(joined_fname_root + '/chains/chain_stats_%06d.dat' % (0), 'wb') first_burn = True for i_run, run in enumerate(run_list): for i_chain in xrange(run.n_chains): fin = open( run.filename_root + '/chains/chain_stats_%06d.dat' % (i_chain), 'rb') if first_burn == False: # Seek past the burn of this chain n_iterations_burn = chain_list[i_run + i_chain].n_iterations_burn byte_seek = (8 * run.n_P * 12) + (8 * 6) if run.flag_autocor_on_file == 1: byte_seek += (run.n_avg - 1) * 8 fin.seek(byte_seek * n_iterations_burn, 1) else: first_burn = False shutil.copyfileobj(fin, fout) fin.close() fout.close() # Concatenate all of the chain trace files together fout = open(joined_fname_root + '/chains/chain_trace_%06d.dat' % (0), 'wb') first_burn = True for run in run_list: for i_chain in xrange(run.n_chains): fin = open( run.filename_root + '/chains/chain_trace_%06d.dat' % (i_chain), 'rb') if first_burn == False: # Seek past the burn of this chain chain = chain_list[i_run + i_chain] n_iterations_burn = chain.n_iterations_burn n_avg = run.n_avg byte_seek = 1 + 8 + (8 * run.n_P) if run.flag_no_map_write == 0: for i_DS in xrange(run.n_DS): byte_seek += 8 * run.n_M[i_DS] fin.seek(byte_seek * n_iterations_burn * n_avg, 1) else: first_burn = False shutil.copyfileobj(fin, fout) fin.close() fout.close()
def __make_dir__(self): utility.mkdir(self.screenShot) utility.mkdir(self.screenShot_keys) utility.mkdir(self.screenShot_result) utility.mkdir(self.__my_dict__)
from bs4 import BeautifulSoup import utility from scraper import Story, HTMLFormatter from utility import Settings settings_file = "settings.dat" settings = Settings(settings_file) # http://forums.spacebattles.com/threads/survival-of-the-fittest-worm-si.297753/ # http://forums.spacebattles.com/threads/to-go-a-viking-asoiaf-au.294304/ url = easygui.enterbox("URL:", "SBDownloader") if not utility.is_site_down(url): doc = utility.get_page(url) else: doc = utility.load("test.html") soup = BeautifulSoup(doc, "html.parser") story = Story.parse(soup) story.download_messages() fmt = HTMLFormatter() doc = fmt.export_story(story, settings.author_only) utility.mkdir(settings.download_path) utility.save(doc, os.path.join(settings.download_path, story.clean_title+".html")) settings.store()
def __enter__(self): utility.mkdir(self.imageFolderName) return self
def get_B_C_breadthFirst(i_view, cf_getBC): time_calcu = False begin = datetime.datetime.now() B = [] C = [] if cf_getBC.shape_or_pose == "sil_for_deep": postfix = "jpg" else: postfix = "bmp" image = get8ImageFromImage(path=cf_getBC.path_video, SID=cf_getBC.i_frame, views=[i_view], postfix=postfix) if time_calcu == True: print("\ni_frame: %d" % cf_getBC.i_frame) print("Time of getting 1 image: " + str((datetime.datetime.now() - begin))) cf_getBC.i_view = i_view cluster_quad_breadthFirst(im=image[0], leftup=cf_getBC.leftup, rightdown=cf_getBC.rightdown, B=B, C=C, cf_getBC=cf_getBC) if time_calcu == True: print("Time of clustering 1 image: " + str((datetime.datetime.now() - begin))) # write to file if 1: pathSave_imageClustered = cf.pathVideo_ + "v%d/" % i_view utility.mkdir(pathSave_imageClustered) # max_cluster = 5000 file = open( pathSave_imageClustered + "image%d_B.txt" % cf_getBC.i_frame, 'w') # i_cluster = 0 for fp in B: file.write(str(fp)) file.write('\n') # i_cluster = i_cluster + 1 # if i_cluster == max_cluster: # break file.close() file = open( pathSave_imageClustered + "image%d_C.txt" % cf_getBC.i_frame, 'w') # i_cluster = 0 for fp in C: file.write(str(fp)) file.write('\n') # i_cluster = i_cluster + 1 # if i_cluster == max_cluster: # break file.close() if time_calcu == True: print("Time of saving 1 cluster of image: " + str((datetime.datetime.now() - begin))) return