def info():
#defining the variables
temperature = []
humidity = []
time = []
dir = "./uploads/"
menu = os.listdir(dir)
# dir = sys.path.append('../modules')
from modules.FileManager import FileManager
stats = FileManager(dir + menu[0])
reading = stats.read()
print(reading)
for temp, humid, timing in reading:
temperature.append(temp)
humidity.append(humid)
time.append(timing)
# print("temperature"+temperature)
print("temperature")
print(temperature)
print("humidity")
print(humidity)
print("time")
print(time)
#print("--------------------------------- raw below ---------------------------------")
reading = stats.read()
return temperature, humidity, time
def get_all_problems(cu: CodeUp):
"""모든 문제"""
print("<< 지금까지 해결한 모든 문제 가져오기(이미 가져온 문제는 제외) >>")
fm = FileManager([SITE_NAME, 'all'])
# 이미 저장된 파일의 목록 가져오기
saved_file_list = fm.get_default_dir_file_list()
# 제외할 problem id 리스트 만들기
exclude_id_list: List[str] = [
file.split('_')[0] for file in saved_file_list
]
# 문제 가져오기
solved_problems = cu.get_solved_problems_all_not_saved(exclude_id_list)
# 파일로 저장
for p in solved_problems:
file_basename = '_'.join([p.id, p.name])
for i, lang_and_source in enumerate(p.lang_and_source):
if i > 0:
fm.write_file(file_basename + f' ({i})', lang_and_source[1],
lang_and_source[0])
else:
fm.write_file(file_basename, lang_and_source[1],
lang_and_source[0])
print(f"{fm.save_cnt} 개의 문제 저장완료."
if fm.save_cnt > 0 else f"이미 모든 문제를 가져왔습니다.")
def input_problem_number(cu: CodeUp):
"""문제번호 입력"""
print("<< 문제번호를 입력하고, 해당 문제 가져오기 >>")
sp = cu.get_solved_problem_by_number()
# 예외 1) 해당 문제번호가 없을 경우: name == '해당 문제를 열람할 수 없습니다!!'
if sp.name == '해당 문제를 열람할 수 없습니다!!':
print('문제번호가 존재하지 않습니다!')
# 예외 2) 해당 문제에 해결한 코드가 없을 경우
elif not sp.lang_and_source:
print('해결한 코드가 없습니다!')
# 파일로 저장
else:
fm = FileManager([SITE_NAME, 'problem_id'])
file_basename = '_'.join([sp.id, sp.name])
for i, lang_and_source in enumerate(sp.lang_and_source):
if i > 0:
fm.write_file(file_basename + f' ({i})', lang_and_source[1],
lang_and_source[0])
else:
fm.write_file(file_basename, lang_and_source[1],
lang_and_source[0])
print(f"{sp.id} 번 문제 저장완료." if fm.save_cnt > 0 else f"이미 저장된 문제입니다.")
def select_problemset(cu: CodeUp):
"""문제집 선택"""
print("<< 문제집 선택하여 해결한 문제만 가져오기 >>")
# 1. 문제집을 선택해서 해결한 문제 가져오기
solved_problems, dirname = cu.get_solved_problems_and_dirname_by_selecting_problemset(
)
# 2. 가져온 문제들을 문제집 폴더에 각각 파일로 저장
fm = FileManager([SITE_NAME, 'problemset', dirname])
is_success, save_cnt = False, 0
for j, p in enumerate(solved_problems, start=1):
file_basename = '_'.join([p.id, p.name])
for i, lang_and_source in enumerate(p.lang_and_source):
is_success = fm.write_file(
file_basename + (f' ({i})' if i > 0 else ''),
lang_and_source[1], lang_and_source[0])
if is_success:
print(f'{file_basename} 저장.')
print(
f"{fm.save_cnt} 개의 문제 저장완료." if fm.save_cnt > 0 else f"이미 저장된 문제집입니다.")
def download(path, ncbi_id, url_list):
db_dir = path + '/homologous_sequences/'
db_dir = FileManager.handle_output_directory(db_dir)
max_pool_size = 3 #API rate limit exceeded, can't go higher
cpus = multiprocessing.cpu_count()
pool = multiprocessing.Pool(
cpus if cpus < max_pool_size else max_pool_size)
for id, url in zip(ncbi_id, url_list):
pool.apply_async(run_process, args=(id, url, db_dir))
pool.close()
pool.join()
file_path = db_dir + '/*'
db_path = path + '/All_homologous_sequences.fna.gz'
os.system('cat {} > {}'.format(file_path, db_path))
return db_path
def polish_genome(assembly, model_path, sketch_path, genus, threads,
output_dir, minimap_args, mash_threshold,
download_contig_nums, debug):
out = []
output_dir = FileManager.handle_output_directory(output_dir)
contig_output_dir_debug = output_dir + '/debug'
contig_output_dir_debug = FileManager.handle_output_directory(
contig_output_dir_debug)
assembly_name = assembly.rsplit('/', 1)[-1]
assembly_name = assembly_name.split('.')[0]
total_start_time = time.time()
for contig in SeqIO.parse(assembly, 'fasta'):
timestr = time.strftime("[%Y/%m/%d %H:%M]")
sys.stderr.write(TextColor.GREEN + str(timestr) + " INFO: RUN-ID: " +
contig.id + "\n" + TextColor.END)
contig_output_dir = contig_output_dir_debug + '/' + contig.id
contig_output_dir = FileManager.handle_output_directory(
contig_output_dir)
contig_name = contig_output_dir + '/' + contig.id + '.fasta'
SeqIO.write(contig, contig_name, "fasta")
if sketch_path:
screen_start_time = time.time()
print_system_log('MASH SCREEN')
mash_file = mash.screen(contig_name, sketch_path, threads,
contig_output_dir, mash_threshold,
download_contig_nums, contig.id)
screen_end_time = time.time()
ncbi_id = mash.get_ncbi_id(mash_file)
if len(
ncbi_id
) < 5: #Would'nt polish if closely-related genomes less than 5
out.append(contig_name)
continue
url_list = download.parser_url(ncbi_id)
if genus:
ncbi_id, url_list = download.parser_genus(genus)
download_start_time = time.time()
print_system_log('DOWNLOAD CONTIGS')
db = download.download(contig_output_dir, ncbi_id, url_list)
download_end_time = time.time()
pileup_start_time = time.time()
print("\n")
print_system_log('PILE UP')
db_npz = alignment.align(contig_name, minimap_args, threads, db,
contig_output_dir)
if db_npz == False:
continue
pileup_end_time = time.time()
align2df_start_time = time.time()
print_system_log('TO DATAFRAME')
df = align2df.todf(contig_name, db_npz, contig_output_dir)
align2df_end_time = time.time()
predict_start_time = time.time()
print_system_log('PREDICT')
df = contig_output_dir + '/' + contig.id + '.feather'
result = predict.predict(df, model_path, threads, contig_output_dir)
predict_end_time = time.time()
polish_start_time = time.time()
print_system_log('POLISH')
finish = polish.stitch(contig_name, result, contig_output_dir)
polish_end_time = time.time()
if sketch_path:
screen_time = get_elapsed_time_string(screen_start_time,
screen_end_time)
print_stage_time('SCREEN', screen_time)
#calculating time
download_time = get_elapsed_time_string(download_start_time,
download_end_time)
pileup_time = get_elapsed_time_string(pileup_start_time,
pileup_end_time)
align2df_time = get_elapsed_time_string(align2df_start_time,
align2df_end_time)
predict_time = get_elapsed_time_string(predict_start_time,
predict_end_time)
polish_time = get_elapsed_time_string(polish_start_time,
polish_end_time)
#print stage time
print_stage_time('DOWNLOAD', download_time)
print_stage_time('PILEUP', pileup_time)
print_stage_time('TO DATAFRAME', align2df_time)
print_stage_time('PREDICT', predict_time)
print_stage_time('POLISH', polish_time)
out.append(finish)
os.system('cat {} > {}/{}_homopolished.fasta'.format(
' '.join(out), output_dir, assembly_name))
if debug:
try:
shutil.rmtree(contig_output_dir_debug)
except OSError as e:
print(e)
else:
return True
total_end_time = time.time()
total_time = get_elapsed_time_string(total_start_time, total_end_time)
print_stage_time('Total', total_time)
def genome_level_parallelization(bam_file, ref_file, vcf_file, output_dir,
max_threads):
"""
This method calls chromosome_level_parallelization for each chromosome.
:param bam_file: BAM file path
:param ref_file: Reference file path
:param vcf_file: VCF file path
:param output_dir: Output directory
:param max_threads: Maximum number of threads to create in chromosome level
:return: Saves a bed file
"""
chr_list = [
"chr1", "chr2", "chr3", "chr4", "chr4", "chr5", "chr6", "chr7", "chr8",
"chr9", "chr10", "chr11", "chr12", "chr13", "chr14", "chr15", "chr16",
"chr17", "chr18", "chr19", "chr20", "chr21", "chr22"
]
program_start_time = time.time()
# chr_list = ["chr1"]
# each chormosome in list
for chr in chr_list:
sys.stderr.write(TextColor.BLUE + "STARTING " + str(chr) +
" PROCESSES" + "\n")
start_time = time.time()
# create dump directory inside output directory
output_dir = create_output_dir_for_chromosome(output_dir, chr)
# do a chromosome level parallelization
chromosome_level_parallelization(chr, bam_file, ref_file, vcf_file,
output_dir, max_threads)
end_time = time.time()
sys.stderr.write(TextColor.PURPLE + "FINISHED " + str(chr) +
" PROCESSES" + "\n")
sys.stderr.write(TextColor.CYAN + "TIME ELAPSED: " +
str(end_time - start_time) + "\n")
# wait for the last process to end before file processing
while True:
if len(multiprocessing.active_children()) == 0:
break
for chr in chr_list:
# here we dumped all the bed files
path_to_dir = output_dir
concatenated_file_name = output_dir + chr + "_labeled.bed"
filemanager_object = FileManager()
# get all bed file paths from the directory
file_paths = filemanager_object.get_file_paths_from_directory(
path_to_dir)
# dump all bed files into one
filemanager_object.concatenate_files(file_paths,
concatenated_file_name)
# delete all temporary files
filemanager_object.delete_files(file_paths)
program_end_time = time.time()
sys.stderr.write(TextColor.RED + "PROCESSED FINISHED SUCCESSFULLY" + "\n")
sys.stderr.write(TextColor.CYAN +
"TOTAL TIME FOR GENERATING ALL RESULTS: " +
str(program_end_time - program_start_time) + "\n")
log.info('STARTED')
app = QApplication(sys.argv)
############################################################################
# CONTEXT
context = Context()
############################################################################
# RESULT TABLE
result_table = ResultTable(context)
############################################################################
# Modules Init
profiler = Profiler()
engine_manager = EngineManager(context, profiler, result_table)
file_manager = FileManager()
############################################################################
# EDITOR
editor = CodeEditor(context)
highlight = CodePainter(editor.document(), STYLES)
window = Main(context, app, editor, engine_manager, file_manager, profiler, result_table)
if context.window['maximized']:
window.showMaximized()
else:
window.show()
############################################################################
# END
sys.exit(app.exec_())