def __init__(self, kromsatel_args):
super().__init__(kromsatel_args)
self.cleaner = NanoporeReadsCleaner(kromsatel_args)
self.reads_fpath = self.kromsatel_args.long_read_fpath
self.chunk_size = self.kromsatel_args.chunk_size
num_reads_total = \
_count_unpaired_reads_verbosely(self.reads_fpath)
self.progress = Progress(num_reads_total)
output_prefix = fs.rm_fastq_extention(
os.path.basename(self.reads_fpath)
)
if kromsatel_args.split_output:
self.binner = SplitUnpairedBinner(
self.kromsatel_args.outdir_path,
output_prefix,
self.kromsatel_args.min_len
)
else:
self.binner = SimpleUnpairedBinner(
self.kromsatel_args.outdir_path,
output_prefix,
self.kromsatel_args.min_len
)
def test_add_item_scraped():
progress = Progress()
progress.init()
progress_result = progress.read_progress()
assert progress_result['items_scraped'] == 0
assert progress_result['total'] == 0
assert len(progress_result['items']) == 0
some_object = {'property': 'value'}
progress.add_item_scraped(some_object)
progress_result = progress.read_progress()
assert len(progress_result['items']) == 1
def run_freida_scraping():
try:
progress = Progress()
progress.init()
progress.save_process_progress(False, False)
browser = setup_browser()
config = FreidaConfig(browser)
url = config.initial_page
main(browser, url, config)
browser.quit()
except Exception:
if browser:
browser.quit()
def run_scraping_task():
try:
progress = Progress()
progress.init()
progress.save_process_progress(False,False)
# TODO Remove if not using selenium
browser = setup()
config = Config(browser) #TODO: Pass browser if the crawling is selenium based
url = config.initial_page
main(browser, config)
browser.quit()
except Exception:
if browser:
browser.quit()
class LongReadKromsatelCore(KromsatelCore):
# TODO: do not save reference to kromsatel args by creating some "BlastArguments" class
def __init__(self, kromsatel_args):
super().__init__(kromsatel_args)
self.cleaner = NanoporeReadsCleaner(kromsatel_args)
self.reads_fpath = self.kromsatel_args.long_read_fpath
self.chunk_size = self.kromsatel_args.chunk_size
num_reads_total = \
_count_unpaired_reads_verbosely(self.reads_fpath)
self.progress = Progress(num_reads_total)
output_prefix = fs.rm_fastq_extention(
os.path.basename(self.reads_fpath)
)
if kromsatel_args.split_output:
self.binner = SplitUnpairedBinner(
self.kromsatel_args.outdir_path,
output_prefix,
self.kromsatel_args.min_len
)
else:
self.binner = SimpleUnpairedBinner(
self.kromsatel_args.outdir_path,
output_prefix,
self.kromsatel_args.min_len
)
# end if
# end def
def run(self):
reads_chunks = src.fastq.fastq_chunks_unpaired(
fq_fpath=self.reads_fpath,
chunk_size=self.chunk_size
)
self.progress.print_status_bar()
self._clean_chunks(reads_chunks)
self.progress.print_status_bar()
print()
# end def
def _clean_chunks(self, reads_chunks):
with mp.Pool(self.threads_num) as pool:
task_iterator = pool.imap(
self._clean_nanopore_chunk,
reads_chunks,
chunksize=1
)
for task in task_iterator:
pass
# end for
# end with
pool.close()
pool.join()
# end def
def _clean_nanopore_chunk(self, reads_chunk):
alignments = parse_alignments_nanopore(
src.blast.blast_align(reads_chunk, self.kromsatel_args)
)
self.cleaner.fill_binner(reads_chunk, alignments, self.binner)
self._write_output()
increment = len(reads_chunk)
self._update_progress(increment)
self._print_progress()
class IlluminaPEKromsatelCore(KromsatelCore):
def __init__(self, kromsatel_args):
super().__init__(kromsatel_args)
self.cleaner = IlluminaPEReadsCleaner(kromsatel_args)
self.frw_read_fpath = self.kromsatel_args.frw_read_fpath
self.rvr_read_fpath = self.kromsatel_args.rvr_read_fpath
self.chunk_size = self.kromsatel_args.chunk_size
num_reads_total = \
_count_paired_reads_verbosely(self.frw_read_fpath)
self.progress = Progress(num_reads_total)
output_prefix = fs.rm_fastq_extention(
os.path.basename(self.frw_read_fpath)
)
if kromsatel_args.split_output:
self.binner = SplitPairedBinner(
self.kromsatel_args.outdir_path,
output_prefix,
self.kromsatel_args.min_len
)
else:
self.binner = SimplePairedBinner(
self.kromsatel_args.outdir_path,
output_prefix,
self.kromsatel_args.min_len
)
# end if
# end def
def run(self):
reads_chunks = src.fastq.fastq_chunks_paired(
frw_read_fpath=self.frw_read_fpath,
rvr_read_fpath=self.rvr_read_fpath,
chunk_size=self.chunk_size
)
self.progress.print_status_bar()
self._clean_chunks(reads_chunks)
self.progress.print_status_bar()
print()
# end def
def _clean_chunks(self, reads_chunks):
with mp.Pool(self.threads_num) as pool:
task_iterator = pool.imap(
self._clean_illumina_pe_chunk,
reads_chunks,
chunksize=1
)
for task in task_iterator:
pass
# end for
# end with
pool.close()
pool.join()
# end def
def _clean_illumina_pe_chunk(self, reads_chunk):
alignments = self._align_read_pairs(reads_chunk)
self.cleaner.fill_binner(reads_chunk, alignments, self.binner)
self._write_output()
increment = len(reads_chunk[0])
self._update_progress(increment)
self._print_progress()
# end def
def _align_read_pairs(self, reads_chunk):
frw_chunk = reads_chunk[0]
frw_alignments = parse_alignments_illumina(
src.blast.blast_align(frw_chunk, self.kromsatel_args)
)
rvr_chunk = reads_chunk[1]
rvr_alignments = parse_alignments_illumina(
src.blast.blast_align(rvr_chunk, self.kromsatel_args)
)
alignments = (frw_alignments, rvr_alignments)
return alignments
import sys
import os
from flask_bootstrap import Bootstrap
from flask import Flask, render_template, jsonify, redirect
from src.progress import Progress
from src.crawling_threading import CrawlingThreading
if getattr(sys, 'frozen', False):
template_folder = os.path.join(sys._MEIPASS, 'templates')
app = Flask(__name__, template_folder=template_folder)
else:
template_folder = os.path.join(os.path.dirname(__file__), 'templates')
app = Flask(__name__, template_folder=template_folder)
Bootstrap(app)
progress = Progress()
progress.init()
@app.route('/')
def app_entrypoint():
global progress
try:
progress_result = progress.read_progress()
except Exception:
progress_result = progress
return render_template('index.html', progress=progress_result)
@app.route('/scrape')
def scrape_beerwulf():
def notify(self, file) -> None:
self.queue.put(Progress(file, self.done.value, self.failed.value, self.files_count))
def test_read_total_number_items():
progress = Progress()
progress.init()
total_number_items = progress.read_total_number_items()
assert total_number_items == 0
def test_save_number_items_scraped_so_far():
progress = Progress()
progress.init()
total_number_items = progress.read_total_number_items()
items_scraped_so_far = progress.read_number_items_scraped_so_far()
assert total_number_items == 0
assert items_scraped_so_far == 0
progress.save_number_items_scraped_so_far(10)
progress.save_total_number_items(5)
items_scraped_so_far = progress.read_number_items_scraped_so_far()
total_number_items = progress.read_total_number_items()
assert items_scraped_so_far == 10
assert total_number_items
def crawl_all_pages_to_end(self, initial_url, file_to_save_results):
try:
url_first_load = initial_url
self.browser.get(
"https://freida.ama-assn.org/Freida/#/programs?program=residencies&specialtiesToSearch=140"
)
self.config.answer_prompt_questions()
# Wait for page to load
wait = WebDriverWait(self.browser,
10,
ignored_exceptions=[
StaleElementReferenceException,
NoSuchElementException
])
items_presence = EC.presence_of_element_located(
(By.CSS_SELECTOR, self.config.next_button_css_selector))
wait.until(items_presence)
self.browser.execute_script('window.open("' + url_first_load +
'");')
# Swith to new tab
self.browser.switch_to.window(self.browser.window_handles[1])
# Create dataframe to hold the data
dataframe_file = Path(file_to_save_results)
if (not dataframe_file.exists()):
dataframe = pd.DataFrame()
else:
dataframe = pd.read_csv(file_to_save_results)
jsonData = self.loadJsonContent(self.browser.page_source)
progress = Progress()
total_number_items = self.total_number_items_to_scrape(jsonData)
progress.save_total_number_items(total_number_items)
progress.save_process_progress(False, False)
for pagination_item in jsonData['solrPagination']:
url_to_parse = urljoin(self.config.host,
pagination_item["url"])
self.browser.get(url_to_parse)
page_json_data = self.loadJsonContent(self.browser.page_source)
for item in page_json_data["searchResults"]:
cleaned_item = self.extract_item_data(item)
item_serie = pd.Series(cleaned_item,
index=cleaned_item.keys())
dataframe = dataframe.append(item_serie,
ignore_index=True,
sort=False)
progress.save_number_items_scraped_so_far(
dataframe.shape[0])
progress.add_item_scraped(cleaned_item)
progress.save_process_progress(False, False)
dataframe.to_csv(file_to_save_results)
progress.save_process_progress(True, False)
return True
except Exception as e:
print(str(e))
progress.save_process_progress(True, True)
return False