class MassScraping(Scrap):
def __init__(self, queries_list):
super().__init__()
self.qb = QueryBuilder()
# queries must be of list type
if type(queries_list) != list and type(queries_list) != dict:
print(
'Error Mass Scraping queries must be of type list, or dictionary',
type(queries_list), 'is given!')
exit()
# set query
self.qb.set_query(queries_list)
self.queries = self.qb.get_query()
def start(self, ask_for_rerun=False):
"""
Start running mass scraping, you can prompt every saved
query and ask for rerun the query again by setting the
"ask_for_rerun" option to True <default: False>
:param ask_for_rerun:
:return:
"""
for query in self.queries:
# query check
if type(query) == dict:
result = self.qb.check_query(query['query'])
else:
result = self.qb.check_query(query)
if result:
print(f'"{query}" query already performed on: {result[2]}')
# Prompt the user for rerun the query
if ask_for_rerun:
rerun = input(
"Do you like to perform this query again?[Yes|No] "
).lower()
while rerun != 'yes' and rerun != 'no':
rerun = input(
"Do you like to perform this query again?[Yes|No] "
).lower()
if rerun == 'yes':
self.run(query)
else:
continue
else:
self.run(query)
class DetailsScraping(Scrap):
def __init__(self, query):
super().__init__()
self.db = QueryBuilder()
# query must be type of string
if type(query) != str:
print('Error Details Scraping query must be of type string,',
type(query), 'given!')
exit()
# set query
self.db.set_query(query)
self.query = self.db.get_query()
def start(self, display=False):
# TODO:: Code the "display data" option.
if self.db.check_query(self.query):
print(f'"{self.query}" query already done!')
else:
self.run(self.query)