Пример #1
0
 def __init__(self, queries_list):
     super().__init__()
     self.qb = QueryBuilder()
     # queries must be of list type
     if type(queries_list) != list and type(queries_list) != dict:
         print('Error Mass Scraping queries must be of type list, or dictionary', type(queries_list) , 'is given!')
         exit()
     # set query
     self.qb.set_query(queries_list)
     self.queries = self.qb.get_query()
Пример #2
0
 def __init__(self, query):
     super().__init__()
     self.db = QueryBuilder()
     # query must be type of string
     if type(query) != str:
         print('Error Details Scraping query must be of type string,',
               type(query), 'given!')
         exit()
     # set query
     self.db.set_query(query)
     self.query = self.db.get_query()
Пример #3
0
class MassScraping(Scrap):
    def __init__(self, queries_list):
        super().__init__()
        self.qb = QueryBuilder()
        # queries must be of list type
        if type(queries_list) != list and type(queries_list) != dict:
            print(
                'Error Mass Scraping queries must be of type list, or dictionary',
                type(queries_list), 'is given!')
            exit()
        # set query
        self.qb.set_query(queries_list)
        self.queries = self.qb.get_query()

    def start(self, ask_for_rerun=False):
        """
        Start running mass scraping, you can prompt every saved
        query and ask for rerun the query again by setting the
        "ask_for_rerun" option to True <default: False>

        :param ask_for_rerun:
        :return:
        """
        for query in self.queries:
            # query check
            if type(query) == dict:
                result = self.qb.check_query(query['query'])
            else:
                result = self.qb.check_query(query)

            if result:
                print(f'"{query}" query already performed on: {result[2]}')

                # Prompt the user for rerun the query
                if ask_for_rerun:
                    rerun = input(
                        "Do you like to perform this query again?[Yes|No] "
                    ).lower()

                    while rerun != 'yes' and rerun != 'no':
                        rerun = input(
                            "Do you like to perform this query again?[Yes|No] "
                        ).lower()

                    if rerun == 'yes':
                        self.run(query)
                    else:
                        continue
            else:
                self.run(query)
Пример #4
0
class DetailsScraping(Scrap):
    def __init__(self, query):
        super().__init__()
        self.db = QueryBuilder()
        # query must be type of string
        if type(query) != str:
            print('Error Details Scraping query must be of type string,',
                  type(query), 'given!')
            exit()
        # set query
        self.db.set_query(query)
        self.query = self.db.get_query()

    def start(self, display=False):
        # TODO:: Code the "display data" option.

        if self.db.check_query(self.query):
            print(f'"{self.query}" query already done!')

        else:
            self.run(self.query)