Exemple #1
0
 def __init__(self, country_manager):
     """
     Creates a Manager. Important, **it gets all urls and descriptions from
     database on creation**.
     args:
         country_manager: db_manager object which represents the specific
         country connection.
     """
     self.__country_manager = country_manager
     self.__db = None
     self.__urls = {}
     self.__descriptions = {}
     self.__sl = slavy.slavy()
     self.__load_check_info()
Exemple #2
0
 def __init__(self, crawling_config, classifieds_table=u'classifieds',
     cities_table=u'location',categories_table=u'cursos_category', checks_in_memory=True):
     """
     args:
         crawling_config: the configuration for crawler.
         classifieds_table: the table name where classifieds are stored.
         cities_table: the table name where cities and provinces are stored.
         checks_in_memory: if True the whole classifieds table will be
             loaded one time and this information will be used to check if
             the classified is already in the database, otherwise this will
             be checked by means on a query on the database.
     """
     
     self.config = crawling_config
     self.classifieds_table = classifieds_table + u"_new"
     self.classifieds_table_old = classifieds_table 
     self.cities_table = cities_table
     self.categories_table = categories_table
     self.checks_in_memory = checks_in_memory
     # Slavy bot to crawl images
     self.__slavy = slavy()
     self.__load_check_info()