def build_queries(self, constraints_dict): """ Build Weboob ``weboob.capabilities.housing.Query`` objects from the constraints defined in the configuration. Each query has at most 3 postal codes, to comply with housing websites limitations. :param constraints_dict: A dictionary of constraints, as defined in the config. :return: A list of Weboob ``weboob.capabilities.housing.Query`` objects. Returns ``None`` if an error occurred. """ queries = [] for postal_codes in tools.batch(constraints_dict["postal_codes"], 3): query = Query() query.cities = [] for postal_code in postal_codes: matching_cities = [] try: for city in self.webnip.do("search_city", postal_code): matching_cities.append(city) except CallErrors as exc: # If an error occured, just log it LOGGER.error(("An error occured while building query for " "postal code %s: %s"), postal_code, str(exc)) if not matching_cities: # If postal code gave no match, warn the user LOGGER.warn( "Postal code %s could not be matched with a city.", postal_code) # Append the matched cities to the query for city in matching_cities: query.cities.append(city) try: query.house_types = [ getattr(Query.HOUSE_TYPES, house_type.upper()) for house_type in constraints_dict["house_types"] ] except AttributeError: LOGGER.error("Invalid house types constraint.") return None try: query.type = getattr( Query, "TYPE_{}".format(constraints_dict["type"].upper())) except AttributeError: LOGGER.error("Invalid post type constraint.") return None query.area_min = constraints_dict["area"][0] query.area_max = constraints_dict["area"][1] query.cost_min = constraints_dict["cost"][0] query.cost_max = constraints_dict["cost"][1] query.nb_rooms = constraints_dict["rooms"][0] queries.append(query) return queries
def query_for_cities(args): cities = [city(line.strip()) for line in args.infile] if args.debug: logging.basicConfig(level=logging.DEBUG) for handler in logging.root.handlers: handler.setLevel(logging.DEBUG) try: with open('leboncoin.json') as json_file: prev_urls = json.load(json_file) except FileNotFoundError: prev_urls = [] proxy = None if args.proxies: proxy = {'https': random.choice(list(args.proxies)).strip()} LeboncoinBrowser.VERIFY = False requests.packages.urllib3.disable_warnings(InsecureRequestWarning) browser = LeboncoinBrowser(proxy=proxy) query = Query() query.type = POSTS_TYPES[args.type] query.house_types = [HOUSE_TYPES[ht] for ht in args.house_types] if query.area_min: query.area_min = args.area_min if query.area_max: query.area_max = args.area_max if query.cost_min: query.cost_min = args.cost_min if query.cost_max: query.cost_max = args.cost_max if query.nb_rooms: query.nb_rooms = args.nb_rooms new_urls = [] for i in range( 0, len(cities), 10 ): # if the frontend tells the truth, the API supports max 10 cities at a time query.cities = cities[i:i + 10] for housing in browser.search_housings(query, None): new_urls.append(housing.url) diff_urls = r'\n'.join(set(new_urls) - set(prev_urls)) print('Saving {} current housing matches to leboncoin.json'.format( len(new_urls))) with open('leboncoin.json', 'w+') as json_file: json.dump(new_urls, json_file) print(diff_urls.replace(r'\n', '\n')) if diff_urls: msg = r'Nouvelle(s) annonce(s) LeBonCoin:\n' + diff_urls if args.alert_cmd: check_output([args.alert_cmd, msg]) if args.alert_phone_number: from send_text_msg_with_twilio import send_text_msg send_text_msg(os.environ['TWILIO_ACCOUNT_SID'], os.environ['TWILIO_AUTH_TOKEN'], args.alert_phone_number, msg)
def test_entreparticuliers_professional(self): query = Query() query.cities = [] for city in self.backend.search_city('lille'): city.backend = self.backend.name query.cities.append(city) query.type = POSTS_TYPES.SALE query.house_types = [HOUSE_TYPES.HOUSE] query.advert_types = [ADVERT_TYPES.PROFESSIONAL] results = list( itertools.islice(self.backend.search_housings(query), 0, 20)) self.assertEqual(len(results), 0)
def test_entreparticuliers(self): query = Query() query.cities = [] for city in self.backend.search_city('lille'): city.backend = self.backend.name query.cities.append(city) query.type = Query.TYPE_SALE query.house_types = [Query.HOUSE_TYPES.HOUSE] results = list(itertools.islice(self.backend.search_housings(query), 0, 20)) self.assertTrue(len(results) > 0) obj = self.backend.fillobj(results[0]) self.assertTrue(obj.area is not None, 'Area for "%s"' % (obj.id))
def build_queries(self, constraints_dict): """ Build WebOOB ``weboob.capabilities.housing.Query`` objects from the constraints defined in the configuration. Each query has at most 3 cities, to comply with housing websites limitations. :param constraints_dict: A dictionary of constraints, as defined in the config. :return: A list of WebOOB ``weboob.capabilities.housing.Query`` objects. Returns ``None`` if an error occurred. """ queries = [] # First, find all matching cities for the postal codes in constraints matching_cities = [] for postal_code in constraints_dict["postal_codes"]: try: for city in self.webnip.do("search_city", postal_code): matching_cities.append(city) except CallErrors as exc: # If an error occured, just log it LOGGER.error(("An error occured while building query for " "postal code %s: %s"), postal_code, str(exc)) if not matching_cities: # If postal code gave no match, warn the user LOGGER.warn( "Postal code %s could not be matched with a city.", postal_code) # Remove "TOUTES COMMUNES" entry which are duplicates of the individual # cities entries in Logicimmo module. matching_cities = [ city for city in matching_cities if not (city.backend == 'logicimmo' and city.name.startswith('TOUTES COMMUNES')) ] # Then, build queries by grouping cities by at most 3 for cities_batch in tools.batch(matching_cities, 3): query = Query() query.cities = list(cities_batch) try: query.house_types = [ getattr(HOUSE_TYPES, house_type.upper()) for house_type in constraints_dict["house_types"] ] except AttributeError: LOGGER.error("Invalid house types constraint.") return None try: query.type = getattr(POSTS_TYPES, constraints_dict["type"].upper()) except AttributeError: LOGGER.error("Invalid post type constraint.") return None query.area_min = constraints_dict["area"][0] query.area_max = constraints_dict["area"][1] query.cost_min = constraints_dict["cost"][0] query.cost_max = constraints_dict["cost"][1] query.nb_rooms = constraints_dict["rooms"][0] queries.append(query) return queries