def build_queries(self, constraints_dict):
"""
Build Weboob ``weboob.capabilities.housing.Query`` objects from the
constraints defined in the configuration. Each query has at most 3
postal codes, to comply with housing websites limitations.
:param constraints_dict: A dictionary of constraints, as defined in the
config.
:return: A list of Weboob ``weboob.capabilities.housing.Query``
objects. Returns ``None`` if an error occurred.
"""
queries = []
for postal_codes in tools.batch(constraints_dict["postal_codes"], 3):
query = Query()
query.cities = []
for postal_code in postal_codes:
matching_cities = []
try:
for city in self.webnip.do("search_city", postal_code):
matching_cities.append(city)
except CallErrors as exc:
# If an error occured, just log it
LOGGER.error(("An error occured while building query for "
"postal code %s: %s"), postal_code, str(exc))
if not matching_cities:
# If postal code gave no match, warn the user
LOGGER.warn(
"Postal code %s could not be matched with a city.",
postal_code)
# Append the matched cities to the query
for city in matching_cities:
query.cities.append(city)
try:
query.house_types = [
getattr(Query.HOUSE_TYPES, house_type.upper())
for house_type in constraints_dict["house_types"]
]
except AttributeError:
LOGGER.error("Invalid house types constraint.")
return None
try:
query.type = getattr(
Query, "TYPE_{}".format(constraints_dict["type"].upper()))
except AttributeError:
LOGGER.error("Invalid post type constraint.")
return None
query.area_min = constraints_dict["area"][0]
query.area_max = constraints_dict["area"][1]
query.cost_min = constraints_dict["cost"][0]
query.cost_max = constraints_dict["cost"][1]
query.nb_rooms = constraints_dict["rooms"][0]
queries.append(query)
return queries
def query_for_cities(args):
cities = [city(line.strip()) for line in args.infile]
if args.debug:
logging.basicConfig(level=logging.DEBUG)
for handler in logging.root.handlers:
handler.setLevel(logging.DEBUG)
try:
with open('leboncoin.json') as json_file:
prev_urls = json.load(json_file)
except FileNotFoundError:
prev_urls = []
proxy = None
if args.proxies:
proxy = {'https': random.choice(list(args.proxies)).strip()}
LeboncoinBrowser.VERIFY = False
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
browser = LeboncoinBrowser(proxy=proxy)
query = Query()
query.type = POSTS_TYPES[args.type]
query.house_types = [HOUSE_TYPES[ht] for ht in args.house_types]
if query.area_min:
query.area_min = args.area_min
if query.area_max:
query.area_max = args.area_max
if query.cost_min:
query.cost_min = args.cost_min
if query.cost_max:
query.cost_max = args.cost_max
if query.nb_rooms:
query.nb_rooms = args.nb_rooms
new_urls = []
for i in range(
0, len(cities), 10
): # if the frontend tells the truth, the API supports max 10 cities at a time
query.cities = cities[i:i + 10]
for housing in browser.search_housings(query, None):
new_urls.append(housing.url)
diff_urls = r'\n'.join(set(new_urls) - set(prev_urls))
print('Saving {} current housing matches to leboncoin.json'.format(
len(new_urls)))
with open('leboncoin.json', 'w+') as json_file:
json.dump(new_urls, json_file)
print(diff_urls.replace(r'\n', '\n'))
if diff_urls:
msg = r'Nouvelle(s) annonce(s) LeBonCoin:\n' + diff_urls
if args.alert_cmd:
check_output([args.alert_cmd, msg])
if args.alert_phone_number:
from send_text_msg_with_twilio import send_text_msg
send_text_msg(os.environ['TWILIO_ACCOUNT_SID'],
os.environ['TWILIO_AUTH_TOKEN'],
args.alert_phone_number, msg)
def test_entreparticuliers_professional(self):
query = Query()
query.cities = []
for city in self.backend.search_city('lille'):
city.backend = self.backend.name
query.cities.append(city)
query.type = POSTS_TYPES.SALE
query.house_types = [HOUSE_TYPES.HOUSE]
query.advert_types = [ADVERT_TYPES.PROFESSIONAL]
results = list(
itertools.islice(self.backend.search_housings(query), 0, 20))
self.assertEqual(len(results), 0)
def test_entreparticuliers(self):
query = Query()
query.cities = []
for city in self.backend.search_city('lille'):
city.backend = self.backend.name
query.cities.append(city)
query.type = Query.TYPE_SALE
query.house_types = [Query.HOUSE_TYPES.HOUSE]
results = list(itertools.islice(self.backend.search_housings(query), 0, 20))
self.assertTrue(len(results) > 0)
obj = self.backend.fillobj(results[0])
self.assertTrue(obj.area is not None, 'Area for "%s"' % (obj.id))
def build_queries(self, constraints_dict):
"""
Build WebOOB ``weboob.capabilities.housing.Query`` objects from the
constraints defined in the configuration. Each query has at most 3
cities, to comply with housing websites limitations.
:param constraints_dict: A dictionary of constraints, as defined in the
config.
:return: A list of WebOOB ``weboob.capabilities.housing.Query``
objects. Returns ``None`` if an error occurred.
"""
queries = []
# First, find all matching cities for the postal codes in constraints
matching_cities = []
for postal_code in constraints_dict["postal_codes"]:
try:
for city in self.webnip.do("search_city", postal_code):
matching_cities.append(city)
except CallErrors as exc:
# If an error occured, just log it
LOGGER.error(("An error occured while building query for "
"postal code %s: %s"), postal_code, str(exc))
if not matching_cities:
# If postal code gave no match, warn the user
LOGGER.warn(
"Postal code %s could not be matched with a city.",
postal_code)
# Remove "TOUTES COMMUNES" entry which are duplicates of the individual
# cities entries in Logicimmo module.
matching_cities = [
city for city in matching_cities
if not (city.backend == 'logicimmo'
and city.name.startswith('TOUTES COMMUNES'))
]
# Then, build queries by grouping cities by at most 3
for cities_batch in tools.batch(matching_cities, 3):
query = Query()
query.cities = list(cities_batch)
try:
query.house_types = [
getattr(HOUSE_TYPES, house_type.upper())
for house_type in constraints_dict["house_types"]
]
except AttributeError:
LOGGER.error("Invalid house types constraint.")
return None
try:
query.type = getattr(POSTS_TYPES,
constraints_dict["type"].upper())
except AttributeError:
LOGGER.error("Invalid post type constraint.")
return None
query.area_min = constraints_dict["area"][0]
query.area_max = constraints_dict["area"][1]
query.cost_min = constraints_dict["cost"][0]
query.cost_max = constraints_dict["cost"][1]
query.nb_rooms = constraints_dict["rooms"][0]
queries.append(query)
return queries