def refreshHousingsList(self):
name = self.ui.queriesList.itemText(self.ui.queriesList.currentIndex())
q = self.config.get('queries', name)
if q is None:
return q
self.ui.housingsList.clear()
self.ui.queriesList.setEnabled(False)
self.ui.bookmarksButton.setEnabled(False)
query = Query()
query.type = int(q.get('type', 0))
query.cities = []
for c in q['cities']:
city = City(c['id'])
city.backend = c['backend']
city.name = c['name']
query.cities.append(city)
query.area_min = int(q['area_min']) or None
query.area_max = int(q['area_max']) or None
query.cost_min = int(q['cost_min']) or None
query.cost_max = int(q['cost_max']) or None
query.nb_rooms = int(q['nb_rooms']) or None
self.process = QtDo(self.weboob,
self.addHousing,
fb=self.addHousingEnd)
self.process.do(self.app._do_complete, 20, None, 'search_housings',
query)
def build_queries(self, constraints_dict):
"""
Build Weboob ``weboob.capabilities.housing.Query`` objects from the
constraints defined in the configuration. Each query has at most 3
postal codes, to comply with housing websites limitations.
:param constraints_dict: A dictionary of constraints, as defined in the
config.
:return: A list of Weboob ``weboob.capabilities.housing.Query``
objects. Returns ``None`` if an error occurred.
"""
queries = []
for postal_codes in tools.batch(constraints_dict["postal_codes"], 3):
query = Query()
query.cities = []
for postal_code in postal_codes:
matching_cities = []
try:
for city in self.webnip.do("search_city", postal_code):
matching_cities.append(city)
except CallErrors as exc:
# If an error occured, just log it
LOGGER.error(("An error occured while building query for "
"postal code %s: %s"), postal_code, str(exc))
if not matching_cities:
# If postal code gave no match, warn the user
LOGGER.warn(
"Postal code %s could not be matched with a city.",
postal_code)
# Append the matched cities to the query
for city in matching_cities:
query.cities.append(city)
try:
query.house_types = [
getattr(Query.HOUSE_TYPES, house_type.upper())
for house_type in constraints_dict["house_types"]
]
except AttributeError:
LOGGER.error("Invalid house types constraint.")
return None
try:
query.type = getattr(
Query, "TYPE_{}".format(constraints_dict["type"].upper()))
except AttributeError:
LOGGER.error("Invalid post type constraint.")
return None
query.area_min = constraints_dict["area"][0]
query.area_max = constraints_dict["area"][1]
query.cost_min = constraints_dict["cost"][0]
query.cost_max = constraints_dict["cost"][1]
query.nb_rooms = constraints_dict["rooms"][0]
queries.append(query)
return queries
def refreshHousingsList(self):
name = self.ui.queriesList.itemText(self.ui.queriesList.currentIndex())
q = self.config.get('queries', name)
if q is None:
return q
self.ui.housingsList.clear()
self.ui.queriesList.setEnabled(False)
self.ui.bookmarksButton.setEnabled(False)
query = Query()
query.type = int(q.get('type', 0))
query.cities = []
for c in q['cities']:
city = City(c['id'])
city.backend = c['backend']
city.name = c['name']
query.cities.append(city)
query.area_min = int(q['area_min']) or None
query.area_max = int(q['area_max']) or None
query.cost_min = int(q['cost_min']) or None
query.cost_max = int(q['cost_max']) or None
query.nb_rooms = int(q['nb_rooms']) or None
self.process = QtDo(self.weboob, self.addHousing, fb=self.addHousingEnd)
self.process.do(self.app._do_complete, 20, (), 'search_housings', query)
def do_search(self, line):
pattern = 'notempty'
query = Query()
query.cities = []
while pattern:
if len(query.cities) > 0:
print '\n%sSelected cities:%s %s' % (
self.BOLD, self.NC, ', '.join(
[c.name for c in query.cities]))
pattern = self.ask('Enter a city pattern (or empty to stop)',
default='')
if not pattern:
break
cities = []
for backend, city in self.do('search_city', pattern):
cities.append(city)
if len(cities) == 0:
print ' Not found!'
continue
if len(cities) == 1:
if city in query.cities:
query.cities.remove(city)
else:
query.cities.append(city)
continue
r = 'notempty'
while r != '':
for i, city in enumerate(cities):
print ' %s%2d)%s [%s] %s' % (self.BOLD, i + 1, self.NC,
'x' if city in query.cities
else ' ', city.name)
r = self.ask(' Select cities (or empty to stop)',
regexp='(\d+|)',
default='')
if not r.isdigit():
continue
r = int(r)
if r <= 0 or r > len(cities):
continue
city = cities[r - 1]
if city in query.cities:
query.cities.remove(city)
else:
query.cities.append(city)
query.area_min = self.ask_int('Enter min area')
query.area_max = self.ask_int('Enter max area')
query.cost_min = self.ask_int('Enter min cost')
query.cost_max = self.ask_int('Enter max cost')
query.nb_rooms = self.ask_int('Enter number of rooms')
self.change_path([u'housings'])
self.start_format()
for backend, housing in self.do('search_housings', query):
self.cached_format(housing)
self.flush()
def query_for_cities(args):
cities = [city(line.strip()) for line in args.infile]
if args.debug:
logging.basicConfig(level=logging.DEBUG)
for handler in logging.root.handlers:
handler.setLevel(logging.DEBUG)
try:
with open('leboncoin.json') as json_file:
prev_urls = json.load(json_file)
except FileNotFoundError:
prev_urls = []
proxy = None
if args.proxies:
proxy = {'https': random.choice(list(args.proxies)).strip()}
LeboncoinBrowser.VERIFY = False
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
browser = LeboncoinBrowser(proxy=proxy)
query = Query()
query.type = POSTS_TYPES[args.type]
query.house_types = [HOUSE_TYPES[ht] for ht in args.house_types]
if query.area_min:
query.area_min = args.area_min
if query.area_max:
query.area_max = args.area_max
if query.cost_min:
query.cost_min = args.cost_min
if query.cost_max:
query.cost_max = args.cost_max
if query.nb_rooms:
query.nb_rooms = args.nb_rooms
new_urls = []
for i in range(
0, len(cities), 10
): # if the frontend tells the truth, the API supports max 10 cities at a time
query.cities = cities[i:i + 10]
for housing in browser.search_housings(query, None):
new_urls.append(housing.url)
diff_urls = r'\n'.join(set(new_urls) - set(prev_urls))
print('Saving {} current housing matches to leboncoin.json'.format(
len(new_urls)))
with open('leboncoin.json', 'w+') as json_file:
json.dump(new_urls, json_file)
print(diff_urls.replace(r'\n', '\n'))
if diff_urls:
msg = r'Nouvelle(s) annonce(s) LeBonCoin:\n' + diff_urls
if args.alert_cmd:
check_output([args.alert_cmd, msg])
if args.alert_phone_number:
from send_text_msg_with_twilio import send_text_msg
send_text_msg(os.environ['TWILIO_ACCOUNT_SID'],
os.environ['TWILIO_AUTH_TOKEN'],
args.alert_phone_number, msg)
def do_search(self, line):
"""
search
Search for housing. Parameters are interactively asked.
"""
pattern = 'notempty'
query = Query()
query.cities = []
while pattern:
if len(query.cities) > 0:
print '\n%sSelected cities:%s %s' % (self.BOLD, self.NC, ', '.join([c.name for c in query.cities]))
pattern = self.ask('Enter a city pattern (or empty to stop)', default='')
if not pattern:
break
cities = []
for backend, city in self.weboob.do('search_city', pattern):
cities.append(city)
if len(cities) == 0:
print ' Not found!'
continue
if len(cities) == 1:
if city in query.cities:
query.cities.remove(city)
else:
query.cities.append(city)
continue
r = 'notempty'
while r != '':
for i, city in enumerate(cities):
print ' %s%2d)%s [%s] %s' % (self.BOLD, i+1, self.NC, 'x' if city in query.cities else ' ', city.name)
r = self.ask(' Select cities (or empty to stop)', regexp='(\d+|)', default='')
if not r.isdigit():
continue
r = int(r)
if r <= 0 or r > len(cities):
continue
city = cities[r-1]
if city in query.cities:
query.cities.remove(city)
else:
query.cities.append(city)
query.area_min = self.ask_int('Enter min area')
query.area_max = self.ask_int('Enter max area')
query.cost_min = self.ask_int('Enter min cost')
query.cost_max = self.ask_int('Enter max cost')
query.nb_rooms = self.ask_int('Enter number of rooms')
self.change_path([u'housings'])
self.start_format()
for backend, housing in self.do('search_housings', query):
self.cached_format(housing)
def do_search(self, line):
"""
search
Search for housing. Parameters are interactively asked.
"""
pattern = 'notempty'
query = Query()
query.cities = []
while pattern:
if len(query.cities) > 0:
print '\n%sSelected cities:%s %s' % (self.BOLD, self.NC, ', '.join([c.name for c in query.cities]))
pattern = self.ask('Enter a city pattern (or empty to stop)', default='')
if not pattern:
break
cities = []
for backend, city in self.weboob.do('search_city', pattern):
cities.append(city)
if len(cities) == 0:
print ' Not found!'
continue
if len(cities) == 1:
if city in query.cities:
query.cities.remove(city)
else:
query.cities.append(city)
continue
r = 'notempty'
while r != '':
for i, city in enumerate(cities):
print ' %s%2d)%s [%s] %s' % (self.BOLD, i+1, self.NC, 'x' if city in query.cities else ' ', city.name)
r = self.ask(' Select cities (or empty to stop)', regexp='(\d+|)', default='')
if not r.isdigit():
continue
r = int(r)
if r <= 0 or r > len(cities):
continue
city = cities[r-1]
if city in query.cities:
query.cities.remove(city)
else:
query.cities.append(city)
r = 'notempty'
while r != '':
for good in Query.HOUSE_TYPES.values:
print ' %s%2d)%s [%s] %s' % (self.BOLD,
Query.HOUSE_TYPES.index[good] + 1,
self.NC,
'x' if good in query.house_types else ' ', good)
r = self.ask(' Select type of house (or empty to stop)', regexp='(\d+|)', default='')
if not r.isdigit():
continue
r = int(r)
if r <= 0 or r > len(Query.TYPE_OF_GOOD.values):
continue
value = Query.TYPE_OF_GOOD.values[r - 1]
if value in query.type_of_good:
query.type_of_good.remove(value)
else:
query.type_of_good.append(value)
_type = None
while _type not in [query.TYPE_RENT, query.TYPE_SALE]:
print ' %s%2d)%s %s' % (self.BOLD,
query.TYPE_RENT,
self.NC,
"Rent")
print ' %s%2d)%s %s' % (self.BOLD,
query.TYPE_SALE,
self.NC,
"Sale")
_type = self.ask_int('Type of query')
query.type = _type
query.area_min = self.ask_int('Enter min area')
query.area_max = self.ask_int('Enter max area')
query.cost_min = self.ask_int('Enter min cost')
query.cost_max = self.ask_int('Enter max cost')
query.nb_rooms = self.ask_int('Enter number of rooms')
save_query = self.ask('Save query (y/n)', default='n')
if save_query.upper() == 'Y':
name = ''
while not name:
name = self.ask('Query name')
self.config.set('queries', name, query)
self.config.save()
self.complete_search(query)
def do_search(self, line):
"""
search
Search for housing. Parameters are interactively asked.
"""
pattern = 'notempty'
query = Query()
query.cities = []
while pattern:
if len(query.cities) > 0:
print('\n%sSelected cities:%s %s' %
(self.BOLD, self.NC, ', '.join(
[c.name for c in query.cities])))
pattern = self.ask('Enter a city pattern (or empty to stop)',
default='')
if not pattern:
break
cities = []
for city in self.weboob.do('search_city', pattern):
cities.append(city)
if len(cities) == 0:
print(' Not found!')
continue
if len(cities) == 1:
if city in query.cities:
query.cities.remove(city)
else:
query.cities.append(city)
continue
r = 'notempty'
while r != '':
for i, city in enumerate(cities):
print(' %s%2d)%s [%s] %s (%s)' %
(self.BOLD, i + 1, self.NC, 'x' if city
in query.cities else ' ', city.name, city.backend))
r = self.ask(' Select cities (or empty to stop)',
regexp='(\d+|)',
default='')
if not r.isdigit():
continue
r = int(r)
if r <= 0 or r > len(cities):
continue
city = cities[r - 1]
if city in query.cities:
query.cities.remove(city)
else:
query.cities.append(city)
r = 'notempty'
while r != '':
for good in Query.HOUSE_TYPES.values:
print(' %s%2d)%s [%s] %s' %
(self.BOLD, Query.HOUSE_TYPES.index[good] + 1, self.NC,
'x' if good in query.house_types else ' ', good))
r = self.ask(' Select type of house (or empty to stop)',
regexp='(\d+|)',
default='')
if not r.isdigit():
continue
r = int(r)
if r <= 0 or r > len(Query.HOUSE_TYPES.values):
continue
value = Query.HOUSE_TYPES.values[r - 1]
if value in query.house_types:
query.house_types.remove(value)
else:
query.house_types.append(value)
_type = None
while _type not in [query.TYPE_RENT, query.TYPE_SALE]:
print(' %s%2d)%s %s' %
(self.BOLD, query.TYPE_RENT, self.NC, "Rent"))
print(' %s%2d)%s %s' %
(self.BOLD, query.TYPE_SALE, self.NC, "Sale"))
_type = self.ask_int('Type of query')
query.type = _type
query.area_min = self.ask_int('Enter min area')
query.area_max = self.ask_int('Enter max area')
query.cost_min = self.ask_int('Enter min cost')
query.cost_max = self.ask_int('Enter max cost')
query.nb_rooms = self.ask_int('Enter number of rooms')
save_query = self.ask('Save query (y/n)', default='n')
if save_query.upper() == 'Y':
name = ''
while not name:
name = self.ask('Query name')
self.config.set('queries', name, query)
self.config.save()
self.complete_search(query)
def do_search(self, line):
"""
search
Search for housing. Parameters are interactively asked.
"""
pattern = "notempty"
query = Query()
query.cities = []
while pattern:
if len(query.cities) > 0:
print "\n%sSelected cities:%s %s" % (self.BOLD, self.NC, ", ".join([c.name for c in query.cities]))
pattern = self.ask("Enter a city pattern (or empty to stop)", default="")
if not pattern:
break
cities = []
for backend, city in self.weboob.do("search_city", pattern):
cities.append(city)
if len(cities) == 0:
print " Not found!"
continue
if len(cities) == 1:
if city in query.cities:
query.cities.remove(city)
else:
query.cities.append(city)
continue
r = "notempty"
while r != "":
for i, city in enumerate(cities):
print " %s%2d)%s [%s] %s" % (
self.BOLD,
i + 1,
self.NC,
"x" if city in query.cities else " ",
city.name,
)
r = self.ask(" Select cities (or empty to stop)", regexp="(\d+|)", default="")
if not r.isdigit():
continue
r = int(r)
if r <= 0 or r > len(cities):
continue
city = cities[r - 1]
if city in query.cities:
query.cities.remove(city)
else:
query.cities.append(city)
query.area_min = self.ask_int("Enter min area")
query.area_max = self.ask_int("Enter max area")
query.cost_min = self.ask_int("Enter min cost")
query.cost_max = self.ask_int("Enter max cost")
query.nb_rooms = self.ask_int("Enter number of rooms")
self.change_path([u"housings"])
self.start_format()
for backend, housing in self.do("search_housings", query):
self.cached_format(housing)
def do_search(self, line):
"""
search
Search for housing. Parameters are interactively asked.
"""
pattern = 'notempty'
query = Query()
query.cities = []
while pattern:
if len(query.cities) > 0:
print('\n%sSelected cities:%s %s' % (self.BOLD, self.NC, ', '.join([c.name for c in query.cities])))
pattern = self.ask('Enter a city pattern (or empty to stop)', default='')
if not pattern:
break
cities = []
for city in self.weboob.do('search_city', pattern):
cities.append(city)
if len(cities) == 0:
print(' Not found!')
continue
if len(cities) == 1:
if city in query.cities:
query.cities.remove(city)
else:
query.cities.append(city)
continue
r = 'notempty'
while r != '':
for i, city in enumerate(cities):
print(' %s%2d)%s [%s] %s (%s)' % (self.BOLD, i+1, self.NC, 'x' if city in query.cities else ' ', city.name, city.backend))
r = self.ask('Select cities (or empty to stop)', regexp='(\d+|)', default='')
if not r.isdigit():
continue
r = int(r)
if r <= 0 or r > len(cities):
continue
city = cities[r-1]
if city in query.cities:
query.cities.remove(city)
else:
query.cities.append(city)
r = 'notempty'
while r != '':
for i, good in enumerate(HOUSE_TYPES, 1):
print(' %s%2d)%s [%s] %s' % (self.BOLD,
i,
self.NC,
'x' if good in query.house_types else ' ', good))
r = self.ask('Select type of house (or empty to stop)', regexp='(\d+|)', default='')
if not r.isdigit():
continue
r = int(r)
if r <= 0 or r > len(HOUSE_TYPES):
continue
value = list(HOUSE_TYPES)[r - 1]
if value in query.house_types:
query.house_types.remove(value)
else:
query.house_types.append(value)
_type = None
posts_types = sorted(POSTS_TYPES, key=lambda e: e.value)
while _type not in range(len(posts_types)):
for i, t in enumerate(posts_types):
print(' %s%2d)%s %s' % (self.BOLD,
i,
self.NC,
t))
_type = self.ask_int('Type of query')
query.type = posts_types[_type]
r = 'notempty'
while r != '':
for i, good in enumerate(ADVERT_TYPES, 1):
print(' %s%2d)%s [%s] %s' % (self.BOLD,
i,
self.NC,
'x' if good in query.advert_types else ' ', good))
r = self.ask('Select type of posts (or empty to stop)', regexp='(\d+|)', default='')
if not r.isdigit():
continue
r = int(r)
if r <= 0 or r > len(ADVERT_TYPES):
continue
value = list(ADVERT_TYPES)[r - 1]
if value in query.advert_types:
query.advert_types.remove(value)
else:
query.advert_types.append(value)
query.area_min = self.ask_int('Enter min area')
query.area_max = self.ask_int('Enter max area')
query.cost_min = self.ask_int('Enter min cost')
query.cost_max = self.ask_int('Enter max cost')
query.nb_rooms = self.ask_int('Enter number of rooms')
save_query = self.ask('Save query (y/n)', default='n')
if save_query.upper() == 'Y':
name = ''
while not name:
name = self.ask('Query name')
self.config.set('queries', name, query)
self.config.save()
self.complete_search(query)
def build_queries(self, constraints_dict):
"""
Build WebOOB ``weboob.capabilities.housing.Query`` objects from the
constraints defined in the configuration. Each query has at most 3
cities, to comply with housing websites limitations.
:param constraints_dict: A dictionary of constraints, as defined in the
config.
:return: A list of WebOOB ``weboob.capabilities.housing.Query``
objects. Returns ``None`` if an error occurred.
"""
queries = []
# First, find all matching cities for the postal codes in constraints
matching_cities = []
for postal_code in constraints_dict["postal_codes"]:
try:
for city in self.webnip.do("search_city", postal_code):
matching_cities.append(city)
except CallErrors as exc:
# If an error occured, just log it
LOGGER.error(("An error occured while building query for "
"postal code %s: %s"), postal_code, str(exc))
if not matching_cities:
# If postal code gave no match, warn the user
LOGGER.warn(
"Postal code %s could not be matched with a city.",
postal_code)
# Remove "TOUTES COMMUNES" entry which are duplicates of the individual
# cities entries in Logicimmo module.
matching_cities = [
city for city in matching_cities
if not (city.backend == 'logicimmo'
and city.name.startswith('TOUTES COMMUNES'))
]
# Then, build queries by grouping cities by at most 3
for cities_batch in tools.batch(matching_cities, 3):
query = Query()
query.cities = list(cities_batch)
try:
query.house_types = [
getattr(HOUSE_TYPES, house_type.upper())
for house_type in constraints_dict["house_types"]
]
except AttributeError:
LOGGER.error("Invalid house types constraint.")
return None
try:
query.type = getattr(POSTS_TYPES,
constraints_dict["type"].upper())
except AttributeError:
LOGGER.error("Invalid post type constraint.")
return None
query.area_min = constraints_dict["area"][0]
query.area_max = constraints_dict["area"][1]
query.cost_min = constraints_dict["cost"][0]
query.cost_max = constraints_dict["cost"][1]
query.nb_rooms = constraints_dict["rooms"][0]
queries.append(query)
return queries