def main(args):
config_file = args.config_file or "/etc/pa.conf"
config = configparser.ConfigParser()
config.read(config_file)
if args.debug or config.getboolean('pa', 'debug', fallback=False):
setup_logging(logging.DEBUG)
else:
setup_logging(logging.INFO)
token_file = args.token_file or config.get('pa', 'token_file')
with open(token_file) as tok:
TOKEN = tok.read().strip()
stomp_host = config.get('stomp', 'host', fallback='localhost')
stomp_port = config.getint('stomp', 'port', fallback=61613)
heartbeat_in = config.getint('stomp', 'heartbeat_in', fallback=0)
heartbeat_out = config.getint('stomp', 'heartbeat_out', fallback=0)
stomp_connection = stomp.Connection([(stomp_host, stomp_port)],
heartbeats=(heartbeat_in,
heartbeat_out))
tg = Tg(TOKEN, stomp_connection)
tg.run_forever()
def __init__(self, task):
"""Constructor"""
self.__task = task # protected
# Инициализируем базу
self.__db = self.__prepare_db(task["database"])
# Инициализируем бота для Telegram
if "proxy" in task:
self.__tg = Tg(task["tgBotKey"], task["proxy"])
else:
self.__tg = Tg(task["tgBotKey"])
class Youla:
"""docstring"""
def __init__(self, task):
"""Constructor"""
self.__task = task # protected
# Инициализируем базу
self.__db = self.__prepare_db(task["database"])
# Инициализируем бота для Telegram
if "proxy" in task:
self.__tg = Tg(task["tgBotKey"], task["proxy"])
else:
self.__tg = Tg(task["tgBotKey"])
def __prepare_db(self, link):
db = records.Database(link)
conn = db.get_connection()
with conn:
conn.query(
'Create table if not exists YOULA (tag VARCHAR(255), link VARCHAR(255),price VARCHAR(20))'
)
return db
def __check_record(self, record):
conn = self.__db.get_connection()
with conn:
rows = conn.query(
"select link from YOULA where tag='{}' and link='{}' and price='{}'"
.format(record['tag'], record['link'], record['price']),
fetchall=True)
if len(rows) == 0:
# Пишем в базу
conn.query(
"insert into YOULA (tag,link,price) values('{}','{}','{}')"
.format(record['tag'], record['link'], record['price']))
return len(rows)
def grab(self):
ua = UserAgent()
header = {'User-Agent': str(ua.chrome), 'Accept-Encoding': 'utf-8'}
task = self.__task
tag = task["tag"]
base_url = "https://youla.io"
searchUrl = task["search"]
result = requests.get(base_url + searchUrl, headers=header)
content = result.text
soup = BeautifulSoup(content, "html.parser")
advert = soup.find_all("li", "product_item")
for ad in advert:
itemLink = base_url + ad.findNext("a")['href']
itemPrice = self.__getText(
ad.find("div", "product_item__description")).replace(' ', '')
# Нашли цену и url, можем писать в базу
record = dict()
record['tag'] = tag
record['link'] = itemLink
record['price'] = itemPrice
print("link: {} price: {}".format(record['link'], record['price']))
if self.__check_record(record) == 0:
print("new record")
message = "{} price: {}".format(itemLink, itemPrice)
self.__tg.send_message(task["tgBotKey"], task["tgChannelId"],
message)
else:
pass
def __getText(self, parent):
return ''.join(parent.find_all(text=True, recursive=False)).strip()
def setUp(self):
self.bot = Tg(sentinel.token, sentinel.connection)
self.mock_request = MagicMock()
self.bot.bot._api_request = self.mock_request
import logging
from tg import Tg
from time import sleep
LOG_LEVEL = logging.DEBUG
logger = logging.getLogger("buildnotifier")
logger.setLevel(LOG_LEVEL)
handler = logging.StreamHandler()
handler.setLevel(LOG_LEVEL)
handler.setFormatter(
logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s'))
logger.addHandler(handler)
logger.info("Starting tg")
tg = Tg(logger=logger)
logger.info("Starting flask")
app = Flask(__name__)
@app.route('/post', methods=['POST'])
def post():
data = request.json
if data['type'] == 'build':
logger.info('Data received: Starting build: ' + data['tag'])
tg.send_build(data['tag'])
if data['type'] == 'stage':
logger.info('Data received: Running stage: ' + data['stage'])
tg.send_stage(data['stage'])
if data['type'] == 'complete':
class Avito:
"""docstring"""
def __init__(self, task):
"""Constructor"""
self.__task = task # protected
# Инициализируем базу
self.__db = self.__prepare_db(task["database"])
# Инициализируем бота для Telegram
if "proxy" in task:
self.__tg = Tg(task["tgBotKey"], task["proxy"])
else:
self.__tg = Tg(task["tgBotKey"])
def __prepare_db(self, link):
db = records.Database(link)
conn = db.get_connection()
with conn:
conn.query(
'Create table if not exists AVITO (tag VARCHAR(255),id VARCHAR(255),link VARCHAR(255),price VARCHAR(20))'
)
return db
def __check_record(self, avito_record):
conn = self.__db.get_connection()
with conn:
rows = conn.query(
"select price from AVITO where tag='{}' and id='{}'".format(
avito_record['tag'], avito_record['id']),
fetchall=True)
if len(rows) == 0:
# new item
conn.query(
"insert into AVITO (tag,id,link,price) values('{}','{}','{}','{}')"
.format(avito_record['tag'], avito_record['id'],
avito_record['link'], avito_record['price']))
res = {'message': "New", 'icon': '✅'}
elif int(rows[0]['price']) > int(avito_record['price']):
# item changed, price down
conn.query(
"update AVITO set price='{}' where tag='{}' and id='{}'".
format(avito_record['price'], avito_record['tag'],
avito_record['id']))
res = {
'message': "Price down",
'icon': '👍',
'oldPrice': rows[0]['price']
}
elif int(rows[0]['price']) < int(avito_record['price']):
# item changed, price up
conn.query(
"update AVITO set price='{}' where tag='{}' and id='{}'".
format(avito_record['price'], avito_record['tag'],
avito_record['id']))
res = {
'message': "Price up",
'icon': '👎',
'oldPrice': rows[0]['price']
}
else:
# price not changed
res = None
return res
def __getText(self, parent):
return ''.join(parent.find_all(text=True, recursive=False)).strip()
def grab(self):
ua = UserAgent()
header = {'User-Agent': str(ua.chrome), 'Accept-Encoding': 'utf-8'}
task = self.__task
tag = task["tag"]
base_url = "https://www.avito.ru"
search_url = task["search"]
result = requests.get(base_url + search_url, headers=header)
content = result.content
print(content)
soup = BeautifulSoup(content, "html.parser")
main_item_div_list = soup.find_all('div', "item")
count_all = 0
count_new = 0
count_change = 0
for mainItemDiv in main_item_div_list:
item_id = mainItemDiv['id']
href_tag = mainItemDiv.find('a', 'snippet-link')
if href_tag is not None:
price_tag = mainItemDiv.find('span', 'snippet-price')
if price_tag is not None:
item_name = href_tag.get('title')
item_price = self.__getText(
price_tag) #price_tag.get('content')
# Нашли цену и url, можем писать в базу
count_all += 1
avito_record = dict()
avito_record['tag'] = tag
avito_record['id'] = item_id
item_link = base_url + href_tag.get('href')
avito_record['link'] = item_link
avito_record['price'] = item_price.replace(' ', '')
message = self.__check_record(avito_record)
if message is not None:
if 'oldPrice' in message:
count_change += 1
formatted_message = "{}{} \n{} \n💰 {}₽ -> {} \n🔗[Перейти]({})".format(
message['icon'],
message['message'],
item_name,
message['oldPrice'],
avito_record['price'],
avito_record['link'],
)
print(formatted_message)
self.__tg.send_message(task["tgChannelId"],
formatted_message)
else:
count_new += 1
formatted_message = "{}{} \n{} \n💰 {}₽ \n🔗[Перейти]({})".format(
message['icon'],
message['message'],
item_name,
avito_record['price'],
avito_record['link'],
)
print(formatted_message)
self.__tg.send_message(task["tgChannelId"],
formatted_message)
else:
print("Empty link for {}".format(item_id))
print("Total: {} new: {} changed: {}".format(count_all, count_new,
count_change))