def create_object(link: str):
driver = Chrome('selenium/chromedriver.exe')
driver.get(link)
try:
# Get script with object info
script_text = ''
scripts = driver.find_elements_by_xpath(
"//script[@type='text/javascript']")
for script in scripts:
if script.get_attribute('innerHTML')[0:15] == ';FE.manageEvent':
script_text = script.get_attribute('innerHTML')
# Fill object
object = Object()
object.id = int(re.findall('"objectId":[0-9]+', script_text)[0][11:])
object.type = driver.find_element_by_xpath(
'//div[@class="col_3c"]/div').text
object.address = re.findall('"address": "[^"]+"',
script_text)[0][12:-1]
object.lat = re.findall('"objectLat": [0-9.]+,', script_text)[0][12:-1]
object.lon = re.findall('"objectLon": [0-9.]+,', script_text)[0][12:-1]
# Push object to DB
session = new_session()
object.push(session)
session.commit()
session.close()
except Exception:
log_error(type='create_object', object_link='link')
pass
driver.close()
def open(self):
# logger.debug("USER_ID")
# logger.debug(self.get_secure_cookie(USER_ID))
with new_session() as self.session:
if self.user:
user_id = self.get_secure_cookie(USER_ID)
app_id = self.get_secure_cookie(APP_ID)
logger.debug('Try to connect from user %s' % user_id)
if app_id in [con.app_id for con in clients]:
logger.debug('Connection with this app_id already exists')
for con in clients:
if con.app_id == app_id:
logger.debug('Close connection hard')
if con.ws_connection:
con.close()
clients.remove(con)
if self not in clients:
logger.debug('self not in clients')
# if self not in clients and app_id not in [con.app_id for con in clients]:
self.app_id = app_id
setattr(self, 'app_id', app_id)
# select and send all messages
apl_messages = self.session.query(ApplicationMessages).filter(and_(ApplicationMessages.application_id == app_id,
ApplicationMessages.delivered == False)).order_by(ApplicationMessages.id)
for app_msg in apl_messages:
msg = self.session.query(RoomMessage).get(app_msg.message_id)
response = {
'push_type': 1,
'messages': [sa_object_to_dict(msg)]
}
logger.debug('Sending after logging. App_id %s, msg %s, delivered %s' % (self.app_id, app_msg.message_id, app_msg.delivered))
self.write_message(json.dumps(response, default=json_serial))
clients.append(self)
logger.debug('WS: New connection from user_id = ' + user_id)
logger.debug('WS Active conections ID:')
for con in clients:
logger.debug(con.user.id)
# clients_log_string = ''
# for c in clients:
# clients_log_string = '{0}, {1}'.format(clients_log_string, str(c.user.id))
#
# logger.debug('Current connections are: {0}'.format(clients_log_string))
# print 'new connection ' + user_id
# logger.debug(user_id)
# self.user = self.session.query(User).get(user_id)
# else:
# logger.debug('This connection is in active clients')
self.pingTimer = tornado.ioloop.IOLoop.current().add_timeout(datetime.timedelta(seconds=30), self._send_ping)
else:
logger.debug('WS: USER_ID is None')
def kraken_bot():
session = new_session()
# noinspection PyBroadException
try:
# Get all strategies
strategies = Strategy.get_all(session)
# Get current prices
pairs = []
for strategy in strategies:
if strategy.pair_name not in pairs:
pairs.append(strategy.pair_name)
current_prices = get_current_price(pairs)
# Update all open orders
orm_orders = Order.get_by_status(session, 'open')
orders_id_list = []
kraken_orders = {}
for orm_order in orm_orders:
orders_id_list.append(orm_order.id)
if len(orders_id_list) > 0:
kraken_orders = query_orders(orders_id_list)
for orm_order in orm_orders:
if orm_order.id in kraken_orders.keys():
kraken_order = kraken_orders[orm_order.id]
orm_order.status = kraken_order['status']
orm_order.open_position = kraken_order['open_position']
orm_order.closed_position = kraken_order['closed_position']
session.commit()
# Call strategies
for strategy in strategies:
bot = Bot(strategy.id, current_prices[strategy.pair_name])
bot.run_robot()
except Exception as e:
# Print error
print('kraken_bot: ' + str(e))
# noinspection PyBroadException
try:
# Send error to DB log
Error.post('kraken_bot: ' + str(e))
# Send error notification
send_telegram_message('Error happened: kraken_bot: ' + str(e))
except Exception as e:
pass
finally:
session.close()
def create_author(element):
# Create and fill author
author = Author()
author.id = int(
re.findall('user_id=[0-9]+', element.get_attribute('href'))[0][8:])
author.full_name = element.text
result = author.id
# Push author to DB
session = new_session()
author.push(session)
session.commit()
session.close()
return result
def read_page(html_page: html.HtmlElement, oktmo_code):
# get all elements
elements = html_page.xpath(
'//div[@class="search-registry-entrys-block"]/div')
# Iterate through all elements
for element in elements:
# If element is an object:
if element.get(
'class'
) == 'search-registry-entry-block box-shadow-search-input':
# Get object link and ID
object_link = 'https://zakupki.gov.ru' + element.xpath(
'.//div[@class="registry-entry__header-mid__number"]/a'
)[0].get('href')
object_id = int(
re.findall('reestrNumber=[0-9]+', object_link)[0][13:])
session = new_session()
# Open general info page
request = requests.get(object_link, headers=request_header)
object_html_page = html.fromstring(request.content)
# Get contract and client info
contract = read_contract(object_html_page, object_id, session)
read_client(object_html_page, contract, oktmo_code, session)
# Get contractors info
read_contractors(object_html_page, contract, session)
# Open target of order page
request = requests.get(
f'https://zakupki.gov.ru/epz/contract/contractCard/payment-info-and-target-of-order.html?reestrNumber={object_id}',
headers=request_header)
target_of_order_page = html.fromstring(request.content)
# Get contract items
read_items(target_of_order_page, contract, session)
# Commit changes and save session
session.commit()
session.close()
def __init__(self, message: Message, context: CallbackContext):
"""
Base class for dialogue that is linked to the database row and handles interactions with user.
:param context: telegram.ext.CallbackContext
:param message: telegram message
"""
self.session = orm.new_session()
self.bot_context = context
self.message = message
self.dialogue_context = orm.Dialogue.get_by_chat_id(
self.session, message.chat_id)
# If there is not such dialogue - create it
if self.dialogue_context is None:
self.dialogue_context = orm.Dialogue(chat_id=self.message.chat_id,
type=self.__class__.__name__,
step='start')
self.dialogue_context.push(self.session)
self.session.commit()
def message_router(update: Update, context: CallbackContext):
# 1. Check if user exists
session = orm.new_session()
user = orm.User.get_by_chat_id(session, update.message.chat_id)
if user is None:
# If not - create user and start registration process
user = orm.User(chat_id=update.message.chat_id)
user.push(session)
session.commit()
session.close()
dialogue = dialogues.Registration(update.message, context)
dialogue.start()
return
# 2. Check if user has a dialogue
if user.current_dialogue is None:
# If not - check if answer is from main menu and start dialogue
if update.message.text in main_menu.keys():
dialogue_class = dialogues.get_dialogue_class(
main_menu[update.message.text])
dialogue = dialogue_class(update.message, context)
dialogue.start()
# If answer is not from menu - ask again
else:
update.message.reply_text(
'Выберите действие:',
reply_markup=dialogues.Dialogue.reply_markups['main_menu'])
return
# 3. Route a message to correct dialogue
dialogue_type = user.current_dialogue.type
session.close()
dialogue_class = dialogues.get_dialogue_class(dialogue_type)
dialogue = dialogue_class(update.message, context)
dialogue.route_message()
def on_message(self, message):
# logger.debug(self.get_secure_cookie(USER_ID))
room_id = None
room_users = None
with new_session() as self.session:
logger.debug(message)
if self.user is None:
logger.debug('WS: USER_ID is None')
self.write_message(json.dumps({
"status": 1,
"info": "User unauthorized"
}))
request_object = json.loads(message)
push_type = int(request_object['push_type'])
if push_type == MessageType.Message:
messages = None
if 'messages' in request_object:
messages = request_object['messages']
room_id = None
if 'room_id' in request_object:
room_id = request_object['room_id']
success_status = request_object.get('success_status', 1)
if int(success_status) == 0:
for msg in messages:
msg_id = msg['id']
# if hasattr(self, 'app_id'):
# change_application_message_status(self, int(msg_id), True)
change_application_message_status(self, int(msg_id), True)
else:
last_message = None
if messages and room_id:
for m in messages:
message_text = None
date_created = None
count_find_temp_id = 0
if 'message_text' in m:
message_text = m['message_text']
# if 'date_created' in m:
# date_created = m['date_created']
if 'temp_id' in m:
temp_id = m['temp_id']
else:
temp_id = None
existing_room_message = None
if temp_id:
existing_room_message = self.session.query(RoomMessage).\
filter(RoomMessage.temp_id == str(temp_id)).first()
if existing_room_message:
m['id'] = existing_room_message.id
m['user_id'] = existing_room_message.user_id
request_object['info'] = "Message with this temp_id already exists."
self.write_message(json.dumps(request_object))
logger.debug(request_object)
return
else:
room_message = RoomMessage()
room_message.room_id = room_id
room_message.user_id = self.user.id
room_message.temp_id = temp_id
room_message.message_text = message_text
if date_created is not None:
room_message.date_created = date_created
self.session.add(room_message)
self.session.commit()
# add this message to all user's applications in this room
msg_room_users = self.session.query(RoomUser).filter(RoomUser.room_id == room_id)
msg_room_users_id = [u.user_id for u in msg_room_users]
applications = self.session.query(UserApplications).filter(UserApplications.user_id.in_(msg_room_users_id))
for application in applications:
if self.session.query(ApplicationMessages).filter(and_(ApplicationMessages.application_id == application.id,
ApplicationMessages.message_id == room_message.id)).count() == 0:
application_message = ApplicationMessages()
application_message.application_id = application.id
application_message.message_id = room_message.id
self.session.add(application_message)
self.session.commit()
last_message = room_message
m['id'] = room_message.id
m['user_id'] = self.user.id
request_object['room_id'] = room_id
request_object['messages'] = [m]
self.write_message(json.dumps(request_object))
# if hasattr(self, 'app_id'):
# change_application_message_status(self, m['id'], True)
change_application_message_status(self, m['id'], True)
logger.debug("User " + str(self.user.id) + " said: " + str(request_object))
response = {
"push_type": request_object['push_type'],
"room_id": request_object['room_id'],
"messages": [
m
]
}
for con in clients:
if int(con.user.id) in msg_room_users_id:
if con != self:
con.write_message(json.dumps(response))
if int(con.user.id) == int(room_message.user_id):
change_application_message_status(self, m['id'], True)
logger.debug("send message from User %s to User %s (room %s)" % (str(self.user.id), str(con.user.id), str(room_id)))
logger.debug(response)
async_push(self, msg_room_users, room_id, m['message_text'])
if last_message:
logger.debug("Room_ID: {0}".format(room_id))
room = self.session.query(Room).get(room_id)
room.last_message_id = last_message.id
room.last_message_text = last_message.message_text
room.updated_at = datetime.datetime.utcnow()
self.session.commit()
# request_object['room_id'] = room_id
# request_object['messages'] = messages
# self.write_message(json.dumps(request_object))
# # if hasattr(self, 'app_id'):
# # change_application_message_status(self, m['id'], True)
# change_application_message_status(self, m['id'], True)
# logger.debug("User " + str(self.user.id) + " said: " + str(request_object))
#
# # send msgs to all online users
# for m in messages:
# response = {
# "push_type": request_object['push_type'],
# "room_id": request_object['room_id'],
# "messages": [
# m
# ]
# }
#
# for con in clients:
# if int(con.user.id) in msg_room_users_id:
# if con != self:
# con.write_message(json.dumps(response))
# if int(con.user.id) == int(room_message.user_id):
# change_application_message_status(self, m['id'], True)
# logger.debug("send message from User %s to User %s (room %s)" % (str(self.user.id), str(con.user.id), str(room_id)))
# logger.debug(response)
#
# async_push(self, msg_room_users, room_id, m['message_text'])
else:
request_object['info'] = "room_id or messages is empty"
self.write_message(json.dumps(request_object))
logger.debug(request_object)
# logger.debug("Room_ID: {0}".format(room_id))
# room = self.session.query(Room).get(room_id)
# room.last_message_id = last_message.id
# room.last_message_text = last_message.message_text
# room.updated_at = datetime.datetime.utcnow()
# self.session.commit()
if push_type == MessageType.StatusChange and 'status_id' in request_object:
status_id = request_object['status_id']
user = self.session.query(User).get(self.user.id)
user.status_id = status_id
self.session.commit()
request_object['user_id'] = self.user.id
for con in clients:
if con != self:
logger.debug(con.user)
con.write_message(json.dumps(request_object))
logger.debug(request_object)
if push_type == MessageType.TheMessageIsRead:
success_status = request_object.get('success_status', 1)
if 'room_id' in request_object:
room_id = request_object['room_id']
messages = None
if 'messages' in request_object:
messages = request_object['messages']
if int(success_status) == 0:
for msg in messages:
msg_id = msg['id']
# if hasattr(self, 'app_id'):
# change_application_message_status(self, int(msg_id), True)
change_application_message_status(self, int(msg_id), True)
else:
if messages:
for msg in messages:
message_id = msg['id']
message = self.session.query(RoomMessage).get(message_id)
if message:
message.status = MessageStatus.Read
logger.debug('push_type3 save in DB: Message Read: msg_id {0}, user_id {1}'.format(message_id, self.user.id))
# if hasattr(self, 'app_id'):
# change_application_message_status(self, message.id, False)
change_application_message_status(self, message.id, False)
self.session.commit()
if message:
for con in clients:
# if int(con.user.id) == int(message.user_id):
if int(con.user.id) == int(message.user_id) or int(con.user.id) == int(self.user.id):
# if (int(con.user.id) == int(message.user_id) or int(con.user.id) == int(self.user.id)) and int(con.user.id) in room_users_id:
con.write_message(json.dumps(request_object))
else:
logger.debug('push_type3: Messages is empty')
def create_report(element, object_id: int):
# Create author
author_element = element.find_element_by_xpath(
'.//div[contains(@class,"m-name")]/a')
author_id = create_author(author_element)
# Create and fill report
report = Report()
report.id = int(element.get_attribute('reqnum'))
report.object_id = object_id
report.author_id = author_id
# These variables will be filled later
report.theme = ''
report.date = ''
report.text = ''
report.image_links = ''
# Get and fill theme
try:
report.theme = element.find_element_by_xpath(
'.//div[@class="themeText bold"]').text
except Exception:
pass
# Get and fill text
try:
report.text = element.find_element_by_xpath(
'.//div[@class="messageText"]/p').text
except Exception:
pass
# Get and fill report date
try:
date_row = element.find_element_by_xpath(
'.//div[@class="m-date"]').text
day = re.findall(' [0-9]{2} ', date_row)[0][1:-1]
month_text = re.findall(
'Января|Февраля|Марта|Апреля|Мая|Июня|Июля|Августа|Сентября|Октября|Ноября|Декабря',
date_row)[0]
month = months[month_text]
year = re.findall(' [0-9]{4} ', date_row)[0][1:-1]
time = re.findall(' [0-9]{2}:[0-9]{2},', date_row)[0][1:-1]
report.date = f'{year}-{month}-{day} {time}'
except Exception:
pass
# Get first image
try:
report.image_links += element.find_element_by_xpath(
'.//div[@class="messageText"]/div[@class="img-mes"]'
'/div[@class="img-mes-bg yug"]').get_attribute('original')
except Exception:
pass
# Get other images
try:
add_images = element.find_elements_by_xpath(
'.//div[@class="messageText"]/div[@class="g-box"]/div')
for image in add_images:
report.image_links += ';' + image.get_attribute('original')
except Exception:
pass
# Push report to DB
session = new_session()
report.push(session)
session.commit()
session.close()
def __init__(self, strategy_id, current_price):
self.session = new_session()
self.strategy = Strategy.find(self.session, strategy_id)
self.current_price = current_price
