def show(self, bot, update, args, me=0):
usernames = " ".join(args).split()
username = usernames[0] if usernames else ""
if not username:
username = update.message.from_user.username
elif username[0] == "@":
username = username[1:]
else:
usage_text = "Usage:\n`/ping_show[_me] @username`\n`/ping_show me`\n`/ping_show`\n"
bot.send_message(chat_id=update.message.chat_id,
parse_mode='markdown',
text=usage_text)
return
with connector(self.config.engine()) as ses:
user_matches = ses.query(Pingers).filter(
Pingers.chat_id == update.message.chat_id,
Pingers.username == username, Pingers.me == me).all()
out_text = ""
for match in user_matches:
out_text += "\n{}".format(match.match)
if out_text == "":
out_text = "No such user"
bot.send_message(chat_id=update.message.chat_id, text=out_text)
log_print('Show pings of "@{0}", by {1}'.format(
username, update.message.from_user.username))
def database_handler(config, bot, update, args):
if update.message.from_user.username not in config.admins():
out_text = "You are not an administrator. The incident will be reported"
command = "not an administrator"
else:
command = " ".join(args)
if command == ".schema":
command = "SELECT sql FROM sqlite_master WHERE type = 'table'"
if command == ".tables":
command = "SELECT name FROM sqlite_master WHERE type = 'table'"
if "%%%chat_id%%%" in command:
command = command.replace("%%%chat_id%%%",
str(update.message.chat_id))
engine = create_engine(config.database())
connector = engine.connect()
try:
out_text = "\n".join([
" | ".join([str(i) for i in i])
for i in engine.execute(command).fetchall()
])
connector.close()
except ResourceClosedError:
out_text = command = "Successfuly"
connector.close()
except:
out_text = command = "Bad command"
connector.close()
if out_text:
bot.send_message(chat_id=update.message.chat_id, text=out_text)
log_print('Manage "{0}"'.format(command),
update.message.from_user.username)
def analyze_offer_details_page(self, url: str) -> dict:
"""
:param url: the url for details page
:return: dict with params that will be used ass attribs into dataclass HouseOffer
"""
try:
page = self.page_downloader.get_page(url)
offer_html = BeautifulSoup(page.content, 'html.parser')
safe_eval = Bs4SafeCodeEvaluator(offer_html, url)
return {
'rooms_number':
safe_eval.get_value_or_none(
"int(self.offer_html.find_all(text='Liczba pokoi')[0].parent.parent.contents[1].text)"
),
'floor':
safe_eval.get_value_or_none(
"self.offer_html.find_all(text='Piętro')[0].parent.parent.contents[1].text"
),
'flat_area':
safe_eval.get_value_or_none(
"self.offer_html.find_all(text='Powierzchnia')[0].parent.parent.contents[3].text"
),
}
except Exception as e:
log_print(f'Impossible to download page {url}. Details: {e}', )
return {}
def scan_site(self, url):
try:
page = self.page_downloader.get_page(url)
parser = None
if 'otodom.pl' in url:
parser = OtoDomParser
if 'olx.pl' in url:
parser = OlxParser
if 'sprzedajemy.pl' in url:
parser = SprzedajemyParser
if 'otomoto.pl' in url:
parser = OtomotoParser
if parser is None:
log_print(
f"[ERROR] couldn't find Page Parser for the site {url}")
return
return parser(downloader=self.page_downloader,
found_offers_set=self.found_offers_set,
message_sender=self.message_sender,
page_url=url).analyze_html_page(page.content)
except Exception as e:
log_print(f"Coudn't fetch the url {url}. The error: {e}",
message_type=1)
print(traceback.format_exc())
def _msg_handle(self):
while True:
log_print('while.txt', 'while is running')
if not self.running_info.empty():
# log_print('handler.txt', '--- print running queue ' + str(list(self.running_info.queue)))
# TODO: below
# Detect if there are available GPUs
try:
if self.allocate_gpu():
info = self.running_info.get()
self.update_running_info(info)
else:
# currently no gpu can be allocated
log_print('scheduler.txt',
'--- current no gpu allocated')
log_print(
'scheduler.txt',
'----current GPU util: ' + str(self.resources))
except:
log_print('while.txt',
'current gpu: ' + str(self.resources))
log_print(
'while.txt', '--- print running queue ' +
str(list(self.running_info.queue)))
elif self.init_job_queue.empty() and self.check_free_gpu() is True:
self.introspect()
time.sleep(5)
def clash_enable(config, bot, update):
send_typing_action(bot, update)
username = update.message.from_user.username
chat_id = update.message.chat_id
message_id = update.message.message_id
if not username:
msg = "The unnamed ones are free from these silly humans vanity"
else:
with connector(config.engine()) as ses:
all_excludes = ses.query(ClashExclude.username).filter(ClashExclude.chat_id == update.message.chat_id).all()
all_excludes = [ x for x in all_excludes for x in x ]
if username in all_excludes:
ses.query(ClashExclude).filter(and_(
ClashExclude.chat_id == chat_id,
ClashExclude.username == username)).delete()
msg = "You will be notified when a new game is created!"
else:
msg = "You've already subscribed"
bot.send_message(chat_id=update.message.chat_id,
reply_to_message_id=message_id,
text=msg,
parse_mode="markdown")
log_print("Enabled",
chat_id=update.message.chat_id,
username=username,
level="INFO",
command="clash_disable")
def classify(input_dir, output_dir, model_dir):
"""
classify the input image to 7 classes
LABELS = ['front', 'front_3_quarter', 'side', 'rear_3_quarter', 'rear', 'interior', 'tire']
Args:
input_dir:
output_dir:
Returns:
list:
result of classification number of images for each labels
"""
if input_dir is None or output_dir is None:
raise Exception(" Input_dir or Output_dir not defined")
if model_dir is None:
model_dir = "./mdoel"
global MODEL_DIR
MODEL_DIR = model_dir
if not os.path.exists(model_dir) or _load_model_coefs(model_dir) == False:
raise Exception(" No such dir or files for getting coefs.")
cnts = _inference(input_dir, output_dir)
logger.log_print(" Classification finished Successfully!\n")
return cnts
def me(config, bot, update, args):
import telegram
send_typing_action(bot, update)
username = update.message.from_user.username
with connector(config.engine()) as ses:
user_matches = ses.query(Pingers).filter(Pingers.chat_id == update.message.chat_id,
Pingers.username == username).all()
out_text = ""
if [x.match for x in user_matches if x.me == 1]:
match = random.choice([x.match for x in user_matches if x.me == 1])
else:
match = username
out_text=italize("{match} {message}".format(
match=match.capitalize(),
message=escape(' '.join(args))))
bot.send_message(chat_id=update.message.chat_id,
text=out_text,
parse_mode=telegram.ParseMode.HTML)
bot.delete_message(chat_id=update.message.chat_id,
message_id=update.message.message_id)
log_print("Me",
chat_id=update.message.chat_id,
match=match,
username=username,
level="INFO",
command="me")
def align_combine(input_dir, output_dir):
"""
mode="combine", size=256
resize the images to be processed
with size(size=256x256) and mode(mode="combined")
Args:
input_dir: path to folder containing images
output_dir: path to folder containing combined/output image
Returns:
number of processed files
"""
if input_dir is None or output_dir is None:
raise Exception(" input_dir or ouput_dir not defined\n")
input_paths = glob.glob(os.path.join(input_dir, "*.jpg"))
if len(input_paths) == 0:
# raise Exception(" input_dir contains no .jpg image files")
logger.log_print(" input_dir contains no .jpg image files\n")
return len(input_paths)
# align and combined for testing
cnts = _align_and_combine(input_dir, output_dir, size=256)
logger.log_print(
" {} files Successfully aligned and combined.\n".format(cnts))
return cnts
def clash_disable(config, bot, update):
send_typing_action(bot, update)
username = update.message.from_user.username
chat_id = update.message.chat_id
message_id = update.message.message_id
if not username:
msg = "The unnamed ones are free from these silly humans vanity"
else:
with connector(config.engine()) as ses:
all_excludes = ses.query(ClashExclude.username).filter(ClashExclude.chat_id == update.message.chat_id).all()
all_excludes = [ x for x in all_excludes for x in x ]
if username in all_excludes:
msg = "You've already disabled notifications"
else:
exclude = ClashExclude(
username=username,
chat_id=chat_id)
ses.add(exclude)
msg = "Now you won't receive any notifications about Clash of Code games"
bot.send_message(chat_id=update.message.chat_id,
reply_to_message_id=message_id,
text=msg)
log_print("Disabled",
chat_id=update.message.chat_id,
username=username,
level="INFO",
command="clash_disable")
def receive_grow(self, info):
# print(info)
log_print('daemon.txt', 'grow ack job id: ' + info['id'])
try:
self.scheduler.grow_ack(info)
except Exception as e:
log_print('daemon.txt', 'exception receive_grow: ' + str(info))
def gpu_shrink(self, job):
log_print('scheduler.txt', '--- gpu shrink, job id: ' + job.id)
shrink_gpu_num = int(job.gpu_num / 2)
# TODO hardcoded here
node = 'localhost'
gpus = []
# TODO No multi node version
log_print('scheduler.txt', '--- print job: ' + str(vars(job)))
# below has error?
for n in job.gpus_loc.keys():
if len(job.gpus_loc[n]) >= shrink_gpu_num:
# node = n
gpus = job.gpus_loc[n][:shrink_gpu_num]
# job.gpus_loc[n] = job.gpus_loc[n][shrink_gpu_num:]
# break
# above is OK
for gpu in gpus:
self.resources[node][gpu] = -2
msg = {'status': 's', 'node': node, 'gpus': gpus}
send_msg(job.address, msg)
self.running_jobs[job.id].status = 'shrinking'
self.running_jobs[job.id].lock = True
self.shrinking_jobs.append(job.id)
return 0
def chat_id(bot, update):
current_chat_id = update.message.chat_id
username = update.message.from_user.username
bot.send_message(chat_id=current_chat_id,
text="`{0}`".format(current_chat_id),
reply_to_message_id=update.message.message_id,
parse_mode='markdown')
log_print('Chat id {0}'.format(current_chat_id), username)
def update_token(self):
"""Refreshes access_token attribute, intended for use when access_token expires."""
old_token = self.access_token
refresh_body = {"grant_type": "refresh_token", "refresh_token": self._refresh_token,
"client_id": spotify_cred.get_id()}
token_data = requests.post(access_token_url, data=refresh_body).json()
self.access_token = token_data["access_token"]
new_token = self.access_token
log_print(f"TOKEN UPDATE: Tokens Different? {old_token == new_token}")
def introspect(self):
""" When there is no job in queue, try to grow-shrink current running jobs.
Args:
Return:
"""
# print('introspect start')
jobs = self.get_running_jobs()
if len(jobs) == 0:
return
available_nodes = self.return_free_gpus()
if self.check_free_gpu() is False:
return
single_node_max = max(
[len(available_nodes[l]) for l in available_nodes.keys()])
schedule_jobs = []
for job in jobs:
if job.id in self.growing_jobs:
continue
# if a job is locked, it is under speed test
if not job.lock:
# utilization
if job.gpu_num == 1:
if self.cal_gpu_util(job) > 0.5:
schedule_jobs.append(job.id)
elif self.cal_gpu_util(job) * job.gpu_num > job.gpu_num - 1:
schedule_jobs.append(job.id)
else:
continue
# sort job by running time per epoch
schedule_jobs = sorted(schedule_jobs,
key=lambda item: self.running_jobs[item].ep_tm)
# if len(schedule_jobs) != 0:
# print('try to introspect jobs')
for jid in schedule_jobs:
if self.running_jobs[jid].gpu_num <= single_node_max:
# allocate GPU to this job
# TODO need to judge whether this job's gpu and allocated gpu in same node
gpus_loc = self.get_gpus(self.running_jobs[jid].gpu_num, 'g')
if gpus_loc[0] is None:
log_print('scheduler.txt',
'Error when allocating GPUs, job id: ' + jid)
self.gpu_grow(self.running_jobs[jid], gpus_loc)
available_nodes = self.return_free_gpus()
if self.check_free_gpu() is False:
single_node_max = 0
else:
single_node_max = max([
len(available_nodes[l])
for l in available_nodes.keys()
])
# TODO lock jobs that assigned gpu
else:
continue
async def answer_parser(config, bot, update):
in_text = prepare_message(update)
with connector(config.engine()) as ses:
out_text = ses.query(Answers.answer).filter(
literal(in_text).contains(Answers.match))
for message in ["".join(i) for i in out_text]:
bot.send_message(chat_id=update.message.chat_id, text=message)
log_print("Answer", update.message.from_user.username)
def _read_offers_ids_from_db(self):
# primitive way of storing data in file
try:
with open(settings.DB_FILE, 'rb') as file:
self.FOUND_OFFERS_IDS = pickle.load(file)
log_print(
f'read offers id from the database {self.FOUND_OFFERS_IDS}',
message_type=3)
except FileNotFoundError:
pass
def send_msg(address, message):
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
ip_address, port = address.split(':')
server_address = (ip_address, int(port))
sock.connect(server_address)
try:
sock.sendall(dict_to_binary(message))
except:
log_print('utils.txt', 'cannot send to address: ' + address + ' content: ' + message)
finally:
sock.close()
def begin(self):
print('generator begins!')
last_tm = time.time()
for each in self.jobs_list:
log_print('generator.txt', ' -------' + str(each))
remain = each[1] - last_tm
print('next job will come in ' + str(remain) + ' sec')
last_tm = each[1]
if remain > 0:
time.sleep(remain)
self.scheduler.receive_running_info(each[0])
def analyze_offer(self, offer_html) -> CarOffer:
safe_eval = Bs4SafeCodeEvaluator(offer_html, self.page_url)
offer_url = safe_eval.get_value_or_none(
"self.offer_html.find('a').attrs['href']")
if 'https://' not in offer_url:
offer_url = 'https://sprzedajemy.pl' + offer_url
offer_data = {
'tittle':
safe_eval.get_value_or_none(
"self.offer_html.find('h2', class_='title').text"),
'price':
safe_eval.get_value_or_none(
"self.offer_html.find('span', class_='price').text"),
'localization':
safe_eval.get_value_or_none(
"self.offer_html.find('strong', class_='city').text"),
'add_date':
safe_eval.get_value_or_none(
"self.offer_html.find('time', class_='time').attrs['datetime']"
),
'image_url':
safe_eval.get_value_or_none(
"self.offer_html.find('img').attrs['src']"),
'offer_url':
offer_url,
'mileage_in_km':
safe_eval.get_value_or_none(
"self.offer_html.find('span', string=re.compile('Przebieg')).parent.text"
),
'portal_offer_id':
offer_html.attrs['id'][len('offer-'):] # strip the 'offer-' prefix
}
internal_id = f"{self.__class__.__name__}_{offer_data['portal_offer_id']}"
engine_desc = safe_eval.get_value_or_none(
"self.offer_html.find('span', string=re.compile('Silnik')).parent.text"
)
engine_size = ''
gas_type = ''
if engine_desc:
try:
engine_size = engine_desc.split()[1]
gas_type = engine_desc.split()[2]
except Exception as e:
log_print(
f'The extracting the value from string {engine_desc} was unable. Details {e}'
)
car_offer = CarOffer(id=internal_id,
engine_size=engine_size,
gas_type=gas_type,
**offer_data)
return car_offer
def get_json_data(self, url: str) -> dict:
"""Pushes GET requests to Spotify API given a valid URL."""
self.update_general_header()
user_request = requests.get(url, headers=self.general_html_header)
try:
json_data = user_request.json()
except JSONDecodeError:
if user_request == "<Response [404]>":
log_print("No Track Playing")
return {}
return json_data
async def ping_parser(config, bot, update):
in_text = prepare_message(update)
with connector(config.engine()) as ses:
in_text_list = in_text.split()
username = update.message.from_user.username
chat_id = update.message.chat_id
try:
ses.query(PingPhrases.phrase).filter(
PingPhrases.phrase.in_(in_text_list)).limit(1).one()
usernames = ses.query(Pingers.username).filter(
and_(Pingers.match.in_(in_text_list),
or_(Pingers.chat_id == chat_id,
Pingers.chat_id == "all"))).distinct().all()
usernames = [i for i in usernames for i in i]
if 'EVERYONE GET IN HERE' in usernames:
try:
ses.query(PingExcludes.match).filter(
PingExcludes.match.in_(in_text_list)).one()
usernames = ses.query(Pingers.username).filter(
and_(
Pingers.username.notin_(usernames),
Pingers.username != username,
or_(Pingers.chat_id == chat_id,
Pingers.chat_id == "all"))).distinct().all()
usernames = [i for i in usernames for i in i]
except NoResultFound:
if ['EVERYONE GET IN HERE'] == usernames:
usernames = ses.query(Pingers.username).filter(
and_(
Pingers.username != 'EVERYONE GET IN HERE',
Pingers.username != username,
or_(Pingers.chat_id == chat_id, Pingers.chat_id
== "all"))).distinct().all()
usernames = [i for i in usernames for i in i]
if usernames:
send_typing_action(bot, update)
if 'EVERYONE GET IN HERE' in usernames:
usernames.remove('EVERYONE GET IN HERE')
out_text = " ".join(["@" + i for i in usernames])
bot.send_message(chat_id=update.message.chat_id,
reply_to_message_id=update.message.message_id,
text=out_text)
log_print("Ping",
chat_id=update.message.chat_id,
username=username,
pinged=out_text.split(),
level="INFO",
command="ping")
except NoResultFound:
pass
def connector(engine):
session = Session(engine)
try:
yield session
session.commit()
except:
error = str(sys.exc_info())
log_print(error, level="ERROR", command="connector")
session.rollback()
raise
finally:
session.close()
def connector(engine):
session = Session(engine)
try:
yield session
session.commit()
except:
error = str(sys.exc_info())
log_print("Error is: ", error)
session.rollback()
raise
finally:
session.close()
def update_running_info(self, info):
log_print('scheduler.txt', 'update job: ' + str(info))
if info['status'] == 'e':
del self.running_jobs[info['id']]
elif info['status'] == 'n':
gpu_tu = self.get_gpus(1, 'n')
info['gpus_loc'] = {gpu_tu[0]: gpu_tu[1]}
new_job = self.generate_new_job_by_info(info)
# get one GPU for it to run
self.running_jobs[info['id']] = new_job
# print('exec job')
self.E.exec(new_job)
def get_last_game_from_file(config, username, chat_id):
last_game = {"clash_id":"", "message_id":"", "users": "", "username": username}
try:
with open("/tmp/clash_{}".format(chat_id), "r") as file:
last_game = json.loads(file.read())
except IOError as io_e:
log_print("Could not read last_game from file",
error=str(io_e),
level="WARN",
func="get_last_game")
return last_game
def clash_start(config, bot, update):
send_typing_action(bot, update)
username = update.message.from_user.username
last_game = get_last_game(config, username, update.message.chat_id)
cookies = clash_get_cookies(config)
if last_game["clash_id"]:
if last_game["username"] == username:
r = requests.post('https://www.codingame.com/services/ClashOfCodeRemoteService/startClashByHandle',
headers={"content-type":"application/json;charset=UTF-8",
"cookie":"rememberMe={remember_me};cgSession={cg_session}".format(
remember_me=cookies["rememberMe"],
cg_session=cookies["cgSession"])},
data='[{user_id}, "{clash_id}"]'.format(user_id=cookies["user_id"],
clash_id=last_game["clash_id"]))
if r.status_code == 200 and "error" not in json.loads(r.text):
message="The game is about to start, hurry up!"
if last_game["users"]:
message = '{users}\n\n{msg}'.format(
users=last_game["users"],
msg=message)
log_print("Started",
chat_id=update.message.chat_id,
username=username,
clash_id=last_game["clash_id"],
level="INFO",
command="clash_start")
else:
message = 'Could not start "{}" Clash game...'.format(
last_game["clash_id"])
log_print("Failed on start",
chat_id=update.message.chat_id,
username=username,
clash_id=last_game["clash_id"],
level="ERROR",
command="clash_start")
else:
last_game["message_id"] = update.message.message_id
message = 'Only @{} is allowed to start the game'.format(last_game["username"])
else:
last_game["clash_id"] = "None"
message = "Could not find the last Clash id"
if last_game["message_id"]:
bot.send_message(chat_id=update.message.chat_id,
reply_to_message_id=last_game["message_id"],
text=message)
else:
bot.send_message(chat_id=update.message.chat_id,
text=message)
def __init__(self, file_path="./config/config.y*ml", database_path=None):
cfg = None
for filename in glob.glob(file_path):
with open(filename, 'r') as ymlfile:
cfg = yaml.load(ymlfile)
if cfg is None:
raise FileNotFoundError("There is no config file")
self.__tg_token = cfg['tokens']['tg_token'] if 'tg_token' in cfg[
'tokens'] else None
self.__clash_login = cfg['clash']['login'] if 'login' in cfg[
'clash'] else None
self.__clash_password = cfg['clash']['password'] if 'password' in cfg[
'clash'] else None
if database_path is None:
self.__db_host = cfg['database']['host'] if 'host' in cfg[
'database'] else None
elif isinstance(database_path, str) and database_path != "":
self.__db_host = database_path
self.__tg_mode = cfg['telegram']['mode'] if 'mode' in cfg[
'telegram'] else None
self.__tg_webhook_port = cfg['telegram'][
'webhook_port'] if 'webhook_port' in cfg['telegram'] else None
self.__tg_webhook_url = cfg['telegram']['webhook_url'].format(
self.__tg_token) if 'webhook_url' in cfg['telegram'] else None
self.__tg_listen_ip = cfg['telegram'][
'listen_ip'] if 'listen_ip' in cfg['telegram'] else None
self.__tg_admins = cfg['admins'] if 'admins' in cfg else []
self.__database = 'sqlite:///{}?check_same_thread=False'.format(
self.__db_host) if self.__db_host is not None else None
self.__engine = create_engine(
self.__database) if self.__database is not None else None
self.__redis_host = cfg['redis']['host'] if 'host' in cfg[
'redis'] else "localhost"
self.__redis_port = cfg['redis']['port'] if 'port' in cfg[
'redis'] else 6379
self.__redis_db = cfg['redis']['db'] if 'db' in cfg['redis'] else 0
try:
self.__redis = redis.StrictRedis(host=self.__redis_host,
port=self.__redis_port,
db=self.__redis_db)
except redis.RedisError as e:
log_print("Could not connect to Redis", error=str(e), level="WARN")
self.__redis = None
self.__log_level = cfg['log'][
'level'] if 'log' in cfg and 'level' in cfg['log'] else "INFO"
def doc2imgs(self, doc_path):
_, ext = os.path.splitext(os.path.basename(doc_path))
file_type = ext[1:].lower()
if file_type in ["pdf"]:
img_paths = self.pdfTojpgs(pdf_path=doc_path)
elif file_type in ["png", "jpg"]:
img_paths = [doc_path]
else:
log.log_print("not supported file type.")
sys.exit(1)
img_paths.sort()
return img_paths
def delete(self, bot, update, args):
usernames = [name[1:] for name in args if name[0] == "@"]
matches = [match.lower() for match in args if match[0] != "@"]
sender_username = update.message.from_user.username
if not usernames:
usernames = [sender_username]
if not matches:
if sender_username not in self.config.admins():
usage_text = "Usage:\n`/ping_delete [match]`"
else:
usage_text = "Usage:\n`/ping_delete [@username] [match]`\n`/ping_delete [match]`"
bot.send_message(chat_id=update.message.chat_id,
parse_mode='markdown',
text=usage_text)
return
if sender_username not in self.config.admins() and (
len(usernames) > 1
or len(list(filter(lambda x: x != sender_username,
usernames))) != 0):
message = "Deleting pings of another users is only allowed for admins."
bot.send_message(chat_id=update.message.chat_id, text=message)
return
answer = ""
with connector(self.config.engine()) as ses:
for username in usernames:
for match in matches:
result = ses.query(Pingers).filter(
and_(Pingers.chat_id == update.message.chat_id,
Pingers.username == username,
Pingers.match == match))
if not result.all():
answer += "Match `{0}` for user `@{1}` not found\n".format(
match, username)
else:
result.delete()
answer += "Match `{0}` for user `@{1}` deleted\n".format(
match, username)
log_print("Match deleted",
chat_id=update.message.chat_id,
ping_username=username,
match=match,
username=update.message.from_user.username,
level="INFO",
command="ping_delete")
bot.send_message(chat_id=update.message.chat_id,
parse_mode='markdown',
text=answer)
