def show(self, bot, update, args, me=0): usernames = " ".join(args).split() username = usernames[0] if usernames else "" if not username: username = update.message.from_user.username elif username[0] == "@": username = username[1:] else: usage_text = "Usage:\n`/ping_show[_me] @username`\n`/ping_show me`\n`/ping_show`\n" bot.send_message(chat_id=update.message.chat_id, parse_mode='markdown', text=usage_text) return with connector(self.config.engine()) as ses: user_matches = ses.query(Pingers).filter( Pingers.chat_id == update.message.chat_id, Pingers.username == username, Pingers.me == me).all() out_text = "" for match in user_matches: out_text += "\n{}".format(match.match) if out_text == "": out_text = "No such user" bot.send_message(chat_id=update.message.chat_id, text=out_text) log_print('Show pings of "@{0}", by {1}'.format( username, update.message.from_user.username))
def database_handler(config, bot, update, args): if update.message.from_user.username not in config.admins(): out_text = "You are not an administrator. The incident will be reported" command = "not an administrator" else: command = " ".join(args) if command == ".schema": command = "SELECT sql FROM sqlite_master WHERE type = 'table'" if command == ".tables": command = "SELECT name FROM sqlite_master WHERE type = 'table'" if "%%%chat_id%%%" in command: command = command.replace("%%%chat_id%%%", str(update.message.chat_id)) engine = create_engine(config.database()) connector = engine.connect() try: out_text = "\n".join([ " | ".join([str(i) for i in i]) for i in engine.execute(command).fetchall() ]) connector.close() except ResourceClosedError: out_text = command = "Successfuly" connector.close() except: out_text = command = "Bad command" connector.close() if out_text: bot.send_message(chat_id=update.message.chat_id, text=out_text) log_print('Manage "{0}"'.format(command), update.message.from_user.username)
def analyze_offer_details_page(self, url: str) -> dict: """ :param url: the url for details page :return: dict with params that will be used ass attribs into dataclass HouseOffer """ try: page = self.page_downloader.get_page(url) offer_html = BeautifulSoup(page.content, 'html.parser') safe_eval = Bs4SafeCodeEvaluator(offer_html, url) return { 'rooms_number': safe_eval.get_value_or_none( "int(self.offer_html.find_all(text='Liczba pokoi')[0].parent.parent.contents[1].text)" ), 'floor': safe_eval.get_value_or_none( "self.offer_html.find_all(text='Piętro')[0].parent.parent.contents[1].text" ), 'flat_area': safe_eval.get_value_or_none( "self.offer_html.find_all(text='Powierzchnia')[0].parent.parent.contents[3].text" ), } except Exception as e: log_print(f'Impossible to download page {url}. Details: {e}', ) return {}
def scan_site(self, url): try: page = self.page_downloader.get_page(url) parser = None if 'otodom.pl' in url: parser = OtoDomParser if 'olx.pl' in url: parser = OlxParser if 'sprzedajemy.pl' in url: parser = SprzedajemyParser if 'otomoto.pl' in url: parser = OtomotoParser if parser is None: log_print( f"[ERROR] couldn't find Page Parser for the site {url}") return return parser(downloader=self.page_downloader, found_offers_set=self.found_offers_set, message_sender=self.message_sender, page_url=url).analyze_html_page(page.content) except Exception as e: log_print(f"Coudn't fetch the url {url}. The error: {e}", message_type=1) print(traceback.format_exc())
def _msg_handle(self): while True: log_print('while.txt', 'while is running') if not self.running_info.empty(): # log_print('handler.txt', '--- print running queue ' + str(list(self.running_info.queue))) # TODO: below # Detect if there are available GPUs try: if self.allocate_gpu(): info = self.running_info.get() self.update_running_info(info) else: # currently no gpu can be allocated log_print('scheduler.txt', '--- current no gpu allocated') log_print( 'scheduler.txt', '----current GPU util: ' + str(self.resources)) except: log_print('while.txt', 'current gpu: ' + str(self.resources)) log_print( 'while.txt', '--- print running queue ' + str(list(self.running_info.queue))) elif self.init_job_queue.empty() and self.check_free_gpu() is True: self.introspect() time.sleep(5)
def clash_enable(config, bot, update): send_typing_action(bot, update) username = update.message.from_user.username chat_id = update.message.chat_id message_id = update.message.message_id if not username: msg = "The unnamed ones are free from these silly humans vanity" else: with connector(config.engine()) as ses: all_excludes = ses.query(ClashExclude.username).filter(ClashExclude.chat_id == update.message.chat_id).all() all_excludes = [ x for x in all_excludes for x in x ] if username in all_excludes: ses.query(ClashExclude).filter(and_( ClashExclude.chat_id == chat_id, ClashExclude.username == username)).delete() msg = "You will be notified when a new game is created!" else: msg = "You've already subscribed" bot.send_message(chat_id=update.message.chat_id, reply_to_message_id=message_id, text=msg, parse_mode="markdown") log_print("Enabled", chat_id=update.message.chat_id, username=username, level="INFO", command="clash_disable")
def classify(input_dir, output_dir, model_dir): """ classify the input image to 7 classes LABELS = ['front', 'front_3_quarter', 'side', 'rear_3_quarter', 'rear', 'interior', 'tire'] Args: input_dir: output_dir: Returns: list: result of classification number of images for each labels """ if input_dir is None or output_dir is None: raise Exception(" Input_dir or Output_dir not defined") if model_dir is None: model_dir = "./mdoel" global MODEL_DIR MODEL_DIR = model_dir if not os.path.exists(model_dir) or _load_model_coefs(model_dir) == False: raise Exception(" No such dir or files for getting coefs.") cnts = _inference(input_dir, output_dir) logger.log_print(" Classification finished Successfully!\n") return cnts
def me(config, bot, update, args): import telegram send_typing_action(bot, update) username = update.message.from_user.username with connector(config.engine()) as ses: user_matches = ses.query(Pingers).filter(Pingers.chat_id == update.message.chat_id, Pingers.username == username).all() out_text = "" if [x.match for x in user_matches if x.me == 1]: match = random.choice([x.match for x in user_matches if x.me == 1]) else: match = username out_text=italize("{match} {message}".format( match=match.capitalize(), message=escape(' '.join(args)))) bot.send_message(chat_id=update.message.chat_id, text=out_text, parse_mode=telegram.ParseMode.HTML) bot.delete_message(chat_id=update.message.chat_id, message_id=update.message.message_id) log_print("Me", chat_id=update.message.chat_id, match=match, username=username, level="INFO", command="me")
def align_combine(input_dir, output_dir): """ mode="combine", size=256 resize the images to be processed with size(size=256x256) and mode(mode="combined") Args: input_dir: path to folder containing images output_dir: path to folder containing combined/output image Returns: number of processed files """ if input_dir is None or output_dir is None: raise Exception(" input_dir or ouput_dir not defined\n") input_paths = glob.glob(os.path.join(input_dir, "*.jpg")) if len(input_paths) == 0: # raise Exception(" input_dir contains no .jpg image files") logger.log_print(" input_dir contains no .jpg image files\n") return len(input_paths) # align and combined for testing cnts = _align_and_combine(input_dir, output_dir, size=256) logger.log_print( " {} files Successfully aligned and combined.\n".format(cnts)) return cnts
def clash_disable(config, bot, update): send_typing_action(bot, update) username = update.message.from_user.username chat_id = update.message.chat_id message_id = update.message.message_id if not username: msg = "The unnamed ones are free from these silly humans vanity" else: with connector(config.engine()) as ses: all_excludes = ses.query(ClashExclude.username).filter(ClashExclude.chat_id == update.message.chat_id).all() all_excludes = [ x for x in all_excludes for x in x ] if username in all_excludes: msg = "You've already disabled notifications" else: exclude = ClashExclude( username=username, chat_id=chat_id) ses.add(exclude) msg = "Now you won't receive any notifications about Clash of Code games" bot.send_message(chat_id=update.message.chat_id, reply_to_message_id=message_id, text=msg) log_print("Disabled", chat_id=update.message.chat_id, username=username, level="INFO", command="clash_disable")
def receive_grow(self, info): # print(info) log_print('daemon.txt', 'grow ack job id: ' + info['id']) try: self.scheduler.grow_ack(info) except Exception as e: log_print('daemon.txt', 'exception receive_grow: ' + str(info))
def gpu_shrink(self, job): log_print('scheduler.txt', '--- gpu shrink, job id: ' + job.id) shrink_gpu_num = int(job.gpu_num / 2) # TODO hardcoded here node = 'localhost' gpus = [] # TODO No multi node version log_print('scheduler.txt', '--- print job: ' + str(vars(job))) # below has error? for n in job.gpus_loc.keys(): if len(job.gpus_loc[n]) >= shrink_gpu_num: # node = n gpus = job.gpus_loc[n][:shrink_gpu_num] # job.gpus_loc[n] = job.gpus_loc[n][shrink_gpu_num:] # break # above is OK for gpu in gpus: self.resources[node][gpu] = -2 msg = {'status': 's', 'node': node, 'gpus': gpus} send_msg(job.address, msg) self.running_jobs[job.id].status = 'shrinking' self.running_jobs[job.id].lock = True self.shrinking_jobs.append(job.id) return 0
def chat_id(bot, update): current_chat_id = update.message.chat_id username = update.message.from_user.username bot.send_message(chat_id=current_chat_id, text="`{0}`".format(current_chat_id), reply_to_message_id=update.message.message_id, parse_mode='markdown') log_print('Chat id {0}'.format(current_chat_id), username)
def update_token(self): """Refreshes access_token attribute, intended for use when access_token expires.""" old_token = self.access_token refresh_body = {"grant_type": "refresh_token", "refresh_token": self._refresh_token, "client_id": spotify_cred.get_id()} token_data = requests.post(access_token_url, data=refresh_body).json() self.access_token = token_data["access_token"] new_token = self.access_token log_print(f"TOKEN UPDATE: Tokens Different? {old_token == new_token}")
def introspect(self): """ When there is no job in queue, try to grow-shrink current running jobs. Args: Return: """ # print('introspect start') jobs = self.get_running_jobs() if len(jobs) == 0: return available_nodes = self.return_free_gpus() if self.check_free_gpu() is False: return single_node_max = max( [len(available_nodes[l]) for l in available_nodes.keys()]) schedule_jobs = [] for job in jobs: if job.id in self.growing_jobs: continue # if a job is locked, it is under speed test if not job.lock: # utilization if job.gpu_num == 1: if self.cal_gpu_util(job) > 0.5: schedule_jobs.append(job.id) elif self.cal_gpu_util(job) * job.gpu_num > job.gpu_num - 1: schedule_jobs.append(job.id) else: continue # sort job by running time per epoch schedule_jobs = sorted(schedule_jobs, key=lambda item: self.running_jobs[item].ep_tm) # if len(schedule_jobs) != 0: # print('try to introspect jobs') for jid in schedule_jobs: if self.running_jobs[jid].gpu_num <= single_node_max: # allocate GPU to this job # TODO need to judge whether this job's gpu and allocated gpu in same node gpus_loc = self.get_gpus(self.running_jobs[jid].gpu_num, 'g') if gpus_loc[0] is None: log_print('scheduler.txt', 'Error when allocating GPUs, job id: ' + jid) self.gpu_grow(self.running_jobs[jid], gpus_loc) available_nodes = self.return_free_gpus() if self.check_free_gpu() is False: single_node_max = 0 else: single_node_max = max([ len(available_nodes[l]) for l in available_nodes.keys() ]) # TODO lock jobs that assigned gpu else: continue
async def answer_parser(config, bot, update): in_text = prepare_message(update) with connector(config.engine()) as ses: out_text = ses.query(Answers.answer).filter( literal(in_text).contains(Answers.match)) for message in ["".join(i) for i in out_text]: bot.send_message(chat_id=update.message.chat_id, text=message) log_print("Answer", update.message.from_user.username)
def _read_offers_ids_from_db(self): # primitive way of storing data in file try: with open(settings.DB_FILE, 'rb') as file: self.FOUND_OFFERS_IDS = pickle.load(file) log_print( f'read offers id from the database {self.FOUND_OFFERS_IDS}', message_type=3) except FileNotFoundError: pass
def send_msg(address, message): sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) ip_address, port = address.split(':') server_address = (ip_address, int(port)) sock.connect(server_address) try: sock.sendall(dict_to_binary(message)) except: log_print('utils.txt', 'cannot send to address: ' + address + ' content: ' + message) finally: sock.close()
def begin(self): print('generator begins!') last_tm = time.time() for each in self.jobs_list: log_print('generator.txt', ' -------' + str(each)) remain = each[1] - last_tm print('next job will come in ' + str(remain) + ' sec') last_tm = each[1] if remain > 0: time.sleep(remain) self.scheduler.receive_running_info(each[0])
def analyze_offer(self, offer_html) -> CarOffer: safe_eval = Bs4SafeCodeEvaluator(offer_html, self.page_url) offer_url = safe_eval.get_value_or_none( "self.offer_html.find('a').attrs['href']") if 'https://' not in offer_url: offer_url = 'https://sprzedajemy.pl' + offer_url offer_data = { 'tittle': safe_eval.get_value_or_none( "self.offer_html.find('h2', class_='title').text"), 'price': safe_eval.get_value_or_none( "self.offer_html.find('span', class_='price').text"), 'localization': safe_eval.get_value_or_none( "self.offer_html.find('strong', class_='city').text"), 'add_date': safe_eval.get_value_or_none( "self.offer_html.find('time', class_='time').attrs['datetime']" ), 'image_url': safe_eval.get_value_or_none( "self.offer_html.find('img').attrs['src']"), 'offer_url': offer_url, 'mileage_in_km': safe_eval.get_value_or_none( "self.offer_html.find('span', string=re.compile('Przebieg')).parent.text" ), 'portal_offer_id': offer_html.attrs['id'][len('offer-'):] # strip the 'offer-' prefix } internal_id = f"{self.__class__.__name__}_{offer_data['portal_offer_id']}" engine_desc = safe_eval.get_value_or_none( "self.offer_html.find('span', string=re.compile('Silnik')).parent.text" ) engine_size = '' gas_type = '' if engine_desc: try: engine_size = engine_desc.split()[1] gas_type = engine_desc.split()[2] except Exception as e: log_print( f'The extracting the value from string {engine_desc} was unable. Details {e}' ) car_offer = CarOffer(id=internal_id, engine_size=engine_size, gas_type=gas_type, **offer_data) return car_offer
def get_json_data(self, url: str) -> dict: """Pushes GET requests to Spotify API given a valid URL.""" self.update_general_header() user_request = requests.get(url, headers=self.general_html_header) try: json_data = user_request.json() except JSONDecodeError: if user_request == "<Response [404]>": log_print("No Track Playing") return {} return json_data
async def ping_parser(config, bot, update): in_text = prepare_message(update) with connector(config.engine()) as ses: in_text_list = in_text.split() username = update.message.from_user.username chat_id = update.message.chat_id try: ses.query(PingPhrases.phrase).filter( PingPhrases.phrase.in_(in_text_list)).limit(1).one() usernames = ses.query(Pingers.username).filter( and_(Pingers.match.in_(in_text_list), or_(Pingers.chat_id == chat_id, Pingers.chat_id == "all"))).distinct().all() usernames = [i for i in usernames for i in i] if 'EVERYONE GET IN HERE' in usernames: try: ses.query(PingExcludes.match).filter( PingExcludes.match.in_(in_text_list)).one() usernames = ses.query(Pingers.username).filter( and_( Pingers.username.notin_(usernames), Pingers.username != username, or_(Pingers.chat_id == chat_id, Pingers.chat_id == "all"))).distinct().all() usernames = [i for i in usernames for i in i] except NoResultFound: if ['EVERYONE GET IN HERE'] == usernames: usernames = ses.query(Pingers.username).filter( and_( Pingers.username != 'EVERYONE GET IN HERE', Pingers.username != username, or_(Pingers.chat_id == chat_id, Pingers.chat_id == "all"))).distinct().all() usernames = [i for i in usernames for i in i] if usernames: send_typing_action(bot, update) if 'EVERYONE GET IN HERE' in usernames: usernames.remove('EVERYONE GET IN HERE') out_text = " ".join(["@" + i for i in usernames]) bot.send_message(chat_id=update.message.chat_id, reply_to_message_id=update.message.message_id, text=out_text) log_print("Ping", chat_id=update.message.chat_id, username=username, pinged=out_text.split(), level="INFO", command="ping") except NoResultFound: pass
def connector(engine): session = Session(engine) try: yield session session.commit() except: error = str(sys.exc_info()) log_print(error, level="ERROR", command="connector") session.rollback() raise finally: session.close()
def connector(engine): session = Session(engine) try: yield session session.commit() except: error = str(sys.exc_info()) log_print("Error is: ", error) session.rollback() raise finally: session.close()
def update_running_info(self, info): log_print('scheduler.txt', 'update job: ' + str(info)) if info['status'] == 'e': del self.running_jobs[info['id']] elif info['status'] == 'n': gpu_tu = self.get_gpus(1, 'n') info['gpus_loc'] = {gpu_tu[0]: gpu_tu[1]} new_job = self.generate_new_job_by_info(info) # get one GPU for it to run self.running_jobs[info['id']] = new_job # print('exec job') self.E.exec(new_job)
def get_last_game_from_file(config, username, chat_id): last_game = {"clash_id":"", "message_id":"", "users": "", "username": username} try: with open("/tmp/clash_{}".format(chat_id), "r") as file: last_game = json.loads(file.read()) except IOError as io_e: log_print("Could not read last_game from file", error=str(io_e), level="WARN", func="get_last_game") return last_game
def clash_start(config, bot, update): send_typing_action(bot, update) username = update.message.from_user.username last_game = get_last_game(config, username, update.message.chat_id) cookies = clash_get_cookies(config) if last_game["clash_id"]: if last_game["username"] == username: r = requests.post('https://www.codingame.com/services/ClashOfCodeRemoteService/startClashByHandle', headers={"content-type":"application/json;charset=UTF-8", "cookie":"rememberMe={remember_me};cgSession={cg_session}".format( remember_me=cookies["rememberMe"], cg_session=cookies["cgSession"])}, data='[{user_id}, "{clash_id}"]'.format(user_id=cookies["user_id"], clash_id=last_game["clash_id"])) if r.status_code == 200 and "error" not in json.loads(r.text): message="The game is about to start, hurry up!" if last_game["users"]: message = '{users}\n\n{msg}'.format( users=last_game["users"], msg=message) log_print("Started", chat_id=update.message.chat_id, username=username, clash_id=last_game["clash_id"], level="INFO", command="clash_start") else: message = 'Could not start "{}" Clash game...'.format( last_game["clash_id"]) log_print("Failed on start", chat_id=update.message.chat_id, username=username, clash_id=last_game["clash_id"], level="ERROR", command="clash_start") else: last_game["message_id"] = update.message.message_id message = 'Only @{} is allowed to start the game'.format(last_game["username"]) else: last_game["clash_id"] = "None" message = "Could not find the last Clash id" if last_game["message_id"]: bot.send_message(chat_id=update.message.chat_id, reply_to_message_id=last_game["message_id"], text=message) else: bot.send_message(chat_id=update.message.chat_id, text=message)
def __init__(self, file_path="./config/config.y*ml", database_path=None): cfg = None for filename in glob.glob(file_path): with open(filename, 'r') as ymlfile: cfg = yaml.load(ymlfile) if cfg is None: raise FileNotFoundError("There is no config file") self.__tg_token = cfg['tokens']['tg_token'] if 'tg_token' in cfg[ 'tokens'] else None self.__clash_login = cfg['clash']['login'] if 'login' in cfg[ 'clash'] else None self.__clash_password = cfg['clash']['password'] if 'password' in cfg[ 'clash'] else None if database_path is None: self.__db_host = cfg['database']['host'] if 'host' in cfg[ 'database'] else None elif isinstance(database_path, str) and database_path != "": self.__db_host = database_path self.__tg_mode = cfg['telegram']['mode'] if 'mode' in cfg[ 'telegram'] else None self.__tg_webhook_port = cfg['telegram'][ 'webhook_port'] if 'webhook_port' in cfg['telegram'] else None self.__tg_webhook_url = cfg['telegram']['webhook_url'].format( self.__tg_token) if 'webhook_url' in cfg['telegram'] else None self.__tg_listen_ip = cfg['telegram'][ 'listen_ip'] if 'listen_ip' in cfg['telegram'] else None self.__tg_admins = cfg['admins'] if 'admins' in cfg else [] self.__database = 'sqlite:///{}?check_same_thread=False'.format( self.__db_host) if self.__db_host is not None else None self.__engine = create_engine( self.__database) if self.__database is not None else None self.__redis_host = cfg['redis']['host'] if 'host' in cfg[ 'redis'] else "localhost" self.__redis_port = cfg['redis']['port'] if 'port' in cfg[ 'redis'] else 6379 self.__redis_db = cfg['redis']['db'] if 'db' in cfg['redis'] else 0 try: self.__redis = redis.StrictRedis(host=self.__redis_host, port=self.__redis_port, db=self.__redis_db) except redis.RedisError as e: log_print("Could not connect to Redis", error=str(e), level="WARN") self.__redis = None self.__log_level = cfg['log'][ 'level'] if 'log' in cfg and 'level' in cfg['log'] else "INFO"
def doc2imgs(self, doc_path): _, ext = os.path.splitext(os.path.basename(doc_path)) file_type = ext[1:].lower() if file_type in ["pdf"]: img_paths = self.pdfTojpgs(pdf_path=doc_path) elif file_type in ["png", "jpg"]: img_paths = [doc_path] else: log.log_print("not supported file type.") sys.exit(1) img_paths.sort() return img_paths
def delete(self, bot, update, args): usernames = [name[1:] for name in args if name[0] == "@"] matches = [match.lower() for match in args if match[0] != "@"] sender_username = update.message.from_user.username if not usernames: usernames = [sender_username] if not matches: if sender_username not in self.config.admins(): usage_text = "Usage:\n`/ping_delete [match]`" else: usage_text = "Usage:\n`/ping_delete [@username] [match]`\n`/ping_delete [match]`" bot.send_message(chat_id=update.message.chat_id, parse_mode='markdown', text=usage_text) return if sender_username not in self.config.admins() and ( len(usernames) > 1 or len(list(filter(lambda x: x != sender_username, usernames))) != 0): message = "Deleting pings of another users is only allowed for admins." bot.send_message(chat_id=update.message.chat_id, text=message) return answer = "" with connector(self.config.engine()) as ses: for username in usernames: for match in matches: result = ses.query(Pingers).filter( and_(Pingers.chat_id == update.message.chat_id, Pingers.username == username, Pingers.match == match)) if not result.all(): answer += "Match `{0}` for user `@{1}` not found\n".format( match, username) else: result.delete() answer += "Match `{0}` for user `@{1}` deleted\n".format( match, username) log_print("Match deleted", chat_id=update.message.chat_id, ping_username=username, match=match, username=update.message.from_user.username, level="INFO", command="ping_delete") bot.send_message(chat_id=update.message.chat_id, parse_mode='markdown', text=answer)