async def check_if_command(self, data: MessageEventData,
msg_id: int) -> None:
msg_obj = Message(self.vk, data)
result = await self.cmd_system.process_command(msg_obj)
if self.LOG_MESSAGES:
who = f"{'конференции' if data.conf else 'ЛС'} {data.peer_id}"
hues.info(f"Сообщение из {who} > {data.body}")
def discover_fonts(path, urlprefix='', picks=[]):
'''Use font-config to scan and list the properties in a dataframe.
In the scanned directory, font-config optionally accepts an output format
string. We use this format for each font files found:
>{family[0]}|{postscriptname}|{file}
I added the `>` specifier to distinguish additional output (to stderr, but
subprocess.getoutput captures both `stdout` and `stderr` together) from
warnings.
[NOTE] 05-09-2019
Modified pattern to family[0], since the font family
could include more than one names. We only want one.
'''
header = ['family', 'postscriptname', 'file']
fc_format = '%{family[0]}|%{postscriptname}|%{file}'
# Uses fc-scan utility from font-config.
cmd = f'fc-scan {path} -b -f ">{fc_format}\\n"'
df = (py_(subprocess.getoutput(cmd)).lines().filter(
lambda s: s[0] == '>').map(lambda s: s[1:]).join('\n').thru(
io.StringIO).thru(lambda b: pd.read_csv(b, sep='|', names=header)).
thru(lambda df: process_fontlist(df, urlprefix, path)).value())
if len(picks) > 0:
hues.info('Picking:', picks)
picked_fonts = [name.lower() for name in picks]
frame_slice = (df.family.str.lower().isin(picked_fonts))
return df[frame_slice]
return df
def searchArchive():
global ids_temp
global urls_temp
hues.info('SCANNING ARCHIVE')
files = glob.glob(
os.path.dirname(os.path.realpath(__file__)) + '/html/*.html')
soup = BeautifulSoup(open(files[0], encoding="utf-8"), 'lxml')
rows = soup.findAll("tr", attrs={"class": "searchResultsItem"})
for row in rows:
rows_string = str(row).split('\n')
# Find the ID of the Ad
id = re.findall('data-id=\"(.*?)\">', rows_string[0], re.DOTALL)
ids_cleaned = str(id).replace('[\'', '')
ids_cleaned = ids_cleaned.replace('\']', '')
ids_temp.append(int(ids_cleaned))
# Find & Clear the URL
url = re.findall('/ilan/(.*?)/detay', rows_string[2], re.DOTALL)
url_string = str(url[0]).replace('[', '')
url_string = str(url[0]).replace(']', '')
#urls = 'https://www.sahibinden.com/ilan/' + url_string + '/detay'
urls_temp.append('https://www.sahibinden.com/ilan/' + url_string +
'/detay')
def init_settings(self):
"""Функция инициализации файла настроек и его создания"""
# Если у нас есть только settings.py.sample
if isfile('settings.py.sample') and not isfile('settings.py'):
try:
shutil.copy('settings.py.sample', 'settings.py')
except Exception:
fatal('Я не могу копировать файлы в текущей папке, '
'проверьте ваши права на неё!')
hues.info('Был создан файл settings.py, '
'не забудьте добавить данные для авторизации!')
exit()
# Если у нас уже есть settings.py
elif isfile('settings.py'):
import settings
try:
self.BLACKLIST = settings.BLACKLIST
self.PREFIXES = settings.PREFIXES
self.LOG_MESSAGES = settings.LOG_MESSAGES
self.LOG_COMMANDS = settings.LOG_COMMANDS
self.APP_ID = settings.APP_ID
self.SCOPE = settings.SCOPE
self.FLOOD_INTERVAL = settings.FLOOD_INTERVAL
self.USERS = settings.USERS
self.PROXIES = settings.PROXIES
if not self.USERS:
fatal("Проверьте, что у есть LOGIN и PASSWORD, или же TOKEN в файле settings.py!"
"Без них бот работать НЕ СМОЖЕТ.")
except (ValueError, AttributeError, NameError):
fatal('Проверьте содержимое файла settings.py, возможно вы удалили что-то нужное!')
# Если не нашли ни settings.py, ни settings.py.sample
else:
fatal("settings.py и settings.py.sample не найдены, возможно вы их удалили?")
def get_photos_urls(vk, user_id, directory):
hues.info("Начинаю поиск фотографий...")
photos_urls = {}
start_from = 0
while True:
results = vk.messages.getHistoryAttachments(
peer_id=user_id, media_type="photo", count=200, start_from=start_from
)
if results["items"]:
for attachment in results["items"]:
photo = attachment["attachment"]["photo"]
photo_name = (
f"{photo['id']}_{photo['owner_id']} "
f"{dt.fromtimestamp(photo['date'], TZ).strftime('%d-%m-%y %H:%M')}.jpg"
)
# example: 373772945_105918493 07-07-15 02:29
photo_save_path = f"{directory}/{photo_name}"
photo_download_url = photo["sizes"][-1]["url"]
photos_urls.update({photo_save_path: photo_download_url})
start_from = results["next_from"]
hues.log(
f"Получено {len(photos_urls)} фото, следующее смещение: {start_from}"
)
else:
hues.info(f"Найдено {len(photos_urls)} фото.")
break
time.sleep(INTERVAL)
return photos_urls
def mserver():
host = '127.0.0.1' # Bind to all interfaces
port = 51500
# Step1: 创建socket对象
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
# Step2: 设置socket选项(可选)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
# Step3: 绑定到某一个端口
s.bind((host, port))
# Step4: 监听该端口上的连接
while 1:
try:
message, address = s.recvfrom(8192)
hues.info("Got data from ", address)
hues.success("Data is:",message)
s.sendto("Data is received succeefully", address) # 告知客户端，信息已收到
except (KeyboardInterrupt, SystemExit):
hues.warn("raise")
raise
except:
hues.warn("traceback")
traceback.print_exc()
def tcpclient():
s=socket.socket(socket.AF_INET,socket.SOCK_STREAM)
s.connect(('10.10.160.11',9999))
hues.info(s.recv(1024))
for data in list('s=socket.socket(socket.AF_INET,socket.SOCK_STREAM)'): #['Michael','Tracy','Sarah']:
s.send(data.encode())
print(s.recv(1024))
s.send(b'exit')
async def check_if_command(self, data: MessageEventData,
msg_id: int) -> None:
msg_obj = Message(self.vk, data)
result = await self.cmd_system.process_command(msg_obj)
if result:
# Если это - команда, то помечаем сообщение прочитанным
# Сделано для того, чтобы бот не читал обычные сообщения
return await self.vk.mark_as_read(msg_id)
if self.LOG_MESSAGES:
who = f"{'конференции' if data.conf else 'ЛС'} {data.peer_id}"
hues.info(f"Сообщение из {who} > {data.body}")
def webfonts_generator(path, urlprefix, output, take, silent):
'''Discover and generate CSS file for the webfonts.'''
fonts = discover_fonts(path, urlprefix, take)
with open(output, 'w') as fp:
fp.write(generate_css(fonts))
if not silent:
hues.success(f'[!] Found {len(fonts)} matching fonts in {path}.')
for name, els in fonts.groupby('fname').groups.items():
hues.info(f' -> {name} [{len(els)} style(s)]')
hues.success(f'<~> CSS File written in {output}')
def tcplink(sock,addr):
hues.info("accept new connection from %s:%s..." % addr)
sock.send("Welcom!".encode())
while True: #通信循环：发送与接收
data=sock.recv(1024)
hues.success("Received [%s] from %s:%s" %(data,addr[0],addr[1]))
time.sleep(5)
if data=='exit' or not data:
break
sock.send("hello: ".encode()+data)
sock.close()
hues.warn("Connection from %s:%s closed." % addr)
def handle(self, *args, **options):
indices = registered_indices()
if len(indices):
hues.info('Discovered', len(indices), 'Indexes')
else:
hues.warn('No search indexes found')
for i, index in enumerate(indices, 1):
hues.info('==> Indexing objects from', index.model, 'in', index)
for obj in tqdm(index.queryset):
doc = index.init_using_pk(obj.pk)
doc.prepare()
doc.save()
hues.success('--> {0}/{1} indexed.'.format(i, len(indices)))
def searchInRange(self,inlist,target):
if target>inlist[-1] or target<inlist[0]:
return -1
low,high=0,len(inlist)-1 # 以数组的位置下标为索引
while True:
mpos=(low+high)/2
mv=inlist[mpos]
if target<mv:
high=len(inlist)/2
elif target>mv:
low=len(inlist)/2
else:
return mpos
hues.info(low,high)
def run(self):
if DataChecker.check():
self.logger.fatal('Please check config.json')
raise SystemExit(-1)
hues.info('Work started')
if not self.user_id:
self.user_id = InputWorker.get_user_id()
for i, user in enumerate(self.user_id):
if not user:
self.logger.fatal("Bad ID")
raise SystemExit(-1)
hues.info(f'User with ID {user} is handling now. ({i + 1}/{len(self.user_id)})')
DataHandler(user).handler()
def compareCSV():
global ids_new
global urls_new
global ids_temp
global urls_temp
global ids_real
global urls_real
# Compare _temp and _real
ids_new = [x for x in ids_temp if x not in ids_real]
urls_new = [x for x in urls_temp if x not in urls_real]
if ids_new:
hues.info('NEW ID COUNT: ' + str(len(ids_new)))
elif not ids_new:
hues.warn('NO NEW ID')
def plugin_init(self):
hues.info("Загрузка плагинов...")
# Подгружаем плагины
self.plugin_system = PluginSystem(self.vk, folder=abspath('plugins'))
self.plugin_system.register_commands()
# Чтобы плагины могли получить список всех плагинов (костыль)
self.vk.get_plugins = self.plugin_system.get_plugins
# Для парсинга команд с пробелом используется
# обратная сортировка, для того, чтобы самые
# длинные команды были первыми в списке
command_names = list(self.plugin_system.commands.keys())
command_names.sort(key=len, reverse=True)
from command import CommandSystem
self.cmd_system = CommandSystem(command_names, self.plugin_system)
self.scheduled_funcs = self.plugin_system.scheduled_events
hues.success("Загрузка плагинов завершена")
def cli():
cli = argparse.ArgumentParser(
description='Parse and validate Solr Synonyms file.')
cli.add_argument('file', help='Solr Synonyms file', nargs='+')
cli.add_argument('-v',
'--verbose',
action='store_true',
help='Print a verbose report')
args = cli.parse_args()
status = 0
for fname in args.file:
hues.info('Validating', fname)
reports = SynParser.parse(filename=fname)
error = print_reports(reports, args.verbose)
if error:
status += 1
sys.exit(status)
async def user(self, username, password, app_id, scope):
self.username = username
self.password = password
self.app_id = app_id
self.scope = scope
retries = 5
for i in range(retries):
self.token = await get_token(username, password, app_id, scope)
if self.token:
break
if not self.token:
return hues.error("Can't get token!")
self_data = await self.execute("return API.account.getProfileInfo();")
hues.info(f"Вошёл как: {self_data['first_name']} {self_data['last_name']} "
f"(https://vk.com/{self_data['screen_name']})")
def locales_combinator(path, output, silent):
'''Combines Locale Strings in yml files at `path` in a json blob at `output`'''
locale_files = list(discover_locales(path))
locales = {l.lang: l.body for l in locale_files}
langs = [l.lang for l in locale_files]
blob = {
'_meta': {
'langs': langs,
'generated': datetime.datetime.utcnow().isoformat(),
},
**locales,
}
with open(output, 'w') as fp:
json.dump(blob, fp, ensure_ascii=False, indent=2, sort_keys=True)
if not silent:
hues.info('-> Discovered locales in languages:',
blob['_meta']['langs'])
hues.success('<~> Wrote locales as json to', output)
def get_peer_id(vk):
while True:
peer_id = input(
"Введите ID (пользователя/беседы/сообщества) для начала загрузки фото: "
)
try:
peer_id = int(peer_id)
except ValueError:
hues.error("ID должен быть числом!")
continue
# Групповая беседа
if peer_id > 2000000000:
hues.info(f"Групповая беседа {peer_id}")
break
# Сообщество
if peer_id < 0:
hues.info(f"Сообщество {peer_id}")
break
# Пользователь
try:
results = vk.users.get(user_ids=peer_id)
hues.info(
f"Найден пользователь {results[0]['first_name']} {results[0]['last_name']}"
)
break
except vk_api.exceptions.ApiError:
hues.error("Неверный id (Пример: 105918493)")
return peer_id
def infos(strs, crlf=0): # 默认前面不换行
#################################### 命令行即时输出
colorstr = "\033[1;34;40m"
closecolor = "\033[0m"
##### 非LINUX不支持彩色日志输出
if platform.system() != "Linux":
colorstr = ""
closecolor = ""
######
print("---------------------------------------------------------")
#print(colorstr + strs + closecolor ) ## 这种模式不兼容 windows , 因此暂时停止使用
### 暂时使用以下模式 #hues.log hues.info hues.error hues.warn hues.success
hues.info(strs)
################################################## 报告输出
#### 获得是否报告的标记位
reports = get_reports_tag()
##### 报告标记位正常
if reports != 0:
doc = documents.doc
# 插入表格
if crlf == 1: #换行
doc.insert_text("\n")
mytable3 = doc.insert_table(1, 1)
doc.table_setattr(mytable3, "A1", "BackColor", 0xcdc9c9) # 灰色
doc.insert_tabletext(mytable3, "A1", strs)
async def check_if_command(self, data: MessageEventData, user) -> None:
if self.LOG_MESSAGES:
who = f"{'конференции' if data.conf else 'ЛС'} {data.peer_id}"
hues.info(f"Сообщение из {who} > {data.body}")
msg_obj = Message(self.vk, data, user)
cmd = Command(msg_obj)
if not cmd.has_prefix:
if self.DO_CHAT and self.IGNORE_PREFIX:
await self.do_chat(msg_obj, user)
return
if self.ONLY_CHAT and self.DO_CHAT:
await self.do_chat(msg_obj, user)
else:
result = await self.cmd_system.process_command(msg_obj, cmd)
if result is False:
await self.do_chat(msg_obj, user)
def run(self) -> None:
hues.info('### Configuring started ###')
token = self.__request('VK token: ', lambda x: len(x) == 85)
delta = self.__request('Delta (hour/day/week/month/year): ',
lambda x: x in ['hour', 'day', 'week', 'month', 'year'])
fps_suj = 0
if delta == 'hour':
fps_suj = 60
elif delta == 'day':
fps_suj = 15
elif delta == 'week':
fps_suj = 7
elif delta == 'month':
fps_suj = 5
elif delta == 'year':
fps_suj = 1
fps = self.__request(f'FPS ({fps_suj} is recommended): ', lambda x: 0 < int(x) < 120)
clean = True if self.__request(f'Clear cache? (y/n): ',
lambda x: x.lower().strip() in ['y', 'n']) == 'y' else False
_display = get_monitors()
if len(_display) > 1:
_d_msg = '\n'.join(
[f'{i + 1}# Display (width={obj.width}, height={obj.height})' for i, obj in enumerate(_display)])
_d_i = self.__request(f'You have {len(_display)} monitors. Choose video resolution:\n{_d_msg}',
lambda x: 1 < int(x) < len(_display) + 1)
display = _display[int(_d_i) - 1]
_display = _display[0]
d_width, d_height = _display.width, _display.height
hues.info(f'Video resolution is ({d_width}, {d_height}) now.')
result = copy(vars())
for x in list(result):
if x in ['self', 'result'] or x.startswith('_'):
result.pop(x)
open('config.json', 'w', encoding='UTF-8').write(json.dumps(result))
hues.info('### Configuring done ###')
if self.restart:
sleep(2)
execl(executable, executable, *argv)
def handle(self, delete, *args, **options):
indices = registered_indices()
connection = connections.get_connection()
hues.info('Using connection', connection)
if len(indices):
hues.info('Discovered', len(indices), 'Indexes')
else:
hues.warn('No search index found')
for i, index in enumerate(indices, 1):
hues.info('==> Initializing', index.__name__)
with index().ensure_closed_and_reopened() as ix:
if delete:
hues.warn('Deleting existing index.')
ix.delete_index()
ix.init()
hues.success('--> Done {0}/{1}'.format(i, len(indices)))
compareCSV()
# 5 - Fetch new single pages
#hues.info('AD PAGES DOWNLOADING')
for urls in urls_new:
fetchSinglePage(urls)
#print('New Ad ID: ' + str(row_temp))
#print('URL: ' + urls_temp[x])
# 5.1 - IDs & URLs to real_estate_data.csv
real_estate_data()
# 5.2 - Write _new to id_url.csv
writeNew()
# 6 - Delete archive.html & single.html, Clear _new and _temp
clearSYSTEM()
if __name__ == '__main__':
while True:
#automator(url)
for x in range(1, 951):
if float(x % 50) == 0:
hues.info('PAGE: ' + str(int(x / 50)))
automator(
'https://www.sahibinden.com/satilik-daire?viewType=List&pagingOffset='
+ str(x) + '&pagingSize=50&sorting=date_desc')
hues.info('SLEEPING')
time.sleep(900) # 15 min
def create_directory(peer_id):
hues.info("Создаю папку...")
directory = f"{peer_id} ({dt.now(TZ).strftime('%d-%m-%y %H:%M:%S')})"
if not os.path.exists(directory):
os.makedirs(directory)
return directory
if err_num == 1:
self.longpoll_values['ts'] = events['ts']
# Коды 2 и 3 - нужно запросить данные нового
# Long Polling сервера
elif err_num in (2, 3):
await self.init_long_polling(err_num)
continue
# Обновляем время, чтобы не приходили старые события
self.longpoll_values['ts'] = events['ts']
for event in events['updates']:
schedule_coroutine(self.check_event(event))
if __name__ == '__main__':
hues.info("Приступаю к запуску DarkVKBOT")
bot = Bot()
main_loop = asyncio.get_event_loop()
main_loop.run_until_complete(set_up_roles(bot))
hues.success("Приступаю к приему сообщений")
try:
main_loop.run_until_complete(bot.run(main_loop))
except (KeyboardInterrupt, SystemExit):
hues.warn("Выключение бота...")
except Exception as ex:
import traceback
import os
import numpy as np
import scipy.io as io
import argparse
import hues
parser = argparse.ArgumentParser()
parser.add_argument("--path", default='./', type=str)
opt, _ = parser.parse_known_args()
if __name__ == "__main__":
for root, dirs, files in os.walk(opt.path, topdown=False):
for name in files:
if os.path.splitext(name)[-1] == '.npy':
hues.info(os.path.join(root, name))
temp = np.load(os.path.join(root, name))
if temp.ndim == 3:
temp = temp.transpose(1, 2, 0)
save_name = os.path.splitext(name)[0] + '.mat'
io.savemat(os.path.join(root, save_name), {'img': temp})
train_psnr_list = []
for i, data in enumerate(train_dataloader):
iter_start_time = time.time()
total_steps += train_opt.batchsize
epoch_iter += train_opt.batchsize
visualizer.reset()
train_model.set_input(data, True)
train_model.optimize_joint_parameters(epoch)
hues.info("[{}/{} in {}/{}]".format(
i, dataset_size // train_opt.batchsize, epoch,
train_opt.niter + train_opt.niter_decay))
train_psnr = train_model.cal_psnr()
train_psnr_list.append(train_psnr)
if epoch % train_opt.print_freq == 0:
losses = train_model.get_current_losses()
t = (time.time() - iter_start_time) / train_opt.batchsize
visualizer.print_current_losses(epoch, epoch_iter, losses, t)
if train_opt.display_id > 0:
visualizer.plot_current_losses(
epoch,
float(epoch_iter) / dataset_size, train_opt, losses)
visualizer.display_current_results(
train_model.get_current_visuals(),
def log(self):
"""Пишет в лог, что была распознана команда"""
pid = self._data.peer_id
who = ("конференции {}" if self._data.conf else "ЛС {}").format(pid)
hues.info(
f"Команда '{self.command}' из {who} с аргументами {self.args}")
def log(self, message: str):
hues.info(f'Плагин {self.name} -> {message}')
# def __init__(self, x):
# self.label = x
# self.next = None
# self.random = None
class Solution(object):
def copyRandomList(self, head):
"""
:type head: RandomListNode
:rtype: RandomListNode
"""
'''
import hues
hues.log('Mission', 42)
hues.info('Finding', 42)
hues.error(41, 'is not', 42)
hues.warn('We are distracted...')
hues.info('Found', 24)
hues.success('Close enough')
class LinkListRandom:
def __init__(self, x):
self.label = x
self.next = None
self.random = None
#插入一个节点
def insert(self, head, val):
node = LinkListRandom(val)
def log(self):
"""Отправка сообщения в лог о том, что была распознана команда"""
pid = self.msg.peer_id
who = ("конференции {}" if self.msg.conf else "ЛС {}").format(pid)
hues.info(f"Команда '{self.command}' из {who} с аргументами {self.args}")
if err_num == 1:
self.longpoll_values['ts'] = events['ts']
# Коды 2 и 3 - нужно запросить данные нового
# Long Polling сервера
elif err_num in (2, 3):
await self.init_long_polling(err_num)
continue
# Обновляем время, чтобы не приходили старые события
self.longpoll_values['ts'] = events['ts']
for event in events['updates']:
schedule_coroutine(self.check_event(event))
if __name__ == '__main__':
hues.info("Приступаю к запуску VBot v5.0")
bot = Bot()
main_loop = asyncio.get_event_loop()
main_loop.run_until_complete(set_up_roles(bot))
hues.success("Приступаю к приему сообщений")
try:
main_loop.run_until_complete(bot.run(main_loop))
except (KeyboardInterrupt, SystemExit):
hues.warn("Выключение бота...")
except Exception as ex:
import traceback
indexes = [int(x) for x in sorted(data['ID'].unique())]
df = pd.DataFrame(columns=['prix','last_seen'], index=indexes)
for i in tqdm(range(len(data))):
ID, time, prix = data.iloc[i]
ID = int(ID)
time0 = df.loc[ID]['last_seen']
if not pd.isnull(time0):
if int(time0) > int(time):
continue
df.loc[ID,'prix'] = int(prix.replace(" ",""))
df.loc[ID,'last_seen'] = int(time)
return df
if __name__ == "__main__":
hues.info("Connecting")
con = sqlite3.connect("./data/raw_data.db")
hues.info("Importing annonces ...")
annonces = pd.read_sql_query("SELECT * FROM annonce", con, index_col='ID')
annonces = annonces[['arrondissement', 'agency_phone']]
annonces.index = [int(x) for x in annonces.index]
hues.info("Importing description ...")
descriptions = pd.read_sql_query("SELECT * FROM description", con, index_col='ID')
descriptions.index = [int(x) for x in descriptions.index]
hues.info("Importing prices ...")
prices = pd.read_sql_query("SELECT * FROM prix", con)
prices = reshape_price(prices)
