def create_connect(self, role):
#建立socket
self.sockfd = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.sockfd.setblocking(False)
#SO_LINGER选项设置
#optval = struct.pack("ii",0,0)
#self.sockfd.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, optval)
#建立connect，根据desc_idx的值，不断切换
if self.desc_idx==0:
self.desc_idx = 1
desc_address = (config.ADDR_1, int(config.PORT_1))
elif self.desc_idx==1:
self.desc_idx = 0
desc_address = (config.ADDR_2, int(config.PORT_2))
try:
self.sockfd.connect(desc_address)
except socket.error, ex:
(code, msg) = ex
if code == 115:
# 当设置了noblock之后，115错误表示服务器忙，其实已经建立连接了
# 所以可以忽略这个错误（当服务器当掉了，也会报这个错误）
config.write_log("the socket.connect error! code:%d"%code+", msg:"+msg, role)
pass
else:
self.sockfd.close()
print "the socket.connect error! code:%d"%code+", msg:"+msg
config.write_log("the socket.connect error! code:%d"%code+", msg:"+msg, role)
sys.exit(1)
def check_free_date(bot, message):
"""
:param our_date: str
:return: str
"""
user = str(message.chat.id)
our_date = str(message.text)
while True:
time.sleep(1)
try:
date = read_date(user)
if our_date != date:
write_result_date(user, "", False)
break
data_list = load_date()["data"]
for data in data_list:
if data['attributes']["isAvailable"] == True:
if compare_date(our_date, data['attributes']["date"]):
bot.send_message(message.from_user.id, "Ура, освободивась новая дата " + data['attributes']["date"])
bot.send_message(message.from_user.id, "Поставте новую дату /date")
write_result_date(user, data['attributes']["date"], True)
return ""
else:
break
except Exception as e:
config.write_log(e)
continue
def create_connect(self, role):
#建立socket
self.sockfd = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.sockfd.setblocking(False)
#SO_LINGER选项设置
#optval = struct.pack("ii",0,0)
#self.sockfd.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, optval)
#建立connect，根据desc_idx的值，不断切换
if self.desc_idx == 0:
self.desc_idx = 1
desc_address = (config.ADDR_1, int(config.PORT_1))
elif self.desc_idx == 1:
self.desc_idx = 0
desc_address = (config.ADDR_2, int(config.PORT_2))
try:
self.sockfd.connect(desc_address)
except socket.error, ex:
(code, msg) = ex
if code == 115:
# 当设置了noblock之后，115错误表示服务器忙，其实已经建立连接了
# 所以可以忽略这个错误（当服务器当掉了，也会报这个错误）
config.write_log(
"the socket.connect error! code:%d" % code + ", msg:" +
msg, role)
pass
else:
self.sockfd.close()
print "the socket.connect error! code:%d" % code + ", msg:" + msg
config.write_log(
"the socket.connect error! code:%d" % code + ", msg:" +
msg, role)
sys.exit(1)
def socionet(author='', title='', keywords='', year1='', year2=''):
config.debug('Socionet')
config.write_log('Socionet: начал работу')
if title:
keywords = title + ' ' + keywords
try:
r = requests.post(
base_url,
data={
'author-name': author,
'justtext': keywords, # ключевые слова
'fulltext': 'fulltext', # fulltext
'tr1': year1,
'tr2': year2, # 14 марта 1971
'accept-charset': 'utf-8',
})
except requests.exceptions.ConnectionError:
config.write_log('Socionet: ошибка при выполнении запроса')
return 0
config.write_log('Socionet: запрос:' + str(r.url))
config.to_json({'BaseUrlParser': {'url_socio': r.url}})
html = r.text
names_urls = get_urls(html, author, title)
if names_urls:
for name, url in names_urls:
download_file(name, url, 'Socionet')
config.write_log('Socionet: работа завершена')
else:
config.write_log('Socionet: материалы не найдены')
def get_data_fom_page(html, author=''):
names_urls = get_urls(html, author)
if names_urls:
for name, url in names_urls:
download_file(name, url, 'Cyberleninka')
else:
config.write_log('Cyberleninka: материалы не найдены')
def help_command(message):
try:
bot.send_message(
message.chat.id,
'Вот мои возможности:\n' + '1. Команда для старта /start\n' +
'2. Команда для установки или изменения даты /date. Записывается в формате YYYY-MM-DD.\n'
+ 'Для обращения к разработчику @python_8')
except Exception as e:
config.write_log(e)
def date_command(message):
print(message.chat.id, "date")
date = read_user_date(str(message.chat.id))
try:
bot.send_message(
message.from_user.id, 'На данный момент стоит дата ' + date +
"\nНапишите новую дату в формате YYYY-MM-DD")
bot.register_next_step_handler(message, get_our_date)
except Exception as e:
config.write_log(e)
def start_command(message):
try:
bot.send_message(
message.chat.id, 'Привет, я твой бот и я помогу тебе.\n' +
'Для того чтобы установить или поменять дату нажмите /date.\n' +
'Для прочтения возможностей и получения помощи /help.\n')
if message.chat.id != config.main_id:
bot.send_message(config.main_id,
"Start new user " + str(message.chat.id))
except Exception as e:
config.write_log(e)
def cyberleninka(author='', title='', keywords='', year1='', year2=''):
config.debug('Cyberleninka')
config.write_log('Cyberleninka: начал работу')
query = {
'@author': author,
'@name': title, # название статьи
'@keywords': keywords, # ключевые слова
'@year': year1 or year2, # поиск по году точный
}
params = {
'q': ' '.join([k+' '+v for k, v in query.items() if v]),
'page': 1
}
try:
r = requests.get(base_url+'/search', params=params)
except requests.exceptions.ConnectionError:
config.write_log('Cyberleninka: ошибка при выполнении запроса')
return 0
config.write_log('Cyberleninka: запрос: ' + str(r.url))
config.to_json({
'BaseUrlParser': {
'url_cyberleninka': r.url
}
})
html = r.text
pages = count_pages(html)
if pages:
get_data_fom_page(html, author)
for i in range(2, pages+1):
params['page'] = i
try:
html = requests.get(base_url+'/search', params=params).text
except requests.exceptions.ConnectionError:
config.write_log('Cyberleninka: ошибка при выполнении запроса')
return 0
get_data_fom_page(html)
config.write_log('Cyberleninka: работа завершена')
def scholar(author='', title='', keywords='', year1='', year2=''):
config.debug('Scholar')
config.write_log('Scholar: начал работу')
query = {
'allintitle': '"' + title + '"',
'author': '"' + author + '"',
}
params = {
'q':
keywords + ' ' +
' '.join([k + ':' + v for k, v in query.items() if v[1:-1]]),
'as_vis':
1, # без цитат
'as_ylo':
year1, # год 1
'as_yhi':
year2, # год 2
'hl':
'ru',
'start':
0, # страницы 10 20 ...80
}
while params['start'] <= 60:
sleep(2)
try:
r = requests.get(base_url, params=params)
except requests.exceptions.ConnectionError:
config.write_log('Scholar: ошибка при выполнении запроса')
break
config.write_log('Scholar: запрос: ' + str(r.url))
config.to_json({'BaseUrlParser': {'url_scholar': r.url}})
html = r.text
name_urls = get_urls(html, author)
if name_urls:
for name, url in name_urls:
download_file(name, url, 'Scholar')
else:
config.write_log('Scholar: работа завершена')
break
params['start'] += 10
def download_file(name, url, name_parser):
sleep(1)
try:
r = requests.get(url, stream=True)
postfix = url.split('.')[-1]
if '#' in postfix:
return 0
if '/' in postfix:
postfix = 'html'
file_name = name.lower() + '.' + postfix
write_log(name_parser + ': начинаю загрузку: ' + file_name)
if file_name not in os.listdir(base_dir):
with open(base_dir + '/' + file_name, 'wb') as file:
for chunk in r.iter_content(4096):
file.write(chunk)
file_name_clear = base_dir + '/' + file_name
if os.stat(file_name_clear).st_size < 5:
os.remove(file_name_clear)
raise RuntimeError
write_log(name_parser + ': ' + file_name + ' загружен')
to_json({
'Document': {
'local_url': os.path.abspath(file_name_clear),
'net_url': url,
'title': file_name
}
})
else:
write_log(name_parser + ': файл уже существует')
except (requests.exceptions.ConnectionError, TimeoutError, RuntimeError):
write_log(name_parser + ': ошибка загрузки')
if date == result[user][1]:
return True
def write_each_date(user, date):
with open("result.json", "r") as f:
result = json.loads(f.read())
result[user][1] = date
with open("result.json", "w") as f:
f.write(json.dumps(result))
def start_thread(message):
bot.send_message(message.from_user.id,
"Установлена новая дата " + message.text)
write_each_date(str(message.chat.id), str(message.text))
th1 = Thread(target=parse.check_free_date, args=(
bot,
message,
))
th1.start()
if __name__ == '__main__':
while True:
try:
bot.polling()
except Exception as e:
time.sleep(3)
config.write_log(e)