def test_delete_1000(self):
""" put (key, value) on random server, run servers, delete value, check if value has been deleted"""
n=100
for x in xrange(n):
# put (key, value) into hash ring
server = random.choice(self.servers)
client = PynamoClient(server.hostname, int(server.external_port))
key = util.get_hash(str(random.random()))
value = util.get_hash(key)
key_hash = util.get_hash(key)
responsible_node_hashes = server.membership_stage.get_responsible_node_hashes(key_hash)
client.put(key, value)
for _ in xrange(10):
self.run_servers()
client._immediate_shutdown()
# delete key from hash ring
server = random.choice(self.servers)
client = PynamoClient(server.hostname, int(server.external_port))
client.delete(key)
for _ in xrange(10):
self.run_servers()
for server in self.servers:
if server.node_hash in responsible_node_hashes:
self.assertFalse(server.persistence_stage.get(key_hash)['value'])
client._immediate_shutdown()
def test_get_1000(self):
""" put random (key, value) on random server, run servers, check if value is on correct nodes"""
n = 100
for x in xrange(n):
server = random.choice(self.servers)
client = PynamoClient(server.hostname, int(server.external_port))
key = util.get_hash(str(random.random()))
value = util.get_hash(key)
client.put(key, value)
for _ in xrange(10):
self.run_servers()
put_reply = client.replies.pop(0)
self.assertEqual(put_reply['error_code'], '\x00')
client._immediate_shutdown()
server = random.choice(self.servers)
client = PynamoClient(server.hostname, int(server.external_port))
client.get(key)
for _ in xrange(10):
self.run_servers()
get_reply = client.replies.pop(0)
self.assertEqual(get_reply['error_code'], '\x00')
self.assertEqual(get_reply['value'], value)
client._immediate_shutdown()
def test_shutdown_keys_propagate(self):
""" Instruct one server to shut down, check if node hash is present on other servers after running servers."""
n = 100
for server in self.servers:
client = PynamoClient(server.hostname, int(server.external_port))
for i in xrange(n/len(self.servers)):
key = util.get_hash(str(random.random()))
value = util.get_hash(key)
client.put(key, value)
for _ in xrange(5):
self.run_servers()
client._immediate_shutdown()
server_down = random.choice(self.servers)
self.client = PynamoClient(server_down.hostname, int(server_down.external_port))
self.client.shutdown()
for _ in xrange(1000):
self.run_servers()
for key_hash in server_down.persistence_stage.keys():
value = server_down.persistence_stage.get(key_hash)['value']
responsible_node_hashes = server_down.membership_stage.get_responsible_node_hashes(key_hash)
for responsible_node_hash in responsible_node_hashes:
for server_up in self.servers:
if server_up.node_hash == responsible_node_hash:
self.assertEqual(value, server_up.persistence_stage.get(key_hash)['value'])
def put_n_times(self, n):
for _ in xrange(n):
server = random.choice(self.servers)
client = PynamoClient(server.hostname, int(server.external_port))
key = util.get_hash(str(random.random()))
value = util.get_hash(key)
client.put(key, value)
for x in xrange(5):
self.run_servers()
client._immediate_shutdown()
self.run_servers()
def test_get_single(self):
""" put (key, value) on random server, run servers, get key from other random server"""
server = random.choice(self.servers)
client = PynamoClient(server.hostname, int(server.external_port))
key = util.offset_hex(server.node_hash, -1)
value = util.get_hash(key)
client.put(key, value)
for _ in xrange(10):
self.run_servers()
put_reply = client.replies.pop(0)
self.assertEqual(put_reply['error_code'], '\x00')
client._immediate_shutdown()
server = random.choice(self.servers)
client = PynamoClient(server.hostname, int(server.external_port))
client.get(key)
for _ in xrange(10):
self.run_servers()
get_reply = client.replies.pop(0)
self.assertEqual(get_reply['error_code'], '\x00')
self.assertEqual(get_reply['value'], value)
client._immediate_shutdown()
def sign(bot, update):
if update.message.from_user.id in config.admin_id or bot.get_chat_member(
update.message.chat_id, update.message.from_user.id).status in [
"creator", "administrator"
]:
if update.message.from_user.name is not None:
user_name = update.message.from_user.name
else:
user_name = str(update.message.from_user.first_name) + " " + str(
update.message.from_user.last_name)
db = MySQLdb.connect(config.database['server'],
config.database['user'],
config.database['password'],
config.database['name'])
db.autocommit(True)
cur = db.cursor()
cur.execute(
"INSERT IGNORE INTO Permissions (User_ID, Group_ID) VALUES (" +
str(update.message.from_user.id) +
", (SELECT ID FROM Groups WHERE Chat_ID = '" +
str(update.message.chat_id) + "'))")
bot.delete_message(update.message.chat_id, update.message.message_id)
cur.close()
db.close()
util.bot_log(
strings.DASH_USER_SIGN %
(str(user_name), str(update.message.from_user.id),
str(update.message.chat.title), str(update.message.chat_id),
util.get_hash(int(update.message.chat_id))), bot)
def text_lang(bot, update):
try:
if update.message is not None:
text = ""
if update.message.text is None:
if update.message.forward_from_chat:
text = update.message.caption
else:
text = update.message.text
if update.message.chat.username is not None:
chat_name = update.message.chat.username
else:
chat_name = update.message.chat.title
if all(is_latin(t) for t in text if t.isalpha()) == False:
util_bot_log(
strings.REPORT_MESSAGE_TEXT %
(html.escape(str(update.message.text)),
str(update.message.from_user.name),
str(chat_name) + " " + str(update.message.chat.username),
util.get_hash(int(chat_id))))
bot.delete_message(update.message.chat_id,
update.message.message_id)
util.log_blocked_content('!non_west', update.message.chat_id,
update.message.from_user.id,
update.message.text)
except AttributeError:
return False
def extract(article):
#print(article.page_name.encode('utf8'), '\n')
# linked entities within article
q = 'article == "{}"'.format(article.page_id)
article_entities = entities['{}'.format(util.get_hash(article.page_id))].query(q)
article_entities = article_entities.loc[article_entities.valid == 'True', 'entity']
article_body = article.to_string()
G = pd.DataFrame(columns=['article', 'entity', 'mention'], dtype='unicode')
for entity in article_entities:
for pair in re.finditer(nlp.get_entity_pattern(entity), article_body):
values = pair.group()[1:].split(']')
mention = values[0]
entity = values[1][1:-1]
G.loc[len(G.index)] = [article.page_name, entity, mention]
# we should strip mentions from ''
G.to_hdf(settings.PATH_DATAOBJECTS + 'stats/' + str(os.getpid()) + '.hdf5', format='table',
append=True, complib='blosc', complevel=3, key='', encoding='utf-8',
min_itemsize={'article': 500, 'entity': 500, 'mention': 500})
def post(self):
"""Verifies the user is registered.
If the user is registered, then he is redirected to the main page,
otherwise the user gets an error message indicating whether the
username or password are incorrect.
"""
# Should not be possible, because only people who are not logged in
# should be able to see the link to sign in.
if self.is_session:
return self.redirect('/')
data = self.json_read()
data['success'] = False
user = data['user']
pwd = data['password']
# verify account exists
account = models.Account.get_by_id(user)
if not account:
data['baduser'] = True
return self.json_write(data)
# verify password is correct
hsh = util.get_hash(account.salt, pwd)
if hsh != account.pwd_hash:
data['badpwd'] = True
return self.json_write(data)
# set session cookies
data['success'] = True
self.response.set_cookie('name', user)
self.response.set_cookie('secret', hsh)
return self.json_write(data)
def post(self):
"""Verifies the user is registered.
If the user is registered, then he is redirected to the main page,
otherwise the user gets an error message indicating whether the
username or password are incorrect.
"""
# Should not be possible, because only people who are not logged in
# should be able to see the link to sign in.
if self.is_session:
return self.redirect('/')
data = self.json_read()
data['success'] = False
user_name = data['user']
pwd = data['password']
# verify account exists
user = User.get_by_id(user_name)
if not user:
data['baduser'] = True
return self.json_write(data)
# verify password is correct
hsh = util.get_hash(user.salt, pwd)
if hsh != user.pwd_hash:
data['badpwd'] = True
return self.json_write(data)
# set session cookies
data['success'] = True
self.response.set_cookie('name', user_name)
self.response.set_cookie('secret', hsh)
return self.json_write(data)
def message_text(bot, update):
if bot.get_chat_member(
update.message.chat_id,
update.message.from_user.id).status not in [
"creator", "administrator"
] or update.message.from_user.id not in config.admin_id:
if update.message is not None:
if update.message.text is None:
if update.message.forward_from_chat:
update.message.text = update.message.caption
if update.message.text is not None:
if any(s in str(update.message.text).lower()
for s in config.spam_words):
if update.message.chat.username is not None:
chat_name = update.message.chat.username
else:
chat_name = update.message.chat.title
chat_id = update.message.chat_id
util.bot_log(
strings.REPORT_MESSAGE_TEXT %
(str(update.message.text),
str(update.message.from_user.name), str(chat_name) +
" " + str(update.message.chat.username),
util.get_hash(int(chat_id))), bot)
bot.delete_message(chat_id, update.message.message_id)
util.log_blocked_content('!spam_word',
update.message.chat_id,
update.message.from_user.id,
update.message.text)
def ban(bot, update, unban=False):
if update.message.from_user.id in config.admin_id or bot.get_chat_member(
update.message.chat_id, update.message.from_user.id).status in [
"creator", "administrator"
]:
if update.message.reply_to_message:
user_id = update.message.reply_to_message.from_user.id
if update.message.reply_to_message.from_user.name is not None:
user_name = update.message.reply_to_message.from_user.name
else:
user_name = str(
update.message.reply_to_message.from_user.first_name
) + " " + str(
update.message.reply_to_message.from_user.last_name)
chat_id = update.message.chat_id
if unban == False:
if user_id != config.bot_id:
bot.send_message(
config.channel_id,
(strings.REPORT_USER_BAN %
(str(update.message.from_user.name), str(user_name),
str(user_id), str(update.message.chat.title),
util.get_hash(int(chat_id)))),
parse_mode=ParseMode.HTML)
bot.kick_chat_member(update.message.chat_id, user_id)
bot.send_message(update.message.chat_id,
(strings.USER_BAN % str(user_name)),
parse_mode=ParseMode.HTML)
util.log_operation('!ban:' + str(user_id), chat_id,
update.message.from_user.id)
else:
bot.send_message(
CONF_.channel_id,
(strings.REPORT_USER_UNBAN %
(str(update.message.from_user.name),
str(update.message.chat.title), str(user_name),
str(user_id), util.get_hash(int(chat_id)))),
parse_mode=ParseMode.HTML)
bot.unban_chat_member(update.messCONF_age.chat_id,
int(user_id))
bot.send_message(update.message.chat_id,
(strings.USER_UNBAN % str(user_name)),
parse_mode=ParseMode.HTML)
util.log_operation('!unban:' + str(user_id), chat_id,
update.message.from_user.id)
bot.delete_message(update.message.chat_id, update.message.message_id)
def test_put_single(self):
""" put (key, value) on random server, run servers, check if value is on correct nodes"""
server = random.choice(self.servers)
self.client = PynamoClient(server.hostname, int(server.external_port))
key = util.offset_hex(server.node_hash, -1)
value = util.get_hash(key)
self.client.put(key, value)
for _ in xrange(5):
self.run_servers()
key_hash = util.get_hash(key)
responsible_node_hashes = server.membership_stage.get_responsible_node_hashes(key_hash)
for server in self.servers:
if server.node_hash in responsible_node_hashes:
self.assertEqual(value, server.persistence_stage.get(key_hash)['value'])
def get_data(bot, update):
if update.message.from_user.id in config.admin_id:
chat_id = update.message.chat_id
if update.message.reply_to_message:
reply = update.message.reply_to_message
if reply.forward_from:
if reply.forward_from.language_code is None:
lang = 'en_US'
else:
lang = str(reply.forward_from.language_code)
bot.send_message(
config.channel_id,
(strings.REPORT_MESSAGE_FORWARD_INFO %
(update.message.from_user.name,
reply.forward_from.username,
reply.forward_from.first_name,
reply.forward_from.last_name, reply.forward_from.id,
lang, util.get_hash(
int(chat_id)), str(reply.forward_from.is_bot))),
parse_mode=ParseMode.HTML)
else:
if reply.from_user.language_code is None:
lang = 'en_US'
else:
lang = str(reply.from_user.language_code)
bot.send_message(
config.channel_id,
(strings.REPORT_MESSAGE_INFO %
(update.message.from_user.name, str(reply.message_id),
reply.from_user.name, reply.from_user.first_name,
reply.from_user.last_name, reply.from_user.id, lang,
str(reply.from_user.is_bot), util.get_hash(
int(chat_id)))),
parse_mode=ParseMode.HTML)
else:
bot.send_message(
config.channel_id,
(strings.REPORT_CHAT_INFO %
(update.message.from_user.name,
str(update.message.chat.username),
str(update.message.chat.title), update.message.chat_id,
update.message.chat.type, util.get_hash(int(chat_id)))),
parse_mode=ParseMode.HTML)
bot.delete_message(update.message.chat_id, update.message.message_id)
def get_one_users1(self, username, password=None): # 根据用户名密码查询账户信息
if password is None:
# 根据用户名查找账户信息
users = self.get_one_object(name=username)
else:
# 根据用户名和密码查询账户信息
users = self.get_one_object(name=username,
password=get_hash(password))
return users
def test_client_put_1000(self):
""" 1000 put requests to single local server"""
num_puts = 1000
for i in xrange(num_puts):
key = util.offset_hex(self.server.node_hash, -i)
value = util.get_hash(key)
self.client.put(key, value)
asyncore.loop(timeout=0.001, count=100)
self.server.process()
asyncore.loop(timeout=0.001, count=100)
self.assertEqual(len(self.client.replies), num_puts)
for index, reply in enumerate(self.client.replies):
request = self.client.requests[index]
value, timestamp = self.server.persistence_stage._persistence_engine.get(util.get_hash(request['key']))
self.assertEqual(reply['error_code'], '\x00')
self.assertEqual(value, request['value'])
def json_output(self):
output = {
'prev_txid':
self.prev_txid,
'prev_tx_out_idx':
self.prev_tx_out_idx,
'signature':
util.get_hash(self.signature) if self.signature != None else "",
'pubkey_hash':
Script.sha160(str(self.pubkey)) if self.pubkey != None else ""
}
return output
def test_client_delete_1000(self):
n = 1000
for i in xrange(n):
key = util.offset_hex(self.server.node_hash, -i)
value = util.get_hash(key)
self.client.put(key, value)
self.client.delete(key)
for _ in xrange(100):
asyncore.loop(timeout=0.001, count=1)
self.server.process()
self.assertEqual(len(self.client.replies), 2 * n)
for index, reply in enumerate(self.client.replies):
request = self.client.requests[index]
if request['command'] == 'put':
self.assertEqual(reply['error_code'], '\x00')
elif request['command'] == 'delete':
self.assertEqual(reply['error_code'], '\x00')
self.assertEqual(self.server.persistence_stage.get(util.get_hash(request['key']))['error_code'], '\x01')
def test_put_1000(self):
""" put 1000 (key, value) on random server, run servers, check if value is on correct nodes"""
n = 100
for server in self.servers:
self.client = PynamoClient(server.hostname, int(server.external_port))
# add an equal number of keys to each server
for i in xrange(n/len(self.servers)):
key = util.offset_hex(server.node_hash, - i )
value = util.get_hash(key)
self.client.put(key, value)
# run servers a few times
for _ in xrange(10):
self.run_servers()
# check key, value pair is on correct replicas
key_hash = util.get_hash(key)
responsible_node_hashes = server.membership_stage.get_responsible_node_hashes(key_hash)
for server in self.servers:
if server.node_hash in responsible_node_hashes:
self.assertEqual(value, server.persistence_stage.get(key_hash)['value'])
self.client._immediate_shutdown()
def __init__(self, server = None, node_addresses=[], wait_time=30):
self.logger = logging.getLogger('{}'.format(self.__class__.__name__))
self.logger.debug('__init__')
self._server = server
self._wait_time = wait_time
self._node_lookup = {util.get_hash(str(node_address)) : node_address for node_address in node_addresses}
self._consistent_hash_ring = ConsistentHashRing(node_hashes = sorted(self._node_lookup.keys()))
self.logger.debug('__init__. node_lookup: {}'.format(self._node_lookup))
self._failed_to_contact_node_hashes = collections.defaultdict(dict)
self._processor = self._handle_membership_checks()
def register_group(bot, update):
chat_id = update.message.chat_id
db = MySQLdb.connect(config.database['server'], config.database['user'],
config.database['password'], config.database['name'])
db.autocommit(True)
cur = db.cursor()
cur.execute("INSERT INTO Groups (Title, Chat_ID) VALUES ('" +
str(update.message.chat.title) + "', '" + str(chat_id) + "')")
bot.send_message(config.channel_id,
(strings.REPORT_NEW_GROUP %
(html.escape(str(update.message.chat.title)),
str(chat_id), util.get_hash(int(chat_id)))),
parse_mode=ParseMode.HTML)
cur.close()
db.close()
def test_unannounced_failure_keys_propagate(self):
n = 100
for server in self.servers:
client = PynamoClient(server.hostname, int(server.external_port))
for i in xrange(n/len(self.servers)):
key = util.get_hash(str(random.random()))
value = util.get_hash(key)
client.put(key, value)
for _ in xrange(5):
self.run_servers()
client._immediate_shutdown()
random.shuffle(self.servers)
server_down = self.servers.pop()
server_down._immediate_shutdown()
server_down.process()
shutdown_server_node_hash = server_down.node_hash
for server in self.servers:
self.assertTrue(shutdown_server_node_hash in server.membership_stage.node_hashes)
for _ in xrange(300):
self.run_servers()
time.sleep(0.1)
for server in self.servers:
self.assertFalse(shutdown_server_node_hash in server.membership_stage.node_hashes)
for key_hash in server_down.persistence_stage.keys():
value = server_down.persistence_stage.get(key_hash)['value']
responsible_node_hashes = server_down.membership_stage.get_responsible_node_hashes(key_hash)
for responsible_node_hash in responsible_node_hashes:
for server_up in self.servers:
if server_up.node_hash == responsible_node_hash:
self.assertEqual(value, server_up.persistence_stage.get(key_hash)['value'])
def process_file(fname):
data = []
with open(fname, 'r') as fp:
rdr = csv.reader(fp)
first_row = next(rdr)
print("Column names: ", first_row)
get = dict(zip(first_row, map(itemgetter, range(len(first_row)))))
for row in csv.reader(fp):
print("read row: ", row)
datum = {}
for k, a in get.items():
datum[k] = a(row)
data.append(datum)
for datum in data:
hash_ = get_hash(datum)
if hash_ in seen:
print("Skipping ", datum)
continue
activity = datum['activity name']
datum['start time'] = string_time_to_unix(datum['start time'])
def process(activity_name, datum=datum):
upload.process_point(datum, activity_name)
print("adding hash: ", hash_)
seen.add(hash_)
heirarchical_activity = in_heirarchal_goal(datum)
if heirarchical_activity:
datum_ = dict(datum,
note='%s -- (%s)' %
(datum['note'], datum['activity name']))
process(heirarchical_activity, datum_)
if matches_sleep(datum):
activity = 'sleepdebt'
datum['duration'] = sleep_handler(datum)
process(activity)
elif activity in upload.GOALS:
process(activity)
def process_file(fname):
data = []
with open(fname, 'r') as fp:
rdr = csv.reader(fp)
first_row = next(rdr)
print ("Column names: ", first_row)
get = dict(zip(first_row,
map(itemgetter, range(len(first_row)))))
for row in csv.reader(fp):
print ("read row: ", row)
datum = {}
for k, a in get.items():
datum[k] = a(row)
data.append(datum)
for datum in data:
hash_ = get_hash(datum)
if hash_ in seen:
print ("Skipping ", datum)
continue
activity = datum['activity name']
datum['start time'] = string_time_to_unix(datum['start time'])
def process(activity_name, datum=datum):
upload.process_point(datum, activity_name)
print ("adding hash: ", hash_)
seen.add(hash_)
heirarchical_activity = in_heirarchal_goal(datum)
if heirarchical_activity:
datum_ = dict(datum,
note='%s -- (%s)' % (datum['note'], datum['activity name']))
process(heirarchical_activity, datum_)
if matches_sleep(datum):
activity = 'sleepdebt'
datum['duration'] = sleep_handler(datum)
process(activity)
elif activity in upload.GOALS:
process(activity)
def post(self):
"""Registers a user.
Checks the username and password are valid, and the username is not
taken. If the username is taken, then the user is prompted for another
username. The user is redirected to the main blog page after
registration is complete.
"""
# Should not be possible, because only people who are not logged in
# should be able to see the link to sign in.
if self.is_session:
return self.redirect('/')
data = self.json_read()
user = data['user']
pwd = data['password']
# Check that username doesn't already exist
account = models.Account.get_by_id(user)
if account:
data['success'] = False
return self.json_write(data)
# Validate password
if not util.process_password(pwd):
data['success'] = False
return self.json_write(data)
# Create account
salt = util.gensalt()
hsh = util.get_hash(salt, pwd)
account = models.Account(id=user, salt=salt, pwd_hash=hsh)
try:
account.put()
except ndb.TransactionFailedError:
data['success'] = False
return self.json_write(data)
data['success'] = True
self.response.set_cookie('name', user)
self.response.set_cookie('secret', hsh)
return self.json_write(data)
def message_links(bot, update):
if bot.get_chat_member(
update.message.chat_id,
update.message.from_user.id).status not in [
"creator", "administrator"
] or update.message.from_user.id not in config.admin_id:
if update.message is not None:
delete = False
chat_id = update.message.chat_id
if update.message.text is None:
if update.message.forward_from_chat:
update.message.text = update.message.caption
else:
return False
if update.message.chat.username is not None:
chat_name = update.message.chat.username
config.safe_urls.append("https://t.me/" + chat_name)
config.safe_urls.append("https://t.me/@" + chat_name)
else:
chat_name = update.message.chat.title
urls = re.findall(config.regex_web_url, update.message.text)
for url in urls:
if url not in config.safe_urls:
if is_spam(url, True, chat_id):
util.bot_log(
strings.REPORT_MESSAGE_URL %
(str(update.message.text),
str(update.message.from_user.name),
str(chat_name), str(url),
util.get_hash(int(chat_id))), bot)
delete = True
if any(tgUrl in update.message.text
for tgUrl in config.telegram_domains):
delete = True
if delete == True:
bot.delete_message(chat_id, update.message.message_id)
util.log_blocked_content('!spam_domain',
update.message.chat_id,
update.message.from_user.id,
update.message.text)
def delete(bot, update):
if update.message.from_user.id in config.admin_id or bot.get_chat_member(
update.message.chat_id, update.message.from_user.id).status in [
"creator", "administrator"
]:
if update.message.reply_to_message:
user_id = update.message.reply_to_message.from_user.id
user_name = update.message.reply_to_message.from_user.name
text = update.message.reply_to_message.text
chat_id = update.message.chat_id
if user_id != config.bot_id:
bot.send_message(config.channel_id,
(strings.REPORT_DELETED_MESSAGE %
(str(update.message.from_user.name),
str(update.message.chat.title), str(text),
util.get_hash(int(chat_id)))),
parse_mode=ParseMode.HTML)
bot.delete_message(update.message.chat_id,
update.message.reply_to_message.message_id)
bot.delete_message(update.message.chat_id, update.message.message_id)
util.log_operation(
'!rm:' + str(update.message.reply_to_message.message_id), chat_id,
update.message.from_user.id)
def post(self):
"""Registers a user.
Checks the username and password are valid, and the username is not
taken. If the username is taken, then the user is prompted for another
username. The user is redirected to the main blog page after
registration is complete.
"""
# Should not be possible, because only people who are not logged in
# should be able to see the link to sign in.
if self.is_session:
return self.redirect('/')
data = self.json_read()
user_name = data['user']
pwd = data['password']
# Check that username doesn't already exist
user = User.get_by_id(user_name)
if user:
data['success'] = False
return self.json_write(data)
# Create account
salt = util.gensalt()
hsh = util.get_hash(salt, pwd)
user = User(id=user_name, salt=salt, pwd_hash=hsh)
try:
user.put()
except ndb.TransactionFailedError:
data['success'] = False
return self.json_write(data)
data['success'] = True
self.response.set_cookie('name', user_name)
self.response.set_cookie('secret', hsh)
return self.json_write(data)
def message_entities(bot, update):
if bot.get_chat_member(
update.message.chat_id,
update.message.from_user.id).status not in [
"creator", "administrator"
] or update.message.from_user.id not in config.admin_id:
if update.message is not None and update.message.entities is not None:
if update.message.forward_from_chat:
update.message.text = update.message.caption
if update.message.chat.username is not None:
if update.message.chat.username not in config.safe_names:
config.safe_names.append('@' +
update.message.chat.username)
for entitie in update.message.entities:
if entitie.type == "mention":
if update.message.text is not None:
if update.message.chat.username is not None:
chat_name = update.message.chat.username
else:
chat_name = update.message.chat.title
e = update.message.text[entitie.offset:entitie.offset +
entitie.length]
if is_spam(e, False, update.message.chat_id):
chat_id = update.message.chat_id
util.bot_log(
strings.REPORT_MESSAGE_TEXT %
(html.escape(str(update.message.text)),
str(update.message.from_user.name),
str(chat_name) + " " +
str(update.message.chat.username),
util.get_hash(int(chat_id))), bot)
bot.delete_message(chat_id,
update.message.message_id)
util.log_blocked_content(
'!spam_telegram', update.message.chat_id,
update.message.from_user.id,
update.message.text)
def new_user_name(bot, update):
if update.message is not None and update.message.new_chat_members is not None:
for new in update.message.new_chat_members:
first = str(new.first_name)
last = str(new.last_name)
if update.message.chat.username is not None:
chat_name = update.message.chat.username
else:
chat_name = update.message.chat.title
if str(last) == "None":
last = ""
if any(s in str(first + " " + last).lower()
for s in mix_spam_list):
chat_id = update.message.chat_id
util.bot_log(
strings.REPORT_USER_NAME %
(str(first + last), str(chat_name),
util.get_hash(int(chat_id))), bot)
bot.kick_chat_member(chat_id, new.id)
bot.delete_message(chat_id, update.message.message_id)
util.log_blocked_content('!spam_in_username',
update.message.chat_id,
update.message.from_user.id,
str(first + last))
def leave(bot, update, force=False, chat_id=False):
if chat_id == False:
if update.message is None:
if update.channel_post is not None:
chat_id = update.channel_post.chat_id
chat_title = update.channel_post.chat.title
f = 0
else:
chat_id = update.message.chat_id
chat_title = update.message.chat.title
f = update.message.from_user.id
else:
f = 0
chat_title = str(chat_id)
if f in config.admin_id or force:
if chat_id != config.channel_id:
bot.send_message(chat_id, strings.LEAVING)
bot.leaveChat(chat_id)
bot.send_message(
config.channel_id,
(strings.REPORT_BOT_REMOVED %
(html.escape(str(chat_title)), str(chat_id),
str(update.message.chat.type), util.get_hash(int(chat_id)))),
parse_mode=ParseMode.HTML)
def _process_message(self, request):
"""
Handles request messages passed from async_chat's found_terminator handler.
-marks message as 'exteral request' and gives it a timestamp. Hashes key before passing the request internally by instantiating an ExternalRequestCoordinator.
Args:
----------
request : JSON
incoming request object.
"""
self.logger.debug('_request_handler.')
request['type'] = 'external request'
request['timestamp'] = util.current_time()
try
request['key'] = util.get_hash(request['key'])
except:
pass
coordinator = ExternalRequestCoordinator(server=self._server, request=request)
self._coordinators.append(coordinator)
self.logger.debug('_request_handler. coordinator appended: {}'.format(coordinator))
def add_to_blacklist(bot, update, remove=False):
if update.message.from_user.id in config.admin_id:
chat_id = update.message.chat_id
if update.message.reply_to_message:
user_id = update.message.reply_to_message.from_user.id
if int(user_id) in config.admin_id:
return False
if update.message.reply_to_message.from_user.name is not None:
user_name = update.message.reply_to_message.from_user.name
else:
user_name = str(
update.message.reply_to_message.from_user.first_name
) + " " + str(
update.message.reply_to_message.from_user.last_name)
bot.send_message(update.message.chat_id,
(strings.USER_BLACKLIST % str(user_name)),
parse_mode=ParseMode.HTML)
else:
user_id = update.message.text[7:]
if int(user_id) in config.admin_id:
return False
user_name = user_id
db = MySQLdb.connect(config.database['server'], config.database['user'],
config.database['password'], config.database['name'])
db.autocommit(True)
cur = db.cursor()
if remove == False:
try:
cur.execute("INSERT INTO Blacklist (User_ID, Chat_ID) VALUES (" +
str(user_id) + ", '" + str(chat_id) + "')")
try:
bot.kick_chat_member(update.message.chat_id, user_id)
except TelegramError:
pass
bot.send_message(config.channel_id, (
strings.REPORT_NEW_USER_IN_BLACKLIST %
(str(update.message.from_user.name), str(user_name),
str(update.message.chat.title), util.get_hash(int(chat_id)))),
parse_mode=ParseMode.HTML)
except MySQLdb.IntegrityError:
bot.send_message(
update.message.chat_id,
(strings.USER_ALREADY_IN_BLACKLIST % str(user_name)))
else:
try:
cur.execute("DELETE FROM Blacklist WHERE User_ID = (" +
str(user_id) + ")")
try:
bot.unban_chat_member(update.message.chat_id, int(user_id))
except TelegramError:
pass
bot.send_message(config.channel_id, (
strings.REPORT_REMOVED_FROM_BLACKLIST %
(str(update.message.from_user.name), str(user_name),
str(update.message.chat.title), util.get_hash(int(chat_id)))),
parse_mode=ParseMode.HTML)
except MySQLdb.IntegrityError:
bot.send_message(update.message.chat_id,
(strings.USER_NOT_IN_BLACKLIST % str(user_name)))
cur.close()
db.close()
bot.delete_message(update.message.chat_id, update.message.message_id)
def check(bot, update):
blacklist=False
try:
for new in update.message.new_chat_members:
if check_in_blacklist(new.id):
user=new
blacklist=True
except AttributeError:
pass
if check_in_blacklist(update.message.from_user.id):
user=update.message.from_user
blacklist=True
if blacklist:
if user.name is not None:
user_name=user.name
else:
user_name=str(user.first_name)+" "+str(user.last_name)
chat_id=update.message.chat_id
util.bot_log(strings.BAN_REPORT % (str(user_name), str(user.id), str(update.message.chat.title), util.get_hash(int(chat_id))), bot)
bot.kick_chat_member(update.message.chat_id, user.id)
bot.delete_message(update.message.chat_id, update.message.message_id)
is_deck = True
if cl != crit[0]:
continue
for i in range((len(crit) - 1) // 3):
cid = crit[3 * i + 1]
op = crit[3 * i + 2]
num = crit[3 * i + 3]
if op == 0: # >=
if cid not in cnt or cnt[cid] < num:
is_deck = False
break
elif op == 1: # <=
if cid in cnt and cnt[cid] > num:
is_deck = False
break
if is_deck:
return deck
return '其它' + classes[cl - 1]
if __name__ == '__main__':
sheet = openpyxl.load_workbook('decklist.xlsx')['Sheet1']
for i in range(8):
output = '{}: '.format(sheet.cell(i + 1, 1).value)
for j in range(3):
(cl,
cnt) = count_cards(util.get_hash(sheet.cell(i + 1, j + 2).value))
output += classify(0, cl, cnt)
if j != 3:
output += ', '
print(output)
def sync(bot, update, trigger=False, chat_id=False):
# Check for admin or trigger
# if no trigger, get chat_id from update
if update.message.from_user.id in config.admin_id or trigger:
if trigger:
sync_message = strings.DASH_SYNCING_BLACKLIST
else:
sync_message = strings.SYNCING_BLACKLIST
chat_id = update.message.chat_id
# send the operation message to the chat and save update for edit
u = bot.send_message(chat_id,
sync_message,
parse_mode=ParseMode.HTML,
disable_web_page_preview=True)
# delete the request message if no trigger
if trigger == False:
try:
bot.delete_message(chat_id, update.message.message_id)
except TelegramError:
pass
# get the blacklist and set a counter to 0
db = MySQLdb.connect(config.database['server'],
config.database['user'],
config.database['password'],
config.database['name'])
db.autocommit(True)
cur = db.cursor()
cur.execute("SELECT User_ID FROM Blacklist")
counter = 0
# for each blacklist item, try to block or fail, wait 1.5s every 10 operations
# then increase counter by one
for row in cur.fetchall():
counter += 1
if counter % 10 == 0:
time.sleep(0)
try:
print("Blocking " + str(row[0]))
bot.kick_chat_member(chat_id, row[0])
print("Success!")
except TelegramError:
print("Fail!")
cur.close()
db.close()
# edit the operation message with result
bot.edit_message_text(text=(strings.SYNC_COMPLETE % str(counter)),
chat_id=u.chat_id,
message_id=u.message_id,
parse_mode=ParseMode.HTML,
disable_web_page_preview=True)
# send a log message to the report channel
bot.send_message(
config.channel_id,
(strings.SYNC_REPORT %
(str(u.chat.title), counter, util.get_hash(int(chat_id)))),
parse_mode=ParseMode.HTML)