async def build_data_profile(self, members, limit=50000):
"""
Used for building a data profile based on a user
Members: list of members we want to import for
Guild: Guild object
Limit: limit of messages to be imported
"""
for guild in self.client.guilds:
for cur_channel in guild.text_channels:
adding = False
for group in enabled_groups:
if str(cur_channel.category).lower() == str(group).lower():
adding = True
break
if adding:
logger.info("Logging from {}".format(cur_channel.name))
counter = 0
already_added = 0
async for message in cur_channel.history(limit=limit,
reverse=True):
if message.author in members:
self.database_tools.add_message_to_db(message)
logger.info(
"{} scraped for {} users - added {} messages, found {} already added"
.format(cur_channel.name, len(members), counter,
already_added))
def __init__(self, client):
self.client = client
self.client_tools = ClientTools(client)
self.database_tools = DatabaseTools(client)
insert_channel = "INSERT INTO channels (channel_id, channel_name) VALUES (%s, %s)"
update_channel = "UPDATE `gssp_logging`.`channels` SET `channel_name`=%s WHERE `channel_id`=%s;"
if not bool(config['discord'].get("skip_scrape")):
for guild in client.guilds:
logger.info("{}: Updating channels".format(str(guild)))
for channel in guild.text_channels:
try:
cursor.execute(
insert_channel,
(channel.id, emoji.demojize(channel.name)))
logger.debug("Inserted {} to DB".format(
emoji.demojize(channel.name)))
except mysql.connector.errors.IntegrityError:
cursor.execute(
update_channel,
(emoji.demojize(channel.name), channel.id))
logger.debug("Updated {}".format(
emoji.demojize(channel.name)))
logger.info("{}: Updating users".format(str(guild)))
for member in tqdm(guild.members,
total=len(guild.members),
desc="Adding users for {}".format(
str(guild))):
self.database_tools.add_user(member)
logger.info("{}: Finished {} users".format(
str(guild), len(guild.members)))
logger.info("{}: Updating roles".format(str(guild)))
for role in guild.roles:
if role.name != "@everyone":
try:
cursor.execute(insert_role,
(role.id, emoji.demojize(role.name),
guild.id, int(role.mentionable)))
except mysql.connector.errors.IntegrityError:
cursor.execute(update_role, (emoji.demojize(
role.name), int(role.mentionable), role.id))
# this is designed to assist with migration, by moving old discord role members over to the new
# system seamlessly
member_ids = []
for member in role.members:
member_ids.append(member.id)
role_db = DbRole(role.id,
role.name,
0,
members=member_ids)
role_db.save_members()
logger.info("{}: Finished {} roles".format(
guild, len(guild.roles)))
cnx.commit()
else:
logger.warn(
"Skipping scraping data from existing servers - data may be out of date"
)
def add_guild(guild=None, guild_id=None):
"""
Create config directory for a new guild
:param guild: Guild object provided by discord
:param guild_id: Raw number ID of the guild you wish to add
"""
if guild is not None and guild_id is None:
guild_id = guild.id
made_change = False
guild_path = "{}/{}".format(base_directory, guild_id)
if not os.path.exists(base_directory):
os.makedirs(base_directory)
made_change = True
if not os.path.exists(guild_path):
os.makedirs("{}/{}".format(base_directory, guild_id))
made_change = True
if not os.path.exists("{}/bad_words.json".format(guild_path)):
with open("{}/bad_words.json".format(guild_path), "w") as bad_words_f:
bad_words_f.write(json.dumps({"words": [], "alert_channel": None}))
made_change = True
if not os.path.exists("{}/settings.json".format(guild_path)):
with open("{}/settings.json".format(guild_path), "w") as settings_f:
settings_f.write(json.dumps({"staff_roles": []}))
if made_change:
logger.info("Created data for {}".format(guild_id))
async def optout_user(self, user):
"""
Opt a user out of experiments, and delete their data
Returns number of messages deleted
"""
logger.info("Deleting data for user ID {}".format(user.id))
cursor.execute("DELETE FROM users WHERE user_id = %s", (user.id, ))
result = cursor.execute(
"DELETE FROM messages_detailed WHERE user_id = %s", (user.id, ))
cnx.commit()
logger.info("Data deleted.")
def load_all_extensions(self):
"""
This will import all of our extensions, and set extension_imported to the code of the imports
"""
self.extension_imported = []
for extension in startup_extensions:
try:
to_load = "{}.{}".format(self.get_path(), extension)
self.extension_imported.append(dict(name=extension, module=importlib.import_module(to_load)))
self.client.load_extension(to_load)
logger.info("Loaded {} (from {})".format(extension, to_load))
del(to_load)
except Exception as e:
exc = '{}: {}'.format(type(e).__name__, e)
logger.error(
'Failed to load extension {}\n{}\n{}'.format(extension, exc, traceback.format_exc()))
async def on_ready():
game = discord.Game("Starting")
await client.change_presence(activity=game)
logger.info("Connected to Discord as {} ({})".format(
client.user.name, client.user.id))
# This needs to be here, so that all the other cogs can be loaded
client.load_extension("ags_experiments.cogs.loader")
await set_activity(client)
for guild in client.guilds:
guild_settings.add_guild(guild)
members = []
if not bool(config['discord'].get("skip_scrape")):
for guild in client.guilds:
if debug:
logger.info("Found guild {} - {} channels".format(
guild.name, len(guild.text_channels)))
for member in guild.members:
name = database_tools.opted_in(user_id=member.id)
if name is not False:
if name not in members:
members.append(member)
logger.info(
"Initialising building data profiles on existing messages. This will take a while."
)
await client_tools.build_data_profile(members, limit=None)
async def sync(self, ctx):
clone_target = self.client.get_guild(
config['discord'].get("clone_server_target"))
def generate_progress_embed(m_text, colour=yellow, url=None):
em = discord.Embed(
title="Server Clone Progress",
description="Status: {text}".format(text=m_text),
colour=colour)
if url is not None:
em.add_field(name="Invite link", value=url)
return em
guild = ctx.guild
# we just need to now create an instant invite to *somewhere* on the server
progress = await ctx.send(embed=generate_progress_embed(
"Dumping existing data from {guild.name}".format(guild=guild)))
channels = []
roles = []
def get_channel_position(old_id=None, new_id=None):
if new_id is None and old_id is None:
raise AttributeError
for x in range(0, len(channels)):
channel = channels[x]
# the and is not None prevent us from returning whatever channel has None as an attribute
if (channel.get("old_id") == old_id and old_id
is not None) or (channel.get("new_id") == new_id
and new_id is not None):
return x
return None
def get_channel(old_id=None, new_id=None):
position = get_channel_position(old_id=old_id, new_id=new_id)
if position is None:
return None
return channels[position]
def add_channel(old_channel, new_channel=None):
to_append = (dict(old_id=old_channel.id, old_channel=old_channel))
if new_channel is None:
to_append['new_id'] = None
to_append['new_channel'] = None
else:
to_append['new_id'] = new_channel.id
to_append['new_channel'] = new_channel
channels.append(to_append)
def set_new_channel(old_channel_id, new_channel):
# we don't use the new_channel id, as not merged yet
position = get_channel_position(old_id=old_channel_id)
new_channel = get_channel_object_dict(new_channel)
channels[position]['new_channel'] = new_channel
channels[position]['new_id'] = new_channel['id']
def get_role_position(old_id=None, new_id=None):
if new_id is None and old_id is None:
return None # means we don't have to do unnecesary searches
if old_id is not None:
old_id = int(old_id)
if new_id is not None:
new_id = int(new_id)
for x in range(0, len(roles)):
role = roles[x]
# the and is not None prevent us from returning whatever channel has None as an attribute
if (role.get("old_id") == old_id and old_id is not None) or (
role.get("new_id") == new_id and new_id is not None):
return x
return None
def get_role(old_id=None, new_id=None):
position = get_role_position(old_id=old_id, new_id=new_id)
if position is None:
return None
return roles[position]
def add_role(old_role, new_role=None):
to_append = (dict(old_id=old_role.id, old_role=old_role))
if new_role is None:
to_append['new_id'] = None
to_append['new_role'] = None
else:
to_append['new_id'] = new_role.id
to_append['new_role'] = new_role
roles.append(to_append)
def set_new_role(old_role_id, new_role):
# we don't use the new_role id, as not merged yet
position = get_role_position(old_id=old_role_id)
roles[position]['new_role'] = new_role
roles[position]['new_id'] = new_role.id
def get_role_object_dict(role):
if type(role) == dict:
return role # if already role, just return it
return dict(id=role.id,
name=role.name,
permissions=role.permissions.value,
colour=role.colour.value,
hoist=role.hoist,
mentionable=role.mentionable)
def get_role_dicts(roles=roles):
backup_roles = roles
role_dicts = []
for role in roles:
if role.get('old_role') is not None:
role['old_role'] = get_role_object_dict(role['old_role'])
if role.get('new_role') is not None:
role['new_role'] = get_role_object_dict(role['new_role'])
role_dicts.append(role)
return role_dicts
def get_channel_type(channel):
if type(channel) == discord.channel.TextChannel:
return "Text"
if type(channel) == discord.channel.VoiceChannel:
return "Voice"
if type(channel) == discord.channel.CategoryChannel:
return "Category"
return "Unknown"
def get_channel_object_dict(channel):
if type(channel) == dict:
return channel # already converted
new_overwrites = []
overwrites = channel.overwrites
for overwrite in overwrites:
allow, deny = overwrite[1].pair()
if type(overwrite[0]) == discord.role.Role:
role = get_role(old_id=overwrite[0].id)
if role is None:
to_append = dict(
grantee=dict(old_id=overwrite[0].id, type="Role"))
else:
to_append = dict(grantee=dict(
old_id=role.get('old_id'),
new_id=role.get('new_id'),
old_name=role.get('old_role', dict()).get('name'),
type="Role"))
else: # user overwrite
to_append = dict(
grantee=dict(id=overwrite[0].id, type="User"))
to_append['allow_permission'] = allow.value
to_append['deny_permission'] = deny.value
new_overwrites.append(to_append)
to_return = dict(id=channel.id,
name=channel.name,
type=get_channel_type(channel),
overwrites=new_overwrites,
position=channel.position)
if to_return['type'] != "Category":
if channel.category is not None:
to_return['category'] = get_channel_object_dict(
channel.category)
else:
to_return['category'] = None
if to_return['type'] == "Text":
# do text
to_return['topic'] = channel.topic
to_return['slowmode_delay'] = channel.slowmode_delay
to_return['nsfw'] = channel.nsfw
elif to_return['type'] == "Voice":
# do voice
to_return['bitrate'] = channel.bitrate
to_return['user_limit'] = channel.user_limit
return to_return
def get_channel_dicts(channels=channels):
backup_channels = channels
channel_dicts = []
for channel in channels:
if channel.get('old_channel') is not None:
channel['old_channel'] = get_channel_object_dict(
channel['old_channel'])
if channel.get('new_channel') is not None:
channel['new_role'] = get_channel_object_dict(
channel['new_channel'])
channel_dicts.append(channel)
channels = backup_channels
return channel_dicts
def generate_overwrites(old_channel):
overwrites = dict()
for overwrite in old_channel['overwrites']:
allow = discord.Permissions(overwrite['allow_permission'])
deny = discord.Permissions(overwrite['deny_permission'])
permission_pair = discord.PermissionOverwrite.from_pair(
allow, deny)
target = None # we do this incase down the road there's a new type of grantee we haven't handled
# if a user isn't in the new server, we can't add overwrites for them
if overwrite['grantee']['type'] == "User":
target = clone_target.get_member(
overwrite['grantee']['id'])
# this is the code which will convert the old_id in the overwrite into the new_id
elif overwrite['grantee']['type'] == "Role":
role = get_role(old_id=overwrite['grantee']['old_id'])
if role is not None:
target = clone_target.get_role(role['new_id'])
else:
old_role = ctx.guild.get_role(
overwrite['grantee']['old_id'])
if old_role.name == "@everyone":
target = clone_target.default_role
else:
print(
"Could not find new role pair for old role with ID {}"
.format(overwrite['grantee']['old_id']))
if target is not None:
overwrites[target] = permission_pair
return overwrites
s_channels = guild.channels
s_channels.sort(key=lambda x: x.position, reverse=False)
for channel in s_channels:
add_channel(channel)
s_roles = guild.roles
s_roles.reverse()
for role in s_roles:
if role.name != "@everyone":
add_role(role)
await progress.edit(embed=generate_progress_embed(
"Wiping roles of {clone_target.name}".format(
clone_target=clone_target)))
for role in clone_target.roles:
try:
await role.delete(reason="Cleaning for copying")
except discord.errors.HTTPException:
pass
await progress.edit(embed=generate_progress_embed(
"Wiping channels of {clone_target.name}".format(
clone_target=clone_target)))
for channel in clone_target.channels:
await channel.delete(reason="Cleaning for copying")
print("Wiped channels")
await progress.edit(embed=generate_progress_embed(
"Creating new roles in {clone_target.name}".format(
clone_target=clone_target)))
for role in get_role_dicts():
old_role = role['old_role']
logger.info("Adding role{id} - {name}".format(
id=role['old_id'], name=role['old_role']['name']))
new_role_to_merge = await clone_target.create_role(
name=old_role['name'],
permissions=discord.Permissions(
permissions=old_role['permissions']),
colour=discord.Colour(old_role['colour']),
hoist=old_role['hoist'],
mentionable=old_role['mentionable'])
set_new_role(old_role['id'], new_role_to_merge)
get_role_dicts(
) # this converts all the new channels into nice dictionary formats
await progress.edit(embed=generate_progress_embed(
"Creating new channels in {clone_target.name}".format(
clone_target=clone_target)))
# first, we add the categories
channels_to_add = get_channel_dicts()
channels.sort(key=lambda x: x['old_channel']['position'],
reverse=False)
for channel in channels_to_add:
old_channel = channel['old_channel']
if old_channel['type'] == "Category":
logger.info("Adding category {id} - {name}".format(
id=old_channel['id'], name=old_channel['name']))
# build overwrites
overwrites = generate_overwrites(old_channel)
channel = await clone_target.create_category_channel(
old_channel['name'],
overwrites=overwrites,
reason="Syncing of channels")
set_new_channel(old_channel['id'], channel)
# this makes sure everything is still in dictionary formats
channels_to_add = get_channel_dicts()
# now we know all our categories are added, we can add all other channels
for channel in channels_to_add:
old_channel = channel['old_channel']
if old_channel['type'] != "Category":
logger.info("Adding {type} channel {id} - {name}".format(
type=old_channel['type'],
id=old_channel['id'],
name=old_channel['name']))
overwrites = generate_overwrites(old_channel)
category = get_channel(
old_id=old_channel['category'].get('id'))
if category is not None:
category_id = category['new_id']
# gets the role object that we need to create the channel
category = clone_target.get_channel(category_id)
if old_channel['type'] == "Text":
channel = await clone_target.create_text_channel(
old_channel['name'],
overwrites=overwrites,
reason="Syncing of channels",
position=old_channel['position'],
topic=old_channel['topic'],
slowmode_delay=old_channel['slowmode_delay'],
nsfw=old_channel['nsfw'],
category=category)
elif old_channel['type'] == "Voice":
channel = await clone_target.create_voice_channel(
old_channel['name'],
overwrites=overwrites,
reason="Syncing of channels",
position=old_channel['position'],
bitrate=old_channel['bitrate'],
user_limit=old_channel['user_limit'],
category=category)
channel = get_channel_object_dict(channel)
set_new_channel(old_channel['id'], channel)
with open(".last_sync.json", "w") as f:
f.write(
json.dumps(dict(roles=get_role_dicts(roles=roles),
channels=get_channel_dicts(channels=channels)),
indent=4))
invite = await clone_target.text_channels[0].create_invite(
max_age=0,
max_uses=0,
unique=True,
reason="Generating invite link to join with")
await progress.edit(embed=generate_progress_embed(
"Done!", colour=green, url=invite.url))
import mysql.connector
import sys
from ags_experiments.logger import logger
from ags_experiments.settings.config import config
port = config['mysql'].get("port")
if port is None:
logger.warn("'mysql'.'port' not found in config - defaulting to 3306")
config['mysql']['port'] = 3306
try:
cnx = mysql.connector.connect(**config['mysql'])
logger.info("Created connection to MySQL")
except mysql.connector.errors.InterfaceError as exception:
logger.error("{} - check your settings".format(exception))
sys.exit(1)
cursor = cnx.cursor(buffered=True)
cursor_dict = cnx.cursor(dictionary=True)
import mysql
from discord.ext import commands
from ags_experiments.settings import guild_settings
from ags_experiments import set_activity
from ags_experiments.client_tools import ClientTools
from ags_experiments.database import cnx, cursor
from ags_experiments.database.database_tools import DatabaseTools, insert_users, insert_settings, insert_role, \
update_role
from ags_experiments.role_c import DbRole
from ags_experiments.settings.config import config, strings
from ags_experiments.logger import logger
if config['discord']['debug'] or bool(
os.environ.get('discord_experiments_debug')):
logger.info("Running in debug mode.")
debug = True
prefix = config['discord']['prefix_debug']
else:
logger.info("Running in production mode.")
debug = False
prefix = config['discord']['prefix']
shard_count = config['discord'].get("shard_count")
if shard_count is None:
logger.warn(
"config['discord']['shard_count'] is not set - defaulting to 1 shard")
shard_count = 1
client = commands.AutoShardedBot(command_prefix=prefix,
owner_id=config['discord']['owner_id'],