def compute_data_files():
'''Sort the data files in increasing order of complexity, then return the n least complex datapoints.'''
data_files = []
print('Loading structure metadata')
structure_to_files = misc.load_file(
'save_state/neutral/structure_to_files.pkl')
all_structure_strs = list(structure_to_files.keys())
# The number of commas in the structure_str is used as a proxy for complexity
i = 0
all_structure_strs.sort(key=lambda x: x.count(','))
'''
for structure_str in all_structure_strs:
data_i=structure_to_files[structure_str]
if ( "SHAPE" in structure_str) and len(data_i)>10000 and len(data_i)<20000:
data_files.extend(data_i)
else:
continue
print(structure_str, len(structure_to_files[structure_str]))
structure_str = '(PROGRESSION,SHAPE,TYPE)'#(XOR,SHAPE,SIZE)
data_files.extend(structure_to_files[structure_str])
print(structure_str, len(structure_to_files[structure_str]))
'''
return all_structure_strs
def _create_train_info(self, args, init_train_info):
import json
train_info_path = os.path.join(self.logdir, 'train_info.txt')
if os.path.isfile(train_info_path):
self.train_info = json.loads(misc.load_file(train_info_path))
else:
self.train_info = init_train_info
def main(args):
# Step 1: init data folders
'''if os.path.exists('save_state/'+args.regime+'/normalization_stats.pkl'):
print('Loading normalization stats')
x_mean, x_sd = misc.load_file('save_state/'+args.regime+'/normalization_stats.pkl')
else:
x_mean, x_sd = preprocess.save_normalization_stats(args.regime)
print('x_mean: %.3f, x_sd: %.3f' % (x_mean, x_sd))'''
# Step 2: init neural networks
print("network is:", args.net)
if args.net == "resnet":
model = resnet50(pretrained=True)
elif args.net == 'wresnet':
model = wresnet50(pretrained=True)
elif args.net == "tmp":
model = tmp()
elif args.net == 'RN_mlp':
model = WildRelationNet()
elif args.net == 'ReasonNet':
model = ReasonNet()
elif args.net == 'ReaP':
model = ReasonNet_p()
elif args.net == 'Reap16':
model = ReasonNet_p16()
elif args.net == 'Reaap':
model = ReasonNet_ap()
elif args.net == 'RN_ap':
model = RN_ap()
elif args.net == 'RN_r':
model = RN_r()
elif args.net == 'esemble':
model = esemble()
elif args.net == 'RNap2':
model = RNap2()
elif args.net == 'rn_mlp':
model = rn_mlp()
elif args.net == 'Reab3p16':
model = Reab3p16()
elif args.net == 'b3pa':
model = b3pa()
elif args.net == "b3_plstm":
model = b3_plstm()
elif args.net == "b3_palstm":
model = b3palstm()
elif args.net == "nmn":
model = nmn()
elif args.net == "b3p3":
model = b3p3()
elif args.net == "multi3":
model = multi3()
elif args.net == "split":
model = b3_split()
if args.gpunum > 1:
model = nn.DataParallel(model, device_ids=range(args.gpunum))
if args.net != 'RN_r':
model.apply(weights_init)
print('weight initial')
weights_path = args.path_weight
if os.path.exists(weights_path) and args.restore:
pretrained_dict = torch.load(weights_path)
model_dict = model.state_dict()
pretrained_dict1 = {}
for k, v in pretrained_dict.items():
if k in model_dict:
pretrained_dict1[k] = v
# print(k)
model_dict.update(pretrained_dict1)
model.load_state_dict(model_dict)
# optimizer.load_state_dict(torch.load(optimizer_path))
print('load weight')
model.cuda()
epoch_count = 1
print(time.strftime('%H:%M:%S', time.localtime(time.time())), 'testing')
print('Loading structure metadata')
structure_to_files = misc.load_file(
'save_state/neutral/structure_to_files.pkl')
all_structure_strs = list(structure_to_files.keys())
# The number of commas in the structure_str is used as a proxy for complexity
accuracy_all = []
for structure_str in all_structure_strs:
data_files = []
data_i = structure_to_files[structure_str]
if ("SHAPE" in structure_str
) and len(data_i) > 10000: # and len(data_i) < 20000:
data_files.extend(data_i)
else:
continue
test_files = [
data_file for data_file in data_files if 'test' in data_file
]
test_loader = torch.utils.data.DataLoader(Dataset(test_files),
batch_size=args.batch_size,
shuffle=True,
num_workers=args.numwork)
since = time.time()
model.eval()
accuracy_epoch = []
for x, y in test_loader:
x, y = Variable(x).cuda(), Variable(y).cuda()
pred = model(x)
pred = pred.data.max(1)[1]
correct = pred.eq(y.data).cpu().sum().numpy()
accuracy = correct * 100.0 / len(y)
accuracy_epoch.append(accuracy)
accuracy_all.append(accuracy)
acc = sum(accuracy_epoch) / len(accuracy_epoch)
print(('epoch:%d, acc:%.1f') % (epoch_count, acc), "test_num:",
len(test_files), (structure_str))
epoch_count += 1
print(('epoch:%d, acc:%.1f') %
(epoch_count, sum(accuracy_all) / len(accuracy_all)))
from typing import Callable
import discord
from discord import PermissionOverwrite
from discord.ext import commands
from discord.ext.commands import Bot
from meme_collections import (add_to_collection, delete_from_collection,
get_from_collection, handle_collections)
from misc import get_messages, load_file, split_word_by_step, turn_into_emoji
# global vars
bot = Bot(command_prefix='~')
# data vars
CONFIG = load_file('data/config.json')
SPECIAL_IDS = load_file('data/ids.json')
SUPER_MODERATOR = SPECIAL_IDS[0]
last_message = ''
def needs_permission(bot: discord.ext.commands.Bot, hidden: bool=False):
"""
Decorator to make a command that forces the user to have sufficient permissions to use the command
"""
def decorator(func: Callable, hidden=hidden):
@bot.command(pass_context=True ,hidden=hidden)
@wraps(func)
async def wrapped(ctx, *args, **kwargs):
if ctx.message.author.id == SUPER_MODERATOR or any(role.name == 'Moderator' for role in ctx.message.author.roles):
return await func(ctx, *args, **kwargs)
await bot.say("You don't have the permissions to run this command!")
def compute_data_files(regime, n, args):
'''Sort the data files in increasing order of complexity, then return the n least complex datapoints.'''
data_dir = '../data/' + regime + '/'
if n == -1:
data_files = os.listdir(data_dir)
data_files = [data_dir + data_file for data_file in data_files]
return data_files
elif n == -2:
test_files = [
data_dir + data_file for data_file in os.listdir(data_dir)
if 'test' in data_file
]
data_files = []
if os.path.exists('save_state/' + regime + '/structure_to_files.pkl'):
print('Loading structure metadata')
structure_to_files = misc.load_file('save_state/' + regime +
'/structure_to_files.pkl')
else:
structure_to_files = save_structure_to_files(regime)
all_structure_strs = list(structure_to_files.keys())
all_structure_strs.sort(key=lambda x: x.count(','))
for structure_str in all_structure_strs:
data_i = structure_to_files[structure_str]
if len(data_i) > 5000:
data_i = data_i[:5000]
data_files.extend(data_i)
#print(structure_str, len(structure_to_files[structure_str]))
data_files = [
data_file for data_file in data_files if 'train' in data_file
]
data_files.extend(test_files)
return data_files
elif n == -3:
data_files = []
if os.path.exists('save_state/' + regime + '/structure_to_files.pkl'):
print('Loading structure metadata')
structure_to_files = misc.load_file('save_state/' + regime +
'/structure_to_files.pkl')
else:
structure_to_files = save_structure_to_files(regime)
all_structure_strs = list(structure_to_files.keys())
all_structure_strs.sort(key=lambda x: x.count(','))
for structure_str in all_structure_strs:
data_i = structure_to_files[structure_str]
if len(data_i) > 20000 or len(data_i) < 10000:
continue
data_files.extend(data_i)
print(structure_str, len(structure_to_files[structure_str]))
return data_files
elif n == -4:
data_files = os.listdir("/home/zkc/reason/andshapecolormask/")
data_files = [
"/home/zkc/reason/andshapecolormask/" + data_file
for data_file in data_files
]
return data_files
else:
data_files = []
if os.path.exists('save_state/' + regime + '/structure_to_files.pkl'):
print('Loading structure metadata')
structure_to_files = misc.load_file('save_state/' + regime +
'/structure_to_files.pkl')
else:
structure_to_files = save_structure_to_files(regime)
all_structure_strs = list(structure_to_files.keys())
# The number of commas in the structure_str is used as a proxy for complexity
i = 0
all_structure_strs.sort(key=lambda x: x.count(','))
for structure_str in all_structure_strs:
data_i = structure_to_files[structure_str]
if args.image_type == "image":
if "SHAPE" in structure_str and "),(" not in structure_str:
data_files.extend(data_i)
print(structure_str, ":", len(data_i))
if "LINE" in structure_str and "),(" not in structure_str:
data_files.extend(data_i)
print(structure_str, ":", len(data_i))
elif args.image_type == "shape_im":
if "SHAPE" in structure_str and "),(" not in structure_str:
data_files.extend(data_i)
print(structure_str, ":", len(data_i))
elif args.image_type == "line_im":
if "LINE" in structure_str and "),(" not in structure_str:
data_files.extend(data_i)
print(structure_str, ":", len(data_i))
return data_files