def prepareTrainParams(args):
print("Read Features from: ", args.data_dir + "/features.txt", file=sys.stderr)
feat = features.Features(args.data_dir + "/features.txt")
print("Loading train data: ", args.data_dir + "/train", file=sys.stderr)
train_data = data.load(args.data_dir + "/train")
print("Loading validation data: ", args.data_dir + "/dev", file=sys.stderr)
validation_data = data.load(args.data_dir + "/dev")
mfp = args.model_dir + "/"
if args.model_dir is None:
mfp = ""
params = trainT.Params(
trainData=train_data,
validationData=validation_data,
modelFile=mfp + args.prefix + '_{epoch:02d}.h5',
hidden=args.hidden,
wordVecSize=args.word_vec_size,
minibatches=minibatches,
gpu=args.use_gpu,
features=feat
)
return params
def fetch_data(args):
from data import data
dataset, _, _ = data.load(args)
args.dataset_dict = dataset
X, Y, G = dataset['features'], dataset['labels'], dataset['hypergraph']
# node features in sparse representation
X = sp.csr_matrix(normalise(np.array(X)), dtype=np.float32)
X = torch.FloatTensor(np.array(X.todense()))
# labels
Y = np.array(Y)
Y = torch.LongTensor(np.where(Y)[1])
X, Y = X.cuda(), Y.cuda()
return X, Y, G
def train(net,
data,
rwd_func,
error_goal=0,
batch=1,
gamma=0.9,
epsilon=0,
d_eps=0.01):
# filepath = \
# "/home/scottgbarnes/Documents/Simulation/Training/Test Set/Data1.mat"
data = data.load(filepath)
error = 100
while (error > error_goal):
for i in range(0, data.shape[2]):
inpt = data[:, :, i] # Import Data of Interest
r_pos = (325, 58) # Set robot's initial position
reward = sim.train(inpt, r_pos, rwd_func)
return net
# Executable file for Keras training
from train import train
from data import data
from keras.models import Sequential
from keras.layers import Dense, Activation, LSTM
from keras.callbacks import History
from keras import optimizers
from goal import goal
import numpy as np
import matplotlib.pyplot as plt
# Import data
print('Loading Data')
path = '/home/scottgbarnes/Documents/Simulation/RLToolbox/Test Set/Data1.mat'
print('...')
data = data.load(path)
print('Data Loaded\n')
# Format Data for Training
lim = data.shape[2]
inpt = np.concatenate((data[:, 1, 0:lim - 1], data[:, 2, 0:lim - 1]), axis=0)
trgt = data[0, 1:3, 1:lim]
inpt = np.transpose(inpt)
trgt = np.transpose(trgt)
inpt = inpt[:, np.newaxis, :]
print(inpt.shape)
print(trgt.shape)
# Create Network Paramaters
print('Creating Network')
print('...')
model = Sequential()
model.add(Dense(9, input_shape=(1, inpt.shape[2])))
from spec.mamba import *
if False:
from data import data
from origen import seven_segment
from origen import seven_segment_data
from origen.puzzles import access
TEST_DATA = data.load(
'data/seven_segment_prototype_test_data.txt',
seven_segment.Glyphs)
with _description('combinations'):
with it('should merge two characters'):
merged = (
seven_segment_data.ALPHABET['F'] |
(seven_segment_data.ALPHABET['l'] >> 1))
expect(merged).to(equal(seven_segment_data.ALPHABET['A']))
with it('should accept ACCESS no-op'):
expect(access.accept(seven_segment_data.ACCESS)).to(be_true)
with it('should accept ACCESS after transformations'):
transformed = (
seven_segment_data.glyphs_from_str('ACCESS') |
seven_segment_data.glyphs_from_str('ALL'))
expect(access.accept(transformed)).to(be_true)
with _description('prototype test data'):
with it('should load test data'):
expect(TEST_DATA).to(have_keys('KEY', 'CORRECT', 'NOPE', 'LOST'))
# Optimization
parser.add_argument('--batch', type=int, default=256)
parser.add_argument('--lr', type=float, default=1e-3)
parser.add_argument('--epochs', type=int, default=30)
parser.add_argument('--times', type=int, default=3)
parser.add_argument('--ensemble', type=int, default=5)
params=parser.parse_args();
assert torch.cuda.is_available(),"You running on CPUs?"
torch.manual_seed(params.seed) #Fix random seed.
torch.cuda.manual_seed(params.seed);
params.argv=sys.argv;
#Load data
data_train=data.load(params.data_train);
data_test=data.load(params.data_test);
params.stuff=data_train.preprocess();
_=data_test.preprocess(params.stuff);
data_train.cuda();
data_test.cuda();
params.dof=data_train.dof;
params.noutput=data_train.noutput;
#Create Networks
arch=importlib.import_module(params.arch);
net=arch.new(params).cuda();
import util.pm as pm_proto
pms=[];
for i in range(0,params.ensemble):
test_accs = []
best_val_accs, best_test_accs = [], []
resultlogger.info(args)
# load data
X, Y, G = fetch_data(args)
for run in range(1, args.n_runs + 1):
run_dir = out_dir / f'{run}'
run_dir.makedirs_p()
# load data
args.split = run
_, train_idx, test_idx = data.load(args)
train_idx = torch.LongTensor(train_idx).cuda()
test_idx = torch.LongTensor(test_idx).cuda()
# model
model, optimizer = initialise(X, Y, G, args)
baselogger.info(f'Run {run}/{args.n_runs}, Total Epochs: {args.epochs}')
baselogger.info(model)
baselogger.info(
f'total_params:{sum(p.numel() for p in model.parameters() if p.requires_grad)}'
)
tic_run = time.time()
from collections import Counter
# Optimization
parser.add_argument('--batch', type=int, default=256)
parser.add_argument('--lr', type=float, default=1e-3)
parser.add_argument('--epochs', type=int, default=30)
parser.add_argument('--times', type=int, default=3)
parser.add_argument('--ensemble', type=int, default=5)
params = parser.parse_args()
assert torch.cuda.is_available(), "You running on CPUs?"
torch.manual_seed(params.seed) #Fix random seed.
torch.cuda.manual_seed(params.seed)
params.argv = sys.argv
#Load data
data_train = data.load(params.data_train)
data_test = data.load(params.data_test)
params.stuff = data_train.preprocess()
_ = data_test.preprocess(params.stuff)
data_train.cuda()
data_test.cuda()
params.dof = data_train.dof
params.noutput = data_train.noutput
#Create Networks
arch = importlib.import_module(params.arch)
net = arch.new(params).cuda()
import util.pm as pm_proto
pms = []
for i in range(0, params.ensemble):
# # set arguments up ([argparse](https://docs.python.org/3/library/argparse.html)) # In[2]: from model import config args = config.setup() log = args.logger # # load data # In[3]: from data import data dataset, splits = data.load(args) # # load model # In[4]: from model import model MPNNR = model.MPNNR(dataset, args) MPNNR.summary(log) # # train, test model # # In[5]:
parser.add_argument('--learn_rate_corsa', type=float, default=0.0005,
help='learning rate of sat and corsa')
parser.add_argument('--drop_prob', type=float, default=0.0,
help='dropout for the decoder')
parser.add_argument('--alpha', type=float, default=1.0,
help='weight of adversarial backprob')
PLP_POWER = 0.90
DATA_DIM = 16
args = parser.parse_args()
fh.random_seed(args.seed)
device = model.which_device()
opt = vars(args)
print('loading the data from folder')
dataset, data_dict, speaker_ids = data.load(opt['data_dir'])
num_speakers = len(np.unique(speaker_ids))
vals = np.unique(speaker_ids)
keys = np.arange(0,(num_speakers))
id_map = mapping = dict(zip(vals, keys))
speaker_ids2 = speaker_ids
for i in range(speaker_ids.shape[0]):
speaker_ids2[i] = id_map[speaker_ids[i]]
dataset = dataset[:, 0:DATA_DIM]
for i in range(DATA_DIM):
dataset[:, i]=dataset[:, i]*pow(PLP_POWER, i)
"""
class CrypticFixture(object):
def __init__(self, name: str, lines: List[str]) -> None:
self.name = name
self.clue, self.method = lines[:2]
with description('with wordnet', 'end2end') as self:
with before.all:
warehouse.save()
warehouse.register('/api/words', word_api.get_api('wordnet'))
self.problems = {}
all_clues = []
fixtures = data.load('data/cryptic_clues.txt', CrypticFixture)
fixtures.update(
data.load_lines(ORIGINAL_UNSOLVED.split('\n'), CrypticFixture))
for fixture in fixtures.values():
all_clues.append(fixture.clue)
self.problems[fixture.name] = cryptic_problem.CrypticProblem(
fixture.name, [fixture.clue])
# expect(all_clues).to(have_len(51)) # Make fail to see output.
warehouse.save()
d = dict([
(fixture.name.lower(), 100000) for fixture in fixtures.values()
])
d['volcanoes'] = 100000 # For GPH Zero Space.
d['micro'] = 10000 # For MUSIC.
d['witch'] = 10000 # For GECKO.
d['wight'] = 10000 # For GECKO.
'''
Oh you expected a real test suite like nose/pytest/unittest/doctest
not today
Just tested if the data is actually there.
TODO:
Implement try/catch. The software will brake under any circumstances
(i.e. no internet, missing data, wrong data input)
without providing much useful information.
'''
from data.data import load
file = load('file')
print(file[0][0][0], '-- is the picture')
print(file[0][0][1], '-- is the page link')
print(file[0][0][2], '-- is the title')
print(file[0][0][3], '-- is the price')
print(file[0][0][4], '-- are the views')
print(file[0][0][5], '-- is the loc')
data.show_data_summary()
joint_train.joint_train(data, old_data, opt)
data.clear_data()
data.save(os.path.join(opt.output, 'data.pkl'))
else:
if os.path.exists(opt.predict):
shutil.rmtree(opt.predict)
os.makedirs(opt.predict)
else:
os.makedirs(opt.predict)
data.load(os.path.join(opt.output, "data.pkl"))
data.MAX_SENTENCE_LENGTH = -1
# for va begin
data.nbest = 1
data.sent_window = 1
data.HP_gpu = opt.gpu
# for va end
data.show_data_summary()
data.fix_alphabet()
data.fix_re_alphabet()
test.predict(opt, data)
with it('ambiguously matches clues with lots of words'):
expect(crossword_problem.CrosswordProblem.score(
['A quick brown fox jumps over the lazy dog'])).to(be_above(.25))
with it('boosts questions with common crossword indicators'):
expect(crossword_problem.CrosswordProblem.score(
['abbreviation of the French honorific "Madame"'])).to(be_above(.75))
with it('matches a real question with punctuation'):
expect(crossword_problem.CrosswordProblem.score(
['Apparatus with "Yes", "No", and "Good Bye"'])).to(be_above(.25))
with it('matches data from fixtures'):
fixtures = data.load(
'data/puzzle_fixtures.txt',
collections.namedtuple('fixture', 'name lines'))
for line in fixtures['crossword_clues'].lines:
expect(call(crossword_problem.CrosswordProblem.score, [line])).to(
be_above(.5))
with it('cleans off address'):
problem = crossword_problem.CrosswordProblem('ex', [' 1. example (3)'])
expect(problem.lines).to(equal(['example (3)']))
with it('address stripping is not fooled by "St. Mark"'):
problem = crossword_problem.CrosswordProblem(
'ex', ['Wandering Magi following St. Mark (6)'])
expect(problem.lines).to(equal(['Wandering Magi following St. Mark (6)']))
with description('constraints'):
dataDir = sys.argv[1]
else:
sys.exit("Data dir argument missing")
if len(sys.argv) > 2:
mPrefix = sys.argv[2]
else:
sys.exit("Model file prefix argument missing")
####################################################################################
# data
####################################################################################
print("Read vocabulary: ", dataDir + "/vocabulary")
vocab = data.readVocabulary(dataDir + "/vocabulary")
print("Loading train data: ", dataDir + "/train")
trainData = data.load(dataDir + "/train")
print("Loading validation data: ", dataDir + "/dev")
validationData = data.load(dataDir + "/dev")
####################################################################################
params = trainT.Params(vocab=vocab,
trainData=trainData,
validationData=validationData,
modelFile=mPrefix + '_{epoch:02d}.h5',
hidden=hidden,
wordVecSize=hidden,
minibatches=minibatches,
gpu=False)
############ training
trainT.trainModel(params)
import os, torch, numpy as np
torch.manual_seed(args.seed)
np.random.seed(args.seed)
os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
os.environ["CUDA_VISIBLE_DEVICES"] = str(args.gpu)
os.environ['PYTHONHASHSEED'] = str(args.seed)
# # load data
# In[ ]:
from data import data
dataset, train, test = data.load(args)
print("length of train is", len(train))
# # initialise HyperGCN
# In[ ]:
from model import model
HyperGCN = model.initialise(dataset, args)
# # train and test HyperGCN
# In[ ]:
HyperGCN = model.train(HyperGCN, dataset, train, args)
self[variant] = None
def __getitem__(self, item):
if super(WordList, self).__getitem__(item) is None:
self[item] = glyphs_from_str(item)
return super(WordList, self).__getitem__(item)
def _enumerate_words(word):
result = []
for candidate in [word, word[0].upper() + word[1:], word.upper()]:
if _VALID_REGEX.match(candidate):
result.append(candidate)
return result
def glyphs_from_str(s):
result = seven_segment.Glyphs(s, '')
for c in s:
result += ALPHABET[c]
result.name = s
return result
ALPHABET = data.load('data/seven_segment_alphabet.txt', seven_segment.Glyphs)
_VALID_REGEX = re.compile('^[%s]+$' % ''.join(ALPHABET.keys()))
ACCESS_WORDS = data.load('data/seven_segment_access_words.txt', WordList)
ACCESS = ACCESS_WORDS['POSITIVE']['ACCESS']
