def load():
"""load the classic Norvig big.txt corpus"""
print("training!")
models.load_models()
print("done training!")
return True
def run_server(port_num=8080):
"""little demo server for demo'ing sake"""
models.load_models()
debug(True)
@route('/<wordA>/<wordB>')
def index(wordA, wordB):
return dict(predict(wordA, wordB))
run(host='localhost', port=port_num)
def main():
product_json_fields = 'product_name manufacturer model family announced-date'.split()
listing_json_fields = 'title manufacturer currency price'.split()
products = list(load_models('data/products.txt', Product, product_json_fields, 'utf-8'))
listings = load_models('data/listings.txt', Listing, listing_json_fields, 'utf-8')
to_review_listings = []
product_buckets = defaultdict(set)
# Group products by the words in the manufacturer field
for product in products:
for token in product.normalized_manufacturer.split():
product_buckets[token].add(product)
for listing in listings:
# Retrieve candidates
candidates = set()
for token in listing.tokens:
candidates.update(product_buckets[token])
potential_products = [
candidate
for candidate in candidates
if is_potential_match(candidate, listing)
]
# If there's more than one match using manufacturer and model, try to disambiguate
# by looking for all product.name tokens in listing.title and listing.manufacturer
if len(potential_products) > 1:
potential_products = filter_by_product_name(potential_products, listing)
if len(potential_products) == 1:
product = potential_products[0]
product.listings.append(listing)
else:
to_review_listings.append(listing)
# Remove listings if their price is under 0.3 of the median price (accessory?)
for product in products:
product.listings, rejected = filter_accessories(
product.listings, 0.3, lambda x: x.price_in_cad
)
if rejected:
to_review_listings.extend(rejected)
# Save results
save_json_by_line('results.txt', (
{
'product_name': product.name,
'listings': [l.as_dict() for l in product.listings]
}
for product in products
), encoding='utf-8')
def configure_app(app, config):
CORS(app) # cross domain
tools = {
'auth': configure_auth(app, config),
'cache': configure_cache(app, config)
}
if config.getboolean('hippo', 'mongo'):
print('connect MongoDB...')
tools['mongo'] = configure_mongo(config)
if config.getboolean('hippo', 'pubsub'):
print('connect Kafka...')
tools['pubsub'] = configure_kafka(config)
models = load_models(config, tools)
apis = load_apis(config, tools, models)
if 'pubsub' in tools:
print('register Subscribers...')
register_subscribers(config, tools, models)
if config.getboolean('hippo', 'refresh_data'):
init_tasks(models)
init_events(models)
return tools, apis
def main():
logging.basicConfig(level=logging.INFO, format='%(message)s')
parser = argparse.ArgumentParser(
description='Create synthetic phrases'
' using trained CRF models and lemma grammar')
parser.add_argument('rev_map', help='reverse inflection map')
parser.add_argument('models',
nargs='+',
help='trained models (category:file)')
parser.add_argument('sgm', help='original sentences + grammar pointers')
parser.add_argument('sgm_lem',
help='original sentences + lemma grammar pointers')
parser.add_argument('out', help='grammar output directory')
args = parser.parse_args()
if not os.path.exists(args.out):
os.mkdir(args.out)
logging.info('Loading reverse inflection map')
with open(args.rev_map) as f:
rev_map = cPickle.load(f)
logging.info('Loading inflection prediction models')
models = load_models(args.models)
logging.info('Generating extended grammars')
data = izip(read_sentences(sys.stdin, skip_empty=False),
read_sgm(args.sgm), read_sgm(args.sgm_lem))
for (source, _, _), (grm_path, sid, left, right),\
(lem_grm_path, lem_sid, lem_left, lem_right) in data:
assert sid == lem_sid and left == lem_left and right == lem_right
# Create grammar file
out_path = os.path.join(args.out, 'grammar.{}.gz'.format(sid))
grammar_file = gzip.open(out_path, 'w')
# Copy original grammar
with gzip.open(grm_path) as f:
for line in f:
grammar_file.write(line)
# Generate synthetic phrases from lemma grammar
for rule in read_grammar(lem_grm_path):
assert not any(src.startswith('[X,')
for src in rule.lhs) # no gaps, please
for match in source_match(rule.lhs, source):
# create (at most) a synthetic rule
for new_rule in synthetic_rule(rev_map, models, rule, source,
match):
grammar_file.write(unicode(new_rule).encode('utf8') + '\n')
grammar_file.close()
# Write sgm
new_left = u'<seg grammar="{}" id="{}">{}</seg>'.format(
out_path, sid, left)
print(u' ||| '.join([new_left] + right).encode('utf8'))
def main(args):
# Set the random seed manually for reproducibility.
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
ae_args, gan_args, idx2word, autoencoder, gan_gen, gan_disc \
= load_models(args.ae_args, args.gan_args, args.vocab_file,
args.ae_model, args.g_model, args.d_model)
word2idx = {v: k for k, v in idx2word.items()}
gan_gen.cuda()
autoencoder.cuda()
gan_gen.eval()
autoencoder.eval()
with open(args.quiz_file) as f:
quizzes = json.load(f)
def gen_batches(data, batch_size=args.batch_size):
for i in range(int(np.ceil(len(data) / batch_size))):
yield data[i:i + batch_size]
def pad_data(data):
maxlen = max(map(len, data))
return [x + (maxlen - len(x)) * [Dictionary.pad] for x in data]
answers = []
i = 0
for batch in tqdm(gen_batches(quizzes),
total=int(np.ceil(len(quizzes) / args.batch_size))):
source_idx = torch.LongTensor(
pad_data([[Dictionary.sos] + [word2idx[w] for w in quiz]
for quiz in batch])).cuda()
source = autoencoder(Variable(source_idx, volatile=True),
noise=False,
encode_only=True)
sentences = []
for _ in range(args.gen_cnt):
hidden = gan_gen(source)
sentences.append(
generate_from_hidden(autoencoder,
hidden,
vocab=idx2word,
sample=args.sample,
maxlen=args.maxlen))
source = hidden
sentences = list(zip(*sentences))
for quiz, ans in zip(batch, sentences):
i += 1
answers.append({'id': i, 'quiz': ''.join(quiz), 'answer': ans})
with open(args.outf, 'w') as f:
json.dump(answers, f, ensure_ascii=False, indent=2)
def run_server(port_num=8080):
if not models.load_models():
#print(os.listdir(os.curdir))
with open(os.path.join(os.path.dirname(__file__), 'big.txt'),
'rb') as f:
print(os.path.join(os.path.dirname(__file__), 'big.txt'))
models.train_models(str(f.read()))
@route('/<wordA>/<wordB>')
def index(wordA, wordB):
return dict(autocomplete.predict(wordA, wordB))
run(host='localhost', port=port_num)
def main():
logging.basicConfig(level=logging.INFO, format='%(message)s')
parser = argparse.ArgumentParser(description='Create synthetic phrases'
' using trained CRF models and lemma grammar')
parser.add_argument('rev_map', help='reverse inflection map')
parser.add_argument('models', nargs='+', help='trained models (category:file)')
parser.add_argument('sgm', help='original sentences + grammar pointers')
parser.add_argument('sgm_lem', help='original sentences + lemma grammar pointers')
parser.add_argument('out', help='grammar output directory')
args = parser.parse_args()
if not os.path.exists(args.out):
os.mkdir(args.out)
logging.info('Loading reverse inflection map')
with open(args.rev_map) as f:
rev_map = cPickle.load(f)
logging.info('Loading inflection prediction models')
models = load_models(args.models)
extracted_tags = ''.join([cat for cat in models.keys()])
logging.info('Inflecting categories: {}'.format(extracted_tags))
lemma_re = re.compile('^(.+)_(['+''+extracted_tags+'])$')
logging.info('Generating extended grammars')
data = izip(read_sentences(sys.stdin, skip_empty=False),
read_sgm(args.sgm), read_sgm(args.sgm_lem))
for (source, _, _), (grm_path, sid, left, right),\
(lem_grm_path, lem_sid, lem_left, lem_right) in data:
assert sid == lem_sid and left == lem_left and right == lem_right
# Create grammar file
out_path = os.path.join(args.out, 'grammar.{}.gz'.format(sid))
grammar_file = gzip.open(out_path, 'w')
# Copy original grammar
with gzip.open(grm_path) as f:
for line in f:
grammar_file.write(line)
# Generate synthetic phrases from lemma grammar
for rule in read_grammar(lem_grm_path):
assert not any(src.startswith('[X,') for src in rule.lhs) # no gaps, please
for match in source_match(rule.lhs, source):
# create (at most) a synthetic rule
for new_rule in synthetic_rule(rev_map, models, rule, source, match, lemma_re):
grammar_file.write(unicode(new_rule).encode('utf8')+'\n')
grammar_file.close()
# Write sgm
new_left = u'<seg grammar="{}" id="{}">{}</seg>'.format(out_path, sid, left)
print(u' ||| '.join([new_left] + right).encode('utf8'))
def main(args):
###########################################################################
# Load the models
###########################################################################
model_args, idx2word, autoencoder, inverter, gan_gen, gan_disc = \
load_models(args.load_path)
# Set the random seed manually for reproducibility.
random.seed(model_args['seed'])
np.random.seed(model_args['seed'])
torch.manual_seed(model_args['seed'])
if torch.cuda.is_available():
torch.cuda.manual_seed(model_args['seed'])
else:
print("Note that our pre-trained models require CUDA to evaluate.")
###########################################################################
# Load data
###########################################################################
corpus = Corpus(model_args['data_path'],
maxlen=model_args['maxlen'],
vocab_size=model_args['vocab_size'],
lowercase=model_args['lowercase'])
if args.test:
eval_batch_size = 1
test_data = batchify(corpus.test, eval_batch_size, shuffle=False)
else:
train_data = batchify(corpus.train, model_args['batch_size'], shuffle=True)
print("Loaded data!")
###########################################################################
# Perturbations
###########################################################################
ring_rng = np.linspace(0., 1., 100)
n_rng = len(test_data) if args.test else len(train_data)
for idx in range(n_rng):
data_batch = test_data[idx] if args.test else train_data[idx]
for l, r in zip(ring_rng, ring_rng[1:]):
flg = perturb(data_batch, autoencoder, idx2word,
model_args['sample'], model_args['maxlen'],
left=l, right=r, n_samples=5, epoch=idx,
gpu=model_args['cuda'])
if flg: break
def main(args):
# Set the random seed manually for reproducibility.
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
if torch.cuda.is_available():
torch.cuda.manual_seed(args.seed)
else:
print("Note that our pre-trained models require CUDA to evaluate.")
###########################################################################
# Load the models
###########################################################################
ae_args, gan_args, idx2word, autoencoder, gan_gen, gan_disc \
= load_models(args.ae_args, args.gan_args, args.vocab_file,
args.ae_model, args.g_model, args.d_model)
###########################################################################
# Generation code
###########################################################################
# Generate sentences
corpus = Corpus(args.data_path, args.dict_file, vocab_size=len(idx2word))
source, _ = next(BatchGen(corpus.get_chunks(size=2), args.ngenerations))
prev_sent = [
decode_idx(corpus.dictionary, sent) for sent in source.tolist()
]
source = Variable(source, volatile=True)
sentences = generate(autoencoder,
gan_gen,
inp=source,
vocab=idx2word,
sample=args.sample,
maxlen=args.maxlen)
if not args.noprint:
print("\nSentence generations:\n")
for prev, sent in zip(prev_sent, sentences):
print(prev)
print(" ", sent)
print("")
with open(args.outf, "w") as f:
f.write("Sentence generations:\n\n")
for prev, sent in zip(prev_sent, sentences):
f.write(prev + '\n')
f.write("-> " + sent + '\n\n')
def test_load_models(self):
load_models(MODELS_JSON_FILE)
self.assertEqual(len(all_models()), 4)
def main():
logging.basicConfig(level=logging.INFO, format="%(message)s")
parser = argparse.ArgumentParser(description="Predict using trained models")
parser.add_argument("rev_map", help="reverse inflection map")
parser.add_argument("models", nargs="+", help="trained models (category:file)")
parser.add_argument("--ambiguous", action="store_true", help="evaluate only lemmas with multiple inflections")
args = parser.parse_args()
logging.info("Loading reverse inflection map")
with open(args.rev_map) as f:
rev_map = cPickle.load(f)
logging.info("Loading inflection prediction models")
models = load_models(args.models)
logging.info("Loaded models for %d categories", len(models))
stats = {cat: [0, 0, 0, 0, 0] for cat in config.EXTRACTED_TAGS}
for source, target, alignment in read_sentences(sys.stdin):
for word, features in extract_instances(source, target, alignment):
gold_inflection, lemma, tag = word
category = tag[0]
gold_tag = tag[1:]
possible_inflections = rev_map.get((lemma, category), [])
if (gold_tag, gold_inflection) not in possible_inflections:
print(u"Expected: {} ({}) not found".format(gold_inflection, gold_tag).encode("utf8"))
continue
if args.ambiguous and len(possible_inflections) == 1:
continue
model = models[category]
scored_inflections = model.score_all(possible_inflections, features)
ranked_inflections = sorted(scored_inflections, reverse=True)
predicted_score, predicted_tag, predicted_inflection = ranked_inflections[0]
gold_rank = 1 + [tag for _, tag, _ in ranked_inflections].index(gold_tag)
gold_score = next((score for score, tag, _ in ranked_inflections if tag == gold_tag))
print(
u"Expected: {} ({}) r={} score={:.3f} |"
" Predicted: {} ({}) score={:.3f}".format(
gold_inflection,
gold_tag,
gold_rank,
gold_score,
predicted_inflection,
predicted_tag,
predicted_score,
).encode("utf8")
)
stats[category][0] += 1
stats[category][1] += 1 / float(gold_rank)
stats[category][2] += gold_inflection == predicted_inflection
stats[category][3] += gold_score
stats[category][4] += len(ranked_inflections)
for category, (n_instances, rrank_sum, n_correct, total_log_prob, n_inflections) in stats.items():
if n_instances == 0:
continue
mrr = rrank_sum / n_instances
accuracy = n_correct / float(n_instances)
ppl = math.exp(-total_log_prob / n_instances)
avg_inflections = n_inflections / float(n_instances)
print(
"Category {}: MRR={:.3f} acc={:.1%} ppl={:.2f} ({} instances; avg #infl={:.2f})".format(
category, mrr, accuracy, ppl, n_instances, avg_inflections
)
)
def load_models(device,
base_folder='./models/BAM/',
specific="bowling_alley",
seed=0,
module="layer3",
experiment="sgd_finetuned",
ratio="0.5",
adv=False,
baseline=False,
epoch=None,
post=False,
multiple=True,
leakage=False,
tcav=False,
force=False,
dataset='bam',
args=None,
ignore_net=False):
'''
if dataset == 'coco' and adv:
class DummyArgs:
num_object = 79
finetune=False
layer='generated_image'
autoencoder_finetune=True
finetune=True
model = balanced_models.ObjectMultiLabelAdv(DummyArgs(), 79, 300, True, 1)
ok = torch.load('model_best.pth.tar', encoding='bytes')
state_dict = {key.decode("utf-8"):ok[b'state_dict'][key] for key in ok[b'state_dict']}
model.load_state_dict(state_dict)
model.to(device)
model.eval()
'''
if leakage:
assert post
if epoch is not None:
epoch = "_" + str(epoch)
else:
epoch = ""
if len(args.custom_end) > 0:
args.custom_end = "_" + str(args.custom_end)
if baseline:
model_end = "resnet_base_"+str(ratio)+epoch+'.pt'
if not post:
n2v_end = "resnet_n2v_base_"+str(ratio)+epoch+'.pt'
else:
n2v_end = "resnet_n2v_base_after_"+str(ratio)+epoch+'.pt'
else:
if not adv:
model_end = "resnet_debias_"+str(ratio)+epoch+'.pt'
if not post:
n2v_end = "resnet_n2v_debias_"+str(ratio)+epoch+'.pt'
else:
n2v_end = "resnet_n2v_debias_after_"+str(ratio)+epoch+str(args.custom_end)+'.pt'
else:
model_end = "resnet_adv_"+str(ratio)+'.pt'
if not post:
n2v_end = "resnet_n2v_adv_"+str(ratio)+'.pt'
else:
n2v_end = "resnet_n2v_adv_after_"+str(ratio)+epoch+'.pt'
if dataset != 'bam' and dataset != 'coco':
model_end = model_end.replace('_'+str(ratio), '')
n2v_end = n2v_end.replace('_'+str(ratio), '')
if dataset == 'bam' or dataset == 'coco':
model_path, n2v_path = utils.get_paths(
base_folder,
seed,
specific,
model_end=model_end,
n2v_end='leakage/' + n2v_end.replace('n2v','mlp') if leakage else n2v_end,
n2v_module=module,
experiment=experiment,
with_n2v=True,
)
else:
model_path = os.path.join(base_folder, str(seed), experiment, module, model_end)
n2v_path = os.path.join(base_folder, str(seed), experiment, module, 'leakage/' + n2v_end.replace('n2v','mlp') if leakage else n2v_end)
if dataset == 'bam':
trainloader, _ = dataload.get_data_loader_SceneBAM(seed=seed,ratio=float(ratio), specific=specific)
_, testloader = dataload.get_data_loader_SceneBAM(seed=seed,ratio=float(0.5), specific=specific)
elif dataset == 'coco':
tmp_args = copy.deepcopy(args)
tmp_args.ratio = ratio
if int(ratio) > 0:
tmp_args.balanced = True
if leakage:
tmp_args.gender_balanced = True
trainloader, testloader = coco_dataload.get_data_loader_coco(
tmp_args
)
else:
trainloader,testloader = dataload.get_data_loader_idenProf('idenprof',train_shuffle=True,
train_batch_size=64,
test_batch_size=64,
exclusive=True)
if not (dataset == 'coco' and adv):
assert os.path.exists(model_path), model_path
if post:
# since we have to run a separate script, might not have finished...
if not leakage:
model_extra = '_adv' if adv else ('_base' if baseline else '_debias')
n2v_extra = model_extra + '_after'
if tcav:
pass
elif force:
post_train.train_net2vec(trainloader,
testloader,
device,
seed,
specific=specific,
p=ratio,
n_epochs=20,
module=module,
lr=.01,
out_file=None,
base_folder=base_folder,
experiment1=experiment,
experiment2=experiment,
model_extra=model_extra,
n2v_extra=n2v_extra,
with_n2v=True,
nonlinear=False, # might want to change this later
model_custom_end=epoch.replace('_',''),
n2v_custom_end=epoch.replace('_',''),
multiple=multiple,
dataset=dataset
)
else:
raise Exception('Run trial again')
elif leakage:
model_extra = '_adv' if adv else ('_base' if baseline else '_debias')
n2v_extra = model_extra + '_after'
if force:
post_train.train_leakage(trainloader,
testloader,
device,
seed,
specific=specific,
p=ratio,
n_epochs=20,
module=module,
lr=5e-5, # leakage model uses adam
out_file=None,
base_folder=base_folder,
experiment1=experiment,
experiment2=experiment,
model_extra=model_extra,
n2v_extra=n2v_extra,
with_n2v=True,
nonlinear=True, # MLP leakage model
model_custom_end='',
n2v_custom_end='',
dataset=dataset
)
else:
raise Exception('Run trial again')
else:
# should've been saved during training if not ported from tianlu
if not (dataset == 'coco' and adv):
assert os.path.exists(n2v_path)
num_attributes = 10 + 9 + 20 if multiple else 12
num_classes=10
if dataset == 'coco':
num_attributes = 81
num_classes = 79
model, net, net_forward, activation_probe = models.load_models(
device,
None if (dataset == 'coco' and adv) else
lambda x,y,z: models.resnet_(
pretrained=True,
custom_path=x,
device=y,
initialize=z,
num_classes=num_classes,
size=50 if (dataset == 'bam') or (dataset == 'coco') else 34
),
model_path=model_path,
net2vec_pretrained=True,
net2vec_path=n2v_path,
module='fc' if leakage else module,
num_attributes=2 if leakage else num_attributes,
model_init = False,
n2v_init = False,
nonlinear = leakage,
ignore_net = ignore_net
)
print(n2v_path)
return model, net, net_forward, activation_probe
rev[f].add(k)
all_zs[k] = noise[i]
pickle.dump((all_sents, all_features, rev, all_zs), open(args.dump, "bw"))
def main(args):
if args.mode == 'gen':
dump_samples(args)
elif args.mode == 'alter':
alter(args)
if __name__ == "__main__":
parser = argparse.ArgumentParser(description='PyTorch experiment')
parser.add_argument('mode', default='gen',
help='choices [gen, alter]')
parser.add_argument('--load_path', type=str, default='',
help='directory to load models from')
parser.add_argument('--dump', type=str, default="features.pkl",
help='path to sample dump')
parser.add_argument('--nbatches', type=int, default=1000)
parser.add_argument('--batch_size', type=int, default=1000)
parser.add_argument('--alter', type=str, default="")
parser.add_argument('--nsent', type=int, default=100)
args = parser.parse_args()
model_args, idx2word, autoencoder, gan_gen, gan_disc \
= load_models(args.load_path)
main(args)
count += 1
else:
previous_state = state
state = STATES["NO_IMAGE"]
if state == STATES["PREDICT"]:
state, pytorch_result, sklearn_result = states.predict(img, pytorch_model, sklearn_model)
if state == STATES["RESULTS"]:
state = states.results(MaskDetectoBot, chat_id, pytorch_result, sklearn_result)
new_offset = update_id + 1
if state == STATES["HELP"]:
states.help(MaskDetectoBot, chat_id, previous_state)
state = previous_state
new_offset = update_id + 1
elif state == STATES["ERROR"] or state == STATES["NO_IMAGE"]:
states.error(MaskDetectoBot, chat_id, state)
state = previous_state
new_offset = update_id + 1
if __name__ == '__main__':
# Load the models here
pytorch_model, sklearn_model = load_models()
try:
main(pytorch_model, sklearn_model)
except KeyboardInterrupt:
exit()
from flask import Flask
from flask_restful import reqparse, abort, Api, Resource
#from flask_cors import CORS
from models import load_models, predict, predict_news_hatespeech
### LOAD APP
application = Flask(__name__)
#CORS(application)
api = Api(application)
### LOAD MODEL
load_models()
### LOAD PARSER
parser = reqparse.RequestParser()
parser.add_argument('content')
parser.add_argument('news_title')
parser.add_argument('headline')
parser.add_argument('model')
parser.add_argument('lang')
parser.add_argument('title_news')
parser.add_argument('lead_news')
parser.add_argument('text_news')
parser.add_argument('authors_news')
def load_models(device,
base_folder='./models/BAM/',
specific="bowling_alley",
seed=0,
module="layer3",
experiment="sgd_finetuned",
ratio="0.5",
adv=False,
baseline=False,
epoch=None,
post=False,
multiple=True,
leakage=False,
tcav=False,
force=False,
dataset='bam'):
if leakage:
assert post
if epoch is not None:
epoch = "_" + str(epoch)
else:
epoch = ""
if baseline:
model_end = "resnet_base_"+str(ratio)+epoch+'.pt'
if not post:
n2v_end = "resnet_n2v_base_"+str(ratio)+epoch+'.pt'
else:
n2v_end = "resnet_n2v_base_after_"+str(ratio)+epoch+'.pt'
else:
if not adv:
model_end = "resnet_debias_"+str(ratio)+epoch+'.pt'
if not post:
n2v_end = "resnet_n2v_debias_"+str(ratio)+epoch+'.pt'
else:
n2v_end = "resnet_n2v_debias_after_"+str(ratio)+epoch+'.pt'
else:
model_end = "resnet_adv_"+str(ratio)+'.pt'
if not post:
n2v_end = "resnet_n2v_adv_"+str(ratio)+'.pt'
else:
n2v_end = "resnet_n2v_adv_after_"+str(ratio)+epoch+'.pt'
if dataset != 'bam':
model_end = model_end.replace('_'+str(ratio), '')
n2v_end = n2v_end.replace('_'+str(ratio), '')
if dataset == 'bam':
model_path, n2v_path = utils.get_paths(
base_folder,
seed,
specific,
model_end=model_end,
n2v_end='leakage/' + n2v_end.replace('n2v','mlp') if leakage else n2v_end,
n2v_module=module,
experiment=experiment,
with_n2v=True,
)
else:
model_path = os.path.join(base_folder, str(seed), experiment, module, model_end)
n2v_path = os.path.join(base_folder, str(seed), experiment, module, 'leakage/' + n2v_end.replace('n2v','mlp') if leakage else n2v_end)
if dataset == 'bam':
trainloader, _ = dataload.get_data_loader_SceneBAM(seed=seed,ratio=float(ratio), specific=specific)
_, testloader = dataload.get_data_loader_SceneBAM(seed=seed,ratio=float(0.5), specific=specific)
else:
trainloader,testloader = dataload.get_data_loader_idenProf('idenprof',train_shuffle=True,
train_batch_size=64,
test_batch_size=64,
exclusive=True)
assert os.path.exists(model_path), model_path
if post:
# since we have to run a separate script, might not have finished...
if not leakage:
model_extra = '_adv' if adv else ('_base' if baseline else '_debias')
n2v_extra = model_extra + '_after'
if tcav:
pass
elif force:
post_train.train_net2vec(trainloader,
testloader,
device,
seed,
specific=specific,
p=ratio,
n_epochs=20,
module=module,
lr=.01,
out_file=None,
base_folder=base_folder,
experiment1=experiment,
experiment2=experiment,
model_extra=model_extra,
n2v_extra=n2v_extra,
with_n2v=True,
nonlinear=False, # might want to change this later
model_custom_end=epoch.replace('_',''),
n2v_custom_end=epoch.replace('_',''),
multiple=multiple,
dataset=dataset
)
else:
raise Exception('Run trial again')
elif leakage:
model_extra = '_adv' if adv else ('_base' if baseline else '_debias')
n2v_extra = model_extra + '_after'
if force:
post_train.train_leakage(trainloader,
testloader,
device,
seed,
specific=specific,
p=ratio,
n_epochs=20,
module=module,
lr=5e-5, # leakage model uses adam
out_file=None,
base_folder=base_folder,
experiment1=experiment,
experiment2=experiment,
model_extra=model_extra,
n2v_extra=n2v_extra,
with_n2v=True,
nonlinear=True, # MLP leakage model
model_custom_end='',
n2v_custom_end='',
dataset=dataset
)
else:
raise Exception('Run trial again')
else:
# should've been saved during training
assert os.path.exists(n2v_path)
num_attributes = 10 + 9 + 20 if multiple else 12
model, net, net_forward, activation_probe = models.load_models(
device,
lambda x,y,z: models.resnet_(pretrained=True, custom_path=x, device=y,initialize=z, size=50 if dataset == 'bam' else 34),
model_path=model_path,
net2vec_pretrained=True,
net2vec_path=n2v_path,
module='fc' if leakage else module,
num_attributes=2 if leakage else num_attributes,
model_init = False,
n2v_init = False,
nonlinear = leakage
)
return model, net, net_forward, activation_probe
def main(args):
# Set the random seed manually for reproducibility.
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
if torch.cuda.is_available():
torch.cuda.manual_seed(args.seed)
else:
print("Note that our pre-trained models require CUDA to evaluate.")
###########################################################################
# Load the models
###########################################################################
model_args, idx2word, autoencoder, gan_gen, gan_disc \
= load_models(args.load_path)
###########################################################################
# Generation code
###########################################################################
# Generate sentences
if args.ngenerations > 0:
noise = torch.ones(args.ngenerations, model_args['z_size'])
noise.normal_()
sentences = generate(autoencoder, gan_gen, z=noise,
vocab=idx2word, sample=args.sample,
maxlen=model_args['maxlen'])
if not args.noprint:
print("\nSentence generations:\n")
for sent in sentences:
print(sent)
with open(args.outf, "w") as f:
f.write("Sentence generations:\n\n")
for sent in sentences:
f.write(sent+"\n")
# Generate interpolations
if args.ninterpolations > 0:
noise1 = torch.ones(args.ninterpolations, model_args['z_size'])
noise1.normal_()
noise2 = torch.ones(args.ninterpolations, model_args['z_size'])
noise2.normal_()
interps = interpolate(autoencoder, gan_gen,
z1=noise1,
z2=noise2,
vocab=idx2word,
steps=args.steps,
sample=args.sample,
maxlen=model_args['maxlen'])
if not args.noprint:
print("\nSentence interpolations:\n")
for interp in interps:
for sent in interp:
print(sent)
print("")
with open(args.outf, "a") as f:
f.write("\nSentence interpolations:\n\n")
for interp in interps:
for sent in interp:
f.write(sent+"\n")
f.write('\n')
def train(
trainloader,
testloader,
device,
seed,
debias_=True,
specific=None,
ratio=0.5, # bias ratio in dataset
n_epochs=5,
model_lr=1e-3,
n2v_lr=1e-3,
combined_n2v_lr=1e-3, # metalearning rate for n2v
alpha=100, # for debias,
beta=0.1, # for adversarial loss
out_file=None,
base_folder="",
results_folder="",
experiment="sgd",
momentum=0,
module="layer4",
finetuned=False,
adversarial=False,
nonlinear=False,
subset=False,
subset_ratio=0.1,
save_every=False,
model_momentum=0,
n2v_momentum=0,
experimental=False,
multiple=False,
debias_multiple=False,
reset=False,
reset_counter=1,
n2v_start=False,
experiment2=None,
adaptive_alpha=False,
n2v_adam=False,
single=False,
imagenet=False,
train_batch_size=64,
constant_resize=False,
adaptive_resize=False,
no_class=False,
gamma=0,
partial_projection=False,
norm='l2',
constant_alpha=False,
jump_alpha=False,
linear_alpha=False,
mean_debias=False,
no_limit=False,
dataset='bam',
parallel=False,
gpu_ids=[],
switch_modes=True):
print("mu", momentum, "debias", debias_, "alpha", alpha, " | ratio:",
ratio)
def get_vg(W):
if single:
return W[-2, :]
else:
return W[-2, :] - W[-1, :]
if dataset == 'bam' or dataset == 'coco':
model_init_path, n2v_init_path = utils.get_paths(
base_folder,
seed,
specific,
model_end="resnet_init" + '.pt',
n2v_end="resnet_n2v_init" + '.pt',
n2v_module=module,
experiment=experiment,
with_n2v=False)
else:
model_init_path = os.path.join(base_folder, str(seed), experiment,
'resnet_init.pt')
n2v_init_path = os.path.join(base_folder, str(seed), experiment,
module, 'resnet_n2v_init.pt')
if finetuned:
if dataset == 'bam' or dataset == 'coco':
model_init_path = utils.get_model_path(
base_folder,
seed,
specific,
"resnet_" + str(ratio) + ".pt",
experiment='post_train'
if not n2v_start else experiment.split('_finetuned')[0])
else:
model_init_path = os.path.join(
base_folder, str(seed), 'post_train' if not n2v_start else
experiment.split('_finetuned')[0], 'resnet.pt')
assert (debias_ and not adversarial) or (
adversarial and not debias_) or (not adversarial and not debias_)
if debias_ and n2v_start:
ext = "_n2v_" if not nonlinear else "_mlp_"
if dataset == 'bam' or dataset == 'coco':
n2v_init_path = utils.get_net2vec_path(
base_folder,
seed,
specific,
module,
"resnet" + str(ext) + str(ratio) + ".pt",
experiment=experiment.split('_finetuned')[0])
else:
n2v_init_path = os.path.join(base_folder, str(seed),
experiment.split('_finetuned')[0],
module,
'resnet' + ext[:-1] + '.pt')
# if we're also doing adversarial, make sure to load the matching n2v as init...
if adversarial:
ext = "_n2v_" if not nonlinear else "_mlp_"
if dataset == 'bam' or dataset == 'coco':
n2v_init_path = utils.get_net2vec_path(base_folder,
seed,
specific,
module,
"resnet" + str(ext) +
str(ratio) + ".pt",
experiment='post_train')
else:
n2v_init_path = os.path.join(base_folder, str(seed),
'post_train', module,
'resnet' + ext[:-1] + '.pt')
num_classes = 10
num_attributes = 12
if nonlinear:
num_attributes = 2
if multiple:
num_attributes = 10 + 9 + 2 * 10
if dataset == 'coco':
num_classes = 79
num_attributes = 81
model, net, net_forward, activation_probe = models.load_models(
device,
lambda x, y, z: models.resnet_(pretrained=True,
custom_path=x,
device=y,
initialize=z,
num_classes=num_classes,
size=50 if (dataset == 'bam' or dataset
== 'coco') else 34),
model_path=model_init_path,
net2vec_pretrained=True,
net2vec_path=n2v_init_path,
module=module,
num_attributes=num_attributes,
# we want to make sure to save the inits if not finetuned...
model_init=True if not finetuned else False,
n2v_init=True if not (finetuned and
(adversarial or
(debias_ and n2v_start))) else False,
loader=trainloader,
nonlinear=nonlinear,
# parameters if we want to initially project probes to have a certain amount of bias
partial_projection=partial_projection,
t=gamma)
print(model_init_path, n2v_init_path)
model_n2v_combined = models.ProbedModel(model,
net,
module,
switch_modes=switch_modes)
if n2v_adam:
combined_optim = torch.optim.Adam(
[{
'params': model_n2v_combined.model.parameters()
}, {
'params': model_n2v_combined.net.parameters()
}],
lr=n2v_lr)
# TODO: allow for momentum training as well
n2v_optim = torch.optim.Adam(net.parameters(), lr=n2v_lr)
else:
combined_optim = torch.optim.SGD(
[{
'params': model_n2v_combined.model.parameters()
}, {
'params': model_n2v_combined.net.parameters(),
'lr': combined_n2v_lr,
'momentum': n2v_momentum
}],
lr=model_lr,
momentum=model_momentum)
# TODO: allow for momentum training as well
n2v_optim = torch.optim.SGD(net.parameters(),
lr=n2v_lr,
momentum=n2v_momentum)
model_optim = torch.optim.SGD(model.parameters(),
lr=model_lr,
momentum=model_momentum)
d_losses = []
adv_losses = []
n2v_train_losses = []
n2v_accs = []
n2v_val_losses = []
class_train_losses = []
class_accs = []
class_val_losses = []
alpha_log = []
magnitudes = []
magnitudes2 = []
unreduced = []
bias_grads = []
loss_shapes = []
loss_shapes2 = []
results = {
"debias_losses": d_losses,
"n2v_train_losses": n2v_train_losses,
"n2v_val_losses": n2v_val_losses,
"n2v_accs": n2v_accs,
"class_train_losses": class_train_losses,
"class_val_losses": class_val_losses,
"class_accs": class_accs,
"adv_losses": adv_losses,
"alphas": alpha_log,
"magnitudes": magnitudes,
"magnitudes2": magnitudes2,
"unreduced": unreduced,
"bias_grads": bias_grads,
"loss_shapes": loss_shapes,
"loss_shapes2": loss_shapes2
}
if debias_:
results_end = str(ratio) + "_debias.pck"
elif adversarial:
results_end = str(ratio) + "_adv.pck"
if nonlinear:
results_end = str(ratio) + "_mlp_adv.pck"
else:
results_end = str(ratio) + "_base.pck"
if dataset == 'bam' or dataset == 'coco':
results_path = utils.get_net2vec_path(
results_folder, seed, specific, module, results_end,
experiment if experiment2 is None else experiment2)
else:
results_path = os.path.join(
results_folder, str(seed),
experiment if experiment2 is None else experiment2, module,
results_end)
if debias_:
model_end = "resnet_debias_" + str(ratio) + '.pt'
n2v_end = "resnet_n2v_debias_" + str(ratio) + '.pt'
elif adversarial:
if not nonlinear:
model_end = "resnet_adv_" + str(ratio) + '.pt'
else:
model_end = "resnet_adv_nonlinear_" + str(ratio) + '.pt'
if not nonlinear:
n2v_end = "resnet_n2v_adv_" + str(ratio) + '.pt'
else:
n2v_end = "resnet_mlp_adv_" + str(ratio) + '.pt'
else:
model_end = "resnet_base_" + str(ratio) + '.pt'
n2v_end = "resnet_n2v_base_" + str(ratio) + '.pt'
if dataset != 'bam' and dataset != 'coco':
model_end = model_end.replace('_' + str(ratio), '')
n2v_end = n2v_end.replace('_' + str(ratio), '')
if dataset == 'bam' or dataset == 'coco':
model_path, n2v_path = utils.get_paths(
base_folder,
seed,
specific,
model_end=model_end,
n2v_end=n2v_end,
n2v_module=module,
experiment=experiment if experiment2 is None else experiment2,
with_n2v=True,
)
else:
model_path = os.path.join(
base_folder, str(seed),
experiment if experiment2 is None else experiment2, module,
model_end)
n2v_path = os.path.join(
base_folder, str(seed),
experiment if experiment2 is None else experiment2, module,
n2v_end)
if hasattr(trainloader.dataset, 'idx_to_class'):
for key in trainloader.dataset.idx_to_class:
if specific is not None and trainloader.dataset.idx_to_class[
key] in specific:
specific_idx = int(key)
else:
specific_idx = 0
train_labels = None if not nonlinear else [-2, -1]
d_last = 0
resize = constant_resize or adaptive_resize
if imagenet:
imagenet_trainloaders, _ = dataload.get_imagenet_tz(
'./datasets/imagenet',
workers=8,
train_batch_size=train_batch_size // 8,
resize=resize,
constant=constant_resize)
imagenet_trainloader = dataload.process_imagenet_loaders(
imagenet_trainloaders)
params = list(model_n2v_combined.parameters())[:-2]
init_alpha = alpha
last_e = 0
# setup training criteria
if dataset == 'coco':
object_weights = torch.FloatTensor(
trainloader.dataset.getObjectWeights())
gender_weights = torch.FloatTensor(
trainloader.dataset.getGenderWeights())
all_weights = torch.cat([object_weights, gender_weights])
probe_criterion = nn.BCEWithLogitsLoss(weight=all_weights.to(device),
reduction='elementwise_mean')
downstream_criterion = nn.BCEWithLogitsLoss(
weight=object_weights.to(device), reduction='elementwise_mean')
else:
probe_criterion = None
downstream_criterion = nn.CrossEntropyLoss()
for e in range(n_epochs):
# save results every epoch...
with open(results_path, 'wb') as f:
print("saving results", e)
print(results_path)
pickle.dump(results, f)
model.eval()
with torch.no_grad():
n2v_acc, n2v_val_loss = utils.net2vec_accuracy(
testloader, net_forward, device, train_labels)
n2v_accs.append(n2v_acc)
n2v_val_losses.append(n2v_val_loss)
if dataset != 'coco':
class_acc, class_val_loss = utils.classification_accuracy(
testloader, model, device)
class_accs.append(class_acc)
class_val_losses.append(class_val_loss)
else:
f1, mAP = utils.detection_results(testloader, model, device)
print("Epoch", e, "| f1:", f1, "| mAP:", mAP)
class_accs.append([f1, mAP])
d_initial = 0
if not adversarial:
curr_W = net.weight.data.clone()
if not multiple:
vg = get_vg(curr_W).reshape(-1, 1)
d_initial = debias.debias_loss(curr_W[:-2], vg, t=0).item()
print("Epoch", e, "bias", str(d_initial), " | debias: ",
debias_)
else:
ds = np.zeros(10)
for i in range(10):
if i == 0:
vg = (curr_W[10, :] - curr_W[11, :]).reshape(-1, 1)
else:
vg = (curr_W[20 + i, :] - curr_W[29 + i, :]).reshape(
-1, 1)
ds[i] = debias.debias_loss(curr_W[:10], vg, t=0).item()
print("Epoch", e, "bias", ds, " | debias: ", debias_)
print("Accuracies:", n2v_acc)
d_initial = ds[0]
else:
print("Epoch", e, "Adversarial", n2v_accs[-1])
if adaptive_alpha and (e == 0 or ((d_last / d_initial) >=
(5 / 2**(e - 1)) or
(0.8 < (d_last / d_initial) < 1.2))):
#alpha = alpha
old_alpha = alpha
# we don't want to increase too much if it's already decreasing
if (e == 0 or (d_last / d_initial) >= (5 / 2**(e - 1))):
alpha = min(
alpha * 2, (15 / (2**e)) / (d_initial + 1e-10)
) # numerical stability just in case d_initial gets really low
#if e > 0 and old_alpha >= alpha:
# alpha = old_alpha # don't update if we're decreasing...
print("Option 1")
if e > 0 and alpha < old_alpha:
# we want to increase if plateaud
alpha = max(
old_alpha * 1.5, alpha
) # numerical stability just in case d_initial gets really low
print("Option 2")
# don't want to go over 1000...
if alpha > 1000:
alpha = 1000
d_last = d_initial
elif not adaptive_alpha and not constant_alpha:
if dataset == 'coco' and jump_alpha:
if e < 2:
alpha = 5e3
elif e >= 2 and e < 4:
alpha = 1e4
else:
alpha = init_alpha
elif jump_alpha and (e - last_e) > 2:
if not mean_debias:
if alpha < 100:
alpha = min(alpha * 2, 100)
last_e = e
else:
# two jumps
# if (e-last_e) >= ((n_epochs - last_e) // 2):
# alpha = 1000
# else:
alpha = 1000
else:
if alpha < 1000:
alpha = min(alpha * 2, 1000)
last_e = e
else:
alpha = 10000
elif linear_alpha and (e - last_e) > 2:
if alpha < 100:
alpha = min(alpha * 2, 100)
last_e = e
else:
alpha += (1000 - 100) / (n_epochs - last_e)
elif not jump_alpha and not linear_alpha:
if (e + 1) % 3 == 0:
# apply alpha schedule?
# alpha = min(alpha * 1.2, max(init_alpha,1000))
alpha = alpha * 1.5
alpha_log.append(alpha)
print("Current Alpha:,", alpha)
if save_every and e % 10 == 0 and e > 0 and seed == 0 and debias_:
torch.save(net.state_dict(),
n2v_path.split('.pt')[0] + '_' + str(e) + '.pt')
torch.save(model.state_dict(),
model_path.split('.pt')[0] + '_' + str(e) + '.pt')
if reset and (e + 1) % reset_counter == 0 and e > 0:
print("resetting")
net, net_forward, activation_probe = net2vec.create_net2vec(
model,
module,
num_attributes,
device,
pretrained=False,
initialize=True,
nonlinear=nonlinear)
n2v_optim = torch.optim.SGD(net.parameters(),
lr=n2v_lr,
momentum=n2v_momentum)
model.train()
ct = 0
for X, y, genders in trainloader:
ids = None
##### Part 1: Update the Embeddings #####
model_optim.zero_grad()
n2v_optim.zero_grad()
labels = utils.merge_labels(y, genders, device)
logits = net_forward(X.to(device), switch_modes=switch_modes)
# Now actually update net2vec embeddings, making sure to use the same batch
if train_labels is not None:
if logits.shape[1] == labels.shape[1]:
logits = logits[:, train_labels]
labels = labels[:, train_labels]
shapes = []
shapes2 = []
if dataset == 'coco':
prelim_loss = probe_criterion(logits, labels)
else:
prelim_loss, ids = utils.balanced_loss(logits,
labels,
device,
0.5,
ids=ids,
multiple=multiple,
specific=specific_idx,
shapes=shapes)
#print("prelim_loss:", prelim_loss.item())
prelim_loss.backward()
# we don't want to update these parameters, just in case
model_optim.zero_grad()
n2v_train_losses.append(prelim_loss.item())
n2v_optim.step()
try:
magnitudes.append(
torch.norm(net.weight.data, dim=1).data.cpu().numpy())
except:
pass
##### Part 2: Update Conv parameters for classification #####
model_optim.zero_grad()
n2v_optim.zero_grad()
class_logits = model(X.to(device))
class_loss = downstream_criterion(class_logits, y.to(device))
class_train_losses.append(class_loss.item())
if debias_:
W_curr = net.weight.data
vg = get_vg(W_curr).reshape(-1, 1)
unreduced.append(
debias.debias_loss(W_curr[:-2], vg, t=0,
unreduced=True).data.cpu().numpy())
loss = class_loss
#### Part 2a: Debias Loss
if debias_:
model_optim.zero_grad()
n2v_optim.zero_grad()
labels = utils.merge_labels(y, genders, device)
o = net.weight.clone()
combined_optim.zero_grad()
with higher.innerloop_ctx(model_n2v_combined,
combined_optim) as (fn2v,
diffopt_n2v):
models.update_probe(fn2v)
logits = fn2v(X.to(device))
if dataset == 'coco':
prelim_loss = probe_criterion(logits, labels)
else:
prelim_loss, ids = utils.balanced_loss(
logits,
labels,
device,
0.5,
ids=ids,
multiple=False,
specific=specific_idx,
shapes=shapes2)
diffopt_n2v.step(prelim_loss)
weights = list(fn2v.parameters())[-2]
vg = get_vg(weights).reshape(-1, 1)
d_loss = debias.debias_loss(weights[:-2],
vg,
t=gamma,
norm=norm,
mean=mean_debias)
# only want to save the actual bias...
d_losses.append(d_loss.item())
grad_of_grads = torch.autograd.grad(
alpha * d_loss,
list(fn2v.parameters(time=0))[:-2],
allow_unused=True)
del prelim_loss
del logits
del vg
del fn2v
del diffopt_n2v
#### Part 2b: Adversarial Loss
if adversarial:
logits = net_forward(
None, forward=True)[:, -2:] # just use activation probe
labels = genders.type(torch.FloatTensor).reshape(
genders.shape[0], -1).to(device)
adv_loss, _ = utils.balanced_loss(logits,
labels,
device,
0.5,
ids=ids,
stable=True)
adv_losses.append(adv_loss.item())
# getting too strong, let it retrain...
if adv_loss < 2:
adv_loss = -beta * adv_loss
loss += adv_loss
loss.backward()
if debias_:
# custom backward to include the bias regularization....
max_norm_grad = -1
param_idx = -1
for ii in range(len(grad_of_grads)):
if (grad_of_grads[ii] is not None
and params[ii].grad is not None
and torch.isnan(grad_of_grads[ii]).long().sum() <
grad_of_grads[ii].reshape(-1).shape[0]):
# just in case some or nan for some reason?
not_nan = ~torch.isnan(grad_of_grads[ii])
params[ii].grad[not_nan] += grad_of_grads[ii][not_nan]
if grad_of_grads[ii][not_nan].norm().item(
) > max_norm_grad:
max_norm_grad = grad_of_grads[ii][not_nan].norm(
).item()
param_idx = ii
bias_grads.append((param_idx, max_norm_grad))
# undo the last step and apply a smaller alpha to prevent stability issues
if not no_limit and ((not mean_debias and max_norm_grad > 100)
or (mean_debias and max_norm_grad > 100)):
for ii in range(len(grad_of_grads)):
if (grad_of_grads[ii] is not None
and params[ii].grad is not None and
torch.isnan(grad_of_grads[ii]).long().sum() <
grad_of_grads[ii].reshape(-1).shape[0]):
# just in case some or nan for some reason?
not_nan = ~torch.isnan(grad_of_grads[ii])
params[ii].grad[not_nan] -= grad_of_grads[ii][
not_nan]
# scale accordingly
# params[ii].grad[not_nan] += grad_of_grads[ii][not_nan] / max_norm_grad
loss_shapes.append(shapes)
loss_shapes2.append(shapes2)
model_optim.step()
#magnitudes2.append(
# torch.norm(net.weight.data, dim=1).data.cpu().numpy()
#)
ct += 1
# save results every epoch...
with open(results_path, 'wb') as f:
print("saving results", e)
print(results_path)
pickle.dump(results, f)
torch.save(net.state_dict(), n2v_path)
torch.save(model.state_dict(), model_path)
def main(args):
# Set the random seed manually for reproducibility.
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
if torch.cuda.is_available():
torch.cuda.manual_seed(args.seed)
else:
print("Note that our pre-trained models require CUDA to evaluate.")
###########################################################################
# Load the models
###########################################################################
model_args, idx2word, autoencoder, gan_gen, gan_disc \
= load_models(args.load_path)
###########################################################################
# Load the data
###########################################################################
corpus = Corpus(args.data_path,
maxlen=100,
vocab_size=model_args['vocab_size'],
lowercase=model_args['lowercase'])
eval_batch_size = 10
test_data = batchify(corpus.test, eval_batch_size, shuffle=False)
###########################################################################
# Prediction code
###########################################################################
autoencoder.eval()
total_loss = 0
ntokens = len(corpus.dictionary.word2idx)
all_accuracies = 0
num_examples = 0
criterion_ce = torch.nn.CrossEntropyLoss(size_average=False)
for i, batch in enumerate(test_data):
source, target, lengths = batch
source = Variable(source, volatile=True)
target = Variable(target, volatile=True)
#source = to_gpu(args.cuda, Variable(source, volatile=True))
#target = to_gpu(args.cuda, Variable(target, volatile=True))
# Generate output.
# output: batch x seq_len x ntokens
hidden = autoencoder(source, lengths, noise=False, encode_only=True)
max_indices = autoencoder.generate(hidden,
model_args['maxlen'],
sample=False)
# Decode in training mode to compute loss.
mask = target.gt(0)
masked_target = target.masked_select(mask)
# examples x ntokens
output_mask = mask.unsqueeze(1).expand(mask.size(0), ntokens)
# output: batch x seq_len x ntokens
output = autoencoder(source, lengths, noise=False)
flattened_output = output.view(-1, ntokens)
masked_output = \
flattened_output.masked_select(output_mask).view(-1, ntokens)
total_loss += criterion_ce(masked_output / model_args['temp'],
masked_target).data
num_examples += source.size()[0]
aeoutf = args.outf
with open(aeoutf, "a") as f:
max_indices = max_indices.view(eval_batch_size,
-1).data.cpu().numpy()
target = target.view(eval_batch_size, -1).data.cpu().numpy()
eos = corpus.dictionary.word2idx['<eos>']
for t, idx in zip(target, max_indices):
# real sentence
length = list(t).index(eos) if eos in t else len(t)
chars = " ".join(
[corpus.dictionary.idx2word[x] for x in t[:length]])
f.write(chars)
f.write("\t")
# autoencoder output sentence
length = list(idx).index(eos) if eos in idx else len(idx)
chars = " ".join(
[corpus.dictionary.idx2word[x] for x in idx[:length]])
f.write(chars)
f.write("\n")
print("Processed {} examples".format(num_examples))
print("Cross-entropy: {:.4f}".format((total_loss / num_examples)[0]))
def train_net2vec(trainloader,
testloader,
device,
seed,
specific=None,
p=0.5,
n_epochs=5,
module='layer4',
lr=0.5,
base_folder="",
out_file=None,
experiment1="",
experiment2="",
model_extra="",
n2v_extra="",
with_n2v=False,
nonlinear=False,
model_custom_end='',
n2v_custom_end='',
multiple=False,
dataset='bam',
parallel=False,
gpu_ids=[]):
if out_file is not None:
f = open(out_file, 'a')
else:
f = None
print("Training N2V | p =", p, file=f)
if not nonlinear:
n2v_extra = "n2v" + str(n2v_extra)
else:
n2v_extra = "mlp" + str(n2v_extra)
if len(model_custom_end) > 0:
model_custom_end = "_" + model_custom_end
if len(n2v_custom_end) > 0:
n2v_custom_end = "_" + n2v_custom_end
if hasattr(trainloader.dataset, 'idx_to_class'):
for key in trainloader.dataset.idx_to_class:
if specific is not None and trainloader.dataset.idx_to_class[
key] in specific:
specific_idx = int(key)
else:
specific_idx = 0
if dataset == 'bam' or dataset == 'coco':
model_path = utils.get_model_path(base_folder,
seed,
specific,
"resnet" + str(model_extra) + "_" +
str(p) + model_custom_end + ".pt",
experiment=experiment1,
with_n2v=with_n2v,
n2v_module=module)
n2v_path = utils.get_net2vec_path(
base_folder,
seed,
specific,
module,
"resnet_" + str(n2v_extra) + "_" + str(p) + n2v_custom_end + ".pt",
experiment=experiment2,
)
else:
if with_n2v:
model_path = os.path.join(
base_folder, str(seed), experiment1, module,
"resnet" + str(model_extra) + model_custom_end + ".pt")
else:
model_path = os.path.join(
base_folder, str(seed), experiment1,
"resnet" + str(model_extra) + model_custom_end + ".pt")
n2v_path = os.path.join(
base_folder, str(seed), experiment2, module,
'resnet_' + str(n2v_extra) + n2v_custom_end + ".pt")
print(model_path, n2v_path)
num_attributes = 12
if nonlinear:
num_attributes = 2
if multiple:
num_attributes = 10 + 9 + 2 * 10
num_classes = 10
if dataset == 'coco':
num_classes = 79
num_attributes = 81
model, net, net_forward, activation_probe = models.load_models(
device,
# load in None means we load in the pretrained weights from Tianlu
None if (dataset == 'coco') and ('adv' in model_extra) else lambda x,
y, z: models.resnet_(pretrained=True,
custom_path=x,
device=y,
num_classes=num_classes,
initialize=z,
size=50 if
(dataset == 'bam' or dataset == 'coco') else 34),
model_path=model_path,
net2vec_pretrained=False,
module=module,
num_attributes=num_attributes,
model_init=False, # don't need to initialize a new one
n2v_init=True,
loader=trainloader,
nonlinear=nonlinear,
parallel=parallel,
gpu_ids=gpu_ids)
if dataset == 'coco':
object_weights = torch.FloatTensor(
trainloader.dataset.getObjectWeights())
gender_weights = torch.FloatTensor(
trainloader.dataset.getGenderWeights())
all_weights = torch.cat([object_weights, gender_weights])
criterion = nn.BCEWithLogitsLoss(weight=all_weights.to(device),
reduction='elementwise_mean')
#criterion = nn.BCEWithLogitsLoss()
else:
criterion = None
net2vec.train_net2vec(
model,
net,
net_forward,
n_epochs,
trainloader,
testloader,
device,
lr=lr,
save_path=n2v_path,
f=f,
train_labels=[-2, -1] if nonlinear else None,
multiple=multiple,
balanced=False if dataset == 'coco' else True,
criterion=criterion,
adam=False, #True if dataset == 'coco' else False,
leakage=False)
if f is not None:
f.close()
def train_leakage(trainloader,
testloader,
device,
seed,
specific=None,
p=0.5,
n_epochs=5,
module='layer4',
lr=0.5,
base_folder="",
out_file=None,
experiment1="",
experiment2="",
model_extra="",
n2v_extra="",
with_n2v=False,
nonlinear=False,
model_custom_end='',
n2v_custom_end='',
multiple=False,
dataset='bam',
parallel=False,
gpu_ids=[]):
if out_file is not None:
f = open(out_file, 'a')
else:
f = None
print("Training Model Leakage | p =", p, file=f)
if not nonlinear:
n2v_extra = "n2v" + str(n2v_extra)
else:
n2v_extra = "mlp" + str(n2v_extra)
if len(model_custom_end) > 0:
model_custom_end = "_" + model_custom_end
if len(n2v_custom_end) > 0:
n2v_custom_end = "_" + n2v_custom_end
if hasattr(trainloader.dataset, 'idx_to_class'):
for key in trainloader.dataset.idx_to_class:
if specific is not None and trainloader.dataset.idx_to_class[
key] in specific:
specific_idx = int(key)
else:
specific_idx = 0
else:
specific_idx = 0
if dataset == 'bam' or dataset == 'coco':
model_path = utils.get_model_path(base_folder,
seed,
specific,
"resnet" + str(model_extra) + "_" +
str(p) + model_custom_end + ".pt",
experiment=experiment1,
with_n2v=with_n2v,
n2v_module=module)
n2v_path = utils.get_net2vec_path(
base_folder,
seed,
specific,
module,
"leakage/resnet_" + str(n2v_extra) + "_" + str(p) +
n2v_custom_end + ".pt",
experiment=experiment2,
)
else:
if with_n2v:
model_path = os.path.join(
base_folder, str(seed), experiment1, module,
"resnet" + str(model_extra) + model_custom_end + ".pt")
else:
model_path = os.path.join(
base_folder, str(seed), experiment1,
"resnet" + str(model_extra) + model_custom_end + ".pt")
n2v_path = os.path.join(
base_folder, str(seed), experiment2, module,
'leakage/resnet_' + str(n2v_extra) + n2v_custom_end + ".pt")
if dataset == 'bam':
if specific is not None and not isinstance(specific, str):
folder_name = '.'.join(sorted(specific))
else:
folder_name = specific
leakage_folder = os.path.join(str(base_folder), str(seed), folder_name,
str(experiment2), str(module), 'leakage')
else:
leakage_folder = os.path.join(str(base_folder), str(seed),
str(experiment2), str(module), 'leakage')
if not os.path.isdir(leakage_folder):
os.mkdir(leakage_folder)
num_classes = 10
if dataset == 'coco':
num_classes = 79
num_attributes = 2
model, net, net_forward, activation_probe = models.load_models(
device,
None if (dataset == 'coco') and ('adv' in model_extra) else lambda x,
y, z: models.resnet_(pretrained=True,
custom_path=x,
device=y,
num_classes=num_classes,
initialize=z,
size=50 if
(dataset == 'bam' or dataset == 'coco') else 34),
model_path=model_path,
net2vec_pretrained=False,
module='fc', # leakage will come from the output logits...
num_attributes=num_attributes,
model_init=False, # don't need to initialize a new one
n2v_init=True,
loader=trainloader,
nonlinear=nonlinear,
parallel=parallel,
gpu_ids=gpu_ids)
def criterion(logits, genders):
return F.cross_entropy(logits,
genders[:, 1].long(),
reduction='elementwise_mean')
net2vec.train_net2vec(model,
net,
net_forward,
n_epochs,
trainloader,
testloader,
device,
lr=lr,
save_path=n2v_path,
f=f,
train_labels=[-2, -1],
balanced=False,
criterion=criterion,
specific=specific_idx,
adam=True,
save_best=True,
leakage=True)
if f is not None:
f.close()
"""Command line tool for trying out the prediction models"""
import argparse
from sklearn.externals import joblib
from haliasdata import get_bird_species
from models import prediction_models, load_models
parser = argparse.ArgumentParser(description='Predict traffic disruptions')
parser.add_argument('var1', help='variable 1', type=float)
parser.add_argument('var2', help='variable 2', type=float)
parser.add_argument('var3', help='variable 3', type=float)
parser.add_argument('var4', help='variable 4', type=float)
args = parser.parse_args()
input_scaler = joblib.load('model/scaler.pkl')
species = get_bird_species()
load_models(prediction_models)
for model in prediction_models:
#print 'Model %s' % (model.model)
value_tuple = input_scaler.transform((args.var1, args.var2, args.var3, args.var4))
prediction = model.predict(value_tuple)
print('Model %s: %s' % (model.name, prediction))
import random
import time
from sklearn import linear_model
import pickle
import threading
import models
import utils_exomodel
from sklearn.metrics import mean_absolute_error, r2_score, mean_squared_error
import utils_model_genetic
importlib.reload(utils_exomodel)
importlib.reload(models)
importlib.reload(utils_model_genetic)
SKLEARN_MODELS = models.load_models()
models_list = list(SKLEARN_MODELS.keys())
def save_obj(obj, name ):
with open( name + '.pkl', 'wb') as f:
pickle.dump(obj, f, pickle.HIGHEST_PROTOCOL)
def load_obj(name ):
with open( name + '.pkl', 'rb') as f:
return pickle.load(f)
def get_lag8columns(df):
lag_columns = [x for x in df.columns if "LAG" in x ]
exclude_columns = list()
for i in range(0,8):
seen.add(sents[i])
a = sents[0].split()
b = sents[i].split()
sm = difflib.SequenceMatcher(a=a, b=b)
for tag, i1, i2, j1, j2 in sm.get_opcodes():
if tag == "equal":
print(" ".join(b[j1:j2]), end=" ")
if tag == "replace":
print(BOLD + " ".join(b[j1:j2]) + ENDC, end=" ")
# print("*" + " ".join(b[j1:j2]) + "*", end=" ")
print()
print()
def gen(vec):
"Generate argmax sentence from vector."
return generate(autoencoder, gan_gen, z=vec,
vocab=idx2word, sample=False,
maxlen=model_args['maxlen'])
if __name__ == "__main__":
parser = argparse.ArgumentParser(description='PyTorch experiment')
parser.add_argument('--load_path', type=str,
help='directory to load models from')
args = parser.parse_args()
model_args, idx2word, autoencoder, gan_gen, gan_disc \
= load_models(args.load_path)
main(args)
def main(args):
# Set the random seed manually for reproducibility.
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
if torch.cuda.is_available():
torch.cuda.manual_seed(args.seed)
else:
print("Note that our pre-trained models require CUDA to evaluate.")
###########################################################################
# Load the models
###########################################################################
model_args, idx2word, autoencoder, gan_gen, gan_disc \
= load_models(args.load_path)
###########################################################################
# Generation code
###########################################################################
# Generate sentences
if args.ngenerations > 0:
noise = torch.ones(args.ngenerations, model_args['z_size'])
noise.normal_()
sentences = generate(autoencoder,
gan_gen,
z=noise,
vocab=idx2word,
sample=args.sample,
maxlen=model_args['maxlen'])
if not args.noprint:
print("\nSentence generations:\n")
for sent in sentences:
print(sent)
with open(args.outf, "w") as f:
f.write("Sentence generations:\n\n")
for sent in sentences:
f.write(sent + "\n")
# Generate interpolations
if args.ninterpolations > 0:
noise1 = torch.ones(args.ninterpolations, model_args['z_size'])
noise1.normal_()
noise2 = torch.ones(args.ninterpolations, model_args['z_size'])
noise2.normal_()
interps = interpolate(autoencoder,
gan_gen,
z1=noise1,
z2=noise2,
vocab=idx2word,
steps=args.steps,
sample=args.sample,
maxlen=model_args['maxlen'])
if not args.noprint:
print("\nSentence interpolations:\n")
for interp in interps:
for sent in interp:
print(sent)
print("")
with open(args.outf, "a") as f:
f.write("\nSentence interpolations:\n\n")
for interp in interps:
for sent in interp:
f.write(sent + "\n")
f.write('\n')
corpus = Corpus(datafiles,
maxlen=args.maxlen,
vocab_size=args.vocab_size,
lowercase=args.lowercase,
vocab=vocabdict)
# save arguments
ntokens = len(corpus.dictionary.word2idx)
print("Vocabulary Size: {}".format(ntokens))
args.ntokens = ntokens
eval_batch_size = 100
en_data = batchify(corpus.data[args.corpus_name],
eval_batch_size,
shuffle=False)
print(len(en_data))
print("Loaded data!")
model_args, idx2word, autoencoder, gan_gen, gan_disc = load_models(
args.outf, args.epochs, twodecoders=True)
if args.cuda:
autoencoder = autoencoder.cuda()
gan_gen = gan_gen.cuda()
gan_disc = gan_disc.cuda()
one = to_gpu(args.cuda, torch.FloatTensor([1]))
mone = one * -1
evaluate_generator(1, False)
vocab_size=len(vocabdict),
lowercase=args.lowercase,
vocab=vocabdict,
debug=args.debug)
eval_batch_size = args.batch_size
test1_data = batchify(corpus.data['test1'], eval_batch_size, shuffle=False)
test2_data = batchify(corpus.data['test2'], eval_batch_size, shuffle=False)
print("Loaded data!")
###############################################################################
# Build the models
###############################################################################
model_args, idx2word, autoencoder, gan_gen, gan_disc = \
load_models(args.load_path, args.epoch, twodecoders=True)
ntokens = len(corpus.dictionary.word2idx)
if args.cuda:
autoencoder = autoencoder.cuda()
autoencoder.gpu = True
###############################################################################
# Training code
###############################################################################
def evaluate_transfer(whichdecoder, data_source, epoch):
# Turn on evaluation mode which disables dropout.
autoencoder.eval()
ntokens = len(corpus.dictionary.word2idx)
def main():
logging.basicConfig(level=logging.INFO, format='%(message)s')
parser = argparse.ArgumentParser(
description='Predict using trained models')
parser.add_argument('rev_map', help='reverse inflection map')
parser.add_argument('models',
nargs='+',
help='trained models (category:file)')
parser.add_argument('--ambiguous',
action='store_true',
help='evaluate only lemmas with multiple inflections')
args = parser.parse_args()
logging.info('Loading reverse inflection map')
with open(args.rev_map) as f:
rev_map = cPickle.load(f)
logging.info('Loading inflection prediction models')
models = load_models(args.models)
logging.info('Loaded models for %d categories', len(models))
stats = {cat: [0, 0, 0, 0, 0] for cat in config.EXTRACTED_TAGS}
for source, target, alignment in read_sentences(sys.stdin):
for word, features in extract_instances(source, target, alignment):
gold_inflection, lemma, tag = word
category = tag[0]
gold_tag = tag[1:]
possible_inflections = rev_map.get((lemma, category), [])
if (gold_tag, gold_inflection) not in possible_inflections:
print(u'Expected: {} ({}) not found'.format(
gold_inflection, gold_tag).encode('utf8'))
continue
if args.ambiguous and len(possible_inflections) == 1: continue
model = models[category]
scored_inflections = model.score_all(possible_inflections,
features)
ranked_inflections = sorted(scored_inflections, reverse=True)
predicted_score, predicted_tag, predicted_inflection = ranked_inflections[
0]
gold_rank = 1 + [tag for _, tag, _ in ranked_inflections
].index(gold_tag)
gold_score = next((score for score, tag, _ in ranked_inflections
if tag == gold_tag))
print(u'Expected: {} ({}) r={} score={:.3f} |'
' Predicted: {} ({}) score={:.3f}'.format(
gold_inflection, gold_tag, gold_rank, gold_score,
predicted_inflection, predicted_tag,
predicted_score).encode('utf8'))
stats[category][0] += 1
stats[category][1] += 1 / float(gold_rank)
stats[category][2] += (gold_inflection == predicted_inflection)
stats[category][3] += gold_score
stats[category][4] += len(ranked_inflections)
for category, (n_instances, rrank_sum, n_correct, total_log_prob,
n_inflections) in stats.items():
if n_instances == 0: continue
mrr = rrank_sum / n_instances
accuracy = n_correct / float(n_instances)
ppl = math.exp(-total_log_prob / n_instances)
avg_inflections = n_inflections / float(n_instances)
print(
'Category {}: MRR={:.3f} acc={:.1%} ppl={:.2f} ({} instances; avg #infl={:.2f})'
.format(category, mrr, accuracy, ppl, n_instances,
avg_inflections))
"""Command line tool for trying out the prediction models"""
import argparse
from models import prediction_models, load_models
load_models(prediction_models)
parser = argparse.ArgumentParser(description='Predict traffic disruptions')
parser.add_argument('temperature', help='Temperature [C]', type=float)
parser.add_argument('rainfall', help='Precipitation [mm]', type=float)
parser.add_argument('windspeed', help='Wind speed [m/s]', type=float)
parser.add_argument('hour', help='Hour of the day', type=int)
args = parser.parse_args()
for model in prediction_models:
#print 'Model %s' % (model.model)
value_tuple = (args.rainfall, args.temperature, args.windspeed, args.hour)
prediction = model.predict(value_tuple[:model.parameters])
print 'Model %s: %s' % (model.name, prediction)
"""
def run_server(port_num=8080):
if not models.load_models():
#print(os.listdir(os.curdir))
with open(os.path.join(os.path.dirname(__file__), 'big.txt'),
'rb') as f:
print(os.path.join(os.path.dirname(__file__), 'big.txt'))
models.train_models(str(f.read()))
@route('/<wordA>/<wordB>')
def index(wordA, wordB):
return dict(autocomplete.predict(wordA, wordB))
run(host='localhost', port=port_num)
if __name__ == "__main__":
"""load the classic Norvig big.txt corpus"""
print("training!")
if not models.load_models():
#print(os.listdir(os.curdir))
with open(os.path.join(os.path.dirname(__file__), 'big.txt'),
'rb') as f:
print(os.path.join(os.path.dirname(__file__), 'big.txt'))
models.train_models(str(f.read()))
print("done training!")
