import json
from collections import namedtuple
from dotmap import DotMap
import mysql.connector
from config import Config
Company = namedtuple('COMPANY', 'CompanyName Modified HiringStatus Comment')
JobDescription = namedtuple(
'JOBDESCRIPTION',
'JobTitle Salary JobResponsibilities TimeCommitment TypeOfContract '
'DateUpload ContractStart ContractDuration Link CompanyName')
_sql = DotMap(Config.sql)
class _Global:
pass
def make_cursor():
db = mysql.connector.connect(host=_sql.server,
user=_sql.username,
passwd=_sql.password,
database=_sql.database,
ssl_ca='cert.pem')
cursor = db.cursor()
_Global.db = db
_Global.cursor = cursor
return cursor
def question_generation(_input):
metadata, output = _input
args = DotMap()
"""
parser = ArgumentParser()
parser.add_argument("--model_type", type=str, default="gpt", help="gpt or gpt2")
parser.add_argument("--model_checkpoint", type=str, default="", help="Path, url or short name of the model")
parser.add_argument("--device", type=str, default="cuda" if torch.cuda.is_available() else "cpu",
help="Device (cuda or cpu)")
parser.add_argument("--filename", type=str, default="data/instances_dev.pkl", help="File to use for decoding")
parser.add_argument("--no_sample", action='store_true', help="Set to use greedy decoding instead of sampling")
parser.add_argument("--max_length", type=int, default=50, help="Maximum length of the output utterances")
parser.add_argument("--min_length", type=int, default=1, help="Minimum length of the output utterances")
parser.add_argument("--seed", type=int, default=42, help="Seed")
parser.add_argument("--temperature", type=int, default=0.7, help="Sampling softmax temperature")
parser.add_argument("--top_k", type=int, default=0, help="Filter top-k tokens before sampling (<=0: no filtering)")
parser.add_argument("--top_p", type=float, default=0.9,
help="Nucleus filtering (top-p) before sampling (<=0.0: no filtering)")
# While using SQUASH in the pipeline mode, prefer using the --key flag
parser.add_argument("--key", type=str, default=None,
help="Override the default settings if the key is set, used in pipeline mode")
args = parser.parse_args()
"""
"""
if args.key is not None:
# Override some the filename and top_p default settings if args.key is set
# This is done when the question generation module is being used in the SQUASH pipeline mode
args.filename = "squash/temp/%s/input.pkl" % args.key
with open("squash/temp/%s/metadata.json" % args.key, "r") as f:
metadata = json.loads(f.read())
args.top_p = metadata["settings"]["top_p"]
args.filename = "squash/temp/%s/input.pkl" % args.key
with open("squash/temp/%s/metadata.json" % args.key, "r") as f:
metadata = json.loads(f.read())
args.top_p = metadata["settings"]["top_p"]
"""
setattr(args, "top_p", metadata["settings"]["top_p"])
args.top_p = metadata["settings"]["top_p"]
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__file__)
logger.info(pformat(args))
args.seed = 42
random.seed(args.seed)
torch.random.manual_seed(args.seed)
torch.cuda.manual_seed(args.seed)
logger.info("Get pretrained model and tokenizer")
# NEW BLOCK
model_checkpoint = "question_generation/gpt2_corefs_question_generation"
model_checkpoint = "/home/gpt2_corefs_question_generation"
model_type = "gpt2"
#model_checkpoint = "https://storage.cloud.google.com/ds-playground/squash/gpt2_qa.tar.gz"
SAVED_MODEL_DIR = "gpt2_corefs_question_generation"
dir_path = os.path.dirname(os.path.realpath(__file__))
model_checkpoint = os.path.join(dir_path, SAVED_MODEL_DIR)
model_checkpoint = "question_generation/gpt2_corefs_question_generation"
tokenizer = GPT2Tokenizer.from_pretrained(model_checkpoint)
model = GPT2LMHeadModel.from_pretrained(model_checkpoint)
""" OLD BLOCK
if args.model_type == 'gpt2':
tokenizer = GPT2Tokenizer.from_pretrained(args.model_checkpoint)
model = GPT2LMHeadModel.from_pretrained(args.model_checkpoint)
else:
tokenizer = OpenAIGPTTokenizer.from_pretrained(args.model_checkpoint)
model = OpenAIGPTLMHeadModel.from_pretrained(args.model_checkpoint)
"""
output_config_file = "/content/squash-generation/question_generation/gpt2_corefs_question_generation/config.json"
output_model_file = "/content/squash-generation/question_generation/gpt2_corefs_question_generation/pytorch_model.bin"
output_vocab_file = "/content/squash-generation/question_generation/gpt2_corefs_question_generation/vocab.json"
merges_file = "/content/squash-generation/question_generation/gpt2_corefs_question_generation/merges.txt"
output_config_file = SparkFiles.get("config.json")
output_model_file = SparkFiles.get("pytorch_model.bin")
output_vocab_file = SparkFiles.get("vocab.json")
merges_file = SparkFiles.get("merges.txt")
config = GPT2Config.from_json_file(output_config_file)
model = GPT2LMHeadModel(config)
state_dict = torch.load(output_model_file,
map_location=torch.device('cpu'))
model.load_state_dict(state_dict)
tokenizer = GPT2Tokenizer(output_vocab_file, merges_file=merges_file)
model.to("cpu")
model.eval()
args.device = "cpu"
args.device = "cpu"
model.to(args.device)
model.eval()
return {"break": "point"}
#data = get_positional_dataset_from_file(tokenizer, args.filename)
data = get_positional_dataset_from_file(tokenizer, output)
final_output_dict = {"version": "squash-2.0", "data": [{"paragraphs": []}]}
question_number = 0
para_cache = {"index": None, "hidden_states": None}
for inst in tqdm.tqdm(data):
with torch.no_grad():
para_index = inst["para_index"]
# Questions from the same paragraph all appear together
# We can re-use the paragraph hidden representations for different questions in the same paragraph
if para_index != para_cache["index"]:
# Since we have moved to a new paragraph, generate its cache
para_cache["hidden_states"] = None
# Ignore the answer and question while building the input
instance, _ = build_para_only_input_from_segments(
inst, tokenizer)
input_ids = torch.tensor(instance['input_ids'],
device=args.device).unsqueeze(0)
token_type_ids = torch.tensor(instance['token_type_ids'],
device=args.device).unsqueeze(0)
# Run a forward pass to generate the para caches
_, para_cache["hidden_states"] = model(
input_ids, token_type_ids=token_type_ids)
# Sample a question using the paragraph cache
output = sample_sequence(inst, tokenizer, model, args, para_cache)
original_paragraph = tokenizer.decode(output['paragraph'])
generated_question = tokenizer.decode(output['question'],
skip_special_tokens=True)
original_answer = tokenizer.decode(output['answer'],
skip_special_tokens=True)
para_index = inst['para_index']
para_cache["index"] = inst['para_index']
# verify whether the answer position is correct, since this will be utilized for filtering
original_ans_position = output["answer_position"]
if original_paragraph[
output["answer_position"]:output["answer_position"] +
len(original_answer)] != original_answer:
# This should never be executed, only used as a last resort
logger.info("Answer mismatch!")
original_ans_position = original_paragraph.index(original_answer)
# Output in a SQUAD-like format with questions clumped together under their parent paragraph
if len(final_output_dict["data"][0]["paragraphs"]) > para_index:
# verify whether the paragraph text is identical
assert original_paragraph == final_output_dict["data"][0][
"paragraphs"][para_index]['context']
# append the question answer pair
final_output_dict["data"][0]["paragraphs"][para_index][
'qas'].append({
'id':
'question_%d' % question_number,
'question':
generated_question,
'answers': [{
'text': original_answer,
'answer_start': original_ans_position,
}],
'class':
output['class'],
'algorithm':
output['algorithm'],
'is_impossible':
False
})
else:
# add a new question to the list of QA pairs
final_output_dict['data'][0]['paragraphs'].append({
'context':
original_paragraph,
'qas': [{
'id':
'question_%d' % question_number,
'question':
generated_question,
'answers': [{
'text': original_answer,
'answer_start': original_ans_position,
}],
'class':
output['class'],
'algorithm':
output['algorithm'],
'is_impossible':
False
}]
})
question_number += 1
#with open("squash/temp/%s/generated_questions.json" % args.key, "w") as f:
# f.write(json.dumps(final_output_dict))
return final_output_dict
def _validate(
self,
input,
seq_lens,
span_token_ids,
target,
target_lens,
decode_strategy,
definition=None,
definition_lens=None,
sentence_mask=None,
):
batch_size, tgt_len = target.shape
# (batch_size,seq_len,hidden_size), (batch_size,hidden_size), (num_layers,batch_size,seq_len,hidden_size)
last_hidden_layer, pooled_representation, all_hidden_layers = self.encoder(
input, attention_mask=sequence_mask(seq_lens), token_type_ids=sentence_mask
)
KLD = None
mu_prime = None
if self.aggregator == "cls":
cls_hidden = last_hidden_layer[:, 0]
cls_hidden_forwarded = self.cls_feed_forward(cls_hidden)
span_representation = cls_hidden_forwarded
hidden_states = last_hidden_layer
else:
span_ids = self._id_extractor(
tokens=span_token_ids, batch=input, lens=seq_lens
)
span_representation, hidden_states = self._span_aggregator(
all_hidden_layers if self.scalar_mix is not None else last_hidden_layer,
sequence_mask(seq_lens),
span_ids,
)
span_representation = self.context_feed_forward(span_representation)
if self.variational:
(
definition_last_hidden_layer,
_,
definition_all_hidden_layers,
) = self.definition_encoder(
definition, attention_mask=sequence_mask(definition_lens)
)
definition_representation = self.definition_feed_forward(
definition_last_hidden_layer[:, 0]
)
post_project = self.w_z_post(
torch.cat([span_representation, definition_representation], -1)
)
prior_project = self.w_z_prior(span_representation)
mu = self.mean_layer(post_project)
logvar = self.logvar_layer(post_project)
mu_prime = self.mean_prime_layer(prior_project)
logvar_prime = self.logvar_prime_layer(prior_project)
hidden_states = last_hidden_layer
KLD = (
kl_divergence(
Normal(mu, torch.exp(logvar * 0.5)),
Normal(mu_prime, torch.exp(logvar_prime * 0.5)),
)
.sum(1)
.mean()
)
span_representation = self.z_project(mu_prime)
memory_bank = hidden_states if self.decoder.attention else None
_, logits = self.decoder(
target, target_lens, span_representation, memory_bank, seq_lens,
)
loss = F.cross_entropy(
logits.view(batch_size * (tgt_len - 1), -1),
target[:, 1:].contiguous().view(-1),
ignore_index=self.embeddings.tgt.padding_idx,
)
ppl = loss.exp()
beam_results = self._strategic_decode(
target,
target_lens,
decode_strategy,
memory_bank,
seq_lens,
span_representation,
)
return DotMap(
{
"predictions": beam_results["predictions"],
"logits": logits.view(batch_size * (tgt_len - 1), -1),
"loss": loss,
"perplexity": ppl,
"kl": KLD,
}
)
def process_search_by_coordinates(cls, x, y, pictures=False):
"""
Searches by coordinate from XML and processes the result.
:param x: longitude
:param y: latitude
:param pictures: True if we want house plan pictures to be scrapped
:return: List of CadasterEntry objects
"""
results = []
xml_dict_map = ScrapperXML.scrap_coord(x, y)
pc1 = None
pc2 = None
if xml_dict_map.consulta_coordenadas.coordenadas.coord.pc != DotMap():
pc1 = xml_dict_map.consulta_coordenadas.coordenadas.coord.pc.pc1
if pc1 == DotMap():
pc1 = None
pc2 = xml_dict_map.consulta_coordenadas.coordenadas.coord.pc.pc2
if pc2 == DotMap():
pc2 = None
if pc1 is not None and pc2 is not None:
entry = ScrapperXML.get_cadaster_entries_by_cadaster(
'', '', ''.join([pc1, pc2]))
picture = None
if entry.consulta_dnp.bico.bi.dt.loine != DotMap():
prov_num = entry.consulta_dnp.bico.bi.dt.loine.cp
city_num = entry.consulta_dnp.bico.bi.dt.cmc
# Parcela
if pictures and prov_num != DotMap() and city_num != DotMap():
picture = Scrapper.scrap_site_picture(
prov_num, city_num, ''.join([pc1, pc2]))
cadaster_entry = CadasterEntryXML(entry, x, y, picture)
cadaster_entry.to_elasticsearch()
results.append(cadaster_entry)
elif entry.consulta_dnp.lrcdnp.rcdnp != DotMap():
# Multiparcela
parcelas = entry.consulta_dnp.lrcdnp.rcdnp
if not isinstance(parcelas, list):
parcelas = [parcelas]
for parcela in parcelas:
prov_num = parcela.dt.loine.cp
city_num = parcela.dt.cmc
cadaster = parcela.rc.pc1 if parcela.rc.pc1 != DotMap(
) else ''
cadaster += parcela.rc.pc2 if parcela.rc.pc2 != DotMap(
) else ''
cadaster += parcela.rc.car if parcela.rc.car != DotMap(
) else ''
cadaster += parcela.rc.cc1 if parcela.rc.cc1 != DotMap(
) else ''
cadaster += parcela.rc.cc2 if parcela.rc.cc2 != DotMap(
) else ''
if pictures and prov_num != DotMap(
) and city_num != DotMap():
picture = Scrapper.scrap_site_picture(
prov_num, city_num, cadaster)
try:
# Try to get info by complete cadaster num
sub_entry = ScrapperXML.get_cadaster_entries_by_cadaster(
'', '', cadaster)
except:
# Cadastro did not return anything by cadaster entry (error? bug?)
# Try to get it by complete address
prov_name = parcela.dt.np
if prov_name is DotMap():
continue
city_name = parcela.dt.np
if city_name is DotMap():
continue
tv = parcela.ldt.locs.lous.lourb.dir.tv
if tv is DotMap():
tv = ''
nv = parcela.ldt.locs.lous.lourb.dir.nv
if nv is DotMap():
nv = ''
num = parcela.ldt.locs.lous.lourb.dir.pnp
if num is DotMap():
num = ''
loint = parcela.dt.locs.lous.lourb.loint
if loint is DotMap():
continue
bl = loint.bl
if bl == DotMap():
bl = ''
es = loint.es
if es == DotMap():
es = ''
pt = loint.pt
if es == DotMap():
pt = ''
pu = loint.pu
if es == DotMap():
pu = ''
sub_entry = ScrapperXML.get_cadaster_entries_by_address(
prov_name, city_name, tv, nv, num, bl, es, pt, pu)
cadaster_entry = CadasterEntryXML(sub_entry, x, y, picture)
cadaster_entry.to_elasticsearch()
results.append(cadaster_entry)
return results
def nested_mapping_to_line(nested_mapping, keys):
# type: (typing.Mapping, typing.Sequence[typing.Text]) -> typing.Text
dotted = DotMap(nested_mapping)
return ','.join([str(recursive_getattr(dotted, key)) for key in keys])
sett_config = DotMap(
native=DotMap(
badger=DotMap(
strategyName="StrategyBadgerRewards",
params=DotMap(
# want = Badger token
# geyser = Special Geyser
performanceFeeStrategist=0,
performanceFeeGovernance=0,
withdrawalFee=0,
),
),
uniBadgerWbtc=DotMap(
strategyName="StrategyBadgerLpMetaFarm",
params=DotMap(
# Note: Will not be able to be deployed until the LP token is created
# want = Uni Badger<>Wbtc LP
performanceFeeStrategist=0,
performanceFeeGovernance=0,
withdrawalFee=0,
),
),
sbtcCrv=DotMap(
strategyName="StrategyCurveGauge",
params=DotMap(
want=pools.sbtcCrv.token,
gauge=pools.sbtcCrv.gauge,
swap=pools.sbtcCrv.swap,
minter=curve.minter,
lpComponent=registry.tokens.wbtc,
performanceFeeStrategist=1000,
performanceFeeGovernance=1000,
withdrawalFee=75,
keepCRV=0,
),
),
renCrv=DotMap(
strategyName="StrategyCurveGauge",
params=DotMap(
want=pools.renCrv.token,
gauge=pools.renCrv.gauge,
swap=pools.renCrv.swap,
minter=curve.minter,
lpComponent=registry.tokens.wbtc,
performanceFeeStrategist=1000,
performanceFeeGovernance=1000,
withdrawalFee=75,
keepCRV=0,
),
),
tbtcCrv=DotMap(
strategyName="StrategyCurveGauge",
params=DotMap(
want=pools.tbtcCrv.token,
gauge=pools.tbtcCrv.gauge,
swap=pools.tbtcCrv.swap,
minter=curve.minter,
lpComponent=registry.tokens.wbtc,
performanceFeeStrategist=1000,
performanceFeeGovernance=1000,
withdrawalFee=75,
keepCRV=0,
),
),
),
pickle=DotMap(renCrv=DotMap(
strategyName="StrategyPickleMetaFarm",
params=DotMap(
want=pools.renCrv.token,
pickleJar=pickle.jars.renCrv,
curveSwap=registry.curve.pool.renCrv.swap,
lpComponent=registry.tokens.wbtc,
pid=pickle.pids.pRenCrv,
performanceFeeStrategist=1000,
performanceFeeGovernance=1000,
withdrawalFee=75,
),
), ),
harvest=DotMap(
renCrv=DotMap(
# Unfinished
strategyName="StrategyHarvestMetaFarm",
params=DotMap(
want=pools.renCrv.token,
harvestVault=registry.harvest.vaults.renCrv,
vaultFarm=registry.harvest.farms.fRenCrv,
metaFarm=registry.harvest.farms.farm,
badgerTree=registry.harvest.badgerTree,
performanceFeeStrategist=1000,
performanceFeeGovernance=1000,
withdrawalFee=75,
),
), ),
uni=DotMap(
uniDiggWbtc=DotMap(
# Unfinished
strategyName="StrategyDiggLpMetaFarm",
params=DotMap(
performanceFeeStrategist=0,
performanceFeeGovernance=0,
withdrawalFee=0,
),
), ),
sushi=DotMap(
sushiBadgerWBtc=DotMap(
# Unfinished
strategyName="StrategySushiBadgerWbtc",
params=DotMap(
# want=pools.renCrv.token,
performanceFeeStrategist=1000,
performanceFeeGovernance=1000,
withdrawalFee=0,
),
),
sushiWethWBtc=DotMap(
# Unfinished
strategyName="StrategySushiBadgerWbtc",
params=DotMap(
# want=pools.renCrv.token,
performanceFeeStrategist=1000,
performanceFeeGovernance=1000,
withdrawalFee=50,
),
),
sushiDiggWBtc=DotMap(
# Unfinished
strategyName="StrategySushiDiggWbtcLpOptimizer",
params=DotMap(
performanceFeeStrategist=1000,
performanceFeeGovernance=1000,
withdrawalFee=0,
),
),
),
)
def process_search_by_provinces(cls,
prov_list,
pictures=False,
start_from='',
matches=None):
"""
Searches by province from XML and processes the result.
:param prov_list: List of province names
:param start_from: Name of the city of the first province to start from
:param pictures: True if we want house plan pictures to be scrapped
:param matches: Max number of matches (for debugging purporses mainly)
:return: List of CadasterEntry objects
"""
times = 0
results = []
for prov_name, prov_num, city_name, city_num, address, tv, nv in Scrapper.get_address_iter(
prov_list, start_from):
if tv == DotMap() or nv == DotMap():
continue
if ElasticSearchUtils.check_if_address_present(
"{} {}".format(tv, nv), city_name, prov_name):
logger.debug(
"Skipping {} {} {} {} because it's been already scrapped.".
format(tv, nv, prov_name, city_name))
continue
num_scrapping_fails = 10
counter = 1
while num_scrapping_fails > 0:
try:
cadaster = ScrapperXML.get_cadaster_by_address(
prov_name, city_name, tv, nv, counter)
res = cls.parse_xml_by_address(cadaster, prov_name,
city_name, tv, nv, counter,
pictures)
if len(res) < 1:
num_scrapping_fails -= 1
else:
num_scrapping_fails = 10
times += 1
results.append(res)
if matches is not None and times >= matches:
return ListUtils.flat(results)
except urllib.error.HTTPError as e:
logger.error("ERROR AT ADDRESS {} {} {} {} {}".format(
tv, nv, counter, prov_name, city_name))
logger.error(
"=============================================")
logger.error(e, exc_info=True)
logger.error("...sleeping due to connection reset...")
logger.debug("...sleeping due to connection reset...")
logger.error(
"=============================================")
''' Could be a service Unavailable or denegation of service'''
num_scrapping_fails -= 1
except Exception as e:
logger.error("ERROR AT ADDRESS {} {} {} {} {}".format(
tv, nv, counter, prov_name, city_name))
logger.error(
"=============================================")
logger.error(e, exc_info=True)
logger.error(
"=============================================")
num_scrapping_fails -= 1
counter += 1
return results
def test_space():
# small search space to see if code works
full_space = DotMap()
full_space.nP = [100, 200] # number of requests per sim time
return full_space
import time
import os
from . import sl_constants
from . import sl_helpers
from . import sl_console
from pathlib import Path
from dotmap import DotMap
from netaddr import IPAddress
from pandas.io.json import json_normalize
__author__ = "Richard Gold"
CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help'])
PROFILE_FILE = os.path.join(str(Path.home()), '.shadowline', 'profile')
settings = DotMap()
@click.group(context_settings=CONTEXT_SETTINGS)
@click.option('--profile',
default='DEFAULT',
help="Name of profile to use. 'DEFAULT' if not specified.")
@click.pass_context
def main(ctx, profile):
"""
ShadowLine: A command-line API for Digital Shadows SearchLight
"""
if ctx.invoked_subcommand == 'setup_profile':
# create_profile method can be called without existing credentials
pass
else:
with open("dependency-artifacts/wbtc/wbtc.json") as f:
wbtc = json.load(f)
artifacts = DotMap(
aragon=DotMap(
Agent=Agent,
CompanyTemplate=CompanyTemplate,
Vault=Vault,
Voting=Voting,
Finance=Finance,
TokenManager=TokenManager,
MiniMeToken=MiniMeToken,
),
gnosis_safe=DotMap(MasterCopy=MasterCopy,
ProxyFactory=ProxyFactory,
GnosisSafe=GnosisSafe),
open_zeppelin=DotMap(
ProxyAdmin=ProxyAdmin,
AdminUpgradeabilityProxy=AdminUpgradeabilityProxy,
TokenTimelock=TokenTimelock,
),
uniswap=DotMap(
UniswapV2Factory=UniswapV2Factory,
UniswapV2Router=UniswapV2Router,
UniswapV2Pair=UniswapV2Pair,
),
multicall=DotMap(multicall={"abi": Multicall}),
wbtc=DotMap(wbtc={"abi": wbtc}),
)
import logging
# start = time.time()
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
# To log file
fh = logging.FileHandler('example.log')
fh.setLevel(logging.DEBUG)
logger.addHandler(fh)
task = Quadratic()
stopCriteria = opto.opto.classes.StopCriteria(maxEvals=10000)
p = DotMap()
p.verbosity = 1
opt = opto.RandomSearch(parameters=p, task=task, stopCriteria=stopCriteria)
opt.optimize()
logs = opt.get_logs()
print('Number evaluations: %d' % logs.get_n_evals())
print('Optimization completed in %f[s]' % (logs.get_final_time()))
# print(logs.get_parameters())
# Parameters
plt.ioff()
# plt.figure()
logs.plot_parameters()
#
plt.figure()
def train_dataloader(self):
train_loader = DataLoader(self.train_dataset,
batch_size=self.arg.batch_size,
shuffle=True)
return train_loader
def val_dataloader(self):
test_loader = DataLoader(self.test_dataset,
batch_size=self.arg.batch_size,
shuffle=False)
return test_loader
def main(arg):
seed_everything(42)
model = PLModel(arg)
wandb_logger = WandbLogger(project="Bachelorarbeit", name=arg.name)
wandb_logger.watch(model)
wandb_logger.log_hyperparams(arg)
trainer = Trainer(gpus=2,
logger=wandb_logger,
distributed_backend='ddp',
deterministic=True,
auto_select_gpus=True,
num_sanity_val_steps=0)
trainer.fit(model)
if __name__ == '__main__':
arg = DotMap(vars(parse_args()))
main(arg)
yamls = yaml.load(crfile)
crfile.close()
yamls['data']['azure_resource_prefix'] = base64.b64encode(
bytes(resource_group, 'utf-8')).decode('utf-8')
yamls['data']['azure_resourcegroup'] = base64.b64encode(
bytes(resource_group, 'utf-8')).decode('utf-8')
with open('./gw/openshift/99_cloud-creds-secret.yaml', 'w') as crout:
yaml.dump(yamls, crout, default_flow_style=False)
crout.close()
with open('./gw/manifests/cloud-provider-config.yaml') as file:
yamlx = yaml.load(file)
file.close()
jsondata = yamlx['data']['config']
jsonx = json.loads(jsondata, object_pairs_hook=OrderedDict)
config = DotMap(jsonx)
config.resourceGroup = resource_group
config.vnetName = "openshiftVnet"
config.vnetResourceGroup = resource_group
config.subnetName = "masterSubnet"
config.securityGroupName = "master1nsg"
config.routeTableName = ""
config.azure_resourcegroup = resource_group
jsondata = json.dumps(dict(**config.toDict()), indent='\t')
jsonstr = str(jsondata)
yamlx['data']['config'] = jsonstr + '\n'
yamlx['metadata']['creationTimestamp'] = None
yamlstr = yaml.dump(yamlx, default_style='\"', width=4096)
yamlstr = yamlstr.replace('!!null "null"', 'null')
with open('./gw/manifests/cloud-provider-config.yaml', 'w') as outfile:
outfile.write(yamlstr)
from dotmap import DotMap
config = dict()
app = DotMap(config) # app globals
def reset():
app.clear()
app.update(config)
def __init__(self, settings):
self.channels = DotMap()
self.channels.rgb = [0, 3]
self.channels.pedestrian_trajectory = 3
self.channels.cars_trajectory = 4
self.channels.pedestrians = 5
self.channels.cars = 6
self.channels.semantic = 7
self.semantic_channels = DotMap()
self.labels_indx = {}
# Local sem mapping
if settings.new_carla_net:
self.semantic_channels.building = 0
self.semantic_channels.fence = 1
self.semantic_channels.static = 2
self.semantic_channels.vegetation = 3
self.semantic_channels.traffic_sign = 4
self.semantic_channels.traffic_light = 5
self.semantic_channels.road = 6
self.semantic_channels.terrain = 7
self.semantic_channels.crosswalk = 8
self.semantic_channels.sidewalk = 9
self.semantic_channels.sidewalk_extra = 10
#
# Buildings and large structures
self.labels_indx[cityscapes_labels_dict[
'building']] = self.semantic_channels.building
self.labels_indx[cityscapes_labels_dict[
'bridge']] = self.semantic_channels.building
self.labels_indx[cityscapes_labels_dict[
'tunnel']] = self.semantic_channels.building
# Fences and guard and individual walls
self.labels_indx[
cityscapes_labels_dict['fence']] = self.semantic_channels.fence
self.labels_indx[cityscapes_labels_dict[
'guard rail']] = self.semantic_channels.fence
self.labels_indx[
cityscapes_labels_dict['wall']] = self.semantic_channels.fence
# All non-moving obstacles
self.labels_indx[cityscapes_labels_dict[
'static']] = self.semantic_channels.static
self.labels_indx[cityscapes_labels_dict[
'dynamic']] = self.semantic_channels.static
self.labels_indx[
cityscapes_labels_dict['pole']] = self.semantic_channels.static
self.labels_indx[cityscapes_labels_dict[
'polegroup']] = self.semantic_channels.static
# Vegetation
self.labels_indx[cityscapes_labels_dict[
'vegetation']] = self.semantic_channels.vegetation
# Traffic sign
self.labels_indx[cityscapes_labels_dict[
'traffic sign']] = self.semantic_channels.traffic_sign
# Traffic light
self.labels_indx[cityscapes_labels_dict[
'traffic light']] = self.semantic_channels.traffic_light
# Road, areas where car drives
self.labels_indx[
cityscapes_labels_dict['ground']] = self.semantic_channels.road
self.labels_indx[
cityscapes_labels_dict['road']] = self.semantic_channels.road
self.labels_indx[cityscapes_labels_dict[
'parking']] = self.semantic_channels.road
# Terrain & rail track
self.labels_indx[cityscapes_labels_dict[
'terrain']] = self.semantic_channels.terrain
self.labels_indx[cityscapes_labels_dict[
'rail track']] = self.semantic_channels.terrain
# Crosswalk
self.labels_indx[cityscapes_labels_dict[
'crosswalk']] = self.semantic_channels.crosswalk
# Sidewalk
self.labels_indx[cityscapes_labels_dict[
'sidewalk']] = self.semantic_channels.sidewalk
# Sidewalk extra
self.labels_indx[cityscapes_labels_dict[
'sidewalk_extra']] = self.semantic_channels.sidewalk_extra
else:
self.semantic_channels.building = 0
self.semantic_channels.fence = 1
self.semantic_channels.static = 2
self.semantic_channels.pole = 3
self.semantic_channels.road = 4
self.semantic_channels.sidewalk = 5
self.semantic_channels.vegetation = 6
self.semantic_channels.wall = 7
self.semantic_channels.traffic_sign = 8
self.labels_indx[cityscapes_labels_dict[
'building']] = self.semantic_channels.building
self.labels_indx[
cityscapes_labels_dict['fence']] = self.semantic_channels.fence
self.labels_indx[cityscapes_labels_dict[
'static']] = self.semantic_channels.static
self.labels_indx[cityscapes_labels_dict[
'bridge']] = self.semantic_channels.static
self.labels_indx[cityscapes_labels_dict[
'tunnel']] = self.semantic_channels.static
self.labels_indx[cityscapes_labels_dict[
'dynamic']] = self.semantic_channels.static
self.labels_indx[cityscapes_labels_dict[
'guard rail']] = self.semantic_channels.static
self.labels_indx[
cityscapes_labels_dict['pole']] = self.semantic_channels.pole
self.labels_indx[cityscapes_labels_dict[
'polegroup']] = self.semantic_channels.pole
self.labels_indx[
cityscapes_labels_dict['ground']] = self.semantic_channels.road
self.labels_indx[
cityscapes_labels_dict['road']] = self.semantic_channels.road
self.labels_indx[cityscapes_labels_dict[
'parking']] = self.semantic_channels.road
self.labels_indx[cityscapes_labels_dict[
'rail track']] = self.semantic_channels.road
self.labels_indx[cityscapes_labels_dict[
'sidewalk']] = self.semantic_channels.sidewalk
self.labels_indx[cityscapes_labels_dict[
'vegetation']] = self.semantic_channels.vegetation
self.labels_indx[cityscapes_labels_dict[
'terrain']] = self.semantic_channels.vegetation
self.labels_indx[
cityscapes_labels_dict['wall']] = self.semantic_channels.wall
self.labels_indx[cityscapes_labels_dict[
'traffic light']] = self.semantic_channels.traffic_sign
self.labels_indx[cityscapes_labels_dict[
'traffic sign']] = self.semantic_channels.traffic_sign
self.carla = settings.carla
self.channels = DotMap()
self.channels.rgb = [0, 3]
self.channels.pedestrian_trajectory = 3
self.channels.cars_trajectory = 4
self.channels.pedestrians = 5
self.channels.cars = 6
self.channels.semantic = 7
self.set_nbr_channels()
super(Seg_2d_min, self).__init__(settings)
intersection_polygon = intersection_polygon.buffer(
intersection_buffer + 0.5)
intersection_safe = not Point(
turning_x, turning_y).within(intersection_polygon)
write_data = DotMap()
write_data.turning.theta = turning_th
write_data.turning.x = turning_x
write_data.turning.y = turning_y
write_data.braking_info.safe = intersection_safe
write_data.braking_info.dist = ego_y - destination
pickle.dump(write_data, open(par_dir + '/data_ego.pickle', 'wb'))
return
PORT = 8888
BUFSIZE = 4096
N_SIM_STEPS = 300
supervisor = Supervisor()
simulation_data = DotMap()
simulation_data.port = PORT
simulation_data.bufsize = BUFSIZE
simulation_data.task = scenic_intersection(N_SIM_STEPS=N_SIM_STEPS,
supervisor=supervisor)
client_task = ClientWebots(simulation_data)
if not client_task.run_client():
print("End of accident scenario generation")
supervisor.simulationQuit(True)
return digg_shares(value)
def as_wei(value):
return value
def as_original(value):
return value
erc20 = DotMap(
balanceOf="balanceOf(address)(uint256)",
totalSupply="totalSupply()(uint256)",
transfer="transfer(address,uint256)()",
safeTransfer="safeTransfer(address,uint256)()",
name="name()(string)",
symbol="symbol()(string)",
decimals="decimals()(uint256)",
)
sett = DotMap(
getPricePerFullShare="getPricePerFullShare()(uint256)",
available="available()(uint256)",
balance="balance()(uint256)",
controller="controller()(address)",
governance="governance()(address)",
strategist="strategist()(address)",
keeper="keeper()(address)",
shares="shares()(uint256)",
)
strategy = DotMap(
def __init__(self):
self._data = DotMap()
'PHENO_predicted.tsv.gz',
'PHENO_metagenome_out/pred_metagenome_unstrat.tsv.gz',
],
),
metacyc=dict(
name='MetaCyc',
title='MetaCyc',
relfp=[
'pathways_out/path_abun_predictions.tsv.gz',
'pathways_out/path_abun_unstrat.tsv.gz',
],
),
),
)
Var = DotMap(_config, _dynamic=False) # app-wide globals container
def reset_Var():
Var.clear()
Var.update(_config)
'''
TSVs are:
'pathways_out/path_abun_unstrat.tsv', # most important to workflow
'pathways_out/path_abun_unstrat_per_seq.tsv',
'pathways_out/path_abun_predictions.tsv',
'EC_predicted.tsv', # 100M
'KO_predicted.tsv', # 358M (Ginormo)
def __init__(self, melodic_model, timbre_model, hparams):
if hparams is None:
hparams = DotMap(get_default_hparams())
self.hparams = hparams
self.melodic_model: Model = melodic_model
self.timbre_model: Model = timbre_model
def parse_xml_by_address(cls,
numerero_map,
prov_name,
city_name,
tv,
nv,
num,
pictures=False):
"""
Parses an XML and crates a CadasterEntry object
:param numerero_map: DotMap obtained from a previous call with information about the address to parse
:param prov_name: Province Name
:param city_name: City Name
:param tv: Kind of way (Tipo de Via) - CL (calle), AV (Avenida) ...
:param nv: Street name (Nombre de via)
:param num: Street number (Numero de via)
:param pictures: True if we want to scrap also house plan pictures. False otherwise.
:return: List of CadasterEntry objects
"""
results = []
if numerero_map.consulta_numerero.lerr.err.cod != DotMap():
return results
numps = numerero_map.consulta_numerero.numerero.nump
if not isinstance(numps, list):
numps = [numps]
for nump in numps:
if nump.num.pnp == DotMap():
continue
num = nump.num.pnp
if nump.pc == DotMap():
continue
if nump.pc.pc1 == DotMap() or nump.pc.pc2 == DotMap():
continue
cadaster_num = nump.pc.pc1 + nump.pc.pc2
coords_map = ScrapperXML.get_coords_from_cadaster(
prov_name, city_name, cadaster_num)
lon = coords_map.consulta_coordenadas.coordenadas.coord.geo.xcen
if lon == DotMap():
lon = None
lat = coords_map.consulta_coordenadas.coordenadas.coord.geo.ycen
if lat == DotMap():
lat = None
''' Adding to tracking file'''
logger.info('{},{}'.format(lon, lat))
entry_map = ScrapperXML.get_cadaster_entries_by_address(
prov_name, city_name, tv, nv, num)
picture = None
if entry_map.consulta_dnp.bico != DotMap():
prov_num = entry_map.consulta_dnp.bico.bi.dt.loine.cp
city_num = entry_map.consulta_dnp.bico.bi.dt.loine.cm
if pictures and prov_num != DotMap() and city_num != DotMap():
picture = Scrapper.scrap_site_picture(
prov_num, city_num, cadaster_num)
# Parcela
cadaster_entry = CadasterEntryXML(entry_map, lon, lat, picture)
results.append(cadaster_entry)
cadaster_entry.to_elasticsearch()
elif entry_map.consulta_dnp.lrcdnp.rcdnp != DotMap():
# Multiparcela
for site in entry_map.consulta_dnp.lrcdnp.rcdnp:
site_map = DotMap(site)
if site_map.rc == DotMap():
continue
# Multiparcela
parcelas = entry_map.consulta_dnp.lrcdnp.rcdnp
if not isinstance(parcelas, list):
parcelas = [parcelas]
for parcela in parcelas:
cadaster = parcela.rc.pc1 if parcela.rc.pc1 != DotMap(
) else ''
cadaster += parcela.rc.pc2 if parcela.rc.pc2 != DotMap(
) else ''
cadaster += parcela.rc.car if parcela.rc.car != DotMap(
) else ''
cadaster += parcela.rc.cc1 if parcela.rc.cc1 != DotMap(
) else ''
cadaster += parcela.rc.cc2 if parcela.rc.cc2 != DotMap(
) else ''
prov_num = parcela.dt.loine.cp
city_num = parcela.dt.cmc
if pictures and prov_num != DotMap(
) and city_num != DotMap():
picture = Scrapper.scrap_site_picture(
prov_num, city_num, cadaster)
try:
# Try to get info by complete cadaster num
sub_entry = ScrapperXML.get_cadaster_entries_by_cadaster(
prov_name, city_name, cadaster)
except:
# Cadastro did not return anything by cadaster entry (error? bug?)
# Try to get it by complete address
loint = parcela.dt.locs.lous.lourb.loint
if loint is DotMap():
continue
bl = loint.bl
if bl == DotMap():
bl = ''
es = loint.es
if es == DotMap():
es = ''
pt = loint.pt
if es == DotMap():
pt = ''
pu = loint.pu
if es == DotMap():
pu = ''
sub_entry = ScrapperXML.get_cadaster_entries_by_address(
prov_name, city_name, tv, nv, num, bl, es, pt,
pu)
cadaster_entry = CadasterEntryXML(
sub_entry, lon, lat, picture)
cadaster_entry.to_elasticsearch()
results.append(cadaster_entry)
return results
def train(self, *, device, n_total_steps, n_warm_up_steps, record_freq,
record, model_training_freq, policy_training_freq, eval_freq,
task_name, model_training_n_batches, train_reward):
""" A single step of interaction with the environment. """
self._setup_if_new()
ex.step_i += 1
behavioral_agent = self.random_agent if ex.step_i <= n_warm_up_steps else self.agent
with torch.no_grad():
action = behavioral_agent.get_action(self.env_loop.state,
deterministic=False).to('cpu')
prev_state = self.env_loop.state.clone().to(device)
if record and (ex.step_i == 1 or ex.step_i % record_freq == 0):
self.env_loop.record_next_episode()
state, next_state, done = self.env_loop.step(
to_np(action), video_file_suffix=ex.step_i)
reward = self.exploitation_task(state, action, next_state).item()
self.buffer.add(state, action, next_state,
torch.from_numpy(np.array([[reward]], dtype=np.float)))
self.stats.add(state, action, next_state, done)
if done:
log_last_episode(self.stats)
tasks_rewards = {
f'{task_name}': self.stats.get_recent_reward(task_name)
for task_name in self.eval_tasks
}
step_stats = dict(
step=ex.step_i,
done=done,
action_abs_mean=action.abs().mean().item(),
reward=self.exploitation_task(state, action, next_state).item(),
action_value=self.agent.get_action_value(prev_state,
action).item(),
)
ex.mlog.add_scalars('main_loop', {**step_stats, **tasks_rewards})
# (Re)train the model on the current buffer
if model_training_freq is not None and model_training_n_batches > 0 and ex.step_i % model_training_freq == 0:
self.model.setup_normalizer(self.buffer.normalizer)
self.reward_model.setup_normalizer(self.buffer.normalizer)
timed(train_model)(self.model,
self.model_optimizer,
self.buffer,
mode='train')
if train_reward:
task = self.exploitation_task
timed(train_reward_model)(self.reward_model,
self.reward_model_optimizer,
self.buffer,
mode='train',
task=task)
# (Re)train the policy using current buffer and model
if ex.step_i >= n_warm_up_steps and ex.step_i % policy_training_freq == 0:
task = self.exploitation_task
self.agent.setup_normalizer(self.buffer.normalizer)
self.agent = timed(train_agent)(self.agent,
self.model,
self.reward_model,
self.buffer,
task=task,
task_name=task_name,
is_done=self.is_done,
mode='train',
context_i={})
# Evaluate the agent
if eval_freq is not None and ex.step_i % eval_freq == 0:
self.last_avg_eval_score = evaluate_on_tasks(agent=self.agent,
model=self.model,
buffer=self.buffer,
task_name=task_name,
context='eval')
experiment_finished = ex.step_i >= n_total_steps
return DotMap(
done=experiment_finished,
avg_eval_score=self.last_avg_eval_score,
action_abs_mean=action.abs().mean().item(
), # This is just for regression tests
step_i=ex.step_i)
def sample_to_features_squad(
sample, tokenizer, max_seq_len, doc_stride, max_query_length, tasks,
):
sample.clear_text = DotMap(sample.clear_text, _dynamic=False)
is_training = sample.clear_text.is_training
unique_id = 1000000000
features = []
query_tokens = tokenizer.tokenize(sample.clear_text.question_text)
if len(query_tokens) > max_query_length:
query_tokens = query_tokens[0:max_query_length]
tok_to_orig_index = []
orig_to_tok_index = []
all_doc_tokens = []
for (i, token) in enumerate(sample.clear_text.doc_tokens):
orig_to_tok_index.append(len(all_doc_tokens))
sub_tokens = tokenizer.tokenize(token)
for sub_token in sub_tokens:
tok_to_orig_index.append(i)
all_doc_tokens.append(sub_token)
tok_start_position = None
tok_end_position = None
if is_training and sample.clear_text.is_impossible:
tok_start_position = -1
tok_end_position = -1
if is_training and not sample.clear_text.is_impossible:
tok_start_position = orig_to_tok_index[sample.clear_text.start_position]
if sample.clear_text.end_position < len(sample.clear_text.doc_tokens) - 1:
tok_end_position = orig_to_tok_index[sample.clear_text.end_position + 1] - 1
else:
tok_end_position = len(all_doc_tokens) - 1
(tok_start_position, tok_end_position) = _SQUAD_improve_answer_span(
all_doc_tokens,
tok_start_position,
tok_end_position,
tokenizer,
sample.clear_text.orig_answer_text,
)
# The -3 accounts for [CLS], [SEP] and [SEP]
max_tokens_for_doc = max_seq_len - len(query_tokens) - 3
# We can have documents that are longer than the maximum sequence length.
# To deal with this we do a sliding window approach, where we take chunks
# of the up to our max length with a stride of `doc_stride`.
_DocSpan = collections.namedtuple( # pylint: disable=invalid-name
"DocSpan", ["start", "length"]
)
doc_spans = []
start_offset = 0
while start_offset < len(all_doc_tokens):
length = len(all_doc_tokens) - start_offset
if length > max_tokens_for_doc:
length = max_tokens_for_doc
doc_spans.append(_DocSpan(start=start_offset, length=length))
if start_offset + length == len(all_doc_tokens):
break
start_offset += min(length, doc_stride)
for (doc_span_index, doc_span) in enumerate(doc_spans):
tokens = []
token_to_orig_map = {}
token_is_max_context = {}
segment_ids = []
tokens.append("[CLS]")
segment_ids.append(0)
for token in query_tokens:
tokens.append(token)
segment_ids.append(0)
tokens.append("[SEP]")
segment_ids.append(0)
for i in range(doc_span.length):
split_token_index = doc_span.start + i
token_to_orig_map[len(tokens)] = tok_to_orig_index[split_token_index]
is_max_context = _SQUAD_check_is_max_context(
doc_spans, doc_span_index, split_token_index
)
token_is_max_context[len(tokens)] = is_max_context
tokens.append(all_doc_tokens[split_token_index])
segment_ids.append(1)
tokens.append("[SEP]")
segment_ids.append(1)
input_ids = tokenizer.convert_tokens_to_ids(tokens)
# The mask has 1 for real tokens and 0 for padding tokens. Only real
# tokens are attended to.
padding_mask = [1] * len(input_ids)
# Zero-pad up to the sequence length.
while len(input_ids) < max_seq_len:
input_ids.append(0)
padding_mask.append(0)
segment_ids.append(0)
assert len(input_ids) == max_seq_len
assert len(padding_mask) == max_seq_len
assert len(segment_ids) == max_seq_len
start_position = 0
end_position = 0
if is_training and not sample.clear_text.is_impossible:
# For training, if our document chunk does not contain an annotation
# we throw it out, since there is nothing to predict.
doc_start = doc_span.start
doc_end = doc_span.start + doc_span.length - 1
out_of_span = False
if not (tok_start_position >= doc_start and tok_end_position <= doc_end):
out_of_span = True
if out_of_span:
start_position = 0
end_position = 0
else:
doc_offset = len(query_tokens) + 2
start_position = tok_start_position - doc_start + doc_offset
end_position = tok_end_position - doc_start + doc_offset
if is_training and sample.clear_text.is_impossible:
start_position = 0
end_position = 0
inp_feat = {}
inp_feat["input_ids"] = input_ids
inp_feat["padding_mask"] = padding_mask # attention_mask
inp_feat["segment_ids"] = segment_ids # token_type_ids
inp_feat["start_position"] = start_position
inp_feat["end_position"] = end_position
inp_feat["is_impossible"] = sample.clear_text.is_impossible
features.append(inp_feat)
unique_id += 1
return features
from helpers.registry.ChainRegistry import ChainRegistry
from dotmap import DotMap
from helpers.registry.WhaleRegistryAction import WhaleRegistryAction
gnosis_safe_registry = DotMap(addresses=DotMap(
proxyFactory="0x76E2cFc1F5Fa8F6a5b3fC4c8F4788F0116861F9B",
masterCopy="0x34CfAC646f301356fAa8B21e94227e3583Fe3F5F",
), )
pancake_registry = DotMap(
cake="0x0e09fabb73bd3ade0a17ecc321fd13a19e81ce82",
symbol="Cake",
syrup="0x009cF7bC57584b7998236eff51b98A168DceA9B0",
masterChef="0x73feaa1eE314F8c655E354234017bE2193C9E24E",
factoryV2="0xBCfCcbde45cE874adCB698cC183deBcF17952812",
routerV2="0x05fF2B0DB69458A0750badebc4f9e13aDd608C7F",
smartChefs="0xe4dD0C50fb314A8B2e84D211546F5B57eDd7c2b9",
chefPairs=DotMap(
bnbBtcb="0x7561EEe90e24F3b348E1087A005F78B4c8453524",
bBadgerBtcb="0x10f461ceac7a17f59e249954db0784d42eff5db5",
bDiggBtcb="0xE1E33459505bB3763843a426F7Fd9933418184ae",
),
chefPids=DotMap(
bnbBtcb=15,
bBadgerBtcb=0,
bDiggBtcb=104,
),
)
multicall_registry = DotMap(
multicall="0xE1dDc30f691CA671518090931e3bFC1184BFa4Aa", )
sushi_registry = DotMap(
sushiToken="",
def _equal_with(_large_rect, _rect2):
_large_rect, rect2 = DotMap(_large_rect), DotMap(_rect2)
if (_large_rect.x == rect2.x):
return True
else:
return False
cosmos_container_client = cosmos_database_client.get_container_client(
getenv("AZURE_COSMOS_CONTAINER", default="images")
)
while True:
print("Receiving messages from queue")
batches = queue_client.receive_messages(
messages_per_page=getenv("AZURE_STORAGE_QUEUE_MSG_COUNT", default="10")
)
for batch in batches.by_page():
for message in batch:
message_json = DotMap(json.loads(message.content))
print("Message received: " + message_json.id)
print("Extracting text from image")
fr_poller = fr_client.begin_recognize_content_from_url(message_json.url)
fr_result = fr_poller.result()
message_json.text = " ".join([line.text for page in fr_result for line in page.lines])
print("Image text: " + message_json.text)
if message_json.text:
print("Analyzing text sentiment")
ta_response = ta_client.analyze_sentiment([message_json.text])
def forward(
self,
input,
seq_lens,
span_token_ids,
target,
target_lens,
definition=None,
definition_lens=None,
classification_labels=None,
sentence_mask=None,
):
batch_size, tgt_len = target.shape
# (batch_size,seq_len,hidden_size), (batch_size,hidden_size), (num_layers,batch_size,seq_len,hidden_size)
last_hidden_layer, sentence_representation, all_hidden_layers = self.encoder(
input, attention_mask=sequence_mask(seq_lens), token_type_ids=sentence_mask
)
cosine_loss = None
loss = None
KLD = None
fake_loss_kl = None
if self.aggregator == "cls":
cls_hidden = last_hidden_layer[:, 0, :].squeeze(1)
span_representation = self.context_feed_forward(cls_hidden)
hidden_states = last_hidden_layer
else:
span_ids = self._id_extractor(
tokens=span_token_ids, batch=input, lens=seq_lens
)
span_representation, hidden_states = self._span_aggregator(
all_hidden_layers if self.scalar_mix is not None else last_hidden_layer,
sequence_mask(seq_lens),
span_ids,
)
span_representation = self.context_feed_forward(span_representation)
if self.variational:
(
definition_last_hidden_layer,
_,
definition_all_hidden_layers,
) = self.definition_encoder(
definition, attention_mask=sequence_mask(definition_lens)
)
definition_representation = self.definition_feed_forward(
definition_last_hidden_layer[:, 0]
)
post_project = self.w_z_post(
torch.cat([span_representation, definition_representation], -1)
)
prior_project = self.w_z_prior(span_representation)
mu = self.mean_layer(post_project)
logvar = self.logvar_layer(post_project)
mu_prime = self.mean_prime_layer(prior_project)
logvar_prime = self.logvar_prime_layer(prior_project)
z = mu + torch.exp(logvar * 0.5) * torch.randn_like(logvar)
span_representation = self.z_project(z)
KLD = kl_divergence(
Normal(mu, torch.exp(logvar * 0.5)),
Normal(mu_prime, torch.exp(logvar_prime * 0.5)),
)
kl_mask = (KLD > (self.target_kl / self.latent_size)).float()
fake_loss_kl = (kl_mask * KLD).sum(dim=1)
predictions, logits = self.decoder(
target, target_lens, span_representation, hidden_states, seq_lens,
)
if self.variational:
loss = (
F.cross_entropy(
logits.view(batch_size * (tgt_len - 1), -1),
target[:, 1:].contiguous().view(-1),
ignore_index=self.embeddings.tgt.padding_idx,
reduction="none",
)
.view(batch_size, tgt_len - 1)
.sum(1)
)
perplexity = F.cross_entropy(
logits.view(batch_size * (tgt_len - 1), -1),
target[:, 1:].contiguous().view(-1),
ignore_index=self.embeddings.tgt.padding_idx,
reduction="mean",
).exp()
else:
loss = F.cross_entropy(
logits.view(batch_size * (tgt_len - 1), -1),
target[:, 1:].contiguous().view(-1),
ignore_index=self.embeddings.tgt.padding_idx,
)
perplexity = loss.exp()
return DotMap(
{
"predictions": predictions,
"logits": logits,
"loss": loss,
"perplexity": perplexity,
"fake_kl": fake_loss_kl,
"kl": KLD,
"cosine_loss": cosine_loss,
}
)
'red_high': HSVColorRange([165, 60, 60], [179, 255, 255]),
'green': HSVColorRange([15, 20, 30], [60, 225, 225]),
},
'processing': {
'rectify_shape': (1920, int(1920 * 0.7)),
'gaussian_blur': (3, 3),
'img_threshold': (100, 255),
'area_threshold': 5000,
'contours_size_threshold': 100,
'polygon_tolerance': 1,
'line_width': 1,
'out_size_inches': (7.87402, 5.51181), # TODO: fix scaling
},
'pdf_output': {
'plate_width_cm': 80,
'plate_height_cm': 50,
'draw_area_width_cm': 20,
'draw_area_height_cm': 14
},
'data': {
'overwrite_output': False,
'output_location': 'desktop' # can be either 'desktop' or 'repository'
}
}
# DotMap: This enables config access like...
# from planvec.config import CONFIG
# x_blur, y_blur = CONFIG.processing.gaussian_blur
# TODO: This is not really satisfactory
planvec_config = DotMap(config_dict)
def __init__(
self,
encoder,
encoder_pretrained,
encoder_frozen,
decoder_hidden,
embeddings,
max_layer=12,
src_pad_idx=0,
encoder_hidden=None,
variational=None,
latent_size=None,
scalar_mix=False,
aggregator="mean",
teacher_forcing_p=0.3,
classification=None,
attentional=False,
definition_encoder=None,
word_dropout_p=None,
decoder_num_layers=None,
):
super(DefinitionProbing, self).__init__()
self.embeddings = embeddings
self.variational = variational
self.encoder_hidden = encoder_hidden
self.decoder_hidden = decoder_hidden
self.decoder_num_layers = decoder_num_layers
self.encoder = encoder
self.latent_size = latent_size
self.src_pad_idx = src_pad_idx
if encoder_pretrained:
self.encoder_hidden = self.encoder.config.hidden_size
if encoder_frozen:
for param in self.encoder.parameters():
param.requires_grad = False
self.max_layer = max_layer
self.aggregator = aggregator
if self.aggregator == "span":
self.span_extractor = SelfAttentiveSpanExtractor(self.encoder_hidden)
self.context_feed_forward = nn.Linear(self.encoder_hidden, self.encoder_hidden)
self.scalar_mix = None
if scalar_mix:
self.scalar_mix = ScalarMix(self.max_layer + 1)
self.global_scorer = GNMTGlobalScorer(
alpha=2, beta=None, length_penalty="avg", coverage_penalty=None
)
self.decoder = LSTM_Decoder(
embeddings.tgt,
hidden=self.decoder_hidden,
encoder_hidden=self.encoder_hidden,
num_layers=self.decoder_num_layers,
word_dropout=word_dropout_p,
teacher_forcing_p=teacher_forcing_p,
attention="general" if attentional else None,
dropout=DotMap({"input": 0.5, "output": 0.5}),
decoder="VDM" if self.variational else "LSTM",
variational=self.variational,
latent_size=self.latent_size,
)
self.target_kl = 1.0
if self.variational:
self.definition_encoder = definition_encoder
self.definition_feed_forward = nn.Linear(
self.encoder_hidden, self.encoder_hidden
)
self.mean_layer = nn.Linear(self.latent_size, self.latent_size)
self.logvar_layer = nn.Linear(self.latent_size, self.latent_size)
self.w_z_post = nn.Sequential(
nn.Linear(self.encoder_hidden * 2, self.latent_size), nn.Tanh()
)
self.mean_prime_layer = nn.Linear(self.latent_size, self.latent_size)
self.logvar_prime_layer = nn.Linear(self.latent_size, self.latent_size)
self.w_z_prior = nn.Sequential(
nn.Linear(self.encoder_hidden, self.latent_size), nn.Tanh()
)
self.z_project = nn.Sequential(
nn.Linear(self.latent_size, self.decoder_hidden), nn.Tanh()
)
def nested_mapping_to_line(nested_mapping: typing.Mapping,
keys: typing.Sequence[typing.Text]) -> typing.Text:
dotted = DotMap(nested_mapping)
return ",".join([str(recursive_getattr(dotted, key)) for key in keys])
