def parse_ncml_file(ncml, id):
# Based on: https://github.com/axiom-data-science/pyncml
if isinstance(ncml, str) and os.path.isfile(ncml):
root = etree.parse(ncml).getroot()
elif isinstance(ncml, str):
root = etree.fromstring(ncml)
elif etree.iselement(ncml):
root = ncml
else:
raise ValueError("Could not parse ncml. \
Did you pass in a valid file path, xml string, or etree Element object?"
)
global_attributes = {}
dimensions = {}
variables = {}
thredds_meta = {}
thredds_xml_meta = root.find('.//{%s}group[@name="THREDDSMetadata"]' %
ncml_namespace)
if thredds_xml_meta is not None:
id = thredds_xml_meta.find('.//{%s}attribute[@name="id"]' %
ncml_namespace).get("value")
od_service = thredds_xml_meta.find(
'.//{%s}attribute[@name="opendap_service"]' %
ncml_namespace).get("value")
thredds_meta = {"opendap_url": od_service, "id": id}
# Variables
for v in root.findall('{%s}variable' % ncml_namespace):
var_name = v.attrib.get("name")
attributes = {}
for a in v.findall('{%s}attribute' % ncml_namespace):
name, value = process_attribute_tag(attributes, a)
attributes[name] = {"value": value}
shape = v.attrib.get("shape").split(" ")
variables[var_name] = DotDict({
"type": v.attrib.get("type"),
"shape": shape,
"attributes": attributes
})
# Global attributes
for a in root.findall('{%s}attribute' % ncml_namespace):
name, value = process_attribute_tag(global_attributes, a)
global_attributes[name] = {"value": value}
# Dimensions
for d in root.findall('{%s}dimension' % ncml_namespace):
dim_name = d.attrib.get('name')
dim_length = d.attrib.get("length")
dimensions[dim_name] = dim_length
"""return DotDict(
{"variables": DotDict(variables), "dimensions": DotDict(dimensions), "attributes": DotDict(global_attributes),
"thredds_meta": DotDict(thredds_meta)})
"""
meta = DatasetMeta(global_attributes, dimensions, variables)
return DatasetInfo(id, meta)
def parse_catalog_ref(element: etree.ElementTree):
xlink_ns = element.nsmap.get("xlink")
href = element.get("{%s}href" % xlink_ns)
title = element.get("{%s}title" % xlink_ns)
id = element.get("ID")
if id is None:
id = title
return DotDict(href=href, title=title, id=id)
def get_args(arguments: Optional[List[str]] = None) -> DotDict:
"Processes command line arguments and configuration."
if arguments is None:
arguments = sys.argv[1:]
possible_models = list(MODELS.keys())
parser = argparse.ArgumentParser(description="Train a model.")
parser.add_argument('--config',
type=str,
choices=['small', 'large'],
default='small',
help="Configuration to use, small or large.")
parser.add_argument('--model',
type=str,
default='zero-trian',
choices=possible_models,
help="Model to run. Use --list-models to get a list of"
" possiblities.")
parser.add_argument('--list-models',
action='store_true',
help="List the possible models to use.")
parser.add_argument('--embedding',
type=str,
default='glove',
choices=['glove', 'bert', 'xlnet'],
help="Embedding to use, GloVe, BERT or XLNet")
parser.add_argument('--cuda',
type=str,
default="0",
help="GPU(s) to use. If multiple, separated by "
"comma. If single, just use the gpu number. If "
"CPU is desired, use -1. Examples: "
"--gpu 0,1,2 --gpu 0.")
parser.add_argument('--name',
type=str,
default=None,
help="Name for this model.")
parser.add_argument('--encoder',
type=str,
default='lstm',
choices=['lstm', 'transformer', 'gru'],
help="Encoder type, one of lstm, gru or transformer")
parser.add_argument('--transformer',
type=str,
default='custom',
choices=['allen', 'custom'],
help="If encoder is transformer, choose which one to "
"use, allen or custom.")
res = parser.parse_args(arguments)
args = DotDict()
if res.list_models:
list_models(possible_models)
exit()
config = res.config
args.MODEL = res.model
args.NER_EMBEDDING_DIM = 8
args.REL_EMBEDDING_DIM = 10
args.POS_EMBEDDING_DIM = 12
args.HANDCRAFTED_DIM = 7
args.EMBEDDING_TYPE = res.embedding
args.CUDA_DEVICE = parse_cuda(res.cuda)
args.MODEL_NAME = res.name
data_folder = Path('data')
# Proper configuration path for the External folder. The data one is
# going to be part of the repo, so this is fine for now, but External isn't
# always going to be.
external_folder = Path('..', 'External')
xlnet_use_window = True
if config == 'large':
# Path to our dataset
args.TRAIN_DATA_PATH = data_folder / 'mctrain-data.json'
args.VAL_DATA_PATH = data_folder / 'mcdev-data.json'
args.TEST_DATA_PATH = data_folder / 'mctest-data.json'
# Path to our embeddings
args.GLOVE_PATH = external_folder / 'glove.840B.300d.txt'
# Size of our embeddings
args.GLOVE_EMBEDDING_DIM = 300
# Size of our hidden layers (for each encoder)
args.HIDDEN_DIM = 100
args.TRANSFORMER_DIM = 512
# Size of minibatch
args.BATCH_SIZE = 24
# Number of epochs to train model
args.NUM_EPOCHS = 30
# Size of the input window for XLNet
if xlnet_use_window:
args.xlnet_window_size = 512
else:
args.xlnet_window_size = None
elif config == 'small':
# Path to our dataset
args.TRAIN_DATA_PATH = data_folder / 'small-train.json'
args.VAL_DATA_PATH = data_folder / 'small-dev.json'
args.TEST_DATA_PATH = data_folder / 'small-test.json'
# Path to our embeddings
args.GLOVE_PATH = external_folder / 'glove.6B.50d.txt'
# Size of our embeddings
args.GLOVE_EMBEDDING_DIM = 50
# Size of our hidden layers (for each encoder)
args.HIDDEN_DIM = 50
args.TRANSFORMER_DIM = 128
# Size of minibatch
args.BATCH_SIZE = 2
# Number of epochs to train model
args.NUM_EPOCHS = 1
# Size of the input window for XLNet
if xlnet_use_window:
args.xlnet_window_size = 128
else:
args.xlnet_window_size = None
args.BERT_PATH = external_folder / 'bert-base-uncased.tar.gz'
args.CONCEPTNET_PATH = external_folder / 'conceptnet.csv'
args.xlnet_vocab_path = data_folder / 'xlnet-base-cased-spiece.model'
args.xlnet_config_path = data_folder / 'xlnet-base-cased-config.json'
args.xlnet_model_path = external_folder / \
'xlnet-base-cased-pytorch_model.bin'
# Path to save the Model and Vocabulary
args.SAVE_FOLDER = Path('experiments')
args.SAVE_PATH = args.SAVE_FOLDER / \
get_experiment_name(args.MODEL, config,
args.EMBEDDING_TYPE, args.MODEL_NAME)
print('Save path', args.SAVE_PATH)
def preprocessed_name(split_type: str) -> str:
"Gets the pre-processed pickle filename from the configuration."
name = get_preprocessed_name(split_type, args.MODEL, config,
args.EMBEDDING_TYPE)
return cast(str, name)
# Path to save pre-processed input
args.TRAIN_PREPROCESSED_NAME = preprocessed_name('train')
args.VAL_PREPROCESSED_NAME = preprocessed_name('val')
args.TEST_PREPROCESSED_NAME = preprocessed_name('test')
args.TRAIN_PREPROCESSED_PATH = (external_folder /
args.TRAIN_PREPROCESSED_NAME)
args.VAL_PREPROCESSED_PATH = external_folder / args.VAL_PREPROCESSED_NAME
args.TEST_PREPROCESSED_PATH = external_folder / args.TEST_PREPROCESSED_NAME
print('Pre-processed data path:', args.TRAIN_PREPROCESSED_PATH)
# Random seed (for reproducibility)
args.RANDOM_SEED = 1234
# Model Configuration
# Use LSTM, GRU or Transformer
args.ENCODER_TYPE = res.encoder
args.WHICH_TRANSFORMER = res.transformer
args.BIDIRECTIONAL = True
args.RNN_LAYERS = 1
args.RNN_DROPOUT = 0.5 if args.ENCODER_TYPE != 'transformer' else 0
args.EMBEDDDING_DROPOUT = 0.5 if args.EMBEDDING_TYPE == 'glove' else 0
# Number of passes the DMN will make over the input
args.DMN_PASSES = 2
# Whether to fine tune the embeddings (specifically BERT and XLNet)
args.finetune_embeddings = False
return args
from util import DotDict
cloaks = DotDict({
'emptycloak': DotDict(name="Empty Cloak", raw="emptycloak", value=0, shop=False, weight=0, protection=0, sort=0),
'leathercloak': DotDict(name="Leather Cloak", raw="leathercloak", value=100, shop=True, weight=1, protection=1, sort=1),
'battlecloak': DotDict(name="Battle Cloak", raw="battlecloak", value=1000, shop=True, weight=2, protection=2, sort=2),
'covercloak': DotDict(name="Cover Cloak", raw="covercloak", value=100, shop=True, weight=1, stealth=1, sort=3),
'covercloak+': DotDict(name="Cover Cloak +", raw="covercloak+", value=110, shop=False, weight=2, protection=1, stealth=1, sort=4),
'darkcloak': DotDict(name="Dark Cloak", raw="darkcloak", value=300, shop=True, weight=1, stealth=2, sort=5),
'darkcloak+': DotDict(name="Dark Cloak +", raw="darkcloak+", value=330, shop=False, weight=2, protection=1, stealth=2, sort=6),
'disguisecloak': DotDict(name="Disguise Cloak", raw="disguisecloak", value=700, shop=True, weight=1, stealth=3, sort=7),
'disguisecloak+': DotDict(name="Disguise Cloak +", raw="disguisecloak+", value=770, shop=False, weight=2, protection=1, stealth=3, sort=8),
'concealcloak': DotDict(name="Conceal Cloak", raw="consealcloak", value=1500, shop=True, weight=1, stealth=4, sort=9),
'concealcloak+': DotDict(name="Conceal Cloak +", raw="consealcloak+", value=1650, shop=False, weight=2, protection=1, stealth=4, sort=10),
'nightcloak': DotDict(name="Night Cloak", raw="nightcloak", value=3100, shop=True, weight=1, stealth=5, sort=11),
'nightcloak+': DotDict(name="Night Cloak +", raw="nightcloak+", value=3410, shop=False, weight=2, protection=1, stealth=5, sort=12),
'stealthcloak': DotDict(name="Stealth Cloak", raw="stealthcloak", value=6300, shop=True, weight=1, stealth=6, sort=13),
'stealthcloak+': DotDict(name="Stealth Cloak +", raw="stealthcloak+", value=6930, shop=False, weight=2, protection=1, stealth=6, sort=14),
'phantomcloak': DotDict(name="Phantom Cloak", raw="phantomcloak", value=12700, shop=True, weight=1, stealth=7, sort=15),
'phantomcloak+': DotDict(name="Phantom Cloak +", raw="phantomcloak+", value=13970, shop=False, weight=2, protection=1, stealth=7, sort=16),
'invisibilitycloak': DotDict(name="Invisibility Cloak", raw="invisibilitycloak", value=25500, shop=True, weight=1, stealth=8, sort=17),
'invisibilitycloak+': DotDict(name="Invisibility Cloak +", raw="invisibilitycloak+", value=28050, shop=False, weight=2, protection=1, stealth=8, sort=18),
'silkcloak': DotDict(name="Silk Cloak", raw="silkcloak", value=2500, shop=True, weight=1, thief=1, sort=19),
'silkcloak+': DotDict(name="Silk Cloak +", raw="silkcloak+", value=2750, shop=False, weight=2, protection=1, thief=1, sort=20),
'thievescloak': DotDict(name="Thieves Cloak", raw="thievescloak", value=5000, shop=True, weight=1, thief=2, sort=21),
'thievescloak+': DotDict(name="Thieves Cloak +", raw="thievescloak+", value=5500, shop=False, weight=2, protection=1, thief=2, sort=22)
})
for cloak in cloaks.values():
from util import DotDict
# prijzen nog bepalen
boots = DotDict({
'emptyboots': DotDict(name="Empty Boots", raw="emptyboots", value=0, shop=False, weight=0, protection=0, sort=0),
'leatherboots': DotDict(name="Leather Boots", raw="leatherboots", value=100, shop=True, weight=1, protection=1, sort=1),
'bronzeboots': DotDict(name="Bronze Boots", raw="bronzeboots", value=200, shop=True, weight=2, protection=2, sort=2),
'ironboots': DotDict(name="Iron Boots", raw="ironboots", value=400, shop=True, weight=3, protection=3, sort=3),
'steelboots': DotDict(name="Steel Boots", raw="steelboots", value=800, shop=True, weight=4, protection=4, sort=4),
'silverboots': DotDict(name="Silver Boots", raw="silverboots", value=1600, shop=True, weight=5, protection=5, sort=5),
'titaniumboots': DotDict(name="Titanium Boots", raw="titaniumboots", value=3200, shop=False, weight=1, protection=5, sort=6),
'bootsofmotion': DotDict(name="Boots of Motion", raw="bootsofmotion", value=1000, shop=True, weight=2, protection=1, movepoints=1, sort=7),
'bootsofmotion+': DotDict(name="Boots of Motion +", raw="bootsofmotion+", value=1100, shop=False, weight=3, protection=2, movepoints=1, sort=8),
'bootsofspeed': DotDict(name="Boots of Speed", raw="bootsofspeed", value=2000, shop=True, weight=2, protection=1, movepoints=2, sort=9),
'bootsofspeed+': DotDict(name="Boots of Speed +", raw="bootsofspeed+", value=2200, shop=False, weight=3, protection=2, movepoints=2, sort=10),
'woodsmansboots': DotDict(name="Woodsman's Boots", raw="woodsmansboots", value=1000, shop=True, weight=2, protection=1, ranger=1, sort=11),
'woodsmansboots+': DotDict(name="Woodsman's Boots +", raw="woodsmansboots+", value=1100, shop=False, weight=3, protection=2, ranger=1, sort=12),
'silenceboots': DotDict(name="Silence Boots", raw="silenceboots", value=1000, shop=True, weight=2, protection=1, stealth=1, sort=13),
'silenceboots+': DotDict(name="Silence Boots +", raw="silenceboots+", value=1100, shop=False, weight=3, protection=2, stealth=1, sort=14)
})
for boot in boots.values():
if 'movepoints' not in boot:
boot.movepoints = None
if 'ranger' not in boot:
boot.ranger = None
if 'stealth' not in boot:
from util import DotDict
helmets = DotDict({
'emptyhelmet': DotDict(name="Empty Helmet", raw="emptyhelmet", value=0, shop=False, weight=0, protection=0, sort=0),
'leathercap': DotDict(name="Leather Cap", raw="leathercap", value=100, shop=True, weight=1, protection=1, sort=1),
'bronzehelmet': DotDict(name="Bronze Helmet", raw="bronzehelmet", value=1225, shop=True, weight=2, protection=2, sort=2),
'ironhelmet': DotDict(name="Iron Helmet", raw="ironhelmet", value=3600, shop=True, weight=3, protection=3, sort=3),
'steelhelmet': DotDict(name="Steel Helmet", raw="steelhelmet", value=7225, shop=True, weight=4, protection=4, sort=4),
'silverhelmet': DotDict(name="Silver Helmet", raw="silverhelmet", value=12100, shop=True, weight=5, protection=5, sort=5),
'titaniumhelmet': DotDict(name="Titanium Helmet", raw="titaniumhelmet", value=24300, shop=False, weight=1, protection=5, sort=6),
'helmofknowledge': DotDict(name="Helm of Knowledge", raw="helmofknowledge", value=5500, shop=True, weight=2, protection=1, intelligence=2, sort=7),
'helmofknowledge+': DotDict(name="Helm of Knowledge +", raw="helmofknowledge+", value=6050, shop=False, weight=3, protection=2, intelligence=2, sort=8),
'helmofwisdom': DotDict(name="Helm of Wisdom", raw="helmofwisdom", value=5500, shop=True, weight=2, protection=1, willpower=2, sort=9),
'helmofwisdom+': DotDict(name="Helm of Wisdom +", raw="helmofwisdom+", value=6050, shop=False, weight=3, protection=2, willpower=2, sort=10),
'helmofcharisma': DotDict(name="Helm of Charisma", raw="helmofcharisma", value=6600, shop=True, weight=2, protection=1, diplomat=1, sort=11),
'helmofcharisma+': DotDict(name="Helm of Charisma +", raw="helmofcharisma+", value=7260, shop=False, weight=3, protection=2, diplomat=1, sort=12),
'helmofinsight': DotDict(name="Helm of Insight", raw="helmofinsight", value=7700, shop=True, weight=2, protection=1, loremaster=1, sort=13),
'helmofinsight+': DotDict(name="Helm of Insight +", raw="helmofinsight+", value=8470, shop=False, weight=3, protection=2, loremaster=1, sort=14),
'helmofcognizance': DotDict(name="Helm of Cognizance", raw="helmofcognizance", value=9900, shop=True, weight=2, protection=1, scientist=1, sort=15),
'helmofcognizance+': DotDict(name="Helm of Cognizance +", raw="helmofcognizance+", value=10890, shop=False, weight=3, protection=2, scientist=1, sort=16),
'helmoftempests': DotDict(name="Helm of Tempests", raw="helmoftempests", value=8800, shop=True, weight=2, protection=1, warrior=1, sort=17),
'helmoftempests+': DotDict(name="Helm of Tempests +", raw="helmoftempests+", value=9680, shop=False, weight=3, protection=2, warrior=1, sort=18)
})
for helmet in helmets.values():
if 'intelligence' not in helmet:
helmet.intelligence = None
if 'willpower' not in helmet:
# value, base hit, damage, sortering
weapon_upgraded = {
"": (1.0, 0, 0, 10),
"+": (1.1, 5, 0, 20),
"++": (1.2, 5, 1, 30)
}
# hoogste damage mogelijk is 30: Silver/Titanium Maul ++
weapons = DotDict(
emptyweapon=DotDict(
name="Empty Weapon",
raw="emptyweapon",
value=0,
shop=False,
skill="Empty",
min_int=0,
min_str=0,
base_hit=0,
damage=0,
sort=0,
col=0,
row=0
))
for key_material, value_material in weapon_material.items():
for key_type, value_type in weapon_type.items():
for key_upgraded, value_upgraded in weapon_upgraded.items():
raw_key_name = (key_material + key_type + key_upgraded).strip().lower().replace(" ", "")
price = int((value_material[0] + value_type[0]) * (value_material[0] + value_type[0]) / 400)
"Heavy": (500, 3, 3, -2, 300)
}
# value, weight, protection, stealth, sortering
armor_upgraded = {
"": (1.0, 0, 0, 0, 10),
"+": (1.1, 0, 0, 1, 20),
"++": (1.2, -1, 0, 2, 30)
}
# hoogste protection mogelijk is 15: Heavy Silver/Titanium Armor /+/++
armors = DotDict(
emptyarmor=DotDict(
name="Empty Armor",
raw="emptyarmor",
value=0,
shop=False,
weight=0,
protection=0,
stealth=0,
sort=0
))
for key_material, value_material in armor_material.items():
for key_type, value_type in armor_type.items():
for key_upgraded, value_upgraded in armor_upgraded.items():
raw_key_name = (key_type + key_material + key_upgraded).strip().lower().replace(" ", "")
price = (value_material[0] + value_type[0]) * (value_material[0] + value_type[0]) / 400
armors[raw_key_name] = DotDict(
name=(key_type + " " + key_material + " " + key_upgraded).strip(),
def __init__(self, index):
self.meta_data = DotDict(index["meta"])
self.datasets = index["datasets"]
self.meta_variables = self.meta_data.variables
self._fill_variables_table()
self._fill_datasets_table()
# value, min str, protection, defense, dexterity, stealth
shield_upgraded = {
"": (1.0, 0, 0, 0, 0, 0, 10),
"+": (1.1, 0, 0, 0, 1, 2, 20),
"++": (1.2, 0, 0, 0, 2, 4, 30)
}
# de hoogste protection/defense mogelijk is 15/30: Large Silver/Titanium Scutum /+/++
shields = DotDict(
emptyshield=DotDict(
name="Empty Shield",
raw="emptyshield",
value=0,
shop=False,
min_str=0,
protection=0,
defense=0,
dexterity=0,
stealth=0,
sort=0,
col=0,
row=0
))
for key_material, value_material in shield_material.items():
for key_type, value_type in shield_type.items():
for key_upgraded, value_upgraded in shield_upgraded.items():
raw_key_name = (key_material + key_type + key_upgraded).strip().lower().replace(" ", "")
price = (value_material[0] + value_type[0]) * (value_material[0] + value_type[0]) / 900