Beispiel #1
0
def parse_ncml_file(ncml, id):
    # Based on: https://github.com/axiom-data-science/pyncml
    if isinstance(ncml, str) and os.path.isfile(ncml):
        root = etree.parse(ncml).getroot()
    elif isinstance(ncml, str):
        root = etree.fromstring(ncml)
    elif etree.iselement(ncml):
        root = ncml
    else:
        raise ValueError("Could not parse ncml. \
                         Did you pass in a valid file path, xml string, or etree Element object?"
                         )

    global_attributes = {}
    dimensions = {}
    variables = {}
    thredds_meta = {}
    thredds_xml_meta = root.find('.//{%s}group[@name="THREDDSMetadata"]' %
                                 ncml_namespace)
    if thredds_xml_meta is not None:
        id = thredds_xml_meta.find('.//{%s}attribute[@name="id"]' %
                                   ncml_namespace).get("value")
        od_service = thredds_xml_meta.find(
            './/{%s}attribute[@name="opendap_service"]' %
            ncml_namespace).get("value")
        thredds_meta = {"opendap_url": od_service, "id": id}

    # Variables
    for v in root.findall('{%s}variable' % ncml_namespace):

        var_name = v.attrib.get("name")
        attributes = {}

        for a in v.findall('{%s}attribute' % ncml_namespace):
            name, value = process_attribute_tag(attributes, a)
            attributes[name] = {"value": value}

        shape = v.attrib.get("shape").split(" ")
        variables[var_name] = DotDict({
            "type": v.attrib.get("type"),
            "shape": shape,
            "attributes": attributes
        })

        # Global attributes
    for a in root.findall('{%s}attribute' % ncml_namespace):
        name, value = process_attribute_tag(global_attributes, a)
        global_attributes[name] = {"value": value}

    # Dimensions
    for d in root.findall('{%s}dimension' % ncml_namespace):
        dim_name = d.attrib.get('name')
        dim_length = d.attrib.get("length")
        dimensions[dim_name] = dim_length
    """return DotDict(
        {"variables": DotDict(variables), "dimensions": DotDict(dimensions), "attributes": DotDict(global_attributes),
         "thredds_meta": DotDict(thredds_meta)})
    """
    meta = DatasetMeta(global_attributes, dimensions, variables)
    return DatasetInfo(id, meta)
Beispiel #2
0
def parse_catalog_ref(element: etree.ElementTree):
    xlink_ns = element.nsmap.get("xlink")
    href = element.get("{%s}href" % xlink_ns)
    title = element.get("{%s}title" % xlink_ns)
    id = element.get("ID")
    if id is None:
        id = title
    return DotDict(href=href, title=title, id=id)
Beispiel #3
0
def get_args(arguments: Optional[List[str]] = None) -> DotDict:
    "Processes command line arguments and configuration."
    if arguments is None:
        arguments = sys.argv[1:]

    possible_models = list(MODELS.keys())

    parser = argparse.ArgumentParser(description="Train a model.")
    parser.add_argument('--config',
                        type=str,
                        choices=['small', 'large'],
                        default='small',
                        help="Configuration to use, small or large.")
    parser.add_argument('--model',
                        type=str,
                        default='zero-trian',
                        choices=possible_models,
                        help="Model to run. Use --list-models to get a list of"
                        " possiblities.")
    parser.add_argument('--list-models',
                        action='store_true',
                        help="List the possible models to use.")
    parser.add_argument('--embedding',
                        type=str,
                        default='glove',
                        choices=['glove', 'bert', 'xlnet'],
                        help="Embedding to use, GloVe, BERT or XLNet")
    parser.add_argument('--cuda',
                        type=str,
                        default="0",
                        help="GPU(s) to use. If multiple, separated by "
                        "comma. If single, just use the gpu number. If "
                        "CPU is desired, use -1. Examples: "
                        "--gpu 0,1,2 --gpu 0.")
    parser.add_argument('--name',
                        type=str,
                        default=None,
                        help="Name for this model.")
    parser.add_argument('--encoder',
                        type=str,
                        default='lstm',
                        choices=['lstm', 'transformer', 'gru'],
                        help="Encoder type, one of lstm, gru or transformer")
    parser.add_argument('--transformer',
                        type=str,
                        default='custom',
                        choices=['allen', 'custom'],
                        help="If encoder is transformer, choose which one to "
                        "use, allen or custom.")

    res = parser.parse_args(arguments)
    args = DotDict()

    if res.list_models:
        list_models(possible_models)
        exit()

    config = res.config
    args.MODEL = res.model

    args.NER_EMBEDDING_DIM = 8
    args.REL_EMBEDDING_DIM = 10
    args.POS_EMBEDDING_DIM = 12
    args.HANDCRAFTED_DIM = 7
    args.EMBEDDING_TYPE = res.embedding

    args.CUDA_DEVICE = parse_cuda(res.cuda)
    args.MODEL_NAME = res.name

    data_folder = Path('data')
    # Proper configuration path for the External folder. The data one is
    # going to be part of the repo, so this is fine for now, but External isn't
    # always going to be.
    external_folder = Path('..', 'External')

    xlnet_use_window = True

    if config == 'large':
        # Path to our dataset
        args.TRAIN_DATA_PATH = data_folder / 'mctrain-data.json'
        args.VAL_DATA_PATH = data_folder / 'mcdev-data.json'
        args.TEST_DATA_PATH = data_folder / 'mctest-data.json'
        # Path to our embeddings
        args.GLOVE_PATH = external_folder / 'glove.840B.300d.txt'
        # Size of our embeddings
        args.GLOVE_EMBEDDING_DIM = 300
        # Size of our hidden layers (for each encoder)
        args.HIDDEN_DIM = 100
        args.TRANSFORMER_DIM = 512
        # Size of minibatch
        args.BATCH_SIZE = 24
        # Number of epochs to train model
        args.NUM_EPOCHS = 30
        # Size of the input window for XLNet
        if xlnet_use_window:
            args.xlnet_window_size = 512
        else:
            args.xlnet_window_size = None
    elif config == 'small':
        # Path to our dataset
        args.TRAIN_DATA_PATH = data_folder / 'small-train.json'
        args.VAL_DATA_PATH = data_folder / 'small-dev.json'
        args.TEST_DATA_PATH = data_folder / 'small-test.json'
        # Path to our embeddings
        args.GLOVE_PATH = external_folder / 'glove.6B.50d.txt'
        # Size of our embeddings
        args.GLOVE_EMBEDDING_DIM = 50
        # Size of our hidden layers (for each encoder)
        args.HIDDEN_DIM = 50
        args.TRANSFORMER_DIM = 128
        # Size of minibatch
        args.BATCH_SIZE = 2
        # Number of epochs to train model
        args.NUM_EPOCHS = 1
        # Size of the input window for XLNet
        if xlnet_use_window:
            args.xlnet_window_size = 128
        else:
            args.xlnet_window_size = None

    args.BERT_PATH = external_folder / 'bert-base-uncased.tar.gz'
    args.CONCEPTNET_PATH = external_folder / 'conceptnet.csv'
    args.xlnet_vocab_path = data_folder / 'xlnet-base-cased-spiece.model'
    args.xlnet_config_path = data_folder / 'xlnet-base-cased-config.json'
    args.xlnet_model_path = external_folder / \
        'xlnet-base-cased-pytorch_model.bin'

    # Path to save the Model and Vocabulary
    args.SAVE_FOLDER = Path('experiments')
    args.SAVE_PATH = args.SAVE_FOLDER / \
        get_experiment_name(args.MODEL, config,
                            args.EMBEDDING_TYPE, args.MODEL_NAME)
    print('Save path', args.SAVE_PATH)

    def preprocessed_name(split_type: str) -> str:
        "Gets the pre-processed pickle filename from the configuration."
        name = get_preprocessed_name(split_type, args.MODEL, config,
                                     args.EMBEDDING_TYPE)
        return cast(str, name)

    # Path to save pre-processed input
    args.TRAIN_PREPROCESSED_NAME = preprocessed_name('train')
    args.VAL_PREPROCESSED_NAME = preprocessed_name('val')
    args.TEST_PREPROCESSED_NAME = preprocessed_name('test')

    args.TRAIN_PREPROCESSED_PATH = (external_folder /
                                    args.TRAIN_PREPROCESSED_NAME)
    args.VAL_PREPROCESSED_PATH = external_folder / args.VAL_PREPROCESSED_NAME
    args.TEST_PREPROCESSED_PATH = external_folder / args.TEST_PREPROCESSED_NAME
    print('Pre-processed data path:', args.TRAIN_PREPROCESSED_PATH)

    # Random seed (for reproducibility)
    args.RANDOM_SEED = 1234

    # Model Configuration
    # Use LSTM, GRU or Transformer
    args.ENCODER_TYPE = res.encoder
    args.WHICH_TRANSFORMER = res.transformer
    args.BIDIRECTIONAL = True
    args.RNN_LAYERS = 1
    args.RNN_DROPOUT = 0.5 if args.ENCODER_TYPE != 'transformer' else 0
    args.EMBEDDDING_DROPOUT = 0.5 if args.EMBEDDING_TYPE == 'glove' else 0

    # Number of passes the DMN will make over the input
    args.DMN_PASSES = 2

    # Whether to fine tune the embeddings (specifically BERT and XLNet)
    args.finetune_embeddings = False

    return args
Beispiel #4
0
from util import DotDict

cloaks = DotDict({
    'emptycloak': DotDict(name="Empty Cloak",                  raw="emptycloak",         value=0,     shop=False, weight=0, protection=0,  sort=0),

    'leathercloak': DotDict(name="Leather Cloak",              raw="leathercloak",       value=100,   shop=True,  weight=1, protection=1,  sort=1),
    'battlecloak': DotDict(name="Battle Cloak",                raw="battlecloak",        value=1000,  shop=True,  weight=2, protection=2,  sort=2),

    'covercloak': DotDict(name="Cover Cloak",                  raw="covercloak",         value=100,   shop=True,  weight=1,                stealth=1, sort=3),
    'covercloak+': DotDict(name="Cover Cloak +",               raw="covercloak+",        value=110,   shop=False, weight=2, protection=1,  stealth=1, sort=4),
    'darkcloak': DotDict(name="Dark Cloak",                    raw="darkcloak",          value=300,   shop=True,  weight=1,                stealth=2, sort=5),
    'darkcloak+': DotDict(name="Dark Cloak +",                 raw="darkcloak+",         value=330,   shop=False, weight=2, protection=1,  stealth=2, sort=6),
    'disguisecloak': DotDict(name="Disguise Cloak",            raw="disguisecloak",      value=700,   shop=True,  weight=1,                stealth=3, sort=7),
    'disguisecloak+': DotDict(name="Disguise Cloak +",         raw="disguisecloak+",     value=770,   shop=False, weight=2, protection=1,  stealth=3, sort=8),
    'concealcloak': DotDict(name="Conceal Cloak",              raw="consealcloak",       value=1500,  shop=True,  weight=1,                stealth=4, sort=9),
    'concealcloak+': DotDict(name="Conceal Cloak +",           raw="consealcloak+",      value=1650,  shop=False, weight=2, protection=1,  stealth=4, sort=10),
    'nightcloak': DotDict(name="Night Cloak",                  raw="nightcloak",         value=3100,  shop=True,  weight=1,                stealth=5, sort=11),
    'nightcloak+': DotDict(name="Night Cloak +",               raw="nightcloak+",        value=3410,  shop=False, weight=2, protection=1,  stealth=5, sort=12),
    'stealthcloak': DotDict(name="Stealth Cloak",              raw="stealthcloak",       value=6300,  shop=True,  weight=1,                stealth=6, sort=13),
    'stealthcloak+': DotDict(name="Stealth Cloak +",           raw="stealthcloak+",      value=6930,  shop=False, weight=2, protection=1,  stealth=6, sort=14),
    'phantomcloak': DotDict(name="Phantom Cloak",              raw="phantomcloak",       value=12700, shop=True,  weight=1,                stealth=7, sort=15),
    'phantomcloak+': DotDict(name="Phantom Cloak +",           raw="phantomcloak+",      value=13970, shop=False, weight=2, protection=1,  stealth=7, sort=16),
    'invisibilitycloak': DotDict(name="Invisibility Cloak",    raw="invisibilitycloak",  value=25500, shop=True,  weight=1,                stealth=8, sort=17),
    'invisibilitycloak+': DotDict(name="Invisibility Cloak +", raw="invisibilitycloak+", value=28050, shop=False, weight=2, protection=1,  stealth=8, sort=18),

    'silkcloak': DotDict(name="Silk Cloak",                    raw="silkcloak",          value=2500,  shop=True,  weight=1,                thief=1, sort=19),
    'silkcloak+': DotDict(name="Silk Cloak +",                 raw="silkcloak+",         value=2750,  shop=False, weight=2, protection=1,  thief=1, sort=20),
    'thievescloak': DotDict(name="Thieves Cloak",              raw="thievescloak",       value=5000,  shop=True,  weight=1,                thief=2, sort=21),
    'thievescloak+': DotDict(name="Thieves Cloak +",           raw="thievescloak+",      value=5500,  shop=False, weight=2, protection=1,  thief=2, sort=22)
})

for cloak in cloaks.values():
Beispiel #5
0
from util import DotDict

# prijzen nog bepalen

boots = DotDict({
    'emptyboots': DotDict(name="Empty Boots",             raw="emptyboots",      value=0,    shop=False, weight=0, protection=0, sort=0),

    'leatherboots': DotDict(name="Leather Boots",         raw="leatherboots",    value=100,  shop=True,  weight=1, protection=1, sort=1),
    'bronzeboots': DotDict(name="Bronze Boots",           raw="bronzeboots",     value=200,  shop=True,  weight=2, protection=2, sort=2),
    'ironboots': DotDict(name="Iron Boots",               raw="ironboots",       value=400,  shop=True,  weight=3, protection=3, sort=3),
    'steelboots': DotDict(name="Steel Boots",             raw="steelboots",      value=800,  shop=True,  weight=4, protection=4, sort=4),
    'silverboots': DotDict(name="Silver Boots",           raw="silverboots",     value=1600, shop=True,  weight=5, protection=5, sort=5),
    'titaniumboots': DotDict(name="Titanium Boots",       raw="titaniumboots",   value=3200, shop=False, weight=1, protection=5, sort=6),

    'bootsofmotion': DotDict(name="Boots of Motion",      raw="bootsofmotion",   value=1000, shop=True,  weight=2, protection=1, movepoints=1, sort=7),
    'bootsofmotion+': DotDict(name="Boots of Motion +",   raw="bootsofmotion+",  value=1100, shop=False, weight=3, protection=2, movepoints=1, sort=8),
    'bootsofspeed': DotDict(name="Boots of Speed",        raw="bootsofspeed",    value=2000, shop=True,  weight=2, protection=1, movepoints=2, sort=9),
    'bootsofspeed+': DotDict(name="Boots of Speed +",     raw="bootsofspeed+",   value=2200, shop=False, weight=3, protection=2, movepoints=2, sort=10),
    'woodsmansboots': DotDict(name="Woodsman's Boots",    raw="woodsmansboots",  value=1000, shop=True,  weight=2, protection=1, ranger=1,     sort=11),
    'woodsmansboots+': DotDict(name="Woodsman's Boots +", raw="woodsmansboots+", value=1100, shop=False, weight=3, protection=2, ranger=1,     sort=12),
    'silenceboots': DotDict(name="Silence Boots",         raw="silenceboots",    value=1000, shop=True,  weight=2, protection=1, stealth=1,    sort=13),
    'silenceboots+': DotDict(name="Silence Boots +",      raw="silenceboots+",   value=1100, shop=False, weight=3, protection=2, stealth=1,    sort=14)
})

for boot in boots.values():
    if 'movepoints' not in boot:
        boot.movepoints = None
    if 'ranger' not in boot:
        boot.ranger = None
    if 'stealth' not in boot:
Beispiel #6
0
from util import DotDict

helmets = DotDict({
    'emptyhelmet': DotDict(name="Empty Helmet",               raw="emptyhelmet",       value=0,     shop=False, weight=0, protection=0, sort=0),

    'leathercap': DotDict(name="Leather Cap",                 raw="leathercap",        value=100,   shop=True,  weight=1, protection=1, sort=1),
    'bronzehelmet': DotDict(name="Bronze Helmet",             raw="bronzehelmet",      value=1225,  shop=True,  weight=2, protection=2, sort=2),
    'ironhelmet': DotDict(name="Iron Helmet",                 raw="ironhelmet",        value=3600,  shop=True,  weight=3, protection=3, sort=3),
    'steelhelmet': DotDict(name="Steel Helmet",               raw="steelhelmet",       value=7225,  shop=True,  weight=4, protection=4, sort=4),
    'silverhelmet': DotDict(name="Silver Helmet",             raw="silverhelmet",      value=12100, shop=True,  weight=5, protection=5, sort=5),
    'titaniumhelmet': DotDict(name="Titanium Helmet",         raw="titaniumhelmet",    value=24300, shop=False, weight=1, protection=5, sort=6),

    'helmofknowledge': DotDict(name="Helm of Knowledge",      raw="helmofknowledge",   value=5500,  shop=True,  weight=2, protection=1, intelligence=2, sort=7),
    'helmofknowledge+': DotDict(name="Helm of Knowledge +",   raw="helmofknowledge+",  value=6050,  shop=False, weight=3, protection=2, intelligence=2, sort=8),
    'helmofwisdom': DotDict(name="Helm of Wisdom",            raw="helmofwisdom",      value=5500,  shop=True,  weight=2, protection=1, willpower=2,    sort=9),
    'helmofwisdom+': DotDict(name="Helm of Wisdom +",         raw="helmofwisdom+",     value=6050,  shop=False, weight=3, protection=2, willpower=2,    sort=10),
    'helmofcharisma': DotDict(name="Helm of Charisma",        raw="helmofcharisma",    value=6600,  shop=True,  weight=2, protection=1, diplomat=1,     sort=11),
    'helmofcharisma+': DotDict(name="Helm of Charisma +",     raw="helmofcharisma+",   value=7260,  shop=False, weight=3, protection=2, diplomat=1,     sort=12),
    'helmofinsight': DotDict(name="Helm of Insight",          raw="helmofinsight",     value=7700,  shop=True,  weight=2, protection=1, loremaster=1,   sort=13),
    'helmofinsight+': DotDict(name="Helm of Insight +",       raw="helmofinsight+",    value=8470,  shop=False, weight=3, protection=2, loremaster=1,   sort=14),
    'helmofcognizance': DotDict(name="Helm of Cognizance",    raw="helmofcognizance",  value=9900,  shop=True,  weight=2, protection=1, scientist=1,    sort=15),
    'helmofcognizance+': DotDict(name="Helm of Cognizance +", raw="helmofcognizance+", value=10890, shop=False, weight=3, protection=2, scientist=1,    sort=16),
    'helmoftempests': DotDict(name="Helm of Tempests",        raw="helmoftempests",    value=8800,  shop=True,  weight=2, protection=1, warrior=1,      sort=17),
    'helmoftempests+': DotDict(name="Helm of Tempests +",     raw="helmoftempests+",   value=9680,  shop=False, weight=3, protection=2, warrior=1,      sort=18)
})

for helmet in helmets.values():
    if 'intelligence' not in helmet:
        helmet.intelligence = None
    if 'willpower' not in helmet:
Beispiel #7
0
#                   value, base hit, damage, sortering
weapon_upgraded = {
    "":              (1.0,        0,      0,        10),
    "+":             (1.1,        5,      0,        20),
    "++":            (1.2,        5,      1,        30)
}
# hoogste damage mogelijk is 30: Silver/Titanium Maul ++

weapons = DotDict(
    emptyweapon=DotDict(
        name="Empty Weapon",
        raw="emptyweapon",
        value=0,
        shop=False,
        skill="Empty",
        min_int=0,
        min_str=0,
        base_hit=0,
        damage=0,
        sort=0,
        col=0,
        row=0
    ))

for key_material, value_material in weapon_material.items():
    for key_type, value_type in weapon_type.items():
        for key_upgraded, value_upgraded in weapon_upgraded.items():

            raw_key_name = (key_material + key_type + key_upgraded).strip().lower().replace(" ", "")
            price = int((value_material[0] + value_type[0]) * (value_material[0] + value_type[0]) / 400)
Beispiel #8
0
    "Heavy":           (500,      3,          3,      -2,       300)
}
#                     value, weight, protection, stealth, sortering
armor_upgraded = {
    "":                (1.0,      0,          0,       0,        10),
    "+":               (1.1,      0,          0,       1,        20),
    "++":              (1.2,     -1,          0,       2,        30)
}
# hoogste protection mogelijk is 15: Heavy Silver/Titanium Armor /+/++

armors = DotDict(
    emptyarmor=DotDict(
        name="Empty Armor",
        raw="emptyarmor",
        value=0,
        shop=False,
        weight=0,
        protection=0,
        stealth=0,
        sort=0
    ))

for key_material, value_material in armor_material.items():
    for key_type, value_type in armor_type.items():
        for key_upgraded, value_upgraded in armor_upgraded.items():

            raw_key_name = (key_type + key_material + key_upgraded).strip().lower().replace(" ", "")
            price = (value_material[0] + value_type[0]) * (value_material[0] + value_type[0]) / 400

            armors[raw_key_name] = DotDict(
                name=(key_type + " " + key_material + " " + key_upgraded).strip(),
Beispiel #9
0
 def __init__(self, index):
     self.meta_data = DotDict(index["meta"])
     self.datasets = index["datasets"]
     self.meta_variables = self.meta_data.variables
     self._fill_variables_table()
     self._fill_datasets_table()
Beispiel #10
0
#                   value, min str, protection, defense, dexterity, stealth
shield_upgraded = {
    "":              (1.0,       0,          0,       0,         0,       0,        10),
    "+":             (1.1,       0,          0,       0,         1,       2,        20),
    "++":            (1.2,       0,          0,       0,         2,       4,        30)
}
# de hoogste protection/defense mogelijk is 15/30: Large Silver/Titanium Scutum /+/++

shields = DotDict(
    emptyshield=DotDict(
        name="Empty Shield",
        raw="emptyshield",
        value=0,
        shop=False,
        min_str=0,
        protection=0,
        defense=0,
        dexterity=0,
        stealth=0,
        sort=0,
        col=0,
        row=0
    ))

for key_material, value_material in shield_material.items():
    for key_type, value_type in shield_type.items():
        for key_upgraded, value_upgraded in shield_upgraded.items():

            raw_key_name = (key_material + key_type + key_upgraded).strip().lower().replace(" ", "")
            price = (value_material[0] + value_type[0]) * (value_material[0] + value_type[0]) / 900