def layout(self):
self._rmtemp()
itemlist = []
for itemfile in file_list('items'):
itemfile = os.path.basename(itemfile)
itemlist.append(Item(load_yaml('items',itemfile)))
inventory_rect = pygame.Rect(self.rect.x,self.rect.y, self.rect.w - 523, self.rect.h)
pack = ContainerDialog(
inventory_rect,
self.frontend,
'Pack:',
layer=self._layer +1,
items=[Item(i) for i in self.char.get('inventory/pack',[])],
onclose=self.update_inventory,
onselect=self.equip_item,
onselect_parms=[],
can_add=False,
can_remove=False,
addfrom=itemlist
)
self._addtemp('Inventory_pack_dialog', pack)
image_x = self.rect.w - 522
image_y = 10
self.image.blit(self.frontend.imagecache['inventory_background.png'], (image_x, image_y))
rects = load_yaml('images','gui_rects.yaml')
debug(self.char())
portrait = self.frontend.imagecache[self.char.get('personal/portrait')]
portrait = pygame.transform.smoothscale(portrait, (256,256))
prect = rects['inventory_portrait']
self.image.blit(portrait,(image_x + prect['x'],image_y + prect['y']))
image_x += self.rect.x
image_y += self.rect.y
for itemtuple in self.char.inventory_generator(['equiped']):
debug(itemtuple)
item, slot = itemtuple[1], itemtuple[2]
irect = rects[slot]
irect = pygame.Rect(image_x + irect['x'], image_y + irect['y'], irect['w'], irect['h'])
debug(slot, irect)
sprite = ButtonSprite(
self.frontend.tilemaps,
irect,
self.frontend.eventstack,
onclick=self.unequip_item,
onclick_params=[slot],
animations=item.getsubtree('animations'),
layer=self._layer + 2,
fps=5,
mouseover=item.displayname(),
frontend=self.frontend,
sendself=True
)
sprite.setanimation('view')
self._addtemp(make_hash(), sprite)
def load_formation(self, form, form_attrs):
"""Create the actors for a formation [form] with the offset (x, y)"""
for item_dict in util.load_yaml("formations", form)["items"]:
kind = item_dict.keys()[0]
attrs = item_dict[kind]
formation_maker = getattr(formations, kind)
new_actors = formation_maker(self, form_attrs["x"], form_attrs["y"], attrs)
if form_attrs.has_key("path"):
for a in new_actors:
a.path = util.path.PathNetwork(util.load_yaml("paths", form_attrs["path"]), a.get_path_origin)
a.path_progress = a.path.starting_progress()
def set_environment(self, env):
if env not in self.region_config[SUPPORTED_ENVS]:
raise ChinaContextError("environment '"+env+"' is not supported in region '" + self.region + "', supported_envs in region config ("+self.region_config_file+"), must be one of: "+str(self.region_config[SUPPORTED_ENVS]))
self.environment = env
# Load the environment configuration, using the region configuration for variable substitution
self.default_environment = util.load_yaml(self.blueprints_dir + "/envs/env.default.yml", self.region_config)
env_yml = self.blueprints_dir + "/envs/env." + env + ".yml"
self.specific_environment = {}
if os.path.isfile(env_yml):
self.specific_environment = util.load_yaml(env_yml, self.region_config)
self.specific_environment['name'] = env
def __init__(self, ctx, unit_name, role_name=None, num_instances=1):
self.region_context = ctx
self.unit_name = unit_name
if role_name is None:
self.role_name = unit_name
else:
self.role_name = role_name
self.num_instances = int(num_instances)
self.env_name = ctx.specific_environment['name']
self.env_group_name = util.env_prefix(ctx) + self.env_name
self.group_name = self.env_group_name + "-" + unit_name
self.context = dict(ctx.region_config.items() + ctx.default_environment.items() + ctx.specific_environment.items())
# print "raw context is "+pformat(self.context)
self.context['unit_name'] = unit_name
self.context['env_name'] = self.env_name
self.context['env_group_name'] = self.env_group_name
self.unit_yml_dir = ctx.blueprints_dir + "/units/" + unit_name
self.yml = self.unit_yml_dir + "/unit.yml"
self.config = util.load_yaml(self.yml, self.context)
print "loaded "+unit_name+" yml: "+pformat(self.config)
if 'override_region_context' in self.config:
for key, value in self.config['override_region_context'].iteritems():
self.region_context.region_config[key] = value
print("config====")
pprint(self.config)
def weapons(self):
"""
>>> char = Character({})
>>> char.weapons[0].displayname()
'Fist'
>>> halberd = char.acquire_item(Item(load_yaml('items','halberd.yaml')))
>>> item = Item(char.get('inventory/pack')[0])
>>> char.equip_item(item)
(True, '[ ] has equiped Halberd')
>>> char.weapons[0].displayname()
'Halberd'
>>> len(char.weapons)
1
"""
equipedweapons = self.equiped_by_type('weapon')
if not equipedweapons:
messages.warning('No weapons equipped - equipping fist')
fist = Item(load_yaml('items', 'ff7b801f5dfa6ad84870ca1ce2d43c74685d9ddbfcdc488e2ad66caa.yaml'))
fist = self.acquire_item(fist)
self.equip_item(fist)
equipedweapons = self.equiped_by_type('weapon')
if equipedweapons and equipedweapons[0].get('slot', "") == 'twohand':
return [equipedweapons[0]]
debug('Equipped weapons', equipedweapons)
return equipedweapons
def loadgame(self, slot):
self.setsavedir(slot)
self.player = Player(load_yaml('player','player.yaml'))
debug(self.player.get_hash())
self.journal = Journal(json.loads(open(file_path('journal', 'journal.yaml')).read()))
self._rmtemp()
def attack_roll(player, target, attack_modifiers, custom_tohit):
custom_tohit = custom_tohit or 0
if custom_tohit:
frontend.campaign.message('Applying custom to-hit modifier of %s' % custom_tohit)
if player.is_casting:
player.interrupt_cast()
frontend.campaign.message('%s is casting. Cast will be interrupted if you attack %s' % (player.displayname(), target.displayname))
attack_mods = load_yaml('adnd2e', 'attack_mods')
total_modifier = custom_tohit
for mod in attack_modifiers:
total_modifier += int(attack_mods[mod])
frontend.campaign.message('Applying modifier %s: %s' % (mod, attack_mods[mod]))
range_modifier = 0
if range_modifier:
frontend.campaign.message('Applying range modifier %s' % range_modifier)
total_modifier += range_modifier
weaponmod = player.to_hit_mod()
frontend.campaign.message('Applying weapon modifier %s' % weaponmod)
total_modifier += weaponmod
frontend.campaign.message('Total modifier: %s<br><br>' % total_modifier)
frontend.campaign.message('%s has a defense modifier of %s and armor class %s' % (target.displayname(), target.def_mod(), target.armor_class()))
target_roll = int(player.thac0 - target.armor_class() - target.def_mod())
target_roll = target_roll - total_modifier
if target_roll <= 0:
frontend.campaign.error('%s is guaranteed to hit %s - no need to roll' % (player.displayname(), target_roll, target.displayname()))
else:
frontend.campaign.error('%s needs to roll %s to hit %s' % (player.displayname(), target_roll, target.displayname()))
return target_roll
def get_registered_accounts_from_yml():
"""
Parses yaml data and returns dictionary with registered users
:return: dictionary with registered users
"""
data = load_yaml('registered_users.yaml')
return data
def set_region(self, region):
if region not in VALID_REGIONS:
raise ChinaContextError("invalid region: " + args.region + ", valid regions are: " + str(VALID_REGIONS.keys()))
self.region = region
self.region_fullname = VALID_REGIONS[region]
# Load the region configuration, using the system environment for variable substitution
self.default_region_config = util.load_yaml(self.blueprints_dir + "/regions/region.default.yml", os.environ)
self.region_config_file = self.blueprints_dir + "/regions/region." + self.region_fullname + ".yml"
self.specific_region_config = util.load_yaml(self.region_config_file, os.environ)
# Merge the specific region config onto the defaults, then let os.environ override anything
self.region_config = dict(self.default_region_config.items() + self.specific_region_config.items() + os.environ.items())
# Now that we've loaded the region config, check if the environment is supported there
if SUPPORTED_ENVS not in self.region_config:
raise ChinaContextError("region '" + self.region + "' has no supported_envs!")
def xp_worth(self):
xpkey = self.get('combat/level-hitdice', 1)
xpvalues = load_yaml('adnd2e', 'creature_xp.objdata')
if str(xpkey) in list(xpvalues.keys()):
xp = xpvalues[str(xpkey)]
elif int(xpkey) > 12:
xp = 3000 + ((int(xpkey) - 13) * 1000)
return int(xp)
def main():
args = parse_args()
config = util.load_yaml(args.config)
files = util.input_files(args.files)
lang = language.get(args.lang, None)["TypeMap"]
files.extend([f"{PRESTO_HOME}/{file}" for file in config.JavaClasses])
classes = defaultdict(util.attrdict)
depends = defaultdict(set)
subclasses = {}
for abstract_name, abstract_value in config.AbstractClasses.items():
classes[abstract_name].class_name = abstract_name
classes[abstract_name].field_name = member_name(abstract_name)
classes[abstract_name].abstract = True
classes[abstract_name].super_class = abstract_value.super
if "comparable" in abstract_value:
classes[abstract_name].comparable = True
classes[abstract_name].subclasses = []
for subclass in abstract_value.subclasses:
subclasses[subclass.name] = util.attrdict(super=abstract_name,
key=subclass.key)
classes[abstract_name].subclasses.append(
util.attrdict(
type=subclass.name,
name=member_name(subclass.name),
key=subclass.key,
))
classes[abstract_name].subclasses[-1]._N = len(
classes[abstract_name].subclasses)
classes[abstract_name].subclasses[-1]._last = True
if "source" in abstract_value:
file = abstract_value.source
process_file(f"{PRESTO_HOME}/{file}", config, lang, subclasses,
classes, depends)
else:
classes[abstract_name].fields = []
add_extra(abstract_name, abstract_name, config, lang, classes,
depends)
for file in files:
process_file(file, config, lang, subclasses, classes, depends)
depends = list(topological({k: list(v) for k, v in depends.items()}))[::-1]
comment = "// This file is generated DO NOT EDIT @" + "generated"
result = [{"comment": comment}]
result += [classes[name] for name in depends if name in classes]
result += [classes[name] for name in config.AddToOutput]
if args.json:
print(util.to_json(result))
def check_config(reporter, source_dir):
"""Check configuration file."""
config_file = os.path.join(source_dir, '_config.yml')
config = load_yaml(config_file)
reporter.check_field(config_file, 'configuration', config, 'kind', 'lesson')
reporter.check_field(config_file, 'configuration', config, 'carpentry', ('swc', 'dc'))
reporter.check_field(config_file, 'configuration', config, 'title')
reporter.check_field(config_file, 'configuration', config, 'email')
def check_config(reporter, source_dir):
"""Check configuration file."""
config_file = os.path.join(source_dir, '_config.yml')
config = load_yaml(config_file)
reporter.check_field(config_file, 'configuration', config, 'kind', 'lesson')
reporter.check_field(config_file, 'configuration', config, 'carpentry', ('swc', 'dc'))
reporter.check_field(config_file, 'configuration', config, 'title')
reporter.check_field(config_file, 'configuration', config, 'email')
def thac0(self):
if self.get('personal/race', '') == "creature":
key = "creature"
else:
key = self.get('class/parent', '')
thac0s = load_yaml("rules", "thac0.yaml")[key]
for key2 in list(thac0s.keys()):
if inrange(self.get('combat/level-hitdice', 1), key2):
return int(thac0s[key2])
def ppd_mod(self):
"""
>>> char = Character(load_yaml('characters', 'bardic_rogue.yaml'))
>>> isinstance(char.ppd_mod(), int)
True
"""
ability_mods = FlattenedDict(load_yaml('rules', 'ability_scores.yaml'))
con = self.get('attributes/con', 1)
return int(ability_mods['con/%s/ppd' % con])
def def_mod(self):
"""
>>> char = Character(load_yaml('characters', 'bardic_rogue.yaml'))
>>> isinstance(char.def_mod(), int)
True
"""
ability_mods = load_yaml('rules', 'ability_scores.yaml')
dex = self.get('attributes/dex', 0)
return int(FlattenedDict(ability_mods).get('/dex/%s/defense' % dex, 0))
def dmg_mod(self):
"""
>>> char = Character(load_yaml('characters', 'bardic_rogue.yaml'))
>>> isinstance(char.dmg_mod(), int)
True
"""
ability_mods = FlattenedDict(load_yaml('rules', 'ability_scores.yaml'))
strength = self.get('attributes/str', 0)
return int(ability_mods.get('/str/%s/dmg' % strength, 0))
def level_up(self):
"""
>>> char = Character(load_yaml('characters', 'bardic_rogue.yaml'))
>>> level = char.get('combat/level-hitdice', 1)
>>> hp = char.get('combat/hitpoints', 1)
>>> max = char.get('combat/max_hp', 1)
>>> debug(char.level_up())
>>> char.get('combat/hitpoints', 1) > hp
True
>>> char.get('combat/max_hp', 1) > max
True
>>> char.get('combat/level-hitdice', 1) == level + 1
True
"""
level = int(self.get('combat/level-hitdice', 1))
level += 1
out = '%s has reached level %s !' % (self.displayname(), level)
self.put('combat/level-hitdice', level)
ability_scores = load_yaml('rules', 'ability_scores.yaml')
con = self.get('attributes/con', 1)
out += '<br>Character constitution: %s' % con
con_bonus = int(FlattenedDict(ability_scores).get('/con/%s/hit' % con,0))
out += '<br>Constitution Bonus: %s' % con_bonus
xp_levels = FlattenedDict(load_yaml('rules', 'xp_levels.yaml'))
pclass = self.get('class/parent', '')
xp_levels = xp_levels.readall(pclass)
hitdice = str(xp_levels.get('%s/%s/hit_dice' % (pclass, level), 1))
debug("Read hitdice as ", hitdice)
if '+' not in hitdice:
hitdice = hitdice + '+0'
hitdice, bonus = hitdice.split('+')
dice = int(xp_levels.get('%s/dice' % pclass, 1))
more_hp, roll = rolldice(numdice=int(hitdice), numsides=dice, modifier=con_bonus)
out += '<br>%s' % roll
more_hp += int(bonus)
current_max = int(self.get('combat/max_hp', 1))
new_max = current_max + more_hp
out += '<br>Maximum hitpoints increased by %s. Maximum hitpoints now: %s' % (more_hp, new_max)
self.put('combat/max_hp', new_max)
new_hp = new_max
out += '<br>Character hitpoints now %s' % new_hp
self.put('combat/hitpoints', new_hp)
self.__init__(self())
return out
def get_build_config(path: str,
image_name: str) -> typing.Optional[ImageBuildConfig]:
'''
Returns an ImageBuildConfig object from the default buildfile
located in the image directory.
:param path: The path of the images directory.
:param name: Name of the image, must exist as a directory.
'''
buildfile_path = get_buildfile_path(path, image_name)
return ImageBuildConfig.from_dict(util.load_yaml(buildfile_path))
def __init__(self, frontend):
self.frontend = frontend
Tempsprites.__init__(self)
self.homedir = os.path.join(os.getenv('HOME'), '.mirthless')
self.player = Player({})
self.cleanuplist = []
template = load_yaml('rules','template_character.yaml')
self.portrait = template['personal/portrait']
self.journal = Journal()
characters = []
def createchar(self):
def value(key):
return self.frontend.sprites[key].value
self.player.put('personal/name/first', value('firstname'))
self.player.put('personal/name/last', value('lastname'))
self.player.put('personal/portrait', self.portrait)
self.player.put('personal/sex', value('sex'))
playerclass = value('playerclass') or choice(PLAYER_CLASSES)
playerclass = playerclass.split(':')
debug(playerclass)
self.player.put('class/parent', playerclass[0])
self.player.put('class/class', playerclass[1])
template = load_yaml('rules','template_character.yaml')
for key in template:
k = None
if key.startswith('conditional/class.parent=%s/' %playerclass[0]):
k = key.replace('conditional/class.parent=%s/' %playerclass[0],'')
elif key.startswith('conditional/class.class=%s/' %playerclass[1]):
k = key.replace('conditional/class.class=%s/' %playerclass[1],'')
if k is not None and k != 'class.class':
self.player.put(k, template[key])
if key.startswith('inventory'):
self.player.put(key, template[key])
if key.startswith('__Yinventory'):
k = key.replace('__Y', '')
self.player.put(k, template[key])
slot = str(len(os.listdir(self.homedir)))
self.setsavedir(slot)
armor = Item(load_yaml('items', 'ab7ed2a7e93bae020aeaab893902702fc0727b0079ecd3a14aa4a57c.yaml'))
armor = self.player.acquire_item(armor)
self.player.equip_item(armor)
debug(self.player())
firstmap = GameMap(load_yaml('maps', 'f1440bb0f6826a470c385218d19515ad937c1d7ab4ad3e0c71206238'))
self.player.moveto(firstmap, 18, 1)
firstmap.savetoslot('maps')
animations = load_yaml('rules', 'default_sprites')
for k,v in animations[playerclass[0]].items():
self.player.put(k, v)
self.player.savetoslot()
self.journal.write('Prison')
self._rmtemp()
def num_attacks(self):
atr_objdata = FlattenedDict(load_yaml('rules', 'various.yaml'))
atr = atr_objdata['various/attacks_per_round']
parentclass = self.get('class/parent', '')
if parentclass not in atr:
myatr = 1
else:
for key in list(atr[parentclass].keys()):
if inrange(self.get('combat/level-hitdice', 1), key):
myatr = int(atr[parentclass][key])
return self.num_weapons() * int(myatr)
def __init__(self, setting_path):
self.setting_path = setting_path
yaml = load_yaml(setting_path)
self.settings = self.Settings(
yaml["project"]["name"],
pathlib.Path(yaml["project"]["resource_path"]).resolve(),
yaml["project"]["kind"], yaml["project"]["domain"],
yaml["project"]["processes"])
self.project = self.Project(self.settings)
self.procedure = self.Procedure(self.settings,
self._validate_processes)
def load_old_params(self):
""" Load parameters from *self.files_spec['previous_QNAS_params']* and replace
*self.train_spec*, *self.QNAS_spec*, and *self.fn_dict* with the file values.
"""
previous_params_file = load_yaml(self.files_spec['previous_QNAS_params'])
self.train_spec = dict(previous_params_file['train'])
self.QNAS_spec = dict(previous_params_file['QNAS'])
self.QNAS_spec['params_ranges'] = eval(self.QNAS_spec['params_ranges'])
self.fn_dict = previous_params_file['fn_dict']
def load(filepath):
Settings.__DATA = {'variables': {}}
Settings.__DATA.update(util.load_yaml(filepath))
Settings.__VARIABLES_EXTENDED = dict(Settings.__DATA['variables'])
root = os.path.abspath('.').replace('\\', '/')
Settings.__VARIABLES_EXTENDED.update({
'rootpath': root,
'rootdir': root,
'cflags': Settings.get('compiler').get('cflags', ''),
'lflags': Settings.get('compiler').get('lflags', '')
})
def load(self):
self._rmtemp()
itemlist = []
for itemfile in file_list(self.dirname):
itemfile = os.path.basename(itemfile)
if self.dirname == 'items':
itemlist.append(Item(load_yaml(self.dirname,itemfile)))
else:
itemlist.append(NPC(load_yaml(self.dirname,itemfile)))
c = ContainerDialog(self.rect,
self.frontend,
'Load %s' %self.dirname,
7,
items=itemlist,
onselect=self.loaditem,
onselect_parms=[],
animation='view',
can_add=False,
can_remove=False
)
self._addtemp(make_hash(), c)
def spell_success(self):
ability_scores = load_yaml('rules', 'ability_scores.yaml')
wis = str(self.get('attributes/wis', 0))
failrate = int(ability_scores["wis/%s/spell_failure" %(wis)].split('%')[0])
out = "Spell failure rate: %s percent" % failrate
roll = rolldice(numdice=1, numsides=100)
out += '<br>%s' % roll[1]
if roll[0] > failrate:
out += '<br>Spell succeeds !'
return (True, out)
else:
out += '<br>Spell fails !'
return(False, out)
def check_config(reporter, source_dir):
"""Check configuration file."""
config_file = os.path.join(source_dir, '_config.yml')
config = load_yaml(config_file)
reporter.check_field(config_file, 'configuration', config, 'kind', 'lesson')
reporter.check_field(config_file, 'configuration', config, 'carpentry', ('swc', 'dc', 'lc'))
reporter.check_field(config_file, 'configuration', config, 'title')
reporter.check_field(config_file, 'configuration', config, 'email')
reporter.check({'values': {'root': '..'}} in config.get('defaults', []),
'configuration',
'"root" not set to ".." in configuration')
def check_config(reporter, filename):
"""
Check YAML configuration file.
"""
config = load_yaml(filename)
kind = config.get('kind', None)
reporter.check(kind == 'workshop', filename,
'Missing or unknown kind of event: {0}', kind)
carpentry = config.get('carpentry', None)
reporter.check(carpentry in ('swc', 'dc', 'lc', 'cp'), filename,
'Missing or unknown carpentry: {0}', carpentry)
def load(self,objtype):
if objtype == 'npc':
data = self.get('npc', False)
if not data:
return None
npc = NPC(load_yaml('characters',data))
if not resaved('characters', data):
npc.set_hash()
npc.savetoslot('characters')
self.add('npc', npc.get_hash())
return npc
current = self.get('items', [])
result = []
itemlist = []
for item in current:
i = Item(load_yaml('items',item))
if not resaved('items', item):
i.set_hash()
i.savetoslot('items')
result.append(i)
itemlist.append(i.get_hash())
self.put('items', itemlist)
return result
def learn_spell(self, spellitem):
spells = self.get('inventory/spells', [])
if isinstance(spells, str):
try:
spells = simpleobjdata.loads(spells)
except:
spells = []
if not isinstance(spells, list):
self.put('inventory/spells', [])
spelltype = spellitem.get('spell_type', 'wizard spells')
parentclass = self.get('class/parent', '')
childclass = self.get('class/class', '')
canlearn = load_yaml('rules', 'various.yaml')["spell progression"]
found = False
for key in canlearn:
if key == parentclass or key == childclass:
debug(key)
found = True
break
if not found:
return "%s cannot learn spells" % self.displayname()
oneline = list(canlearn[key].keys())[0]
if spelltype not in canlearn[key][oneline]:
return "%s cannot learn %s, failed to learn spell %s" % (self.displayname(), spelltype, spellitem.displayname())
intelect = str(self.get('attributes/int', 1))
chance = FlattenedDict(load_yaml('rules', 'ability_scores.yaml'))
chance = chance['/int/%s/spell_learn' % intelect]
out = "<strong>%s has a %s chance to learn a new spell</strong>" % (self.displayname(), chance)
chance = int(chance.replace('%', ''))
roll = rolldice(numdice=1, numsides=100, modifier=0)
out += '<br>%s' % roll[1]
if roll[0] > chance:
return '%s<br><strong>%s has failed to learn %s!</strong>' % (out, self.displayname(), spellitem.displayname())
spellitem.identify()
self()['core']['inventory']['spells'].append(spellitem())
self.autosave()
return "%s<br><strong>%s has learned %s</strong>" % (out, self.displayname(), spellitem.displayname())
def saving_throws(self):
"""
>>> char = Character(load_yaml('characters', 'bardic_rogue.yaml'))
>>> isinstance(char.saving_throws, dict)
True
"""
key = self.get('class/parent', '')
sts = FlattenedDict(load_yaml("rules", "saving_throws.yaml"))
sts = FlattenedDict(sts.getsubtree(key))
hitdice = self.get('combat/level-hitdice', 0)
for key2 in sts.subkeys():
if inrange(hitdice, key2):
st = sts.getsubtree(key2)
st['ppd'] = int(st['ppd']) + self.ppd_mod()
return(st)
def attack(player, target, attack_modifiers, custom_tohit, custom_dmg):
custom_tohit = custom_tohit or 0
if custom_tohit:
frontend.campaign.message('Applying custom to-hit modifier of %s' % custom_tohit)
frontend.campaign.message('%s is attacking %s' % (player.displayname(), target.displayname()))
target_alive = True
attack_number = 1
attack_mods = load_yaml('adnd2e', 'attack_mods')
num_attacks = player.num_attacks()
debug("COMBAT: num_attacks:", num_attacks)
while attack_number <= num_attacks and target_alive:
frontend.campaign.message('<br><Br><strong>Attack %s of %s</strong><br>' % (attack_number, num_attacks))
total_modifier = custom_tohit
for mod in attack_modifiers:
total_modifier += int(attack_mods[mod])
frontend.campaign.message('Applying modifier %s: %s' % (mod, attack_mods[mod]))
debug("Attack number", attack_number, "Out of", num_attacks, "Target alive", target_alive)
weapon = player.current_weapon()
weapon.onattack(player, target)
frontend.campaign.message('Attacking with weapon %s' % weapon.displayname())
range_modifier = range_mod(player, target, weapon)
if range_modifier:
frontend.campaign.message('Applying range modifier %s' % range_modifier)
total_modifier += range_modifier
frontend.campaign.message('Attack number %s out of %s' % (attack_number, num_attacks))
attack_number += 1
weaponmod = player.to_hit_mod()
frontend.campaign.message('Applying weapon modifier %s' % weaponmod)
total_modifier += weaponmod
frontend.campaign.message('Total modifier: %s<br><br>' % total_modifier)
attack_roll = player.attack_roll(target, total_modifier)
frontend.campaign.message('Attack roll: %s %s %s' % (attack_roll[0], attack_roll[1], attack_roll[2]))
if attack_roll[1] == 'Critical Hit !':
frontend.campaign.message('Critical hit ! %s gain an extra attack.' % player.displayname())
num_attacks += 1
if attack_roll[1] == "Critical Miss !":
frontend.campaign.message('Critical miss ! %s loses an attack.' % player.displayname())
num_attacks - 1
if "hit" in attack_roll[1].lower():
if target.is_casting:
target.interrupt_cast()
frontend.campaign.message('%s was casting but it was interrupted by a successfull hit' % target.displayname)
weapon.onstrike(player, target)
damage_result = calc_damage(player, target, custom_dmg)
target_alive = damage_result is True
for char in [player, target]:
char.autosave()
def next_level(self):
parentclass = self.get('class/parent', '')
childclass = self.get('class/class', '')
if childclass == 'paladin':
childclass = 'ranger'
debug('Checking next level for %s' % self.displayname())
nl = int(self.get('combat/level-hitdice', 1)) + 1
if nl > 20:
return -1
xp_levels = FlattenedDict(load_yaml('rules', 'xp_levels.yaml')).readall('/%s/%s' % (parentclass, str(nl)))
if '%s/%s/all' %(parentclass, str(nl)) in xp_levels:
next_xp = int(xp_levels['%s/%s/all' %(parentclass, str(nl))])
else:
next_xp = int(xp_levels['%s/%s/%s' %(parentclass, str(nl),childclass)])
return next_xp
def load_params(exp_path, generation=None, individual=0):
""" Load the parameters from *exp_path/log_params_evolution.txt* and the data from
*exp_path/data_QNAS.txt*. The data loaded is the network encoded by individual
*individual* of generation *generation*.
Args:
exp_path: (str) path to the directory containing evolution files.
generation: (int) the generation number of the individual to be profiled.
If *None*, the last generation will be used.
individual: (int) the number of the individual in *generation* to be profiled.
Returns:
dict holding all the necessary parameters and data.
"""
log_file_path = os.path.join(exp_path, 'log_params_evolution.txt')
log_data_path = os.path.join(exp_path, 'data_QNAS.pkl')
params = load_yaml(log_file_path)
log_data = load_pkl(log_data_path)
input_shape = (1, params['train_data_info']['height'],
params['train_data_info']['width'],
params['train_data_info']['num_channels'])
# Load last generation, if it is not specified
if generation is None:
generation = max(log_data.keys())
log_data = log_data[generation]
nets = log_data['net_pop']
net = QChromosomeNetwork(
fn_list=params['QNAS']['fn_list'],
max_num_nodes=params['QNAS']['max_num_nodes']).decode(nets[individual])
loaded_params = {
'individual_id_str':
f"Generation {generation} - individual {individual}",
'individual_id': (generation, individual),
'net_list': net,
'input_shape': input_shape,
'num_classes': params['train_data_info']['num_classes'],
'fn_dict': params['fn_dict'],
'fn_list': params['QNAS']['fn_list']
}
return loaded_params
def _get_evolution_params(self):
""" Get specific parameters for the evolution phase. """
config_file = load_yaml(self.args['config_file'])
self._check_vars(config_file) # Checking if config file contains valid information.
self.train_spec = dict(config_file['train'])
self.QNAS_spec = dict(config_file['QNAS'])
# Get the parameters lower and upper limits
ranges = self._get_ranges(config_file)
self.QNAS_spec['params_ranges'] = OrderedDict(sorted(ranges.items()))
self._get_fn_spec()
self.train_spec['experiment_path'] = self.args['experiment_path']
def _get_continue_params(self):
""" Get parameters for the continue evolution phase. The evolution parameters are loaded
from previous evolution configuration, except from the maximum number of generations
(*max_generations*).
"""
self.files_spec['continue_path'] = self.args['continue_path']
self.files_spec['previous_QNAS_params'] = os.path.join(
self.files_spec['continue_path'], 'log_params_evolution.txt')
self.files_spec['previous_data_file'] = os.path.join(self.args['continue_path'],
'data_QNAS.pkl')
self.load_old_params()
self.QNAS_spec['max_generations'] = load_yaml(
self.args['config_file'])['QNAS']['max_generations']
self.train_spec['experiment_path'] = self.args['experiment_path']
def remove(self, obj, objtype):
if objtype == 'npc':
self.put('npc', '')
return
if objtype == 'player':
self.put('player', False)
return
counter = 0
todel = None
current = self.get('items', [])
for item in current:
hash = Item(load_yaml('items',item)).get_hash()
if obj.get_hash() == hash:
todel = counter
counter += 1
if todel is not None:
del current[todel]
self.put('items', current)
def moveto(self, map, x, y):
if not mapname:
return
if not isinstance(x, int) or not isinstance(y, int):
try:
x = int(x)
y = int(y)
except:
return
if not map.tile(x,y).canenter():
return
current = self.location()
if current.get('map') and x and y:
gamemap = GameMap(load_yaml('maps', current['map']))
gamemap.removefromtile(current['x'], current['y'],self,'npc')
map.addtotile(x, y, 'npc', self)
if map.tile(x,y).revealed():
messages.warning('%s moves to %sx%s' %(self.displayname(),x, y))
def check_config(reporter, filename):
"""
Check YAML configuration file.
"""
config = load_yaml(filename)
kind = config.get('kind', None)
reporter.check(kind == 'workshop',
filename,
'Missing or unknown kind of event: {0}',
kind)
carpentry = config.get('carpentry', None)
reporter.check(carpentry in ('swc', 'dc'),
filename,
'Missing or unknown carpentry: {0}',
carpentry)
def check_config(reporter, source_dir):
"""Check configuration file."""
config_file = os.path.join(source_dir, '_config.yml')
config = load_yaml(config_file)
reporter.check_field(config_file, 'configuration',
config, 'kind', 'lesson')
reporter.check_field(config_file, 'configuration',
config, 'carpentry', ('swc', 'dc', 'lc', 'cp', 'cms'))
reporter.check_field(config_file, 'configuration', config, 'title')
reporter.check_field(config_file, 'configuration', config, 'email')
for defaults in [
{'values': {'root': '.', 'layout': 'page'}},
{'values': {'root': '..', 'layout': 'episode'}, 'scope': {'type': 'episodes', 'path': ''}},
{'values': {'root': '..', 'layout': 'page'}, 'scope': {'type': 'extras', 'path': ''}}
]:
reporter.check(defaults in config.get('defaults', []),
'configuration',
'"root" not set to "." in configuration')
def save(self, path):
"""
Save model as a Wrapper class (pyfunc)
Parameters
----------
path : str
path where the model object will be loaded.
Returns
-------
None
"""
path_artifacts = path + '_artifacts.pkl'
dump(self.artifacts, path_artifacts)
content = load_json("config/arquivos.json")
conda_env = load_yaml(content["path_yaml"])
mlflow.pyfunc.save_model(path=path,
python_model=self,
artifacts={'model': path_artifacts},
conda_env=conda_env)
test_plan = os.environ.get('TEST_PLAN', 'release').lower()
# 'Debug' only differs from 'Release' but NOT uploading results to QA FTP
root_folder_name = test_plan
debug = False
if test_plan == 'debug':
debug = True
test_plan = 'release'
assert options.res_dir is not None, "use --res_dir option to select resources for jobs"
base_template = os.path.join(os.path.dirname(__file__), 'res',
'template.yaml')
print("*** Loading %s ***" % base_template)
# first load 'base' template
template = load_yaml(base_template)
# then extend it with template specific to release type (ai/gg-ult-fab/...)
res_template = os.path.abspath(
os.path.join(options.res_dir, 'template.yaml'))
print("*** Loading %s ***" % res_template)
template.extend(load_yaml(res_template))
tiden_job = None
for k, v in enumerate(template):
if 'job-template' in template[k]:
if 'id' in template[k]['job-template']:
if 'job-tiden' == template[k]['job-template']['id']:
tiden_job = deepcopy(template[k])
break
import scipy.stats
import torch
import torch.utils.data
from sklearn import metrics
try:
from sklearn.externals import joblib
except ImportError:
import joblib
# Local application/library specific imports.
import util
from pytorch_model import AutoEncoder
# Load configuration from YAML file.
CONFIG = util.load_yaml("./config.yaml")
# String constant: "cuda:0" or "cpu"
DEVICE = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
def load_model(config, machine_type):
"""
Load model file
"""
model_file = "{model}/model_{machine_type}.hdf5".format(
model=CONFIG["model_directory"], machine_type=machine_type)
if not os.path.exists(model_file):
print("{} model not found ".format(machine_type))
sys.exit(-1)
category)
# Check whether we have missing or too many categories
seen_categories = set(header.keys())
check_categories(reporter, REQUIRED, seen_categories,
'Missing categories')
check_categories(reporter, seen_categories, REQUIRED.union(OPTIONAL),
'Superfluous categories')
def check_config(reporter, filename):
"""
Check YAML configuration file.
"""
config = load_yaml(filename)
kind = config.get('kind', None)
reporter.check(kind == 'workshop',
filename,
'Missing or unknown kind of event: {0}',
kind)
carpentry = config.get('carpentry', None)
reporter.check(carpentry in ('swc', 'dc'),
filename,
'Missing or unknown carpentry: {0}',
carpentry)
def main():
def create_cluster(mem_count, ebs_count, func_count, sched_count, route_count,
bench_count, cfile, ssh_key, cluster_name, kops_bucket,
aws_key_id, aws_key):
# create the cluster object with kops
util.run_process(
['./create_cluster_object.sh', cluster_name, kops_bucket, ssh_key])
client, apps_client = util.init_k8s()
# create the kops pod
print('Creating management pods...')
kops_spec = util.load_yaml('yaml/pods/kops-pod.yml')
env = kops_spec['spec']['containers'][0]['env']
util.replace_yaml_val(env, 'AWS_ACCESS_KEY_ID', aws_key_id)
util.replace_yaml_val(env, 'AWS_SECRET_ACCESS_KEY', aws_key)
util.replace_yaml_val(env, 'KOPS_STATE_STORE', kops_bucket)
util.replace_yaml_val(env, 'FLUENT_CLUSTER_NAME', cluster_name)
client.create_namespaced_pod(namespace=util.NAMESPACE, body=kops_spec)
# wait for the kops pod to start
kops_ip = util.get_pod_ips(client, 'role=kops', is_running=True)[0]
# copy kube config file to kops pod, so it can execute kubectl commands
kops_podname = kops_spec['metadata']['name']
kcname = kops_spec['spec']['containers'][0]['name']
os.system('cp %s kvs-config.yml' % cfile)
util.copy_file_to_pod(client, '/home/ubuntu/.kube/config', kops_podname,
'/root/.kube/', kcname)
util.copy_file_to_pod(client, ssh_key, kops_podname, '/root/.ssh/', kcname)
util.copy_file_to_pod(client, ssh_key + '.pub', kops_podname,
'/root/.ssh/', kcname)
util.copy_file_to_pod(client, 'kvs-config.yml', kops_podname,
'/fluent/conf/', kcname)
# start the monitoring pod
mon_spec = util.load_yaml('yaml/pods/monitoring-pod.yml')
util.replace_yaml_val(mon_spec['spec']['containers'][0]['env'], 'MGMT_IP',
kops_ip)
client.create_namespaced_pod(namespace=util.NAMESPACE, body=mon_spec)
util.get_pod_ips(client, 'role=monitoring')
# copy config file into monitoring pod -- wait till we create routing pods,
# so we're sure that the monitoring nodes are up and running
util.copy_file_to_pod(client, 'kvs-config.yml',
mon_spec['metadata']['name'], '/fluent/conf/',
mon_spec['spec']['containers'][0]['name'])
os.system('rm kvs-config.yml')
print('Creating %d routing nodes...' % (route_count))
add_nodes(client, apps_client, cfile, ['routing'], [route_count], True)
util.get_pod_ips(client, 'role=routing')
print('Creating %d memory, %d ebs node(s)...' % (mem_count, ebs_count))
add_nodes(client, apps_client, cfile, ['memory', 'ebs'],
[mem_count, ebs_count], True)
print('Creating routing service...')
service_spec = util.load_yaml('yaml/services/routing.yml')
client.create_namespaced_service(namespace=util.NAMESPACE,
body=service_spec)
print('Adding %d scheduler nodes...' % (sched_count))
add_nodes(client, apps_client, cfile, ['scheduler'], [sched_count], True)
util.get_pod_ips(client, 'role=scheduler')
print('Adding %d function serving nodes...' % (func_count))
add_nodes(client, apps_client, cfile, ['function'], [func_count], True)
print('Creating function service...')
service_spec = util.load_yaml('yaml/services/function.yml')
client.create_namespaced_service(namespace=util.NAMESPACE,
body=service_spec)
print('Adding %d benchmark nodes...' % (bench_count))
add_nodes(client, apps_client, cfile, ['benchmark'], [bench_count], True)
print('Finished creating all pods...')
os.system('touch setup_complete')
util.copy_file_to_pod(client, 'setup_complete', kops_podname, '/fluent',
kcname)
os.system('rm setup_complete')
sg_name = 'nodes.' + cluster_name
sg = ec2_client.describe_security_groups(Filters=[{
'Name': 'group-name',
'Values': [sg_name]
}])['SecurityGroups'][0]
print('Authorizing ports for routing service...')
permission = [{
'FromPort': 6200,
'IpProtocol': 'tcp',
'ToPort': 6203,
'IpRanges': [{
'CidrIp': '0.0.0.0/0'
}]
}]
ec2_client.authorize_security_group_ingress(GroupId=sg['GroupId'],
IpPermissions=permission)
routing_svc_addr = util.get_service_address(client, 'routing-service')
function_svc_addr = util.get_service_address(client, 'function-service')
print('The routing service can be accessed here: \n\t%s' %
(routing_svc_addr))
print('The function service can be accessed here: \n\t%s' %
(function_svc_addr))
ys: Dict[str, torch.Tensor] = {}
for name, conv in self.convs.items():
ys[name] = conv(mid).squeeze()
return ys
@property
def module(self) -> nn.Module:
return self
if __name__ == "__main__":
from train import load_yaml
configs = util.load_yaml("configs/debug.yaml")
latent_vars = build_latent_variables(configs["latent_variables"])
g = Generator(latent_vars)
zs = g.sample_latent_vars(2)
for k, v in zs.items():
print(k, v.shape)
x = g.infer(list(zs.values()))
print("x:", x.shape)
d = Discriminator(configs["models"]["dis"])
d_head, q_head = DHead(), QHead(latent_vars)
mid = d(x)
y, c = d_head(mid), q_head(mid)
def add_nodes(client, apps_client, cfile, kinds, counts, create=False):
for i in range(len(kinds)):
print('Adding %d %s server node(s) to cluster...' %
(counts[i], kinds[i]))
# get the previous number of nodes of type kind that are running
prev_count = util.get_previous_count(client, kinds[i])
# we only add new nodes if we didn't pass in a node IP
util.run_process(['./modify_ig.sh', kinds[i],
str(counts[i] + prev_count)])
util.run_process(['./validate_cluster.sh'])
kops_ip = util.get_pod_ips(client, 'role=kops')[0]
route_ips = util.get_pod_ips(client, 'role=routing')
if len(route_ips) > 0:
seed_ip = random.choice(route_ips)
else:
seed_ip = ''
mon_str = ' '.join(util.get_pod_ips(client, 'role=monitoring'))
route_str = ' '.join(route_ips)
sched_str = ' '.join(util.get_pod_ips(client, 'role=scheduler'))
route_addr = util.get_service_address(client, 'routing-service')
function_addr = util.get_service_address(client, 'function-service')
# create should only be true when the DaemonSet is being created for the
# first time -- i.e., when this is called from create_cluster
if create:
for i in range(len(kinds)):
kind = kinds[i]
fname = 'yaml/ds/%s-ds.yml' % kind
yml = util.load_yaml(fname)
for container in yml['spec']['template']['spec']['containers']:
env = container['env']
util.replace_yaml_val(env, 'ROUTING_IPS', route_str)
util.replace_yaml_val(env, 'ROUTE_ADDR', route_addr)
util.replace_yaml_val(env, 'SCHED_IPS', sched_str)
util.replace_yaml_val(env, 'FUNCTION_ADDR', function_addr)
util.replace_yaml_val(env, 'MON_IPS', mon_str)
util.replace_yaml_val(env, 'MGMT_IP', kops_ip)
util.replace_yaml_val(env, 'SEED_IP', seed_ip)
apps_client.create_namespaced_daemon_set(namespace=util.NAMESPACE,
body=yml)
# wait until all pods of this kind are running
res = []
while len(res) != counts[i]:
res = util.get_pod_ips(client, 'role='+kind, is_running=True)
created_pods = []
pods = client.list_namespaced_pod(namespace=util.NAMESPACE,
label_selector='role=' +
kind).items
for pod in pods:
pname = pod.metadata.name
for container in pod.spec.containers:
cname = container.name
created_pods.append((pname, cname))
os.system('cp %s ./kvs-config.yml' % cfile)
for pname, cname in created_pods:
util.copy_file_to_pod(client, 'kvs-config.yml', pname,
'/fluent/conf/', cname)
os.system('rm ./kvs-config.yml')
def __init__(self, path=DEFAULT_CONFIG_PATH):
self.globel_app_conf = util.load_yaml(path)
import requests
from aiohttp import web
from jwt import JWT, jwk_from_dict
from util import load_yaml, rndstr, url_encode
AUTH_ENDPOINT = "https://id.twitch.tv/oauth2/authorize"
TOKEN_ENDPOINT = "https://id.twitch.tv/oauth2/token"
config = load_yaml("config.yaml")
jwt_instance = JWT()
jwt_key = jwk_from_dict(
requests.get("https://id.twitch.tv/oauth2/keys").json()["keys"][0])
session = {}
async def login(request):
session["state"] = rndstr()
session["nonce"] = rndstr()
args = {
"client_id": config["client_id"],
"response_type": "code",
"scope": "openid",
"nonce": session["nonce"],
"redirect_uri": config["api_url"] + "/callback",
"state": session["state"],
}
raise web.HTTPFound(AUTH_ENDPOINT + url_encode(args))
async def login_callback(request):
def __init__(self, db_path, sql_path):
self.__conn = SqliteConnection(convert_path(db_path))
self.__sqls = load_yaml(convert_path(sql_path))
def __init__(self, path):
self.__subtitles = load_yaml(convert_path(path))
def add_nodes(client,
apps_client,
cfile,
kinds,
counts,
management_ip,
aws_key_id=None,
aws_key=None,
create=False,
prefix=None):
for i in range(len(kinds)):
print('Adding %d %s server node(s) to cluster...' %
(counts[i], kinds[i]))
prev_count = util.get_previous_count(client, kinds[i])
util.run_process(
['./modify_ig.sh', kinds[i],
str(counts[i] + prev_count)])
util.run_process(['./validate_cluster.sh'])
replica_str = ' '.join(util.get_node_ips(client, 'role=aft'))
# Create should only be true when the DaemonSet is being created for the
# first time -- i.e., when this is called from create_cluster. After that,
# we can basically ignore this because the DaemonSet will take care of
# adding pods to created nodes.
if create:
for i in range(len(kinds)):
kind = kinds[i]
fname = 'yaml/ds/%s-ds.yml' % kind
yml = util.load_yaml(fname, prefix)
for container in yml['spec']['template']['spec']['containers']:
env = container['env']
util.replace_yaml_val(env, 'REPLICA_IPS', replica_str)
util.replace_yaml_val(env, 'MANAGER', management_ip)
util.replace_yaml_val(env, 'AWS_ACCESS_KEY_ID', aws_key_id)
util.replace_yaml_val(env, 'AWS_SECRET_ACCESS_KEY', aws_key)
apps_client.create_namespaced_daemon_set(namespace=util.NAMESPACE,
body=yml)
# Wait until all pods of this kind are running
res = []
while len(res) != counts[i]:
res = util.get_pod_ips(client, 'role=' + kind, is_running=True)
created_pods = []
pods = client.list_namespaced_pod(namespace=util.NAMESPACE,
label_selector='role=' +
kind).items
# Generate list of all recently created pods.
for pod in pods:
pname = pod.metadata.name
for container in pod.spec.containers:
cname = container.name
created_pods.append((pname, cname))
pod.metadata.labels['aftReady'] = 'isready'
client.patch_namespaced_pod(pod.metadata.name,
util.NAMESPACE, pod)
# Copy the KVS config into all recently created pods.
os.system('cp %s ./aft-config.yml' % cfile)
for pname, cname in created_pods:
util.copy_file_to_pod(
client, 'aft-config.yml', pname,
'/go/src/github.com/Alchem-Lab/aft/config', cname)
os.system('rm ./aft-config.yml')
def create_cluster(txn_count, keynode_count, rtr_count, worker_count, lb_count,
benchmark_count, config_file, branch_name, ssh_key,
cluster_name, kops_bucket, aws_key_id, aws_key,
anna_config_file):
prefix = './'
util.run_process(['./create_cluster_object.sh', kops_bucket, ssh_key],
'kops')
client, apps_client = util.init_k8s()
print('Creating Monitor Node...')
add_nodes(client, apps_client, config_file, "monitor", 1, aws_key_id,
aws_key, True, prefix, branch_name)
print('Creating %d Anna Routing Nodes...' % (rtr_count))
add_nodes(client, apps_client, anna_config_file, "routing", rtr_count,
aws_key_id, aws_key, True, prefix, branch_name)
print('Creating routing service...')
service_spec = util.load_yaml('yaml/services/routing.yml', prefix)
client.create_namespaced_service(namespace=util.NAMESPACE,
body=service_spec)
util.get_service_address(client, 'routing-service')
print('Creating %d Key Nodes...' % (keynode_count))
add_nodes(client, apps_client, config_file, "keynode", keynode_count,
aws_key_id, aws_key, True, prefix, branch_name)
print('Creating %d Worker Nodes...' % (worker_count))
add_nodes(client, apps_client, config_file, "worker", worker_count,
aws_key_id, aws_key, True, prefix, branch_name)
print('Creating Worker Service...')
service_spec = util.load_yaml('yaml/services/worker.yml', prefix)
client.create_namespaced_service(namespace=util.NAMESPACE,
body=service_spec)
util.get_service_address(client, 'worker-service')
print('Creating %d TASC nodes...' % (txn_count))
add_nodes(client, apps_client, config_file, 'tasc', txn_count, aws_key_id,
aws_key, True, prefix, branch_name)
print('Creating %d Load Balancers...' % (lb_count))
add_nodes(client, apps_client, config_file, 'lb', lb_count, aws_key_id,
aws_key, True, prefix, branch_name)
print('Creating TASC Load Balancing service...')
service_spec = util.load_yaml('yaml/services/tasc.yml', prefix)
client.create_namespaced_service(namespace=util.NAMESPACE,
body=service_spec)
print('Creating %d Benchmark nodes...' % (benchmark_count))
add_nodes(client, apps_client, config_file, 'benchmark', benchmark_count,
aws_key_id, aws_key, True, prefix, branch_name)
benchmark_ips = util.get_node_ips(client, 'role=benchmark', 'ExternalIP')
with open('../cmd/benchmark/benchmarks.txt', 'w+') as f:
for ip in benchmark_ips:
f.write(ip + '\n')
print('Finished creating all pods...')
sg_name = 'nodes.' + cluster_name
sg = ec2_client.describe_security_groups(Filters=[{
'Name': 'group-name',
'Values': [sg_name]
}])['SecurityGroups'][0]
print("Authorizing Ports for TASC...")
permission = [{
'FromPort': 0,
'IpProtocol': 'tcp',
'ToPort': 65535,
'IpRanges': [{
'CidrIp': '0.0.0.0/0'
}]
}]
ec2_client.authorize_security_group_ingress(GroupId=sg['GroupId'],
IpPermissions=permission)
print('Registering Key Nodes...')
keynode_pod_ips = util.get_pod_ips(client, 'role=keynode', is_running=True)
register(client, keynode_pod_ips)
print("\nThe TASC ELB Endpoint: " +
util.get_service_address(client, "tasc-service") + "\n")
print('Finished!')
def create_cluster(replica_count, gc_count, lb_count, bench_count, cfile,
ssh_key, cluster_name, kops_bucket, aws_key_id, aws_key):
prefix = './'
util.run_process(['./create_cluster_object.sh', kops_bucket, ssh_key])
client, apps_client = util.init_k8s()
print('Creating management pod')
# management_spec = util.load_yaml('yaml/pods/management-pod.yml')
# env = management_spec['spec']['containers'][0]['env']
# util.replace_yaml_val(env, 'AWS_ACCESS_KEY_ID', aws_key_id)
# util.replace_yaml_val(env, 'AWS_SECRET_ACCESS_KEY', aws_key)
#
# client.create_namespaced_pod(namespace=util.NAMESPACE,
# body=management_spec)
# management_ip = util.get_pod_ips(client, 'role=management',
# is_running=True)[0]
management_ip = ""
print('Creating standby replicas...')
util.run_process(['./modify_ig.sh', 'standby', '1'])
util.run_process(['./validate_cluster.sh'])
print('Creating %d load balancer, %d GC replicas...' %
(lb_count, gc_count))
add_nodes(client, apps_client, cfile, ['lb', 'gc'], [lb_count, gc_count],
management_ip, aws_key_id, aws_key, True, prefix)
lb_pods = client.list_namespaced_pod(namespace=util.NAMESPACE,
label_selector="role=lb").items
kubecfg = os.path.join(os.environ['HOME'], '.kube/config')
for pod in lb_pods:
util.copy_file_to_pod(client, kubecfg, pod.metadata.name,
'/root/.kube', 'lb-container')
replica_ips = util.get_node_ips(client, 'role=gc', 'ExternalIP')
with open('gcs.txt', 'w') as f:
for ip in replica_ips:
f.write(ip + '\n')
# Wait until the monitoring pod is finished creating to get its IP address
# and then copy KVS config into the monitoring pod.
print('Creating %d Aft replicas...' % (replica_count))
add_nodes(client, apps_client, cfile, ['aft'], [replica_count],
management_ip, aws_key_id, aws_key, True, prefix)
util.get_pod_ips(client, 'role=aft')
replica_ips = util.get_node_ips(client, 'role=aft', 'ExternalIP')
with open('replicas.txt', 'w') as f:
for ip in replica_ips:
f.write(ip + '\n')
os.system('cp %s aft-config.yml' % cfile)
management_pname = management_spec['metadata']['name']
management_cname = management_spec['spec']['containers'][0]['name']
util.copy_file_to_pod(client, 'aft-config.yml', management_pname,
'/go/src/github.com/tajshaik24/aft/config',
management_cname)
util.copy_file_to_pod(client, 'replicas.txt', management_pname,
'/go/src/github.com/tajshaik24/aft',
management_cname)
util.copy_file_to_pod(client, 'gcs.txt', management_pname,
'/go/src/github.com/tajshaik24/aft',
management_cname)
util.copy_file_to_pod(client, kubecfg, management_pname, '/root/.kube/',
management_cname)
os.system('rm aft-config.yml')
os.system('rm gcs.txt')
# Copy replicas.txt to all Aft pods.
aft_pod_list = client.list_namespaced_pod(namespace=util.NAMESPACE,
label_selector="role=aft").items
aft_pod_list = list(map(lambda pod: pod.metadata.name, aft_pod_list))
for pname in aft_pod_list:
util.copy_file_to_pod(client, 'replicas.txt', pname,
'/go/src/github.com/tajshaik24/aft',
'aft-container')
gc_pod_list = client.list_namespaced_pod(namespace=util.NAMESPACE,
label_selector="role=gc").items
gc_pod_list = list(map(lambda pod: pod.metadata.name, gc_pod_list))
for pname in gc_pod_list:
util.copy_file_to_pod(client, 'replicas.txt', pname,
'/go/src/github.com/tajshaik24/aft',
'gc-container')
os.system('rm replicas.txt')
print('Adding %d benchmark nodes...' % (bench_count))
add_nodes(client, apps_client, cfile, ['benchmark'], [bench_count],
management_ip, aws_key_id, aws_key, True, prefix)
print('Finished creating all pods...')
print('Creating Aft service...')
service_spec = util.load_yaml('yaml/services/aft.yml', prefix)
client.create_namespaced_service(namespace=util.NAMESPACE,
body=service_spec)
sg_name = 'nodes.' + cluster_name
sg = ec2_client.describe_security_groups(Filters=[{
'Name': 'group-name',
'Values': [sg_name]
}])['SecurityGroups'][0]
print('Authorizing ports for Aft replicas...')
permission = [{
'FromPort': 7654,
'IpProtocol': 'tcp',
'ToPort': 7656,
'IpRanges': [{
'CidrIp': '0.0.0.0/0'
}]
}, {
'FromPort': 7777,
'IpProtocol': 'tcp',
'ToPort': 7782,
'IpRanges': [{
'CidrIp': '0.0.0.0/0'
}]
}, {
'FromPort': 8000,
'IpProtocol': 'tcp',
'ToPort': 8003,
'IpRanges': [{
'CidrIp': '0.0.0.0/0'
}]
}]
ec2_client.authorize_security_group_ingress(GroupId=sg['GroupId'],
IpPermissions=permission)
print('Finished!')
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
"--config",
"-c",
default="configs/default.yaml",
help="training configuration file",
)
args = parser.parse_args()
configs = util.load_yaml(args.config)
dataset_name = configs["dataset"]["name"]
dataset_path = Path(configs["dataset"]["path"]) / dataset_name
# prepare dataset
if dataset_name == "mnist":
dataset = new_mnist_dataset(dataset_path)
elif dataset_name == "fashion-mnist":
dataset = new_fashion_mnist_dataset(dataset_path)
else:
raise NotImplementedError
dataloader = DataLoader(
dataset,
batch_size=configs["dataset"]["batchsize"],
num_workers=configs["dataset"]["n_workers"],
shuffle=True,
drop_last=True,
pin_memory=True,
worker_init_fn=worker_init_fn,
)
# prepare models
latent_vars = build_latent_variables(configs["latent_variables"])
gen, dis = Generator(latent_vars), Discriminator(configs["models"]["dis"])
dhead, qhead = DHead(), QHead(latent_vars)
models = {"gen": gen, "dis": dis, "dhead": dhead, "qhead": qhead}
# prepare optimizers
opt_gen = create_optimizer([gen, qhead], **configs["optimizer"]["gen"])
opt_dis = create_optimizer([dis, dhead], **configs["optimizer"]["dis"])
opts = {"gen": opt_gen, "dis": opt_dis}
# prepare directories
log_path = Path(configs["log_path"])
log_path.mkdir(parents=True, exist_ok=True)
tb_path = Path(configs["tensorboard_path"])
tb_path.mkdir(parents=True, exist_ok=True)
# initialize logger
logger = Logger(log_path, tb_path)
# initialize losses
losses = {
"adv": loss.AdversarialLoss(),
"info": loss.InfoGANLoss(latent_vars)
}
# start training
trainer = Trainer(dataloader, latent_vars, models, opts, losses,
configs["training"], logger)
trainer.train()
def main():
args = parse_args()
config = util.load_yaml(args.config)
thrift = util.load_yaml(args.thrift)
protocol = util.load_yaml(args.protocol)
pmap = {}
for item in protocol:
if "class_name" in item:
pmap[item.class_name] = item
comment = "// This file is generated DO NOT EDIT @" + "generated"
result = [{"comment": comment}]
for thrift_item in thrift:
config_item = None
if "class_name" in thrift_item and thrift_item.class_name in pmap:
protocol_item = pmap[thrift_item.class_name]
special = False
if "struct" in thrift_item:
if thrift_item.class_name in config.StructMap:
config_item = config.StructMap[thrift_item.class_name]
thrift_item["proto_name"] = config_item.class_name
special = True
for field in thrift_item.fields:
if (
config_item is not None
and field.field_name in config_item.fields
):
field["proto_name"] = config_item.fields[
field.field_name
].field_name
else:
field["proto_name"] = field.field_name
if "struct" in protocol_item:
thrift_field_set = {t.proto_name for t in thrift_item.fields}
protocol_field_set = {p.field_name for p in protocol_item.fields}
valid_fields = thrift_field_set.intersection(protocol_field_set)
for field in thrift_item.fields:
if field.field_name in valid_fields:
field["convert"] = True
if len((thrift_field_set - protocol_field_set)) != 0:
eprint(
"Missing protocol fields: "
+ thrift_item.class_name
+ " "
+ str(thrift_field_set - protocol_field_set)
)
if len((protocol_field_set - thrift_field_set)) != 0:
eprint(
"Missing thrift fields: "
+ thrift_item.class_name
+ " "
+ str(protocol_field_set - thrift_field_set)
)
else:
hfile = "./special/" + thrift_item.class_name + ".hpp.inc"
special = special_file(hfile, special, thrift_item, "hinc")
cfile = "./special/" + thrift_item.class_name + ".cpp.inc"
special = special_file(cfile, special, thrift_item, "cinc")
if not special:
eprint(
"Thrift struct missing from presto_protocol: "
+ thrift_item.class_name
)
else:
eprint("Thrift item missing from presto_protocol: " + item.class_name)
result.extend(thrift)
print(util.to_json(result))
def add_nodes(client, apps_client, cfile, kind, count, aws_key_id=None,
aws_key=None, create=False, prefix=None, branch="master"):
print('Adding %d %s server node(s) to cluster...' % (count, kind))
prev_count = util.get_previous_count(client, kind)
util.run_process(['./modify_ig.sh', kind, str(count + prev_count)], 'kops')
util.run_process(['./validate_cluster.sh'], 'kops')
if create:
fname = 'yaml/ds/%s-ds.yml' % kind
yml = util.load_yaml(fname, prefix)
for container in yml['spec']['template']['spec']['containers']:
env = container['env']
util.replace_yaml_val(env, 'BRANCH', branch)
util.replace_yaml_val(env, 'AWS_ACCESS_KEY_ID', aws_key_id)
util.replace_yaml_val(env, 'AWS_SECRET_ACCESS_KEY', aws_key)
if kind == "tasc":
routing_svc = util.get_service_address(client, 'routing-service')
util.replace_yaml_val(env, 'ROUTING_ILB', routing_svc)
monitor_ip = util.get_node_ips(client, 'role=monitor', 'ExternalIP')[0]
util.replace_yaml_val(env, 'MONITOR', monitor_ip)
worker_svc = util.get_service_address(client, 'worker-service')
util.replace_yaml_val(env, 'WORKER_ILB', worker_svc)
if kind == "keynode":
monitor_ip = util.get_node_ips(client, 'role=monitor', 'ExternalIP')[0]
util.replace_yaml_val(env, 'MONITOR', monitor_ip)
if kind == 'worker':
monitor_ip = util.get_node_ips(client, 'role=monitor', 'ExternalIP')[0]
util.replace_yaml_val(env, 'MONITOR', monitor_ip)
routing_svc = util.get_service_address(client, 'routing-service')
util.replace_yaml_val(env, 'ROUTING_ILB', routing_svc)
apps_client.create_namespaced_daemon_set(namespace=util.NAMESPACE,
body=yml)
# Wait until all pods of this kind are running
res = []
while len(res) != count:
res = util.get_pod_ips(client, 'role=' + kind, is_running=True)
created_pods = []
pods = client.list_namespaced_pod(namespace=util.NAMESPACE,
label_selector='role=' +
kind).items
# Send kube config to lb
if kind == 'lb':
kubecfg = os.path.join(os.environ['HOME'], '.kube/config')
for pod in pods:
cname = pod.spec.containers[0].name
util.copy_file_to_pod(client, kubecfg, pod.metadata.name,
'/root/.kube', cname)
# Generate list of all recently created pods.
created_pod_ips = []
for pod in pods:
created_pod_ips.append(pod.status.pod_ip)
pname = pod.metadata.name
for container in pod.spec.containers:
cname = container.name
created_pods.append((pname, cname))
# Copy the KVS config into all recently created pods.
cfile_name = './tasc-config.yml' if kind != 'routing' else './anna-config.yml'
cfile_dir = '/go/src/github.com/saurav-c/tasc/config' if kind != 'routing' else 'hydro/anna/conf'
os.system(str('cp %s ' + cfile_name) % cfile)
for pname, cname in created_pods:
util.copy_file_to_pod(client, cfile_name[2:], pname,
cfile_dir, cname)
os.system('rm ' + cfile_name)
def using_remote_theme(source_dir):
config_file = os.path.join(source_dir, '_config.yml')
config = load_yaml(config_file)
return 'remote_theme' in config
