def create_character_data_dict(self): data = { "CharacterName": names.get_first_name(), "XP": "0", "ProfBonus": sheet_maths.plus_or_minus(self.clss.proficiency_bonus), "Alignment": random.choice(random.choice(self.sheet_stuff["alignment"])), "Initiative": sheet_maths.plus_or_minus(self.mods["DEXmod"]), "HPMax": self.clss.hit_dice + self.mods["CONmod"], "HPCurrent": self.clss.hit_dice + self.mods["CONmod"], "Equipment": ", ".join(self.bg.equipment + self.clss.equipment), "ProficienciesLang": self.final_languages + "\n\n" + self.final_profs, "Features and Traits": utils.dict_to_string(self.race.features) + utils.dict_to_string(self.clss.features), "Passive": str(self.pp), "AC": self.ac, } # Checking proficient skill boxes for skill in self.final_skills: data[self.sheet_stuff["skills"][skill]["checkbox"]] = "Yes" for skill in self.sheet_stuff["skills"]: if skill in self.final_skills: data[skill] = sheet_maths.plus_or_minus( self.mods[self.sheet_stuff["skills"][skill]["mod"]] + self.sheet_stuff["prof"]) else: data[skill] = sheet_maths.plus_or_minus( self.mods[self.sheet_stuff["skills"][skill]["mod"]]) data.update(self.stats) data.update(sheet_maths.plus_or_minus_dict(self.mods)) data.update(sheet_maths.plus_or_minus_dict(self.saves)) data.update(self.race.race_to_dict()) data.update(self.bg.bg_to_dict()) data.update(self.clss.class_to_dict()) data.update(self.weapons) return data
def build_kernel_options(self, system, profile, distro, image, arch, kickstart_path): """ Builds the full kernel options line. """ management_interface = None if system is not None: blended = utils.blender(self.api, False, system) # find the first management interface try: for intf in system.interfaces.keys(): if system.interfaces[intf]["management"]: management_interface = intf break except: # just skip this then pass elif profile is not None: blended = utils.blender(self.api, False, profile) else: blended = utils.blender(self.api, False, image) append_line = "" kopts = blended.get("kernel_options", dict()) # since network needs to be configured again (it was already in netboot) when kernel boots # and we choose to do it dinamically, we need to set 'ksdevice' to one of # the interfaces' MAC addresses in ppc systems. # ksdevice=bootif is not useful in yaboot, as the "ipappend" line is a pxe feature. if system and arch and "ppc" in arch: for intf in system.interfaces.keys(): # use first interface with defined IP and MAC, since these are required # fields in a DHCP entry mac_address = system.interfaces[intf]['mac_address'] ip_address = system.interfaces[intf]['ip_address'] if mac_address and ip_address: kopts['ksdevice'] = mac_address break # support additional initrd= entries in kernel options. if "initrd" in kopts: append_line = ",%s" % kopts.pop("initrd") hkopts = utils.dict_to_string(kopts) append_line = "%s %s" % (append_line, hkopts) # kickstart path rewriting (get URLs for local files) if kickstart_path is not None and kickstart_path != "": # FIXME: need to make shorter rewrite rules for these URLs try: ipaddress = socket.gethostbyname_ex(blended["http_server"])[2][0] except socket.gaierror: ipaddress = blended["http_server"] if system is not None and kickstart_path.startswith("/"): kickstart_path = "http://%s/cblr/svc/op/ks/system/%s" % (ipaddress, system.name) elif kickstart_path.startswith("/"): kickstart_path = "http://%s/cblr/svc/op/ks/profile/%s" % (ipaddress, profile.name) if distro.breed is None or distro.breed == "redhat": append_line = "%s ks=%s" % (append_line, kickstart_path) gpxe = blended["enable_gpxe"] if gpxe: append_line = append_line.replace('ksdevice=bootif', 'ksdevice=${net0/mac}') elif distro.breed == "suse": append_line = "%s autoyast=%s" % (append_line, kickstart_path) elif distro.breed == "debian" or distro.breed == "ubuntu": append_line = "%s auto-install/enable=true priority=critical url=%s" % (append_line, kickstart_path) if management_interface: append_line += " netcfg/choose_interface=%s" % management_interface elif distro.breed == "freebsd": append_line = "%s ks=%s" % (append_line, kickstart_path) # rework kernel options for debian distros translations = {'ksdevice': "interface", 'lang': "locale"} for k, v in translations.iteritems(): append_line = append_line.replace("%s=" % k, "%s=" % v) # interface=bootif causes a failure append_line = append_line.replace("interface=bootif", "") elif distro.breed == "vmware": if distro.os_version.find("esxi") != -1: # ESXi is very picky, it's easier just to redo the # entire append line here since append_line = " ks=%s %s" % (kickstart_path, hkopts) # ESXi likes even fewer options, so we remove them too append_line = append_line.replace("kssendmac", "") else: append_line = "%s vmkopts=debugLogToSerial:1 mem=512M ks=%s" % \ (append_line, kickstart_path) # interface=bootif causes a failure append_line = append_line.replace("ksdevice=bootif", "") elif distro.breed == "xen": if distro.os_version.find("xenserver620") != -1: img_path = os.path.join("/images", distro.name) append_line = "append %s/xen.gz dom0_max_vcpus=2 dom0_mem=752M com1=115200,8n1 console=com1,vga --- %s/vmlinuz xencons=hvc console=hvc0 console=tty0 install answerfile=%s --- %s/install.img" % (img_path, img_path, kickstart_path, img_path) return append_line if distro is not None and (distro.breed in ["debian", "ubuntu"]): # Hostname is required as a parameter, the one in the preseed is # not respected, so calculate if we have one here. # We're trying: first part of FQDN in hostname field, then system # name, then profile name. # In Ubuntu, this is at least used for the volume group name when # using LVM. domain = "local.lan" if system is not None: if system.hostname is not None and system.hostname != "": # If this is a FQDN, grab the first bit hostname = system.hostname.split(".")[0] _domain = system.hostname.split(".")[1:] if _domain: domain = ".".join(_domain) else: hostname = system.name else: # ubuntu at the very least does not like having underscores # in the hostname. # FIXME: Really this should remove all characters that are # forbidden in hostnames hostname = profile.name.replace("_", "") # At least for debian deployments configured for DHCP networking # this values are not used, but specifying here avoids questions append_line = "%s hostname=%s" % (append_line, hostname) append_line = "%s domain=%s" % (append_line, domain) # A similar issue exists with suite name, as installer requires # the existence of "stable" in the dists directory append_line = "%s suite=%s" % (append_line, distro.os_version) elif distro is not None and distro.breed == 'nexenta': append_line = "-B iso_nfs_path=%s:/var/www/cobbler/links/%s,auto_install=1" % (blended['next_server'], distro.name) # append necessary kernel args for arm architectures if arch is not None and arch.startswith("arm"): append_line = "%s fixrtc vram=48M omapfb.vram=0:24M" % append_line # do variable substitution on the append line # promote all of the ksmeta variables if "ks_meta" in blended: blended.update(blended["ks_meta"]) append_line = self.templar.render(append_line, utils.flatten(blended), None) # FIXME - the append_line length limit is architecture specific if len(append_line) >= 255: self.logger.warning("warning: kernel option length exceeds 255") return append_line
def main(): work_root = os.environ['WORK'] config = get_config(default_config) print_with_rank(str(config)) # Read hyper parameters group_count = config['number_of_reservoirs'] feature_count = config['number_of_features'] lsp = config['overlap_size'] predict_length = config['prediction_size'] train_length = config['training_size'] approx_res_size = config['reservoir_size'] sigma = config['sigma'] radius = config['radius'] beta = config['beta'] degree = config['degree'] # Check preconditions for running if feature_count % group_count != 0: comm.Abort() return if group_count % size != 0: comm.Abort() return if rank == master_node_rank: all_data = load_data(work_root, 'data/QG_everydt_avgu.csv') else: all_data = None for shift in shifts: config['shift'] = shift if rank == master_node_rank: data = all_data[:, shift:train_length + shift] else: data = None model = ESNParallel(group_count, feature_count, lsp, train_length, predict_length, approx_res_size, radius, sigma, random_state=42, beta=beta, degree=degree).fit_reservoir(data) for beta in beta_optimization: model._beta = beta output = model.fit_output().predict() if rank == master_node_rank: config['beta'] = beta shift_folder = dict_to_string( {k: v for k, v in config.items() if k != 'shift'}) directory = os.path.join(work_root, 'results/beta_shift_results', shift_folder) if not os.path.exists(directory): os.makedirs(directory) result_path = os.path.join( directory, 'data=QG-' + dict_to_string(config) + '.txt') np.savetxt(result_path, output) print_with_rank("Saved to " + result_path)
def main(): param_grid = { 'group_count': [11], 'feature_count': [88], 'lsp': [7], 'train_length': [100000], 'predict_length': [1000], 'approx_res_size': [1000], 'radius': list(np.linspace(0.0001, 1, endpoint=False, num=1000)), 'sigma': list(np.linspace(0.0001, 1, num=1000)), 'random_state': [42], 'beta': [0.003], 'degree': [7], 'alpha': list(np.linspace(0.0001, 1, num=1000)), } shifts = list( range(0, param_grid['predict_length'][0] * shift_count, param_grid['predict_length'][0])) if rank == master_node_rank: work_root = os.environ['WORK'] all_data = load_data(work_root, 'data/QG_everydt_avgu.csv') else: all_data = None work_root = None max_evals = 500000 for i in range(max_evals): if rank == master_node_rank: params = {k: random.sample(v, 1)[0] for k, v in param_grid.items()} print_with_rank(str(params)) else: params = None params = comm.bcast(params, master_node_rank) for shift in shifts: if rank == master_node_rank: data = all_data[:, shift:params['train_length'] + shift] else: data = None output = ESNParallel(**params).fit(data).predict() if rank == master_node_rank: params['shift'] = shift shift_folder = dict_to_string( {k: v for k, v in params.items() if k != 'shift'}) directory = os.path.join(work_root, 'results/random_shift_results', shift_folder) if not os.path.exists(directory): os.makedirs(directory) result_path = os.path.join( directory, 'data=QG-' + dict_to_string(params) + '.txt') np.savetxt(result_path, output) print_with_rank("Saved to " + result_path) del params['shift']
def build_kernel_options(self, system, profile, distro, image, arch, kickstart_path): """ Builds the full kernel options line. """ management_interface = None if system is not None: blended = utils.blender(self.api, False, system) # find the first management interface try: for intf in system.interfaces.keys(): if system.interfaces[intf]["management"]: management_interface = intf break except: # just skip this then pass elif profile is not None: blended = utils.blender(self.api, False, profile) else: blended = utils.blender(self.api, False, image) append_line = "" kopts = blended.get("kernel_options", dict()) # since network needs to be configured again (it was already in netboot) when kernel boots # and we choose to do it dinamically, we need to set 'ksdevice' to one of # the interfaces' MAC addresses in ppc systems. # ksdevice=bootif is not useful in yaboot, as the "ipappend" line is a pxe feature. if system and arch and "ppc" in arch: for intf in system.interfaces.keys(): # use first interface with defined IP and MAC, since these are required # fields in a DHCP entry mac_address = system.interfaces[intf]['mac_address'] ip_address = system.interfaces[intf]['ip_address'] if mac_address and ip_address: kopts['ksdevice'] = mac_address break # support additional initrd= entries in kernel options. if "initrd" in kopts: append_line = ",%s" % kopts.pop("initrd") hkopts = utils.dict_to_string(kopts) append_line = "%s %s" % (append_line, hkopts) # kickstart path rewriting (get URLs for local files) if kickstart_path is not None and kickstart_path != "": # FIXME: need to make shorter rewrite rules for these URLs try: ipaddress = socket.gethostbyname_ex(blended["http_server"])[2][0] except socket.gaierror: ipaddress = blended["http_server"] if system is not None and kickstart_path.startswith("/"): kickstart_path = "http://%s/cblr/svc/op/ks/system/%s" % (ipaddress, system.name) elif kickstart_path.startswith("/"): kickstart_path = "http://%s/cblr/svc/op/ks/profile/%s" % (ipaddress, profile.name) if distro.breed is None or distro.breed == "redhat": append_line = "%s ks=%s" % (append_line, kickstart_path) gpxe = blended["enable_gpxe"] if gpxe: append_line = append_line.replace('ksdevice=bootif', 'ksdevice=${net0/mac}') elif distro.breed == "suse": append_line = "%s autoyast=%s" % (append_line, kickstart_path) elif distro.breed == "debian" or distro.breed == "ubuntu": append_line = "%s auto-install/enable=true priority=critical url=%s" % (append_line, kickstart_path) if management_interface: append_line += " netcfg/choose_interface=%s" % management_interface elif distro.breed == "freebsd": append_line = "%s ks=%s" % (append_line, kickstart_path) # rework kernel options for debian distros translations = {'ksdevice': "interface", 'lang': "locale"} for k, v in translations.iteritems(): append_line = append_line.replace("%s=" % k, "%s=" % v) # interface=bootif causes a failure append_line = append_line.replace("interface=bootif", "") elif distro.breed == "vmware": if distro.os_version.find("esxi") != -1: # ESXi is very picky, it's easier just to redo the # entire append line here since append_line = " ks=%s %s" % (kickstart_path, hkopts) # ESXi likes even fewer options, so we remove them too append_line = append_line.replace("kssendmac", "") else: append_line = "%s vmkopts=debugLogToSerial:1 mem=512M ks=%s" % \ (append_line, kickstart_path) # interface=bootif causes a failure append_line = append_line.replace("ksdevice=bootif", "") elif distro.breed == "xen": if distro.os_version.find("xenserver620") != -1: img_path = os.path.join("/images", distro.name) append_line = "append %s/xen.gz dom0_max_vcpus=2 dom0_mem=752M com1=115200,8n1 console=com1,vga --- %s/vmlinuz xencons=hvc console=hvc0 console=tty0 install answerfile=%s --- %s/install.img" % (img_path, img_path, kickstart_path, img_path) return append_line if distro is not None and (distro.breed in ["debian", "ubuntu"]): # Hostname is required as a parameter, the one in the preseed is # not respected, so calculate if we have one here. # We're trying: first part of FQDN in hostname field, then system # name, then profile name. # In Ubuntu, this is at least used for the volume group name when # using LVM. domain = "local.lan" if system is not None: if system.hostname is not None and system.hostname != "": # If this is a FQDN, grab the first bit hostname = system.hostname.split(".")[0] _domain = system.hostname.split(".")[1:] if _domain: domain = ".".join(_domain) else: hostname = system.name else: # ubuntu at the very least does not like having underscores # in the hostname. # FIXME: Really this should remove all characters that are # forbidden in hostnames hostname = profile.name.replace("_", "") # At least for debian deployments configured for DHCP networking # this values are not used, but specifying here avoids questions append_line = "%s hostname=%s" % (append_line, hostname) append_line = "%s domain=%s" % (append_line, domain) # A similar issue exists with suite name, as installer requires # the existence of "stable" in the dists directory append_line = "%s suite=%s" % (append_line, distro.os_version) elif distro is not None and distro.breed == 'nexenta': append_line = "-B iso_nfs_path=%s:/var/www/cobbler/links/%s,auto_install=1" % (blended['next_server'], distro.name) # append necessary kernel args for arm architectures if arch is not None and arch.startswith("arm"): append_line = "%s fixrtc vram=48M omapfb.vram=0:24M" % append_line # do variable substitution on the append line # promote all of the ksmeta variables if "ks_meta" in blended: blended.update(blended["ks_meta"]) append_line = self.templar.render(append_line, utils.flatten(blended), None) # FIXME - the append_line length limit is architecture specific if len(append_line) >= 255: self.logger.warning("warning: kernel option length exceeds 255") return append_line
def run_graph( self, workflow_file, full_result, comment, main, graph_args, disable_inmemory_cache, disable_file_cache, frozens_id, create_frozens, use_frozens, use_frozen_only_if_exists, cleanup, perfomance_logging, ): self.workflow_file = workflow_file self.comment = comment start_time = time.time() with open(workflow_file) as f: self.graph = f.read() self.db_register() message_delay = 60 * float( config.get('cuber', 'message_delay', fallback=3)) job_descritpion = '{}; {}'.format(workflow_file, self.comment) try: cube.Cube.checkpoints_dir = self.checkpoints_dir logging.info('Checkpoints dir: {}'.format( cube.Cube.checkpoints_dir)) wf = workflow.Workflow( workflow_file, main=main, graph_args=graph_args, frozens_dir=self.frozens_dir, frozens_id=frozens_id, create_frozens=create_frozens, use_frozens=use_frozens, use_frozen_only_if_exists=use_frozen_only_if_exists, ) self.db_update_status('running') data = wf.run( disable_inmemory_cache=disable_inmemory_cache, disable_file_cache=disable_file_cache, cleanup=cleanup, perfomance_logging=perfomance_logging, ) res = utils.dict_to_string(data, full=full_result) if time.time() - start_time >= message_delay: logging.critical( 'Calculation is done: {} (graph id: {})\n{}'.format( job_descritpion, self.db_id, res)) else: logging.info( 'Calculation is done: {} (graph id: {})\n{}'.format( job_descritpion, self.db_id, res)) self.db_save_result(res) self.db_update_status('done') except KeyboardInterrupt: if time.time() - start_time >= message_delay: logging.critical( 'Calculation is cancelled: {} (graph id: {})'.format( job_descritpion, self.db_id)) else: logging.error( 'Calculation is cancelled: {} (graph id: {})'.format( job_descritpion, self.db_id)) self.db_save_result('candelled') self.db_update_status('cancelled') except: import traceback traceback.print_exc() if time.time() - start_time >= message_delay: logging.critical( 'Calculation is failed: {} (graph id: {})'.format( job_descritpion, self.db_id)) else: logging.error( 'Calculation is failed: {} (graph id: {})'.format( job_descritpion, self.db_id)) self.db_update_status('failed')
def __run_graph(self, graph_, disable_inmemory_cache, disable_file_cache, cleanup, perfomance_logging): ''' TODO: improve excprions for incorrect graph ''' logger.debug('Graph to do: {}'.format(graph_)) graph_ = self.__fold_graph(graph_) graph_descriptor = graph_['name'] if 'name' in graph_ else str(graph_) graph_id = graph_[ 'name'] if 'name' in graph_ else utils.universal_hash(graph_) for key in {'module', 'class'}: if key not in graph_: raise GraphErrorSpecifiedSubgraph( 'Cube description must have {} parameter.'.format(key), subgraph=graph_descriptor) for key in graph_.keys(): graph_possible_params = { 'attrs', 'deps', 'class', 'module', 'comment', 'name', 'frozen', 'disable_inmemory_cache', 'disable_file_cache' } if key not in graph_possible_params: raise GraphErrorSpecifiedSubgraph('Cube description has param {} that is not allowed. Check for typos. Possible values: {}' \ .format(key, graph_possible_params), subgraph = graph_descriptor) def get_frozen_path(): frozen_path = os.path.join(self.frozens_dir, self.frozens_id, '{}.pkl'.format(graph_id)) frozen_path_dir = os.path.join(self.frozens_dir, self.frozens_id) logger.info('Frozen path: {}'.format(frozen_path)) return frozen_path, frozen_path_dir if utils.parse_bool(graph_.get('frozen', 'false')) and self.use_frozens and \ not os.path.isfile(get_frozen_path()[0]): if not self.use_frozen_only_if_exists: raise GraphErrorSpecifiedSubgraph( 'Frozen {} does not exists, but frozens are enabled and flag "use_frozens_only_if_exists" is not enabled.' .format(get_frozen_path()[0]), subgraph=graph_descriptor) if utils.parse_bool(graph_.get('frozen', 'false')) and self.use_frozens and \ os.path.isfile(get_frozen_path()[0]): logger.info('Loading from frozen') with open(get_frozen_path()[0], 'rb') as f: return pickle.load(f) attrs = copy.deepcopy(graph_.get('attrs', {})) attrs = self.__substitute_graph_args(attrs) for i, dep_ in enumerate(graph_.get('deps', {})): dep = dep_ if isinstance(dep_, dict) else {'graph': dep_} dep_descriptor = dep['name'] if isinstance( dep, dict) and 'name' in dep else '{}-th dep (zero-based)'.format(i) for key in {'graph'}: if key not in dep: raise GraphErrorSpecifiedDep( 'Dep description must have {} parameter.'.format(key), subgraph=graph_descriptor, dep=dep_descriptor) for key in dep.keys(): dep_possible_params = { 'fields', 'graph', 'prefix', 'comment', 'name', 'enable_if' } if key not in dep_possible_params: raise GraphErrorSpecifiedDep('Dep description has param {} that is not allowed. Check for typos. Possible values: {}' \ .format(key, dep_possible_params), subgraph = graph_descriptor, dep = dep_descriptor) if 'enable_if' in dep: if not self.eval_expression(dep['enable_if']): logger.info( 'Skip dependecy "{}" of "{}" because if clause is false' .format(dep_descriptor, graph_descriptor)) continue res = self.__run_graph( dep['graph'], disable_inmemory_cache=disable_inmemory_cache, disable_file_cache=disable_file_cache, cleanup=cleanup, perfomance_logging=perfomance_logging, ) if not isinstance(res, dict): raise GraphErrorSpecifiedDep('You may not use non-dict-result cube as a dependency. Result data ({}): {}.' \ .format(type(res), res), subgraph = graph_descriptor, dep = dep_descriptor) if 'fields' not in dep: for key in res: attr_key = dep.get('prefix', '') + key if attr_key in attrs: raise GraphErrorSpecifiedDep('Argument "{}" for is not unique.' \ .format(attr_key), subgraph = graph_descriptor, dep = dep_descriptor) attrs[attr_key] = res[key] else: for new_key, old_key_ in dep['fields'].iteritems(): attr_key = dep.get('prefix', '') + new_key pack_to_dict = None if isinstance(old_key_, basestring): old_key = old_key_ if old_key_ != '$' else new_key elif isinstance(old_key_, dict): old_key = old_key_['source_field'] if old_key_[ 'source_field'] != '$' else new_key pack_to_dict = old_key_.get('pack_to_dict', None) if pack_to_dict is None: if attr_key in attrs: raise GraphErrorSpecifiedDep('Argument "{}" for is not unique.' \ .format(attr_key), subgraph = graph_descriptor, dep = dep_descriptor) if old_key not in res: raise GraphErrorSpecifiedDep( 'Field "{}" is not got from dependency. Got: {}' .format(old_key, ', '.join(res.keys())), subgraph=graph_descriptor, dep=dep_descriptor) attrs[attr_key] = res[old_key] else: if pack_to_dict not in attrs: attrs[pack_to_dict] = {} if attr_key in attrs[pack_to_dict]: raise GraphErrorSpecifiedDep('Argument "{}" for is not unique for packing dict "{}".' \ .format(attr_key, pack_to_dict), subgraph = graph_descriptor, dep = dep_descriptor) if old_key not in res: raise GraphErrorSpecifiedDep( 'Field "{}" is not got from dependency. Got: {}' .format(old_key, ', '.join(res.keys())), subgraph=graph_descriptor, dep=dep_descriptor) attrs[pack_to_dict][attr_key] = res[old_key] module = importlib.import_module(graph_['module']) logger.debug('Attrs keys: {}'.format(attrs.keys())) try: cube_init = getattr(module, graph_['class'])(**attrs) except Exception as e: logging.error('Faild to init cube:\nCube: {cube}\nGraph part: {graph_part}\nAttrs: {attrs}\nError: {error}\nTraceback: {tb}' \ .format( cube = graph_['module'], graph_part = str(graph_), attrs = utils.dict_to_string(attrs, brackets = True), error = str(e), tb = traceback.format_exc(), ) ) raise try: res = cube_init.get( disable_inmemory_cache=disable_inmemory_cache or utils.parse_bool( graph_.get('disable_inmemory_cache', 'false')), disable_file_cache=disable_file_cache or utils.parse_bool(graph_.get('disable_file_cache', 'false')), cleanup=cleanup, perfomance_logging=perfomance_logging, ) except Exception as e: logging.error('Faild to cube.get():\nCube: {cube}\nGraph part: {graph_part}\nAttrs: {attrs}\nError: {error}\nTraceback: {tb}' \ .format( cube = graph_['module'], graph_part = str(graph_), attrs = utils.dict_to_string(attrs, brackets = True), error = str(e), tb = traceback.format_exc(), ) ) raise if utils.parse_bool(graph_.get('frozen', 'false')) and self.create_frozens: frozen_path, frozen_path_dir = get_frozen_path() if not os.path.isdir(frozen_path_dir): os.makedirs(frozen_path_dir) with open(frozen_path, 'wb') as f: pickle.dump(res, f) logger.info('Frozen point created') return res