def test_unicode_input(unicode_filename, verbose=False): data = open(unicode_filename, 'rb').read().decode('utf-8') value = ' '.join(data.split()) output = yaml.full_load(_unicode_open(StringIO.StringIO(data.encode('utf-8')), 'utf-8')) assert output == value, (output, value) for input in [data, data.encode('utf-8'), codecs.BOM_UTF8+data.encode('utf-8'), codecs.BOM_UTF16_BE+data.encode('utf-16-be'), codecs.BOM_UTF16_LE+data.encode('utf-16-le')]: if verbose: print "INPUT:", repr(input[:10]), "..." output = yaml.full_load(input) assert output == value, (output, value) output = yaml.full_load(StringIO.StringIO(input)) assert output == value, (output, value)
def _load_yaml(self, stream): if not yaml: raise DataError('Using YAML variable files requires PyYAML module ' 'to be installed. Typically you can install it ' 'by running `pip install pyyaml`.') if yaml.__version__.split('.')[0] == '3': return yaml.load(stream) return yaml.full_load(stream)
def kubernetes_api_client(rancher_client, cluster_name): c = rancher_client.by_id_cluster(cluster_name) kc = c.generateKubeconfig() loader = KubeConfigLoader(config_dict=yaml.full_load(kc.config)) client_configuration = type.__call__(Configuration) loader.load_and_set(client_configuration) k8s_client = ApiClient(configuration=client_configuration) return k8s_client
def test_unicode_input_errors(unicode_filename, verbose=False): data = open(unicode_filename, 'rb').read().decode('utf-8') for input in [data.encode('utf-16-be'), data.encode('utf-16-le'), codecs.BOM_UTF8+data.encode('utf-16-be'), codecs.BOM_UTF8+data.encode('utf-16-le')]: try: yaml.full_load(input) except yaml.YAMLError, exc: if verbose: print exc else: raise AssertionError("expected an exception") try: yaml.full_load(StringIO.StringIO(input)) except yaml.YAMLError, exc: if verbose: print exc
def _load(self): """Load or reload the bot's main configuration file (config.yml).""" filename = path.join(self._base_dir, "config.yml") try: with open(filename) as fp: self._data = yaml.full_load(fp) except (OSError, yaml.error.YAMLError) as exc: if exc.errno == errno.ENOENT: # Ignore missing file; use defaults return err = "Couldn't read config file ({}):\n{}" raise ConfigError(err.format(filename, exc)) from None
def load(self, stream): meta_data = [] meta_opened = False for next_line in stream: if next_line.startswith(DELIMITER): if meta_opened: break # all meta is read, stop reading else: meta_opened = True # meta started continue elif not meta_opened: break # no meta found else: meta_data.append(next_line) if meta_data: self.update(yaml.full_load(''.join(meta_data)))
def __init__(self, mapfile, log=None): self.meta_replacements_in = {} self.meta_replacements_out = {} # This is a list of function closures self.input_mapping = [None] * 1280 self.output_mapping = {} self.log = log self.sendcount = 0 # Open and parse the YAML mapping file passed to us try: f = open(mapfile) except: self.log.error("Unable to Open Mapfile - {}".format(mapfile)) raise maps = yaml.full_load(f) f.close() # dictionaries used for converting meta data strings from db to canfix and back self.meta_replacements_in = maps['meta replacements'] self.meta_replacements_out = {v:k for k,v in self.meta_replacements_in.items()} # We really just assign all the outputs to a dictionary for the main # plugin code to use to assign callbacks. for each in maps['outputs']: output = {'canid':each['canid'], 'index':each['index'], 'owner':each['owner'], 'exclude':False, 'lastValue':None} self.output_mapping[each['fixid']] = output # each input mapping item := [CANID, Index, FIX DB ID, Priority] for each in maps['inputs']: p = canfix.protocol.parameters[each["canid"]] # Parameters start at 0x100 so we subtract that offset to index the array ix = each["canid"] - 0x100 if self.input_mapping[ix] is None: self.input_mapping[ix] = [None] * 256 self.input_mapping[ix][each["index"]] = self.getInputFunction(each["fixid"])
def test_recursive(recursive_filename, verbose=False): context = globals().copy() exec(open(recursive_filename, 'rb').read(), context) value1 = context['value'] output1 = None value2 = None output2 = None try: output1 = yaml.dump(value1) value2 = yaml.full_load(output1) output2 = yaml.dump(value2) assert output1 == output2, (output1, output2) finally: if verbose: print("VALUE1:", value1) print("VALUE2:", value2) print("OUTPUT1:") print(output1) print("OUTPUT2:") print(output2)
def detect_suspicious_activity(config, files): """ Looks for suspicious IPs from AWS config = string path to config.yaml file containing whitelisted ips files = string path to cloudtrail log files """ suspicious = [] api_calls = {} associate_ips = [] with open(config) as f: whitelist = yaml.full_load(f) for file in sorted(files): f = None log.info(f'Checking File: {file}') if file.endswith('.gz'): f = gzip.open(file, 'r') else: f = open(file, 'r') try: cloudtrail = json.load(f) except Exception as e: log.error(f'Invalid JSON file: {file} - {e}') continue records = sorted(cloudtrail['Records'], key=lambda x: datetime.strptime( x['eventTime'], '%Y-%m-%dT%H:%M:%SZ'), reverse=False) for record in records: try: if record['eventName'].lower() == 'assumerole': session_name = record['requestParameters'][ 'roleSessionName'] arn = record['requestParameters']['roleArn'] account = record['requestParameters']['roleArn'].split( ':')[4] role = record['requestParameters']['roleArn'].split( '/')[-1] assume_role_session = f'arn:aws:sts::{account}:assumed-role/{role}/{session_name}' if not api_calls.get(session_name, None): api_calls[session_name] = { 'source_ip': [], 'arn': assume_role_session, 'ttl': int(time.time() + 28800) } else: # Set a TTL. This is most useful in DynamoDB api_calls[session_name]['ttl'] = int(time.time() + 28800) if record['userIdentity'].get('type', '') == 'AssumedRole': session = record['userIdentity']['arn'].split('/')[-1] # Check for open access in repository check_policy(TARGET_PROFILE) if api_calls.get(session, None): if 'amazonaws' not in record[ 'sourceIPAddress'] and not whitelisted_ip( whitelist.get('aws_whitelist_ips', []), record['sourceIPAddress']): log.info( f"Outside IP address: {record['sourceIPAddress']} - from eventName: {record['eventName']}" ) # if this is the first call, we can add this IP to the list if len(api_calls[session].get('source_ip', [])) == 0: api_calls[session]['source_ip'].append( record['sourceIPAddress']) else: if record['sourceIPAddress'] not in api_calls[ session].get('source_ip', []): if private_ip_check( record['sourceIPAddress']): for ip in api_calls[session].get( 'source_ip', []): if private_ip_check(ip): log.info( f"Multiple IPs for this credential: {assume_role_session} - sourceIP: {record['sourceIPAddress']}" ) log.debug(record) suspicious.append(record) api_calls[session]['source_ip'].append( record['sourceIPAddress']) continue # see if there was a call to change the IP if session not in associate_ips: log.info( f"Call to change IP for this credential: {assume_role_session} - sourceIP: {record['sourceIPAddress']}" ) log.debug(record) suspicious.append(record) except Exception as e: log.fatal(f'Unknown error on record - {record}') log.fatal(f'Error - {e}') f.close() return suspicious
def load_metadata(self): """Load metadata from storage""" with self.metadata_base_store.open("rb") as fd: return yaml.full_load(fd)
def lint_assessment(self, file_path, assessment): problems = [] # check assessment with open(file_path, "r") as f: f = yaml.full_load(f) # check only if there is content in the file if f is not None: # check openpatch status openpatch = f.get("openpatch", {}) if (openpatch is None or not isinstance(openpatch, dict) or len(openpatch.keys()) == 0): problems.append( Problem( key="openpatch", rule=Rule.NO_OPENPATCH, desc="No OpenPatch found", level=Level.ERROR, )) else: status = openpatch.get("status") if not status or status not in openpatch_status: problems.append( Problem( key="openpatch.status", rule=Rule.INVALID_OPENPATCH_STATUS, desc= f"{status} is not allowed. Use one of {openpatch_status}", level=Level.ERROR, )) elif status == "approved" and not openpatch.get("url"): problems.append( Problem( key="openpatch.url", rule=Rule.NO_OPENPATCH_URL, desc="No OpenPatch url", level=Level.WARNING, )) # check classifications classifications = f.get("classifications", {}) if classifications is None or len(classifications.keys()) == 0: problems.append( Problem( key="classifications", rule=Rule.NO_CLASSIFICATION, desc="No classification found", level=Level.WARNING, )) else: for ( classification_id, classification, ) in classifications.items(): if "_meta" in classification_id: continue if classification_id not in self.classifications: problems.append( Problem( key="classifications", rule=Rule.CLASSIFICATION_NOT_FOUND, desc=f"{classification_id} not found", )) else: problems.extend( self.lint_classification( classification, classification_id)) # check authors authors = f.get("authors", []) if len(authors) == 0: problems.append( Problem( key="authors", rule=Rule.NO_AUTHOR, desc="No author found", level=Level.WARNING, )) else: for author in authors: if author not in self.authors: problems.append( Problem( key="authors", rule=Rule.AUTHOR_NOT_FOUND, desc=f"{author} not found", level=Level.ERROR, )) # check items items = f.get("items", None) if items is None: problems.append( Problem( key="items", rule=Rule.NO_ITEMS, desc=f"No items found", level=Level.WARNING, )) else: count = items.get("count") if not count: problems.append( Problem( key="items", rule=Rule.NO_ITEMS_COUNT, level=Level.WARNING, )) elif not isinstance(count, int) and count != "?": problems.append( Problem( key="items.count", rule=Rule.INVALID_ITEMS_COUNT, desc= f"invalid type {type(count)} should be int or '?'", level=Level.ERROR, )) # check papers papers = f.get("papers", []) if len(papers) == 0: problems.append( Problem( key="papers", rule=Rule.NO_PAPER, desc="No paper found", level=Level.WARNING, )) else: for paper in papers: if isinstance(paper, str): problems.append( Problem( key="papers", rule=Rule.PAPER_NEEDS_TO_BE_A_MAPPING, level=Level.ERROR, desc= f"{paper} needs to be a mapping (id, category)", )) elif paper.get("id") not in self.papers: problems.append( Problem( key="papers", rule=Rule.PAPER_NOT_FOUND, desc=f"{paper['id']} not found", level=Level.ERROR, )) elif paper.get("category") is None: problems.append( Problem( key="papers", rule=Rule.NO_PAPER_CATEGORY, desc=f"{paper['id']} has no category", level=Level.WARNING, )) elif paper.get( "category") not in self.categories["papers"]: problems.append( Problem( key="papers", rule=Rule.PAPER_CATEGORY_NOT_FOUND, level=Level.ERROR, desc= f"{paper['category']} for {paper['id']} not found ({[key for key in self.categories['papers'].keys() if not key.startswith('_')]})", )) return problems
def get_config(config_name): with open('../configs.yml', 'r') as config_file: config = yaml.full_load(config_file) config = config[config_name] return config
def policy_yaml(self): return yaml.full_load(self.policy)
def train(self, rank, start_time, return_dict): device = torch.device("cuda:" + str(rank)) print('Running on device: ', device) torch.cuda.set_device(device) torch.set_default_tensor_type(torch.FloatTensor) writer = None if not self.args.cross_validate_hp: writer = SummaryWriter(logdir=os.path.join(self.save_dir, 'logs')) # posting parameters param_string = "" for k, v in vars(self.args).items(): param_string += ' ' * 10 + k + ': ' + str(v) + '\n' writer.add_text("params", param_string) self.setup(rank, self.args.num_processes) if self.cfg.MC_DQL: transition = namedtuple('Transition', ('episode')) else: transition = namedtuple( 'Transition', ('state', 'action', 'reward', 'next_state', 'done')) memory = TransitionData_ts(capacity=self.args.t_max, storage_object=transition) env = SpGcnEnv(self.args, device, writer=writer, writer_counter=self.global_writer_quality_count, win_event_counter=self.global_win_event_count) # Create shared network # model = GcnEdgeAC_1(self.cfg, self.args.n_raw_channels, self.args.n_embedding_features, 1, device, writer=writer) model = GcnEdgeAC(self.cfg, self.args, device, writer=writer) # model = GcnEdgeAC(self.cfg, self.args.n_raw_channels, self.args.n_embedding_features, 1, device, writer=writer) model.cuda(device) shared_model = DDP(model, device_ids=[model.device], find_unused_parameters=True) # dloader = DataLoader(MultiDiscSpGraphDsetBalanced(no_suppix=False, create=False), batch_size=1, shuffle=True, pin_memory=True, # num_workers=0) dloader = DataLoader(SpgDset(), batch_size=self.cfg.batch_size, shuffle=True, pin_memory=True, num_workers=0) # Create optimizer for shared network parameters with shared statistics # optimizer = CstmAdam(shared_model.parameters(), lr=self.args.lr, betas=self.args.Adam_betas, # weight_decay=self.args.Adam_weight_decay) ###################### self.action_range = 1 self.device = torch.device(device) self.discount = 0.5 self.critic_tau = self.cfg.critic_tau self.actor_update_frequency = self.cfg.actor_update_frequency self.critic_target_update_frequency = self.cfg.critic_target_update_frequency self.batch_size = self.cfg.batch_size self.log_alpha = torch.tensor(np.log(self.cfg.init_temperature)).to( self.device) self.log_alpha.requires_grad = True # set target entropy to -|A| ###################### # optimizers OptimizerContainer = namedtuple('OptimizerContainer', ('actor', 'critic', 'temperature')) actor_optimizer = torch.optim.Adam( shared_model.module.actor.parameters(), lr=self.cfg.actor_lr, betas=self.cfg.actor_betas) critic_optimizer = torch.optim.Adam( shared_model.module.critic.parameters(), lr=self.cfg.critic_lr, betas=self.cfg.critic_betas) temp_optimizer = torch.optim.Adam([self.log_alpha], lr=self.cfg.alpha_lr, betas=self.cfg.alpha_betas) optimizers = OptimizerContainer(actor_optimizer, critic_optimizer, temp_optimizer) for param in model.fe_ext.parameters(): param.requires_grad = False if self.args.model_name != "": shared_model.load_state_dict( torch.load(os.path.join(self.save_dir, self.args.model_name))) elif self.args.model_fe_name != "": shared_model.module.fe_ext.load_state_dict( torch.load(os.path.join(self.save_dir, self.args.model_fe_name))) elif self.args.fe_extr_warmup: print('loaded fe extractor') shared_model.module.fe_ext.load_state_dict( torch.load(os.path.join(self.save_dir, 'agent_model_fe_extr'))) dist.barrier() if not self.args.test_score_only: quality = self.args.stop_qual_scaling + self.args.stop_qual_offset best_quality = np.inf while self.global_count.value() <= self.args.T_max: if self.global_count.value() == 78: a = 1 self.update_env_data(env, dloader, device) # waff_dis = torch.softmax(env.edge_features[:, 0].squeeze() + 1e-30, dim=0) # waff_dis = torch.softmax(env.gt_edge_weights + 0.5, dim=0) # waff_dis = torch.softmax(torch.ones_like(env.b_sg_gt_edge_weights), dim=0) # loss_weight = torch.softmax(env.b_sg_gt_edge_weights + 1, dim=0) env.reset() # self.target_entropy = - float(env.gt_edge_weights.shape[0]) self.target_entropy = -self.args.s_subgraph env.stop_quality = self.stop_qual_rule.apply( self.global_count.value(), quality) if self.cfg.temperature_regulation == 'follow_quality': self.alpha = self.eps_rule.apply(self.global_count.value(), quality) print(self.alpha.item()) with open(os.path.join(self.save_dir, 'runtime_cfg.yaml')) as info: args_dict = yaml.full_load(info) if args_dict is not None: if 'safe_model' in args_dict: self.args.safe_model = args_dict['safe_model'] args_dict['safe_model'] = False if 'add_noise' in args_dict: self.args.add_noise = args_dict['add_noise'] if 'critic_lr' in args_dict and args_dict[ 'critic_lr'] != self.cfg.critic_lr: self.cfg.critic_lr = args_dict['critic_lr'] adjust_learning_rate(critic_optimizer, self.cfg.critic_lr) if 'actor_lr' in args_dict and args_dict[ 'actor_lr'] != self.cfg.actor_lr: self.cfg.actor_lr = args_dict['actor_lr'] adjust_learning_rate(actor_optimizer, self.cfg.actor_lr) if 'alpha_lr' in args_dict and args_dict[ 'alpha_lr'] != self.cfg.alpha_lr: self.cfg.alpha_lr = args_dict['alpha_lr'] adjust_learning_rate(temp_optimizer, self.cfg.alpha_lr) with open(os.path.join(self.save_dir, 'runtime_cfg.yaml'), "w") as info: yaml.dump(args_dict, info) if self.args.safe_model: best_quality = quality if rank == 0: if self.args.model_name_dest != "": torch.save( shared_model.state_dict(), os.path.join(self.save_dir, self.args.model_name_dest)) else: torch.save( shared_model.state_dict(), os.path.join(self.save_dir, 'agent_model')) state = env.get_state() while not env.done: # Calculate policy and values post_input = True if ( self.global_count.value() + 1) % 15 == 0 and env.counter == 0 else False round_n = env.counter # sample action for data collection distr = None if self.global_count.value() < self.cfg.num_seed_steps: action = torch.rand_like(env.sg_current_edge_weights) else: distr, _, _, action = self.agent_forward( env, shared_model, state, grad=False, post_input=post_input) logg_dict = {'temperature': self.alpha.item()} if distr is not None: logg_dict['mean_loc'] = distr.loc.mean().item() logg_dict['mean_scale'] = distr.scale.mean().item() if self.global_count.value( ) >= self.cfg.num_seed_steps and memory.is_full(): self._step(memory, optimizers, env, shared_model, self.global_count.value(), writer=writer) self.global_writer_loss_count.increment() next_state, reward, quality = env.execute_action( action, logg_dict) if self.args.add_noise: noise = torch.randn_like(reward) * self.alpha.item() reward = reward + noise memory.push(self.state_to_cpu(state), action.cpu(), reward.cpu(), self.state_to_cpu(next_state), env.done) # Train the network # self._step(memory, shared_model, env, optimizer, loss_weight, off_policy=True, writer=writer) # reward = self.args.reward_clip and min(max(reward, -1), 1) or reward # Optionally clamp rewards # done = done or episode_length >= self.args.max_episode_length # Stop episodes at a max length state = next_state self.global_count.increment() if "self_reg" in self.args.eps_rule and quality <= 2: break dist.barrier() if rank == 0: if not self.args.cross_validate_hp and not self.args.test_score_only and not self.args.no_save: # pass if self.args.model_name_dest != "": torch.save( shared_model.state_dict(), os.path.join(self.save_dir, self.args.model_name_dest)) print('saved') else: torch.save(shared_model.state_dict(), os.path.join(self.save_dir, 'agent_model')) self.cleanup()
def test_k8s_run_launcher_config(template: HelmTemplate): job_namespace = "namespace" image_pull_policy = "Always" load_incluster_config = True env_config_maps = [{"name": "env_config_map"}] env_secrets = [{"name": "secret"}] env_vars = ["ENV_VAR"] volume_mounts = [ { "mountPath": "/opt/dagster/dagster_home/dagster.yaml", "name": "dagster-instance", "subPath": "dagster.yaml", }, { "name": "test-volume", "mountPath": "/opt/dagster/test_mount_path/volume_mounted_file.yaml", "subPath": "volume_mounted_file.yaml", }, ] volumes = [ { "name": "test-volume", "configMap": { "name": "test-volume-configmap" } }, { "name": "test-pvc", "persistentVolumeClaim": { "claimName": "my_claim", "readOnly": False } }, ] labels = {"my_label_key": "my_label_value"} helm_values = DagsterHelmValues.construct( runLauncher=RunLauncher.construct( type=RunLauncherType.K8S, config=RunLauncherConfig.construct( k8sRunLauncher=K8sRunLauncherConfig.construct( jobNamespace=job_namespace, imagePullPolicy=image_pull_policy, loadInclusterConfig=load_incluster_config, envConfigMaps=env_config_maps, envSecrets=env_secrets, envVars=env_vars, volumeMounts=volume_mounts, volumes=volumes, labels=labels, )), )) configmaps = template.render(helm_values) instance = yaml.full_load(configmaps[0].data["dagster.yaml"]) run_launcher_config = instance["run_launcher"] assert run_launcher_config["module"] == "dagster_k8s" assert run_launcher_config["class"] == "K8sRunLauncher" assert run_launcher_config["config"]["job_namespace"] == job_namespace assert run_launcher_config["config"][ "load_incluster_config"] == load_incluster_config assert run_launcher_config["config"][ "image_pull_policy"] == image_pull_policy assert run_launcher_config["config"]["env_config_maps"][1:] == [ configmap["name"] for configmap in env_config_maps ] assert run_launcher_config["config"]["env_secrets"] == [ secret["name"] for secret in env_secrets ] assert run_launcher_config["config"]["env_vars"] == env_vars assert run_launcher_config["config"]["volume_mounts"] == volume_mounts assert run_launcher_config["config"]["volumes"] == volumes assert run_launcher_config["config"]["labels"] == labels assert not "fail_pod_on_run_failure" in run_launcher_config["config"]
type=str, help="root directory for all datasets") parser.add_argument('--preprocesseddir', type=str, help="preprocessed data directory") parser.add_argument('--traininglogsdir', type=str, help="training logs directory") parser.add_argument('--modelsdir', type=str, help="models directory") args = parser.parse_args() run = Run.get_context() print("GPUs available:") print(tf.config.experimental.list_physical_devices('GPU')) # Update paths of input data in config to represent paths on blob. cfg = yaml.full_load(open(os.getcwd() + "./config.yml", 'r')) # Load config data cfg['PATHS']['RAW_DATA'] = args.rawdatadir cfg['PATHS']['PROCESSED_DATA'] = args.preprocesseddir cfg['PATHS']['TRAIN_SET'] = cfg['PATHS']['PROCESSED_DATA'] + '/' + cfg[ 'PATHS']['TRAIN_SET'].split('/')[-1] cfg['PATHS']['VAL_SET'] = cfg['PATHS']['PROCESSED_DATA'] + '/' + cfg['PATHS'][ 'VAL_SET'].split('/')[-1] cfg['PATHS']['TEST_SET'] = cfg['PATHS']['PROCESSED_DATA'] + '/' + cfg['PATHS'][ 'TEST_SET'].split('/')[-1] # Set paths to run's ./output/ directory cfg['PATHS']['LOGS'] = args.traininglogsdir cfg['PATHS']['MODEL_WEIGHTS'] = args.modelsdir # Set logs directory according to datetime cur_date = datetime.datetime.now().strftime('%Y%m%d-%H%M%S')
def get_yaml_data1(): file1 = open('stu_loginR.yaml', 'r', encoding='utf-8') file2 = yaml.full_load(file1) file1.close() return file2
if next_page_token == 0: more_subgroups = False # Bail out if there aren't any matching subgroups in the group if not groupsio_subgroups: sys.exit() ### Compare local subgroup membership against groups.io, resolve deltas ### all_local_valid_members = dict() # Open the local .yml file with subgroup definitions with open (subgroup_membership_filename,'r') as subgroup_membership_file: local_subgroups_and_members = yaml.full_load(subgroup_membership_file) if not local_subgroups_and_members: print('WARN: No lists defined. Exiting') sys.exit() # Walk through definitions for local_subgroup, local_members in local_subgroups_and_members.items(): # Protect main and the unified list if local_subgroup in [main_list,unified_list]: print('INFO: You cannot modify %s. Ignoring.' % local_subgroup) continue
""" Test file for printing session configuration """ import pprint import yaml from config import constants print(f'{constants.DECORATOR}\nSESSION CONFIGURATION:\n{constants.DECORATOR}') pprint.pprint(yaml.full_load(open('config.yml'))) print(f'{constants.DECORATOR}')
def get_item_yaml_requirements(directory: str): with open(f"{directory}/item.yaml", "r") as f: item = yaml.full_load(f) return item.get("spec", {}).get("requirements", [])
def questionReader(name): name = "".join(e for e in name if e.isalnum()) with open("./data/tmp/" + name + "/questions.yml", "r") as yml: document = yaml.full_load(yml) return document
@app.route('/') def ready(): return 'ready' # Variables for input and output data pathConfig = "/home/config/config.yml" pathApicLoginTemplate = "/home/templates/apicLoginTemplate.json" pathSubscriptionIds = "/home/internal/subscriptionIds.json" basePathLogs = "/home/data/logs/" basePathOutput = "/home/data/output/" loginToken = "" # Load config file, it is required in multiple follwing functions with open(pathConfig, "r") as handle: config = yaml.full_load(handle) # Disable warning for APIC Self-Signed Certificate requests.packages.urllib3.disable_warnings() # Write Tool logs to local logfile, can be mapped to host volume # Messages are provided by the calling functions def writeLog(message): Path(basePathLogs).mkdir(parents=True, exist_ok=True) logPath = basePathLogs + time.strftime('%Y-%m-%d', time.localtime()) + ".txt" timestamp = time.strftime("%d/%m/%Y-%H:%M:%S", time.localtime()) old_stdout = sys.stdout sys.stdout = open(logPath, "a+")
def load_yaml(file): """If pyyaml > 5.1 use full_load to avoid warning""" if hasattr(yaml, "full_load"): return yaml.full_load(file) else: return yaml.load(file)
from jinja2 import Environment, FileSystemLoader import yaml env = Environment(loader=FileSystemLoader('.'), trim_blocks=True, lstrip_blocks=True) template = env.get_template("nexus.txt.j2") with open("config.yaml") as file: interface_file = yaml.full_load(file) open('output.txt', 'w').close() file = open('output.txt', 'a') file.write(template.render(interface_file)) file.close()
def read_yaml(path='devices.yml'): with open(path) as f: devices = yaml.full_load(f) return devices
def startServer(self, cmd): '''启动minecraft服务器''' os.chdir(self.config.game_version_server_dir()) # 如果没有eula.txt文件,则启动服务器生成 if not self.checkEULA(): if self.config.isForge: self.generateForgeServerEULA() else: os.system(cmd) # 同意eula with io.open(self.config.game_version_server_eula_file_path(), 'r', encoding='utf-8') as f: eula = f.read() checkEULA = eula.replace('false', 'true') with io.open(self.config.game_version_server_eula_file_path(), 'w', encoding='utf-8') as f: f.write(checkEULA) # 修改commands.yaml中关于命令方块指令使用Mojang,不会被Essentials插件覆盖命令 bukkit_command_yaml_file_path = self.config.game_version_server_bukkit_command_yaml_file_path( ) if os.path.exists(bukkit_command_yaml_file_path): with io.open(bukkit_command_yaml_file_path, 'r', encoding='utf-8') as f: commands_cfg = yaml.full_load(f) commands_cfg['command-block-overrides'] = ['*'] with io.open(bukkit_command_yaml_file_path, 'w', encoding='utf-8') as f: yaml.dump(commands_cfg, f) RichText.info('commands.yaml has been changed!') # 启动服务器 if self.config.debug: print(cmd) os.system(cmd) # 设置服务器属性为离线模式 with io.open(self.config.game_version_server_properties_file_path(), 'r', encoding='utf-8') as f: properties = f.read() if 'online-mode=false' in properties: offline_properties = None else: offline_properties = properties.replace( 'online-mode=true', 'online-mode=false') if offline_properties != None: with io.open( self.config.game_version_server_properties_file_path(), 'w', encoding='utf-8') as f: f.write(offline_properties) print( ColorString.confirm( 'Setting the server to offline mode, next launch this setting take effect!!!' )) # 为服务器核心数据文件创建符号链接 self.symlink_server_core_files_if_need()
def test_celery_k8s_run_launcher_config(template: HelmTemplate): image = { "repository": "test_repo", "tag": "test_tag", "pullPolicy": "Always" } configSource = { "broker_transport_options": { "priority_steps": [9] }, "worker_concurrency": 1, } workerQueues = [ { "name": "dagster", "replicaCount": 2 }, { "name": "extra-queue-1", "replicaCount": 1 }, ] volume_mounts = [ { "mountPath": "/opt/dagster/dagster_home/dagster.yaml", "name": "dagster-instance", "subPath": "dagster.yaml", }, { "name": "test-volume", "mountPath": "/opt/dagster/test_mount_path/volume_mounted_file.yaml", "subPath": "volume_mounted_file.yaml", }, ] volumes = [ { "name": "test-volume", "configMap": { "name": "test-volume-configmap" } }, { "name": "test-pvc", "persistentVolumeClaim": { "claimName": "my_claim", "readOnly": False } }, ] labels = {"my_label_key": "my_label_value"} image_pull_secrets = [{"name": "IMAGE_PULL_SECRET"}] helm_values = DagsterHelmValues.construct( imagePullSecrets=image_pull_secrets, runLauncher=RunLauncher.construct( type=RunLauncherType.CELERY, config=RunLauncherConfig.construct( celeryK8sRunLauncher=CeleryK8sRunLauncherConfig.construct( image=image, configSource=configSource, workerQueues=workerQueues, volumeMounts=volume_mounts, volumes=volumes, labels=labels, )), ), ) configmaps = template.render(helm_values) instance = yaml.full_load(configmaps[0].data["dagster.yaml"]) run_launcher_config = instance["run_launcher"] assert run_launcher_config["module"] == "dagster_celery_k8s" assert run_launcher_config["class"] == "CeleryK8sRunLauncher" assert run_launcher_config["config"]["config_source"] == configSource assert run_launcher_config["config"]["broker"] == { "env": "DAGSTER_CELERY_BROKER_URL" } assert run_launcher_config["config"]["backend"] == { "env": "DAGSTER_CELERY_BACKEND_URL" } assert run_launcher_config["config"]["volume_mounts"] == volume_mounts assert run_launcher_config["config"]["volumes"] == volumes assert run_launcher_config["config"]["labels"] == labels assert run_launcher_config["config"][ "image_pull_secrets"] == image_pull_secrets assert run_launcher_config["config"]["image_pull_policy"] == "Always" assert run_launcher_config["config"][ "service_account_name"] == "release-name-dagster" assert not "fail_pod_on_run_failure" in run_launcher_config["config"] helm_values_with_image_pull_policy = DagsterHelmValues.construct( runLauncher=RunLauncher.construct( type=RunLauncherType.CELERY, config=RunLauncherConfig.construct( celeryK8sRunLauncher=CeleryK8sRunLauncherConfig.construct( image=image, configSource=configSource, workerQueues=workerQueues, volumeMounts=volume_mounts, volumes=volumes, imagePullPolicy="IfNotPresent", )), ), ) configmaps = template.render(helm_values_with_image_pull_policy) instance = yaml.full_load(configmaps[0].data["dagster.yaml"]) run_launcher_config = instance["run_launcher"] assert run_launcher_config["config"]["image_pull_policy"] == "IfNotPresent" helm_values_with_fail_pod_on_run_failure = DagsterHelmValues.construct( runLauncher=RunLauncher.construct( type=RunLauncherType.CELERY, config=RunLauncherConfig.construct( celeryK8sRunLauncher=CeleryK8sRunLauncherConfig.construct( image=image, configSource=configSource, workerQueues=workerQueues, failPodOnRunFailure=True, )), ), ) configmaps = template.render(helm_values_with_fail_pod_on_run_failure) instance = yaml.full_load(configmaps[0].data["dagster.yaml"]) run_launcher_config = instance["run_launcher"] assert run_launcher_config["config"]["fail_pod_on_run_failure"]
def load_file(file_name): with open(file_name, 'r') as fh: return yaml.full_load(fh)
def parse_yaml(filename='./static/my_resume.yaml'): with open(filename) as file: cv = yaml.full_load(file) cv['cachebuster'] = lambda: md5(str(time()).encode('utf-8')).hexdigest( ) return cv
def load_system_template(): with open(f'{template_path}/system_stub.yml') as file: documents = yaml.full_load(file) return documents
def load_preferred_entries_file(fpath): with open(fpath) as h: return yaml.full_load(h)
from skeleton import skeleton import pickle from tqdm import tqdm import src.utils as utils import src.stat as stats import yaml import sys G, color_edge = skeleton() #skeleton of fly legtips = [4, 9, 14, 19, 24, 29] print('making video') usr_input = sys.argv[-1] #load global parameters par = yaml.full_load(open(usr_input, "rb")) #predictions data = torch.load(par['data_dir'] + '/test_results.pth.tar') out_offset, inp_offset = pickle.load(open('joint_locations.pkl','rb')) #output targets_1d = torch.load(par['data_dir'] + '/stat_3d.pth.tar')['targets_1d'] out_mean = torch.load(par['data_dir'] + '/stat_3d.pth.tar')['mean'] out_std = torch.load(par['data_dir'] + '/stat_3d.pth.tar')['std'] out = utils.unNormalize(data['output'], out_mean[targets_1d], out_std[targets_1d]) #inputs targets_2d = torch.load(par['template_dir'] + '/stat_2d.pth.tar')['targets_2d'] inp_mean = torch.load(par['template_dir'] + 'stat_2d.pth.tar')['mean'] inp_std = torch.load(par['template_dir'] + 'stat_2d.pth.tar')['std']
def main(): """The main function""" parser = argparse.ArgumentParser( description='Test runner', formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument( '--app', default='./core', help='Path to the executable (vast/core)') parser.add_argument( '-s', '--set', type=Path, help='Run the testset from this test definition YAML file') parser.add_argument( '-T', '--tag', nargs='+', help='The tag for which tests will be run') parser.add_argument( '-t', '--test', nargs='+', help='The test(s) to run (runs all tests if unset)') parser.add_argument( '-u', '--update', action='store_true', help='Update baseline for tests') parser.add_argument( '-d', '--directory', default='run_<current_ISO_timestamp>', type=Path, help='The basedir for the test runs') parser.add_argument( '-K', '--keep', action='store_true', help='Keep artifacts of successful runs') parser.add_argument( '--timeout', type=int, default=0, help='Test timeout in seconds') parser.add_argument( '-l', '--list', nargs='*', help='Return a list of available tests optionally filtered with tags') parser.add_argument( '-L', '--list-tags', action='store_true', help='Return a list of all available tags') parser.add_argument( '--flamegraph', action='store_true', help='Generate a flamegraph of the test run') parser.add_argument( '--flamegraph_path', default='scripts/flamegraph', type=Path, help='Path to flamegraph script') parser.add_argument( '-v', '--verbosity', default='DEBUG', help='Set the logging verbosity') args = parser.parse_args() # Setup logging. LOGGER.setLevel(args.verbosity) fmt = "%(asctime)s %(levelname)-8s %(message)s" colored_formatter = coloredlogs.ColoredFormatter(fmt) plain_formatter = logging.Formatter(fmt) formatter = colored_formatter if sys.stdout.isatty() else plain_formatter ch = logging.StreamHandler() ch.setLevel(args.verbosity) ch.setFormatter(formatter) LOGGER.addHandler(ch) # Create a new handler for log level CRITICAL. class ShutdownHandler(logging.Handler): def emit(self, record): logging.shutdown() signal_subprocs(signal.SIGTERM) sys.exit(1) # Register this handler with log level CRITICAL (which equals to 50). sh = ShutdownHandler(level=50) sh.setFormatter(formatter) LOGGER.addHandler(sh) # Load test set. if not args.set: args.set = Path(__file__).resolve().parent / 'default_set.yaml' args.set = args.set.resolve() LOGGER.debug(f'resolved test set path to {args.set}') test_file = open(args.set, 'r') test_dict = yaml.full_load(test_file) test_dec = validate(test_dict, args.set.parent) # Print tests. if args.list is not None: selection = tagselect(args.list, test_dec['tests']) for test in selection.keys(): print(test) return # Print test tags. if args.list_tags: tags = set().union( *[set(t.tags) for _, t in test_dec['tests'].items()]) for tag in tags: print(tag) return # Create working directory. if args.directory.name == 'run_<current_ISO_timestamp>': timestamp = datetime.now().isoformat(timespec='seconds') args.directory = Path(f'run_{timestamp}') LOGGER.debug(f'keeping state in {args.directory}') if not args.directory.exists(): args.directory.mkdir(parents=True) # Setup signal handlers and run. signal.alarm(args.timeout) success = run(args, test_dec) signal.alarm(0) with suppress(OSError): args.directory.rmdir() retcode = 0 if success else 1 sys.exit(retcode)
def __init__(self, app=None, name=None, base_dir='~/.config/', template_file=None, template=None, envvars={}, defaults={}, cli_args={}, path_override=None, file_override=None): """ Automatically load an app config with environment variable and default overrides. Config file path will be {base_dir}/{app}/{name}.[yaml|yml|json] :param str app: Application name :param str name: Name of app sub-config :param str base_dir: Base config dir. Defaults to '~/.config/` :param str template_file: Template file to copy to given location if none exists :param str/dict template: YAML/JSON string or dict to write use as template if no config exists. :param dict envvars: Map of environment variables to map to keys. { key: APP_ENV_VAR } These override anything in the config file :param dict defaults: Default values for config keys :param dict cli_args: Config key overrides likely from the app CLI. These override any config file or env var values. :param str path_override: Specify direct path instead of default base dir :param str file_override: Direct path to config file to read :raises exceptions.IOError: when permissions denied """ self.filename = None self.full_path = None if file_override is not None: self.full_path = file_override if not os.path.isfile(self.full_path): raise Exception('Unable to find specified file {}'.format( self.full_path)) else: if path_override: self.base_dir = os.path.expanduser(path_override) else: self.base_dir = os.path.expanduser(os.path.join(base_dir, app)) if not os.path.isdir(self.base_dir): os.makedirs(self.base_dir) for ext in VALID_EXT: for _ext in [ext, ext.upper()]: filename = '{}.{}'.format(name, _ext) cfg = os.path.join(self.base_dir, filename) if os.path.isfile(cfg): self.full_path = cfg self.filename = filename break if self.filename is None: self.filename = name + '.' + VALID_EXT[0] self.full_path = os.path.join(self.base_dir, self.filename) if template_file: shutil.copyfile(template_file, self.full_path) else: with open(self.full_path, 'w') as f: if template: if isinstance(template, string_types): f.write(template) elif isinstance(template, dict): yaml.dump(template, f, default_flow_style=False) else: yaml.dump({}, f, default_flow_style=False) self._data = defaults with open(self.full_path, 'r') as f: data = yaml.full_load(f) self._data.update(data) # load envvars if given and override for k, v in envvars.items(): if v in os.environ: self._data[k] = os.environ[v] # finally, override with given cli args for k, v in cli_args.items(): if k not in self._data or v is not None: self._data[k] = v
def load_client_template(type='default'): with open(f'{template_path}/client_stub_{type}.yml') as file: documents = yaml.full_load(file) return documents
import ZCBF_module import two_dof_module import yaml import sys #======================================== # Define simlation scenario sim_file = 'ZCBF_control_'+ sys.argv[1] + '.yaml' # 'ZCBF_cont_control_expi.yaml' print('Evaluating bounds for experiment: '+ sim_file) #======================================== # Load parameter file with open(sim_file) as file: data = yaml.full_load(file) # Define system constraints q_max = data['q_max'] q_min = data['q_min'] v_max = data['v_max'] v_min = data['v_min'] u_max = data['u_max'] u_min = data['u_min'] # Define 2DOF model parameters m_list = data['m_list']# link masses (kg) l_list = data['l_list']# link lengths(m) f_list = data['f_list']# damping friction term
import logging.config from pathlib import Path import yaml DIR = Path(__file__).parent.absolute() LOGPATH = DIR.parent.parent / 'logs' def configureLogging(): . Path(LOGPATH).mkdir(exist_ok=True) mainfilename = LOGPATH / 'main.log' debugfilename = LOGPATH / 'debug.log' with open(DIR / 'logging.yaml', 'r') as f: log_cfg = yaml.full_load(f) log_cfg['handlers']['file_handler']['filename'] = mainfilename log_cfg['handlers']['rotating_handler']['filename'] = debugfilename print(log_cfg) logging.config.dictConfig(log_cfg) # Set ERROR level logging on verbose modules modules = ['botocore', 'urllib3', 'googleapiclient'] for module in modules: logging.getLogger(module).setLevel(logging.ERROR)
import os import yaml from codecs import encode, decode from haruka import LOGGER from haruka.modules.sql.translation import prev_locale LANGUAGES = ['en', 'id', 'ru'] strings = {} for i in LANGUAGES: strings[i] = yaml.full_load( open(os.path.dirname(__file__) + f"/{i}/string.yml", "r")) print(f"Loaded {i}/string.yml") def tld(chat_id, t, show_none=True): LANGUAGE = prev_locale(chat_id) if LANGUAGE: LOCALE = LANGUAGE.locale_name if LOCALE in ('en') and t in strings['en']: result = decode( encode(strings['en'][t], 'latin-1', 'backslashreplace'), 'unicode-escape') return result elif LOCALE in ('id') and t in strings['id']: result = decode( encode(strings['id'][t], 'latin-1', 'backslashreplace'), 'unicode-escape')
def main(): arg_parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter) # helm mode with scheduler arg_parser.add_argument("--image-version", default="", help="image version") arg_parser.add_argument("--schedule-conf", metavar='FILE', default='', help="load test schedule from FILE") # local mode arg_parser.add_argument('--local', action='store_true', help='use local milvus server') arg_parser.add_argument('--host', help='server host ip param for local mode', default='127.0.0.1') arg_parser.add_argument('--port', help='server port param for local mode', default=config.SERVER_PORT_DEFAULT) arg_parser.add_argument('--suite', metavar='FILE', help='load test suite from FILE', default='') arg_parser.add_argument('--server-config', metavar='FILE', help='load server config from FILE', default='') args = arg_parser.parse_args() if args.schedule_conf: if args.local: raise Exception( "Helm mode with scheduler and other mode are incompatible") if not args.image_version: raise Exception("Image version not given") env_mode = "helm" image_version = args.image_version with open(args.schedule_conf) as f: schedule_config = full_load(f) f.close() helm_path = os.path.join(os.getcwd(), "..//milvus-helm-charts/charts/milvus-ha") for item in schedule_config: server_host = item["server"] if "server" in item else "" server_tag = item["server_tag"] if "server_tag" in item else "" deploy_mode = item[ "deploy_mode"] if "deploy_mode" in item else config.DEFAULT_DEPLOY_MODE suite_params = item["suite_params"] for suite_param in suite_params: suite_file = "suites/" + suite_param["suite"] with open(suite_file) as f: suite_dict = full_load(f) f.close() logger.debug(suite_dict) run_type, run_params = parser.operations_parser(suite_dict) collections = run_params["collections"] image_type = suite_param["image_type"] image_tag = get_image_tag(image_version) for suite in collections: # run test cases milvus_config = suite[ "milvus"] if "milvus" in suite else None server_config = suite[ "server"] if "server" in suite else None logger.debug(milvus_config) logger.debug(server_config) helm_params = { "server_name": server_host, "server_tag": server_tag, "server_config": server_config, "milvus_config": milvus_config, "image_tag": image_tag, "image_type": image_type } env_params = { "deploy_mode": deploy_mode, "helm_path": helm_path, "helm_params": helm_params } # job = back_scheduler.add_job(run_suite, args=[run_type, suite, env_mode, env_params], # misfire_grace_time=36000) # logger.info(job) # logger.info(job.id) elif args.local: # for local mode deploy_params = args.server_config deploy_params_dict = None if deploy_params: with open(deploy_params) as f: deploy_params_dict = full_load(f) f.close() logger.debug(deploy_params_dict) deploy_mode = utils.get_deploy_mode(deploy_params_dict) server_tag = utils.get_server_tag(deploy_params_dict) env_params = { "host": args.host, "port": args.port, "deploy_mode": deploy_mode, "server_tag": server_tag, "deploy_opology": deploy_params_dict } suite_file = args.suite with open(suite_file) as f: suite_dict = full_load(f) f.close() logger.debug(suite_dict) run_type, run_params = parser.operations_parser(suite_dict) collections = run_params["collections"] if len(collections) > 1: raise Exception("Multi collections not supported in Local Mode") # ensure there is only one case in suite # suite = {"run_type": run_type, "run_params": collections[0]} suite = collections[0] timeout = suite["timeout"] if "timeout" in suite else None env_mode = "local" return run_suite(run_type, suite, env_mode, env_params, timeout=timeout)
def read_yaml(self, k, v): with open(self.file, encoding='utf8') as f: data = yaml.full_load(f) return data[k][v]