def __init__( self, reid_network_weights_path, obj_detect_model_path, tracktor_config_path, reid_config_path, ): super(Tracktor, self).__init__("Tracktor") with open(tracktor_config_path) as config_file: tracktor = yaml.unsafe_load(config_file)["tracktor"] with open(reid_config_path) as config_file: reid = yaml.unsafe_load(config_file)["reid"] # set all seeds torch.manual_seed(tracktor["seed"]) torch.cuda.manual_seed(tracktor["seed"]) np.random.seed(tracktor["seed"]) torch.backends.cudnn.deterministic = True ########################## # Initialize the modules # ########################## # object detection obj_detect = FRCNN_FPN(num_classes=2) obj_detect.load_state_dict( torch.load( obj_detect_model_path, map_location=lambda storage, loc: storage, ) ) obj_detect.eval() obj_detect.cuda() # reid reid_network = resnet50(pretrained=False, **reid["cnn"]) reid_network.load_state_dict( torch.load( reid_network_weights_path, map_location=lambda storage, loc: storage ) ) reid_network.eval() reid_network.cuda() # tracktor if "oracle" in tracktor: self.tracker = OracleTracker( obj_detect, reid_network, tracktor["tracker"], tracktor["oracle"] ) else: self.tracker = Tracker(obj_detect, reid_network, tracktor["tracker"]) self.transforms = ToTensor()
def load(self): super().load() self._build_url_paths self.load_pages() log(f"{self.__class__.__name__}: Loading status (file={self.STATUS_FILE})" ) self.page_status = yaml.unsafe_load(self.STATUS_FILE.open()) log(f"{self.__class__.__name__}: Loading groups (file={self.GROUPS_FILE})" ) self.groups = yaml.unsafe_load(self.GROUPS_FILE.open())
def load_beamtime(directory=None): """ Load a Beamtime and associated objects. Expected directory structure: <glbl['yaml_dir']>/ bt_bt.yml glbl.yml samples/ scanplans/ """ if directory is None: directory = glbl_dict["yaml_dir"] # leave room for multi-beamtime known_uids = {} beamtime_fn = os.path.join(directory, "bt_bt.yml") sample_fns = [fn for fn in os.listdir(os.path.join(directory, "samples"))] scanplan_fns = [ fn for fn in os.listdir(os.path.join(directory, "scanplans")) ] with open(beamtime_fn, "r") as f: bt = load_yaml(f, known_uids) # most recent scanplan order scanplan_order_fn = os.path.join( glbl_dict["config_base"], ".scanplan_order.yml" ) if os.path.isfile(scanplan_order_fn): with open(scanplan_order_fn) as f: scanplan_order = yaml.unsafe_load(f) for fn in sorted( scanplan_fns, key=list(scanplan_order.values()).index ): with open(os.path.join(directory, "scanplans", fn), "r") as f: load_yaml(f, known_uids) # most recent sample order sample_order_fn = os.path.join( glbl_dict["config_base"], ".sample_order.yml" ) if os.path.isfile(sample_order_fn): with open(sample_order_fn) as f: sample_order = yaml.unsafe_load(f) for fn in sorted(sample_fns, key=list(sample_order.values()).index): with open(os.path.join(directory, "samples", fn), "r") as f: load_yaml(f, known_uids) return bt
def main(args): if '.' not in sys.path: sys.path.insert(0, '.') if args.config: with open(args.config) as fle: config = yaml.unsafe_load(fle) else: config = {AMQP_URI_CONFIG_KEY: args.broker} if args.logging_config_file: logging.config.fileConfig(args.logging_config_file, disable_existing_loggers=False) elif 'LOGGING' in config: logging.config.dictConfig(config['LOGGING']) else: logging.basicConfig(level=logging.INFO, format='%(message)s') services = [] for path in args.services: services.extend(import_service(path)) kwargs = {'backdoor_port': args.backdoor_port} if config.get(AUTORELOAD_CONFIG_KEY): logger.info('autoreload enabled') autoreload.make_autoreload(run, args=(services, config), kwargs=kwargs) else: run(services, config, **kwargs)
def get_payment_items(yaml_path): with open(yaml_path, "r") as stream: result = yaml.unsafe_load(stream.read()) payment_items = result["payment_items"] payments = result["payments"] print(payments) for payment_item in payment_items: due_datetime = payment_item["due_datetime"] invoice_datetime = payment_item["invoice_datetime"] if not due_datetime: pass else: match_obj = re.search(MATCHER, due_datetime) if match_obj: days = int(match_obj.groups()[0]) due_datetime = datetime.utcnow() + timedelta(days=days) else: due_datetime = datetime.utcnow() payment_item.update(due_datetime=due_datetime.isoformat()) if not invoice_datetime: pass else: match_obj = re.search(MATCHER, invoice_datetime) if match_obj: days = int(match_obj.groups()[0]) invoice_datetime = datetime.utcnow() + timedelta(days=days) else: invoice_datetime = datetime.utcnow() payment_item.update(invoice_datetime=invoice_datetime.isoformat()) return payment_items
def load_yaml(f, known_uids=None): """ Recreate a ScanPlan, Experiment, or Beamtime object from a YAML file. If its linked objects have already been created, re-link to them. If they have not yet been created, create them now. """ if known_uids is None: known_uids = {} data = yaml.unsafe_load(f) # If f is a file handle, 'rewind' it so we can read it again. if not isinstance(f, str): f.seek(0) if isinstance(data, dict) and "bt_uid" in data: obj = Beamtime.from_yaml(f) known_uids[obj["bt_uid"]] = obj return obj elif isinstance(data, list) and "sa_uid" in data[0]: beamtime = known_uids.get(data[1]["bt_uid"]) obj = Sample.from_yaml(f, beamtime=beamtime) known_uids[obj["sa_uid"]] = obj elif isinstance(data, list) and len(data) == 2: # elif isinstance(data, list) and 'sp_uid' in data[0]: beamtime = known_uids.get(data[1]["bt_uid"]) obj = ScanPlan.from_yaml(f, beamtime=beamtime) known_uids[obj["sp_uid"]] = obj else: raise ValueError("File does not match a recognized specification.") return obj
def update_dynamic_catalog(arn): # This microservice re-uses the SNS topic created with the `add_notifications` command # Subscribe sat-api ingest function to SNS topic. with open(sls_config_path, 'r') as f: sls_config = yaml.unsafe_load(f) if notification_topic_name not in sls_config['resources']['Resources']: sls_config['resources']['Resources'].update({ notification_topic_name: resources.sns_topic(notification_topic_name) }) sls_config['provider']['environment'].update( {'NOTIFICATION_TOPIC': notification_topic_name}) sns_subscription, policy = resources.subscribe_lambda_to_sns( arn, notification_topic_name) sls_config['resources']['Resources'].update({ 'satApiIngestSubscription': sns_subscription, 'satApiIngestPolicy': policy }) with open(sls_config_path, 'w') as outf: yaml.dump(sls_config, outf, indent=1)
def check_dask_settings(cnf=None): if cnf is None: cnf_file = 'ede_config.yaml' try: opts, args = getopt.getopt( cnf, "he:tf:m:vx:d:lq:", ["endpoint=", "file=", "method=", "export=", "detect=", "query="]) except getopt.GetoptError: logger.warning( '[%s] : [WARN] Invalid argument received exiting', datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S')) print("ede.py -f <filelocation>, -t -m <method> -v -x <modelname>") sys.exit(0) for opt, arg in opts: if opt in ("-f", "--file"): cnf_file = arg try: with open(cnf_file) as cf: readCnf = yaml.unsafe_load(cf) SchedulerEndpoint = readCnf['Connector']['Dask']['SchedulerEndpoint'] Scale = readCnf['Connector']['Dask']['Scale'] SchedulerPort = readCnf['Connector']['Dask']['SchedulerPort'] EnforceCheck = readCnf['Connector']['Dask']['EnforceCheck'] except Exception: SchedulerEndpoint = 0 Scale = 0 SchedulerPort = 0 EnforceCheck = 0 return SchedulerEndpoint, Scale, SchedulerPort, EnforceCheck
def load_config_yaml(filepath): """load yaml config and return the config object dict""" try: with open(filepath, 'r', encoding='utf-8') as f: cfg = f.read() cfg = yaml.unsafe_load(cfg) cfg_map = {} if 'general' in cfg: cfg_map['general'] = GeneralConfiguration(**cfg['general']) if 'dataset' in cfg: cfg_map['dataset'] = DatasetConfiguration(**cfg['dataset']) if 'model' in cfg: cfg_map['model'] = ModelConfiguration(**cfg['model']) if 'train' in cfg: cfg_map['train'] = TrainConfiguration(**cfg['train']) if 'test' in cfg: cfg_map['test'] = TestConfiguration(**cfg['test']) if 'log' in cfg: cfg_map['log'] = LogConfiguration(**cfg['log']) if 'rl' in cfg: cfg_map['rl'] = RLConfiguration(**cfg['rl']) return cfg_map except Exception as e: raise Exception("yaml config loading encountered error: {}".format(e))
def parse_config_option(text): import yaml if '=' in text: key, value = text.strip().split('=', 1) return key, yaml.unsafe_load(value) else: return text, True
def load_yaml_with_base(filename: str, allow_unsafe: bool = False) -> None: """ Just like `yaml.load(open(filename))`, but inherit attributes from its `_BASE_`. Args: filename (str): the file name of the current config. Will be used to find the base config file. allow_unsafe (bool): whether to allow loading the config file with `yaml.unsafe_load`. Returns: (dict): the loaded yaml """ with PathManager.open(filename, "r") as f: try: cfg = yaml.safe_load(f) except yaml.constructor.ConstructorError: if not allow_unsafe: raise logger = logging.getLogger(__name__) logger.warning( "Loading config {} with yaml.unsafe_load. Your machine may " "be at risk if the file contains malicious content.". format(filename)) f.close() with open(filename, "r") as f: cfg = yaml.unsafe_load(f) # pyre-ignore # pyre-ignore def merge_a_into_b(a: Dict[Any, Any], b: Dict[Any, Any]) -> None: # merge dict a into dict b. values in a will overwrite b. for k, v in a.items(): if isinstance(v, dict) and k in b: assert isinstance( b[k], dict), "Cannot inherit key '{}' from base!".format(k) merge_a_into_b(v, b[k]) else: b[k] = v if BASE_KEY in cfg: base_cfg_file = cfg[BASE_KEY] if base_cfg_file.startswith("~"): base_cfg_file = os.path.expanduser(base_cfg_file) if not any( map(base_cfg_file.startswith, ["/", "https://", "http://"])): # the path to base cfg is relative to the config file itself. base_cfg_file = os.path.join(os.path.dirname(filename), base_cfg_file) base_cfg = CfgNode.load_yaml_with_base(base_cfg_file, allow_unsafe=allow_unsafe) del cfg[BASE_KEY] # pyre-fixme[6]: Expected `Dict[typing.Any, typing.Any]` for 2nd param # but got `None`. merge_a_into_b(cfg, base_cfg) return base_cfg return cfg
def uploadFile(): print(request.form) print(request.files) if 'flightfile' not in request.files: return redirect(url_for('home')) file = request.files['flightfile'] text = file.read() if not allowed_file(file.filename): return redirect(url_for('home')) stuff = yaml.unsafe_load(text) print(stuff) for flight in stuff: reqs = ['code', 'airline', 'from', 'to', 'deptTime', 'arriveTime'] for i in reqs: if i not in flight: print(i) return "Error, incorrect structure. Please inlucde: code, airline, from, to, depttime, arriveTime for each flight" code = flight['code'] airline = flight['airline'] departsFrom = flight['from'] goingTo = flight['to'] departTime = flight['deptTime'] arriveTime = flight['arriveTime'] db.execute_sql( ''' INSERT INTO flights (code, airline, departsFrom, goingTo, departTime, arriveTime) VALUES (?, ?, ?, ?, ?, ?) ''', (code, airline, departsFrom, goingTo, departTime, arriveTime)) return redirect(url_for('home'))
def indices(name): with open(sls_config_path, 'r+') as f: config = yaml.unsafe_load(f) for indice in name: indice = indice.lower() lambda_func = { "handler": "handler.indices", "environment": { "INDICE_NAME": indice }, "layers": [rasterio_layer_arn], "timeout": 150, "memorySize": 1536 } if not hasattr(StacIndices, indice): raise AttributeError(f"'{indice}' is not a supported indice.") if not config['functions']: config['functions'] = { f'stacDerivatives_{indice}': lambda_func } else: config['functions'].update( {f'stacDerivatives_{indice}': lambda_func}) # https://github.com/vincentsarago/lambda-pyskel/blob/master/lambda_pyskel/scripts/cli.py f.seek(0) f.write(yaml.dump(config, default_flow_style=False))
def gather_detectron(self): glog.info('Gathering Detectron') if not exists(join(self.path_to_dataset, 'detectron')): os.mkdir(join(self.path_to_dataset, 'detectron')) detectron_file = join(self.path_to_dataset, 'metadata', 'detectron.p') if exists(detectron_file): glog.info('Loading coarse detections from: {0}'.format(detectron_file)) with open(detectron_file, 'rb') as f: self.detectron = pickle.load(f) else: for i, basename in enumerate(tqdm(self.frame_basenames)): with open(join(self.path_to_dataset, 'detectron', '{0}.yml'.format(basename)), 'rb') as stream: #data = yaml.load(stream) data = yaml.unsafe_load(stream) boxes, classes, segms = data['boxes'], data['classes'], data['segms'] self.detectron[basename] = {'boxes': boxes, 'segms': segms, 'keyps': None, 'classes': classes} with open(detectron_file, 'wb') as f: pickle.dump(self.detectron, f)
def run(stdout, file, interface, port, dialogue_file): """Plays the specified movie file, either via stdout (if the --stdout) flag is used, or as a Telnet server (the default) There are a few environment variables that are used by this process, for various purposes: \b To make gmail notifications work on server standup, termination, and client connection: * NOTIFICATION_USERNAME: The GMAIL username (without the @gmail.com) * NOTIFICATION_PASSWORD: The GMAIL password * DESTINATION_EMAIL_ADDRESS: The email address to receive the notifications * APP_NAME: The name of the app to include in the notifications If any of the above environment variables are not set, gmail notification will not happen. \b To update DNS entry (for example, when using FreeDNS or similar service): DNS_UPDATE_URL: The url to send a GET request to in order to update the DNS A Record If this is not set, DNS records will not be updated """ if dialogue_file: with open(dialogue_file) as f: result = yaml.unsafe_load(f) dialogue = result['dialogue'] try: if stdout: runStdOut(file, dialogue) else: print("Running TCP server on {0}:{1}".format(interface, port)) print("Playing movie {0}".format(file)) runTcpServer(interface, port, file, dialogue) except KeyboardInterrupt: print("Ascii Player Quit.")
def add_logging(es_host): # Add the ES_LOGGING lambda function (cloudwatch trigger). # Add es_domain to ES_LOGGING lambda as environment variable. # Update IAM permissions (es:*, arn:Aws:es:*) with open(sls_config_path, 'r') as f: sls_config = yaml.unsafe_load(f) # Create lambda function service_name = sls_config['custom']['service-name'] service_stage = sls_config['custom']['stage'] collection_names = [ x.split('_')[0] for x in list(sls_config['functions']) if x not in ['kickoff', 'es_log_ingest'] ] func = resources.lambda_cloudwatch_trigger("es_log_ingest", service_name, service_stage, collection_names) func.update({'environment': {'ES_HOST': es_host}}) sls_config['functions'].update({'es_log_ingest': func}) # Expanding IAM role if 'es:*' not in sls_config['provider']['iamRoleStatements'][0][ 'Action']: sls_config['provider']['iamRoleStatements'][0]['Action'].append( 'es:*') if 'arn:aws:es:*' not in sls_config['provider']['iamRoleStatements'][ 0]['Resource']: sls_config['provider']['iamRoleStatements'][0]['Resource'].append( 'arn:aws:ecs:*') with open(sls_config_path, 'w') as outf: yaml.dump(sls_config, outf, indent=1)
def _load_beamline_config(beamline_config_fp, verif="", test=False): if not os.path.isfile(beamline_config_fp): raise xpdAcqException("WARNING: can not find long term beamline " "configuration file. Please contact the " "beamline scientist ASAP") pp = pprint.PrettyPrinter() os_type = platform.system() if os_type == "Windows": editor = "notepad" else: editor = os.environ.get("EDITOR", "vim") if not test: while verif.upper() != ("Y" or "YES"): with open(beamline_config_fp, "r") as f: beamline_config = yaml.unsafe_load(f) pp.pprint(beamline_config) verif = input("\nIs this configuration correct? y/n: ") if verif.upper() == ("N" or "NO"): print("Edit, save, and close the configuration file.\n") subprocess.call([editor, beamline_config_fp]) beamline_config["Verified by"] = input("Please input your initials: ") timestamp = datetime.datetime.now() beamline_config["Verification time"] = timestamp.strftime( "%Y-%m-%d %H:%M:%S") with open(beamline_config_fp, "w") as f: yaml.dump(beamline_config, f) else: beamline_config = _verify_within_test(beamline_config_fp, verif) return beamline_config
def from_yaml(cls, f): """method to reload object from local yaml""" d = yaml.unsafe_load(f) instance = cls.from_dict(d) if not isinstance(f, str): instance.filepath = os.path.abspath(f.name) return instance
def from_yaml_function(filename, transform_lists_to_nparray=None): """ Function to read a object from a yaml file. """ if transform_lists_to_nparray is None: transform_lists_to_nparray = [] with open(filename, 'r') as f: dic = yaml.unsafe_load(f) dic = serialize_from_json(dic, transform_lists_to_nparray) return dic
def _auto_load_calibration_file(in_scan=True): """function to load the most recent calibration file in config_base Returns ------- calib_dict : dict dictionary contains calibration parameters computed by pyFAI and file name of the most recent calibration. If no calibration file exits in xpdUser/config_base, returns None. """ config_dir = glbl["config_base"] if not os.path.isdir(config_dir): raise xpdAcqException( "WARNING: Required directory {} doesn't" " exist, did you accidentally delete it?".format(config_dir)) calib_yaml_name = os.path.join(config_dir, glbl["calib_config_name"]) # no calib, skip if not os.path.isfile(calib_yaml_name): if in_scan: print("INFO: No calibration file found in config_base.\n" "Scan will still keep going on....") return else: with open(calib_yaml_name) as f: calib_dict = yaml.unsafe_load(f) if in_scan: print("INFO: This scan will append calibration parameters " "recorded in {}".format(calib_yaml_name)) return calib_dict
def parse_config_option(text): import yaml if '=' in text: key, value = text.strip().split('=', 1) return key, yaml.unsafe_load(value) else: return text, True
def make_handlers(): import yaml with open("app.yaml.template", "r") as fp: info = yaml.unsafe_load(fp) with open("app.handlers.json", "w") as fp: json.dump(info["handlers"], fp, indent=2)
def read_yaml(file, safe=True): with open(file, "r") as f: if safe: data = yaml.safe_load(f) else: data = yaml.unsafe_load(f) return data
def test_config(project_root): config_file = os.path.join(project_root, "config.yml") setup_yaml_parser() with open(config_file) as stream: config = yaml.unsafe_load(stream.read()) with nameko.config.patch(config, clear=True): yield
def save(cfg, filename: str): """ Save a config object to a yaml file. Note that when the config dictionary contains complex objects (e.g. lambda), it can't be saved to yaml. In that case we will print an error and attempt to save to a pkl file instead. Args: cfg: an omegaconf config object filename: yaml file name to save the config file """ logger = logging.getLogger(__name__) try: cfg = deepcopy(cfg) except Exception: pass else: # if it's deep-copyable, then... def _replace_type_by_name(x): if "_target_" in x and callable(x._target_): try: x._target_ = _convert_target_to_string(x._target_) except AttributeError: pass # not necessary, but makes yaml looks nicer _visit_dict_config(cfg, _replace_type_by_name) save_pkl = False try: dict = OmegaConf.to_container(cfg, resolve=False) dumped = yaml.dump(dict, default_flow_style=None, allow_unicode=True, width=9999) with PathManager.open(filename, "w") as f: f.write(dumped) try: _ = yaml.unsafe_load(dumped) # test that it is loadable except Exception: logger.warning( "The config contains objects that cannot serialize to a valid yaml. " f"{filename} is human-readable but cannot be loaded.") save_pkl = True except Exception: logger.exception("Unable to serialize the config to yaml. Error:") save_pkl = True if save_pkl: new_filename = filename + ".pkl" try: # retry by pickle with PathManager.open(new_filename, "wb") as f: cloudpickle.dump(cfg, f) logger.warning( f"Config is saved using cloudpickle at {new_filename}.") except Exception: pass
def init_fastavro() -> None: for identifier, schema_root in cerializer.utils.iterate_over_schemata(): fastavro._schema_common.SCHEMA_DEFS[ identifier] = cerializer.utils.parse_schema( # mypy things yaml has no attribute unsafe_load, which is not true yaml.unsafe_load(os.path.join(schema_root, 'schema.yaml')) # type: ignore )
def _load_metadata(self, groups=None): """Load meta data""" if groups is None: groups = self._groups_ return { g: yaml.unsafe_load(self.fh[g][()]) for g in groups if g in self.fh }
def __init__(self, cfg=None, filename=None): if filename: with open(filename, "r") as f: cfg = yaml.unsafe_load(f) for key, val in cfg.items(): if isinstance(val, dict): val = config(cfg=val) setattr(self, key, val)
def load(self): try: if self._file is not None and os.path.isfile(self._file): with open(self._file, 'r') as stream: self._data = yaml.unsafe_load(stream) else: self.empty() except (PermissionError, ValueError) as ex: raise StorageException(ex)
def unsafe_load(): if uploaded_file["file"]: yamlObj.clear() try: yamlObj["file"] = yaml.unsafe_load(uploaded_file["file"]) except yaml.YAMLError as e: yamlObj["error"] = e return redirect(url_for('index'))
def load_rules(): if not os.path.exists(RULES_YAML): return else: with open(RULES_YAML, 'r', encoding='utf-8') as f: from update_sub import group_current rules = yaml.unsafe_load(f) rules = rules[group_current] if group_current in rules else None return rules
def test_unhandled_recursion( self, yaml_config, env_vars, expected_config ): # pragma: no cover setup_yaml_parser() with patch.dict('os.environ'): for key, val in env_vars.items(): os.environ[key] = val results = yaml.unsafe_load(yaml_config) assert results == expected_config
def test_environment_vars_in_config( self, yaml_config, env_vars, expected_config ): setup_yaml_parser() with patch.dict('os.environ'): for key, val in env_vars.items(): os.environ[key] = val results = yaml.unsafe_load(yaml_config) assert results == expected_config
def __init__(self, source_file, category_id=0, create_users=None, verbose=False, force=False): self.source_file = source_file self.category_id = category_id self.create_users = create_users self.verbose = verbose self.force = force self.archive = tarfile.open(fileobj=source_file) self.data = yaml.unsafe_load(self.archive.extractfile('data.yaml')) self.id_map = {} self.user_map = {} self.event_id = None self.system_user_id = User.get_system_user().id self.spec = self._load_spec() self.deferred_idrefs = defaultdict(set)
def test_cannot_recurse(self): setup_yaml_parser() yaml_config = """ FOO: ${VAR1} BAR: - 1 - 2 - 3 """ with patch.dict('os.environ'): os.environ["VAR1"] = "${VAR1}" results = yaml.unsafe_load(yaml_config) assert results == {'FOO': "${VAR1}", 'BAR': [1, 2, 3]}
def main(args): if args.config: with open(args.config) as fle: config = yaml.unsafe_load(fle) broker_from = " (from --config)" else: config = {AMQP_URI_CONFIG_KEY: args.broker} broker_from = "" banner = 'Nameko Python %s shell on %s\nBroker: %s%s' % ( sys.version, sys.platform, config[AMQP_URI_CONFIG_KEY], broker_from ) ctx = {} ctx['n'] = make_nameko_helper(config) runner = ShellRunner(banner, ctx) runner.start_shell(name=args.interface)
def main(args): if '.' not in sys.path: sys.path.insert(0, '.') if args.config: with open(args.config) as fle: config = yaml.unsafe_load(fle) else: config = { AMQP_URI_CONFIG_KEY: args.broker } if "LOGGING" in config: logging.config.dictConfig(config['LOGGING']) else: logging.basicConfig(level=logging.INFO, format='%(message)s') services = [] for path in args.services: services.extend( import_service(path) ) run(services, config, backdoor_port=args.backdoor_port)
def main(args): with open(args.config) as fle: config = yaml.unsafe_load(fle) print(yaml.dump(config, default_flow_style=False))