def tester_task(job_setting_id: int, template_id: int):
js: JobSetting = JobSetting.objects.get(pk=job_setting_id)
if js is None:
return (1, 'job setting not exists')
setting_dict = js.get_dict()
if setting_dict is None:
return (1, 'job setting is not valid')
tp: OpenclashTemplate = OpenclashTemplate.objects.get(pk=template_id)
if tp is None:
return (1, 'Openclash template is not valid')
template = yaml.load(tp.template, Loader=yaml.Loader)
sp = cf_speed()
sp.load_config_from_dict(setting_dict)
sp.load_ip_list()
sp.ping()
sp.speed_test()
clash = sp.generate_openclash_config(template)
print('clash=', clash)
file = djangoSettings.MEDIA_ROOT + 'openclash_generated.yaml'
with open(file, 'w+', encoding='utf-8') as writer:
#yaml.dump(clash, writer, indent=4, mapping=2, sequence=4)
yaml_dump = YAML()
yaml_dump.indent(mapping=2, sequence=4, offset=2)
yaml_dump.compact(seq_seq=True)
yaml_dump.dump(clash, writer)
def dump_yaml(cache: MultCache, out=sys.stdout, compact=False) -> None:
table = reformat_cache(cache)
yaml = YAML()
if compact:
yaml.compact(seq_seq=False, seq_map=False)
else:
yaml.explicit_start = True # type: ignore
yaml.dump(table, out)
def dump_yaml(foo, outfile, no_anchors=False):
if no_anchors:
pyyaml.add_representer(int, hexint_presenter)
pyyaml.dump(foo, outfile, Dumper=NoAliasDumper)
else:
yaml = YAML(typ="rt")
yaml.default_flow_style = False
yaml.allow_unicode = True
yaml.compact(seq_seq=False, seq_map=False)
yaml.indent = 4
yaml.block_seq_indent = 2
yaml.dump(foo, outfile)
def test():
d = {}
ll = ruamel.yaml.comments.CommentedSeq()
ll.fa.set_flow_style()
ll.append('asdf1')
ll.append('asdf23')
ll.append('asdf2')
d['arr'] = ll
mm = {}
mm['bb'] = '122'
mm['cc'] = '23'
d['dd'] = mm
yaml_dump = YAML()
yaml_dump.indent(mapping=2, sequence=4, offset=2)
yaml_dump.compact(seq_seq=False)
yaml_dump.dump(d, sys.stdout)
def load_yaml(foo, no_anchors=False):
if no_anchors:
yaml = YAML(typ="safe")
else:
yaml = YAML(typ="rt")
yaml.default_flow_style = False
yaml.allow_unicode = True
yaml.compact(seq_seq=False, seq_map=False)
yaml.indent = 4
yaml.block_seq_indent = 2
try:
with open(foo, "r") as file:
return yaml.load(file)
except ruamel.yaml.constructor.DuplicateKeyError as msg:
logger = logging.getLogger(__name__)
error = "\n".join(str(msg).split("\n")[2:-7])
logger.error(error)
raise SystemExit
#!/usr/bin/env python3
### handle the non-prowgen jobs at migration
### example to run the command:
### release_repo=/path_to_release_repo bash -c 'find ${release_repo} -name "*openshift-origin-master-presubmits.yaml" -exec python3 hack/duplicate_prowgen_jobs_on_build01.py {} \;'
import copy
import logging
from ruamel.yaml import YAML
from ruamel.yaml.scalarstring import DoubleQuotedScalarString
import sys
filename = sys.argv[1]
yaml = YAML()
yaml.compact(seq_seq=False)
yaml.preserve_quotes = True
with open(filename) as f:
all = yaml.load(f)
for t in ("presubmits", "postsubmits", "periodics"):
if t not in filename:
continue
for repo in all[t]:
jobs = []
for job in all[t][repo]:
jobs.append(job)
if "labels" in job and "ci-operator.openshift.io/prowgen-controlled" in job[
"labels"] and job["labels"][
"ci-operator.openshift.io/prowgen-controlled"] == "true":
build01_job = copy.deepcopy(job)
import logging
import argparse
import operator
import ruamel
from ruamel.yaml import YAML
yaml = YAML(typ="rt")
yaml.default_flow_style = False
yaml.allow_unicode = True
yaml.compact(seq_seq=False, seq_map=False)
def load_yaml(foo):
try:
with open(foo, "r") as file:
return yaml.load(file)
except ruamel.yaml.constructor.DuplicateKeyError as msg:
logger = logging.getLogger(__name__)
error = "\n".join(str(msg).split("\n")[2:-7])
logger.error(error)
raise SystemExit
class ColoredFormatter(logging.Formatter):
"""
Class to create a log output which is colored based on level.
"""
def __init__(self, *args, **kwargs):
super(ColoredFormatter, self).__init__(*args, **kwargs)
self.colors = {
pass
Representer.add_representer(OrderedDict, Representer.represent_dict)
def wrap_yaml_string(s, width=100):
ss = (l.rstrip() for l in s.splitlines())
ss = (l for l in ss if l)
#ss = textwrap.wrap('\n'.join(ss), width=width, drop_whitespace=False, tabsize=2)
return PreservedScalarString('\n'.join(ss))
yaml = YAML(typ='rt')
yaml.Representer = Representer
yaml.compact()
yaml.default_flow_style = False
def yaml_dumps(document):
stream = StringIO()
yaml.dump(document, stream)
return stream.getvalue()
def write_yaml(dir_, fn, data):
if not os.path.exists(dir_):
os.makedirs(dir_)
with open(os.path.join(dir_, fn), 'w') as f:
yaml.dump(data, f)
class Factory:
"""Helper class to load HermesPy simulation scenarios from YAML configuration files."""
extensions: Set[str] = ['.yml', '.yaml', '.cfg']
"""List of recognized filename extensions for serialization files."""
__yaml: YAML
__clean: bool
__purge_regex_alpha: Pattern
__purge_regex_beta: Pattern
__db_regex: Pattern
__restore_regex_alpha: Pattern
__registered_classes: Set[Type[Serializable]]
__registered_tags: Set[str]
def __init__(self) -> None:
# YAML dumper configuration
self.__yaml = YAML(typ='safe', pure=True)
self.__yaml.default_flow_style = False
self.__yaml.compact(seq_seq=False, seq_map=False)
self.__yaml.encoding = None
self.__yaml.indent(mapping=4, sequence=4, offset=2)
self.__clean = True
self.__registered_classes = set()
self.__registered_tags = set()
# Browse the current environment for packages within the 'hermespy' namespace
for finder, name, ispkg in iter_modules(hermes.__path__, "hermespy."):
module = import_module(name)
for _, serializable_class in getmembers(module):
if not isclass(serializable_class) or not issubclass(
serializable_class, Serializable):
continue
self.__registered_classes.add(serializable_class)
self.__yaml.register_class(serializable_class)
if serializable_class.yaml_tag is not None:
self.__registered_tags.add(serializable_class.yaml_tag)
if issubclass(serializable_class, SerializableArray):
array_constructor = partial(Factory.__construct_matrix,
serializable_class)
self.__yaml.constructor.add_multi_constructor(
serializable_class.yaml_tag, array_constructor)
# Add constructors for untagged classes
self.__yaml.constructor.add_constructor('tag:yaml.org,2002:map',
self.__construct_map)
# self.__yaml.constructor.add_constructor('tag:yaml.org,2002:seq', self.__construct_sequence)
# Construct regular expressions for purging
self.__purge_regex_alpha = compile(r': !<.*')
self.__purge_regex_beta = compile(r"- !<([^']+)>")
self.__restore_regex_alpha = compile(r"([ ]*)([a-zA-Z]+):\n$")
self.__restore_regex_beta = compile(r"([ ]*)- ([^\s]+)([^']*)\n$")
self.__range_regex = compile(
r'([0-9.e-]*)[ ]*,[ ]*([0-9.e-]*)[ ]*,[ ]*\.\.\.[ ]*,[ ]*([0-9.e-]*)'
)
self.__db_regex = compile(r"\[([ 0-9.,-]*)\][ ]*dB")
@property
def clean(self) -> bool:
"""Access clean flag.
Returns:
bool: Clean flag.
"""
return self.__clean
@clean.setter
def clean(self, flag: bool) -> None:
"""Modify clean flag.
Args:
flag (bool): New clean flag.
"""
self.__clean = flag
@property
def registered_classes(self) -> Set[Type[Serializable]]:
"""Classes registered for serialization within the factory."""
return self.__registered_classes.copy()
@property
def registered_tags(self) -> Set[str]:
"""Read registered YAML tags.
Returns:
Set[str]: Set of registered YAML tags.
"""
return self.__registered_tags
def load(self, path: str) -> List[Serializable]:
"""Load a serialized executable configuration from a filesystem location.
Args:
path (str): Path to a file or a folder featuring serialization files.
Returns:
executables (List[Serializable]):
Serializable HermesPy objects.
Raises:
RuntimeError: If `path` does not contain an executable object.
RuntimeError: If `path` contains more than one executable object.
"""
# Recover serialized objects
hermes_objects: List[Any] = self.from_path(path)
executables: List[Serializable] = []
for hermes_object in hermes_objects:
if isinstance(hermes_object, Serializable):
executables.append(hermes_object)
# Return fully configured executable
return executables
@staticmethod
def __construct_matrix(cls: Any, constructor: SafeConstructor, tag_suffix: str, node: Any)\
-> Tuple[Any, Tuple[int, ...]]:
"""Construct a matrix node from YAML.
Args:
cls (Any):
The type of class to be constructed. This argument will be managed by ruamel.
The class `cls` must define a `from_yaml` routine.
constructor (SafeConstructor):
A handle to the constructor extracting the YAML information.
tag_suffix (str):
Tag suffix in the YAML config describing the channel position within the matrix.
node (Node):
YAML node representing the `cls` serialization.
Returns:
cls:
Newly created `cls` instance.
int:
First dimension position within the matrix.
int:
Second dimension within the matrix.
"""
indices: List[str] = re.split(' |_', tag_suffix)
if indices[0] == '':
indices.pop(0)
indices: Tuple[int] = tuple([int(idx) for idx in indices])
return cls.from_yaml(constructor, node), indices
@staticmethod
def __construct_map(constructor: SafeConstructor,
node: MappingNode) -> Mapping[MappingNode, Any]:
"""A custom map generator.
Hacks ruamel to accept node names as tags.
Args:
constructor (SafeConstructor): Handle to the constructor.
node (MappingNode): A YAML map node.
Returns:
Mapping[MappingNode, Any]: A sequence of objects created from `node`.
"""
tag = node.value[0][0].value
if tag in constructor.yaml_constructors:
return constructor.yaml_constructors[tag](constructor,
node.value[0][1])
else:
return constructor.construct_mapping(node, deep=True)
@staticmethod
def __construct_sequence(constructor: SafeConstructor,
node: SequenceNode) -> Sequence[Any]:
"""A custom sequence generator.
Hacks ruamel to accept node names as tags.
Args:
constructor (SafeConstructor): Handle to the constructor.
node (SequenceNode): A YAML sequence node.
Returns:
Sequence[Any]: A sequence of objects created from `node`.
"""
sequence = []
for node in node.value:
if node.tag in constructor.yaml_constructors:
sequence.append(constructor.yaml_constructors[node.tag](
constructor, node))
else:
sequence.append(
constructor.construct_non_recursive_object(node))
return sequence
def __purge_tags(self, serialization: str) -> str:
"""Callback to remove explicit YAML tags from serialization stream.
Args:
serialization (str): The serialization sequence to be purged.
Returns:
str: The purged sequence.
"""
cleaned_sequence = ''
for line in serialization.splitlines(True):
cleaned_line = self.__purge_regex_alpha.sub(r':', line)
cleaned_line = self.__purge_regex_beta.sub(r'- \1', cleaned_line)
cleaned_line = cleaned_line.replace('%20', " ")
cleaned_sequence += cleaned_line
return cleaned_sequence
def refurbish_tags(self, serialization: str) -> str:
"""Callback to restore explicit YAML tags to serialization streams."""
pass
@staticmethod
def __decibel_conversion(match: re.Match) -> str:
"""Convert linear series to decibel series.
Args:
match (re.Match): The serialization sequence to be converted.
Returns:
str: The purged sequence.
"""
linear_values = [
db2lin(float(str_rep))
for str_rep in match[1].replace(' ', '').split(',')
]
string_replacement = "["
for linear_value in linear_values:
string_replacement += str(linear_value) + ', '
string_replacement += "]"
return string_replacement
def from_path(self, paths: Union[str, Set[str]]) -> List[Any]:
"""Load a configuration from an arbitrary file system path.
Args:
paths (Union[str, Set[str]]): Paths to a file or a folder featuring .yml config files.
Returns:
List[Any]: List of serializable objects recalled from `paths`.
Raises:
ValueError: If the provided `path` does not exist on the filesystem.
"""
# Convert single path to a set if required
if isinstance(paths, str):
paths = [paths]
hermes_objects = []
for path in paths:
if not os.path.exists(path):
raise ValueError(f"Lookup path '{path}' not found")
if os.path.isdir(path):
hermes_objects += self.from_folder(path)
elif os.path.isfile(path):
hermes_objects += self.from_file(path)
else:
raise ValueError(
"Lookup location '{}' not recognized".format(path))
return hermes_objects
def from_folder(self,
path: str,
recurse: bool = True,
follow_links: bool = False) -> List[Any]:
"""Load a configuration from a folder.
Args:
path (str): Path to the folder configuration.
recurse (bool, optional): Recurse into sub-folders within `path`.
follow_links (bool, optional): Follow links within `path`.
Returns:
List[Any]: List of serializable objects recalled from `path`.
Raises:
ValueError: If `path` is not a directory.
"""
if not os.path.exists(path):
raise ValueError("Lookup path '{}' not found".format(path))
if not os.path.isdir(path):
raise ValueError(
"Lookup path '{}' is not a directory".format(path))
hermes_objects: List[Any] = []
for directory, _, files in os.walk(path, followlinks=follow_links):
for file in files:
_, extension = os.path.splitext(file)
if extension in self.extensions:
hermes_objects += self.from_file(
os.path.join(directory, file))
if not recurse:
break
return hermes_objects
def to_folder(self, path: str, *args: Any) -> None:
"""Dump a configuration to a folder.
Args:
path (str): Path to the folder configuration.
*args (Any):
Configuration objects to be dumped.
"""
pass
def from_str(self, config: str) -> List[Any]:
"""Load a configuration from a string object.
Args:
config (str): The configuration to be loaded.
Returns:
List[Any]: List of serialized objects within `path`.
"""
stream = StringIO(config)
return self.from_stream(stream)
def to_str(self, *args: Any) -> str:
"""Dump a configuration to a folder.
Args:
*args (Any): Configuration objects to be dumped.
Returns:
str: String containing full YAML configuration.
Raises:
RepresenterError: If objects in ``*args`` are unregistered classes.
"""
stream = StringIO()
self.to_stream(stream, args)
return stream.getvalue()
def from_file(self, file: str) -> List[Any]:
"""Load a configuration from a single YAML file.
Args:
file (str): Path to the folder configuration.
Returns:
List[Any]: List of serialized objects within `path`.
"""
with open(file, mode='r') as file_stream:
try:
return self.from_stream(file_stream)
# Re-raise constructor errors with the correct file name
except ConstructorError as constructor_error:
constructor_error.problem_mark.name = file
raise constructor_error
def to_file(self, path: str, *args: Any) -> None:
"""Dump a configuration to a single YML file.
Args:
path (str): Path to the configuration file.
*args (Any): Configuration objects to be dumped.
Raises:
RepresenterError: If objects in ``*args`` are unregistered classes.
"""
pass
def __restore_callback_alpha(self, m: Match) -> str:
"""Internal regular expression callback.
Args:
m (Match): Regular expression match.
Returns:
str: The processed match line.
"""
if m.group(2) in self.registered_tags:
return m.group(1) + m.group(2) + ": !<" + m.group(2) + ">\n"
else:
return m.string
def __restore_callback_beta(self, m: Match) -> str:
"""Internal regular expression callback.
Args:
m (Match): Regular expression match.
Returns:
str: The processed match line.
"""
if m.group(2) in self.registered_tags:
indices = m.group(3).replace(" ", "%20")
return m.group(1) + "- !<" + m.group(2) + indices + ">\n"
else:
return m.string
@staticmethod
def __range_restore_callback(m: Match) -> str:
"""Internal regular expression callback.
Args:
m (Match): Regular expression match.
Returns:
str: The processed match line.
"""
# Extract range parameters
start = float(m.group(1))
step = float(m.group(2)) - start
stop = float(m.group(3)) + step
range = np.arange(start=start, stop=stop, step=step)
replacement = ''
for step in range[:-1]:
replacement += str(step) + ', '
replacement += str(range[-1])
return replacement
def from_stream(self, stream: TextIOBase) -> List[Any]:
"""Load a configuration from an arbitrary text stream.
Args:
stream (TextIOBase): Text stream containing the configuration.
Returns:
List[Any]: List of serialized objects within `stream`.
Raises:
ConstructorError: If YAML parsing fails.
"""
if not self.__clean:
return self.__yaml.load(stream)
clean_stream = ''
for line in stream.readlines():
clean_line = self.__range_regex.sub(self.__range_restore_callback,
line)
clean_line = self.__db_regex.sub(self.__decibel_conversion,
clean_line)
clean_stream += clean_line
hermes_objects = self.__yaml.load(StringIO(clean_stream))
if hermes_objects is None:
return []
if isinstance(hermes_objects, Iterable):
return hermes_objects
else:
return [hermes_objects]
def to_stream(self, stream: TextIOBase, *args: Any) -> None:
"""Dump a configuration to an arbitrary text stream.
Args:
stream (TextIOBase): Text stream to the configuration.
*args (Any): Configuration objects to be dumped.
Raises:
RepresenterError: If objects in ``*args`` are unregistered classes.
"""
for serializable_object in args:
if self.__clean:
self.__yaml.dump(*serializable_object,
stream,
transform=self.__purge_tags)
else:
self.__yaml.dump(*serializable_object, stream)
def create_yaml():
yaml = YAML(typ="rt")
yaml.indent(mapping=2, sequence=4, offset=2)
yaml.compact(seq_seq=False, seq_map=False)
return yaml
