def send(self, value):
"""
Sends a message by serializing, compressing and wrapping to a QByteArray, then streaming over the TCP socket.
:param value: The message to send.
"""
if not self.is_connected():
raise RuntimeError('Try to send on unconnected socket.')
logger.debug('socket send: %s', value)
# serialize value to yaml
stream = StringIO()
yaml.dump(value, stream)
serialized = stream.getvalue()
# encode to utf-8 bytes and compress
compressed = zlib.compress(serialized.encode())
# wrap in QByteArray
bytearray = QtCore.QByteArray(compressed)
# write using a data stream
writer = QtCore.QDataStream(self.socket)
writer.setVersion(QtCore.QDataStream.Qt_5_5)
writer << bytearray
def dump(self, data, stream=None, **kw):
inefficient = False
if stream is None:
inefficient = True
stream = StringIO()
YAML.dump(self, data, stream, **kw)
if inefficient:
return stream.getvalue()
def object_to_yaml(data: JSON_TYPE) -> str:
"""Create yaml string from object."""
yaml = YAML(typ='rt')
yaml.indent(sequence=4, offset=2)
stream = StringIO()
try:
yaml.dump(data, stream)
result = stream.getvalue() # type: str
return result
except YAMLError as exc:
_LOGGER.error("YAML error: %s", exc)
raise HomeAssistantError(exc)
def test_unicode_input(unicode_filename, verbose=False):
yaml = YAML(typ='safe', pure=True)
with open(unicode_filename, 'rb') as fp:
data = fp.read().decode('utf-8')
value = ' '.join(data.split())
output = yaml.load(data)
assert output == value, (output, value)
output = yaml.load(StringIO(data))
assert output == value, (output, value)
for input in [
data.encode('utf-8'),
codecs.BOM_UTF8 + data.encode('utf-8'),
codecs.BOM_UTF16_BE + data.encode('utf-16-be'),
codecs.BOM_UTF16_LE + data.encode('utf-16-le'),
]:
if verbose:
print('INPUT:', repr(input[:10]), '...')
output = yaml.load(input)
assert output == value, (output, value)
output = yaml.load(BytesIO(input))
assert output == value, (output, value)
def test_component_dumping_with_defaults_and_comments(make_classes_2):
A, B = make_classes_2
txt = """!A
akw1: 8
# Comment Here
akw2: !B
bkw1: 1
bkw2: !!str test
"""
txt_expected = """!A
akw1: 8
akw2: !B
bkw1: 1
bkw2: test
bkw3: 99
"""
a_schema = yaml.load(txt)
a = a_schema()
with StringIO() as stream:
yaml.dump(a, stream)
assert txt_expected == stream.getvalue()
def round_trip(ge):
stream = StringIO()
Y.dump(ge, stream)
tmp1 = stream.getvalue()
ge2 = Y.load(tmp1)
stream2 = StringIO()
Y.dump(ge2, stream2)
tmp2 = stream2.getvalue()
if tmp1 == tmp2:
print("Round-trip test passed.")
else:
print("TMP1: \n" + tmp1)
print("TMP2: \n" + tmp2)
return tmp1 == tmp2
def execute(self, cluster_runnable, extensions: Dict[str, str],
new_secrets: str, force: bool) -> None:
"""Execute a ClusterRunnable in the cluster.
It will first upload the runnable file + extensions to the
orchestrator (under $HOME/flambe.yaml) and then it will
execute it based on the provided secrets
Parameters
----------
cluster_runnable: ClusterRunnable
The ClusterRunnable to run in the cluster
extensions: Dict[str, str]
The extensions for the ClusterRunnable
new_secrets: str
The path (relative to the orchestrator) where
the secrets are located.
IMPORTANT: previous to calling this method, the secrets
should have been uploaded to the orchestrator
force: bool
The force parameter provided when running flambe locally
"""
if not self.orchestrator:
raise man_errors.ClusterError(
"Orchestrator instance was not loaded.")
orch_exp = (f"{self.orchestrator.get_home_path()}/flambe.yaml")
with tempfile.NamedTemporaryFile("w") as t:
with StringIO() as s:
yaml.dump_all([extensions, cluster_runnable], s)
t.write(s.getvalue())
t.flush()
self.orchestrator.send_rsync(t.name, orch_exp)
logger.info(cl.BL("Remote runnable file sent to orchestrator"))
self.orchestrator.launch_flambe(orch_exp, new_secrets, force)
def test_unicode_transfer(unicode_filename, verbose=False):
with open(unicode_filename, 'rb') as fp:
data = fp.read().decode('utf-8')
for encoding in [None, 'utf-8', 'utf-16-be', 'utf-16-le']:
input = data
if PY3:
if encoding is not None:
input = ('\ufeff' + input).encode(encoding)
output1 = yaml.emit(yaml.parse(input), allow_unicode=True)
if encoding is None:
stream = StringIO()
else:
stream = BytesIO()
yaml.emit(yaml.parse(input), stream, allow_unicode=True)
output2 = stream.getvalue()
assert isinstance(output1, str), (type(output1), encoding)
if encoding is None:
assert isinstance(output2, str), (type(output1), encoding)
else:
assert isinstance(output2, bytes), (type(output1), encoding)
output2.decode(encoding)
else:
if encoding is not None:
input = (u'\ufeff' + input).encode(encoding)
output1 = yaml.emit(yaml.parse(input), allow_unicode=True)
stream = StringIO()
yaml.emit(yaml.parse(input),
_unicode_open(stream, 'utf-8'),
allow_unicode=True)
output2 = stream.getvalue()
if encoding is None:
assert isinstance(output1,
unicode), (type(output1), encoding) # NOQA
else:
assert isinstance(output1, str), (type(output1), encoding)
output1.decode(encoding)
assert isinstance(output2, str), (type(output2), encoding)
output2.decode('utf-8')
def discover_files(walk_dir):
print('walk_dir = ' + walk_dir)
print('walk_dir (absolute) = ' + os.path.abspath(walk_dir))
for root, subdirs, files in os.walk(walk_dir):
if (root.startswith(walk_dir + 'images')
or root.startswith(walk_dir + '_site')
or root.startswith(walk_dir + 'downloads')
or root.startswith(walk_dir + '.grunt')
or root.startswith(walk_dir + 'scripts')
or root.startswith(walk_dir + '_sass')
or root.startswith(walk_dir + '_plugins')
or root.startswith(walk_dir + 'bower_components')
or root.startswith(walk_dir + 'node_modules')
or root.startswith(walk_dir + '.github')):
continue
for filename in files:
_, extension = os.path.splitext(filename)
if extension is None:
continue
if extension.lower() not in ['.html', '.md']:
continue
if filename.lower() in ['readme.md', 'yaml']:
continue
file_path = os.path.join(root, filename)
# print('\t- file %s (full path: %s)' % (filename, file_path))
with open(file_path,
'r') as original, open(file_path + '.bak',
'w') as updated:
try:
front_matter = yaml.load(pluck_yaml(original))
except Exception as e:
print(e)
os.remove(file_path + '.bak')
continue
if front_matter is None:
print('skipping {}'.format(original))
# os.rename(file_path + '.bak', file_path)
os.remove(file_path + '.bak')
continue
front_matter = prune_keys(front_matter)
tags = prune_tags(front_matter)
if tags is not None:
front_matter['tags'] = tags
categories = prune_categories(front_matter)
if categories is not None:
front_matter['categories'] = categories
stream = StringIO()
yaml.dump((front_matter), stream)
front_matter = stream.getvalue()
content = pluck_content(original)
updated.write('---\n')
updated.write(front_matter)
updated.write('---\n')
updated.write(content)
os.rename(file_path + '.bak', file_path)
def dump(obj: dict, default_flow_style=False) -> str:
yaml = YAML()
yaml.default_flow_style = default_flow_style
stream = StringIO()
yaml.dump(obj, stream)
return stream.getvalue()
def get_yaml(self):
yaml_dump = YAML()
yaml_dump.indent(mapping=2, sequence=4, offset=2)
stream = StringIO()
yaml_dump.dump(self.get_dict(), stream)
return stream.getValue()
def to_yaml(obj):
stream = StringIO()
yaml_parser.dump(cattr_converter.unstructure(obj), stream)
return stream.getvalue()
def to_string(self, data):
stream = StringIO()
RuamelYaml.dump(self, data, stream)
return stream.getvalue()
def dump_all(documents,
stream=None,
Dumper=Dumper,
default_style=None,
default_flow_style=None,
canonical=None,
indent=None,
width=None,
allow_unicode=None,
line_break=None,
encoding=enc,
explicit_start=None,
explicit_end=None,
version=None,
tags=None,
block_seq_indent=None,
top_level_colon_align=None,
prefix_colon=None):
# type: (Any, StreamType, Any, Any, Any, bool, Union[None, int], Union[None, int], bool, Any, Any, Union[None, bool], Union[None, bool], Any, Any, Any, Any, Any) -> Union[None, str] # NOQA
"""
Serialize a sequence of Python objects into a YAML stream.
If stream is None, return the produced string instead.
"""
getvalue = None
if top_level_colon_align is True:
top_level_colon_align = max([len(str(x)) for x in documents[0]])
if stream is None:
if encoding is None:
stream = StringIO()
else:
stream = BytesIO()
getvalue = stream.getvalue
dumper = Dumper(
stream,
default_style=default_style,
default_flow_style=default_flow_style,
canonical=canonical,
indent=indent,
width=width,
allow_unicode=allow_unicode,
line_break=line_break,
encoding=encoding,
explicit_start=explicit_start,
explicit_end=explicit_end,
version=version,
tags=tags,
block_seq_indent=block_seq_indent,
top_level_colon_align=top_level_colon_align,
prefix_colon=prefix_colon,
)
try:
dumper._serializer.open()
for data in documents:
try:
dumper._representer.represent(data)
except AttributeError:
# print(dir(dumper._representer))
raise
dumper._serializer.close()
finally:
dumper._emitter.dispose()
if getvalue is not None:
return getvalue()
return None
def export(self, metadata, **kwargs):
stream = StringIO()
self.myyaml.dump(data=metadata, stream=stream)
metadata = stream.getvalue()
metadata = metadata[:-1]
return u(metadata)
def get_batch_data_and_markers(
self,
batch_spec: BatchSpec) -> Tuple[Any, BatchMarkers]: # batch_data
# We need to build a batch_markers to be used in the dataframe
batch_markers: BatchMarkers = BatchMarkers({
"ge_load_time":
datetime.datetime.now(
datetime.timezone.utc).strftime("%Y%m%dT%H%M%S.%fZ")
})
batch_data: PandasBatchData
if isinstance(batch_spec, RuntimeDataBatchSpec):
# batch_data != None is already checked when RuntimeDataBatchSpec is instantiated
if isinstance(batch_spec.batch_data, pd.DataFrame):
df = batch_spec.batch_data
elif isinstance(batch_spec.batch_data, PandasBatchData):
df = batch_spec.batch_data.dataframe
else:
raise ValueError(
"RuntimeDataBatchSpec must provide a Pandas DataFrame or PandasBatchData object."
)
batch_spec.batch_data = "PandasDataFrame"
elif isinstance(batch_spec, S3BatchSpec):
if self._s3 is None:
raise ge_exceptions.ExecutionEngineError(
f"""PandasExecutionEngine has been passed a S3BatchSpec,
but the ExecutionEngine does not have a boto3 client configured. Please check your config."""
)
s3_engine = self._s3
s3_url = S3Url(batch_spec.path)
reader_method: str = batch_spec.reader_method
reader_options: dict = batch_spec.reader_options or {}
s3_object = s3_engine.get_object(Bucket=s3_url.bucket,
Key=s3_url.key)
logger.debug("Fetching s3 object. Bucket: {} Key: {}".format(
s3_url.bucket, s3_url.key))
reader_fn = self._get_reader_fn(reader_method, s3_url.key)
df = reader_fn(
StringIO(s3_object["Body"].read().decode(
s3_object.get("ContentEncoding", "utf-8"))),
**reader_options,
)
elif isinstance(batch_spec, PathBatchSpec):
reader_method: str = batch_spec.reader_method
reader_options: dict = batch_spec.reader_options
path: str = batch_spec.path
reader_fn: Callable = self._get_reader_fn(reader_method, path)
df = reader_fn(path, **reader_options)
else:
raise BatchSpecError(
f"batch_spec must be of type RuntimeDataBatchSpec, PathBatchSpec, or S3BatchSpec, not {batch_spec.__class__.__name__}"
)
df = self._apply_splitting_and_sampling_methods(batch_spec, df)
if df.memory_usage().sum() < HASH_THRESHOLD:
batch_markers["pandas_data_fingerprint"] = hash_pandas_dataframe(
df)
typed_batch_data = PandasBatchData(execution_engine=self, dataframe=df)
return typed_batch_data, batch_markers
def test_component_dumping_factory(make_instances):
a = make_instances()
config = "!A {}\n"
with StringIO() as stream:
yaml.dump(a, stream)
assert config == stream.getvalue()
def build_toc(content_folder, filename_split_char='_'):
"""Auto-generate a Table of Contents from files/folders.
Parameters
----------
content_folder : str
Path to the folder where content exists. The TOC will be generated
according to the alphanumeric sort of these files/folders.
filename_split_char : str
The character used in inferring spaces in page names from filenames.
"""
if not op.isdir(content_folder):
raise ValueError(f"Could not find the provided content folder\n{content_folder}")
# Generate YAML from the directory structure
out = [YAML_TOP, YAML_WARN]
toc_pages = []
for ii, (ifolder, folders, ifiles) in enumerate(sorted(os.walk(content_folder))):
if ".ipynb_checkpoints" in ifolder:
continue
path_rel_to_content = ifolder.replace(content_folder, '')
if ii == 0:
# Create a dictionary of top-level folders we'll append to
top_level_dict = {folder: [] for folder in folders if len(folder) > 0}
# Append files for the top-most folder
for ifile in ifiles:
if any(ifile.endswith(ii) for ii in [".ipynb", ".md"]):
if ifile == "LICENSE.md":
continue
suff = os.path.splitext(ifile)[-1]
i_title = _filename_to_title(ifile, filename_split_char)
i_url = os.path.join(path_rel_to_content, os.path.basename(ifile)).replace(suff, '')
toc_pages.append({'title': i_title, 'url': i_url})
else:
# Grab the top-most folder to choose which list we'll append to
folder = path_rel_to_content.lstrip('/').split(os.sep)[0]
# If the file ends in ipynb or md, add it to this section
for ifile in ifiles:
suff = os.path.splitext(ifile)[-1]
if not any(ii == suff for ii in [".ipynb", ".md"]):
continue
# Convert to Jupyter-book ready names
i_title = _filename_to_title(ifile, filename_split_char)
i_url = os.path.join(path_rel_to_content, os.path.basename(ifile)).replace(suff, '')
top_level_dict[folder].append({'title': i_title, 'url': i_url})
# Iterate through our top level dict and convert to yaml-style dict
top_level_dict = {key: val for key, val in top_level_dict.items() if len(val) > 0}
out_children = []
for folder, subsections in top_level_dict.items():
name = _filename_to_title(folder)
out_children.append("## REPLACE ##")
out_children.append({'header': name})
out_children += subsections
toc_pages += out_children
# Convert the dictionary into YAML and append it to our output
yaml = YAML()
string = StringIO()
yaml.dump(toc_pages, string)
out.append(string.getvalue().replace("- '## REPLACE ##'", ''))
return '\n'.join(out)
def object_to_yaml_str(obj):
output_str: str
with StringIO() as string_stream:
yaml.dump(obj, string_stream)
output_str = string_stream.getvalue()
return output_str
def stringify_dict(as_dict: dict) -> str:
as_dict = format_node(as_dict, node_path=[])
stream = StringIO()
yaml.dump(as_dict, stream)
return stream.getvalue()
def assert_graphql_resp_expected(resp_orig,
exp_response_orig,
query,
resp_hdrs={},
skip_if_err_msg=False,
skip_assertion=False,
exp_resp_hdrs={}):
print('Reponse Headers: ', resp_hdrs)
print(exp_resp_hdrs)
# Prepare actual and expected responses so comparison takes into
# consideration only the ordering that we care about:
resp = collapse_order_not_selset(resp_orig, query)
exp_response = collapse_order_not_selset(exp_response_orig, query)
matched = equal_CommentedMap(
resp,
exp_response) and (exp_resp_hdrs or {}).items() <= resp_hdrs.items()
if PytestConf.config.getoption("--accept"):
print('skipping assertion since we chose to --accept new output')
else:
yml = yaml.YAML()
# https://yaml.readthedocs.io/en/latest/example.html#output-of-dump-as-a-string :
dump_str = StringIO()
test_output = {
# Keep strict received order when displaying errors:
'response':
resp_orig,
'expected':
exp_response_orig,
'diff':
(lambda diff: "(results differ only in their order of keys)"
if diff == {} else diff)(stringify_keys(
jsondiff.diff(exp_response, resp))),
'query':
query
}
if 'x-request-id' in resp_hdrs:
test_output['request id'] = resp_hdrs['x-request-id']
if exp_resp_hdrs:
diff_hdrs = {
key: val
for key, val in resp_hdrs.items() if key in exp_resp_hdrs
}
test_output['headers'] = {
'actual': dict(resp_hdrs),
'expected': exp_resp_hdrs,
'diff': (stringify_keys(jsondiff.diff(exp_resp_hdrs,
diff_hdrs)))
}
yml.dump(test_output, stream=dump_str)
if not skip_if_err_msg:
if skip_assertion:
return resp, matched
else:
assert matched, '\n' + dump_str.getvalue()
elif matched:
return resp, matched
else:
def is_err_msg(msg):
return any(msg.get(x) for x in ['error', 'errors'])
def as_list(x):
return x if isinstance(x, list) else [x]
# If it is a batch GraphQL query, compare each individual response separately
for (exp, out) in zip(as_list(exp_response), as_list(resp)):
matched_ = equal_CommentedMap(exp, out)
if is_err_msg(exp) and is_err_msg(out):
if not matched_:
warnings.warn(
"Response does not have the expected error message\n"
+ dump_str.getvalue())
return resp, matched
else:
if skip_assertion:
return resp, matched_
else:
assert matched_, '\n' + dump_str.getvalue()
return resp, matched # matched always True unless --accept
def getConfigHash(self):
s = StringIO()
yaml.dump(self.get_config(), s)
return hashlib.md5(s.getvalue().encode('utf-8')).hexdigest()
def discover_files(walk_dir):
print('walk_dir = ' + walk_dir)
print('walk_dir (absolute) = ' + os.path.abspath(walk_dir))
for root, subdirs, files in os.walk(walk_dir):
if (root.startswith(walk_dir + 'images') or root.startswith(walk_dir + '_site') or root.startswith(walk_dir + 'downloads') or
root.startswith(walk_dir + '.grunt') or root.startswith(walk_dir + 'scripts') or root.startswith(walk_dir + '_sass') or
root.startswith(walk_dir + '_plugins') or root.startswith(walk_dir + 'bower_components') or root.startswith(walk_dir + 'node_modules') or
root.startswith(walk_dir + '.github')):
continue
for filename in files:
_, extension = os.path.splitext(filename)
if extension is None:
continue
if extension.lower() not in ['.html', '.md']:
continue
if filename.lower() in ['readme.md', 'yaml']:
continue
file_path = os.path.join(root, filename)
# print('\t- file %s (full path: %s)' % (filename, file_path))
with open(file_path, 'r') as original, open(file_path + '.bak', 'w') as updated:
try:
front_matter = yaml.load(pluck_yaml(original))
except Exception as e:
print(e)
os.remove(file_path + '.bak')
continue
if front_matter is None:
print('skipping {}'.format(original))
# os.rename(file_path + '.bak', file_path)
os.remove(file_path + '.bak')
continue
front_matter = prune_keys(front_matter)
tags = prune_tags(front_matter)
if tags is not None:
front_matter['tags'] = tags
categories = prune_categories(front_matter)
if categories is not None:
front_matter['categories'] = categories
stream = StringIO()
yaml.dump((front_matter), stream)
front_matter = stream.getvalue()
content = pluck_content(original)
updated.write('---\n')
updated.write(front_matter)
updated.write('---\n')
updated.write(content)
os.rename(file_path + '.bak', file_path)
def yaml_to_string(v: dict):
s = StringIO()
fast_load.dump(v, s)
return s.getvalue()
