def send(self, value): """ Sends a message by serializing, compressing and wrapping to a QByteArray, then streaming over the TCP socket. :param value: The message to send. """ if not self.is_connected(): raise RuntimeError('Try to send on unconnected socket.') logger.debug('socket send: %s', value) # serialize value to yaml stream = StringIO() yaml.dump(value, stream) serialized = stream.getvalue() # encode to utf-8 bytes and compress compressed = zlib.compress(serialized.encode()) # wrap in QByteArray bytearray = QtCore.QByteArray(compressed) # write using a data stream writer = QtCore.QDataStream(self.socket) writer.setVersion(QtCore.QDataStream.Qt_5_5) writer << bytearray
def dump(self, data, stream=None, **kw): inefficient = False if stream is None: inefficient = True stream = StringIO() YAML.dump(self, data, stream, **kw) if inefficient: return stream.getvalue()
def object_to_yaml(data: JSON_TYPE) -> str: """Create yaml string from object.""" yaml = YAML(typ='rt') yaml.indent(sequence=4, offset=2) stream = StringIO() try: yaml.dump(data, stream) result = stream.getvalue() # type: str return result except YAMLError as exc: _LOGGER.error("YAML error: %s", exc) raise HomeAssistantError(exc)
def test_unicode_input(unicode_filename, verbose=False): yaml = YAML(typ='safe', pure=True) with open(unicode_filename, 'rb') as fp: data = fp.read().decode('utf-8') value = ' '.join(data.split()) output = yaml.load(data) assert output == value, (output, value) output = yaml.load(StringIO(data)) assert output == value, (output, value) for input in [ data.encode('utf-8'), codecs.BOM_UTF8 + data.encode('utf-8'), codecs.BOM_UTF16_BE + data.encode('utf-16-be'), codecs.BOM_UTF16_LE + data.encode('utf-16-le'), ]: if verbose: print('INPUT:', repr(input[:10]), '...') output = yaml.load(input) assert output == value, (output, value) output = yaml.load(BytesIO(input)) assert output == value, (output, value)
def test_component_dumping_with_defaults_and_comments(make_classes_2): A, B = make_classes_2 txt = """!A akw1: 8 # Comment Here akw2: !B bkw1: 1 bkw2: !!str test """ txt_expected = """!A akw1: 8 akw2: !B bkw1: 1 bkw2: test bkw3: 99 """ a_schema = yaml.load(txt) a = a_schema() with StringIO() as stream: yaml.dump(a, stream) assert txt_expected == stream.getvalue()
def round_trip(ge): stream = StringIO() Y.dump(ge, stream) tmp1 = stream.getvalue() ge2 = Y.load(tmp1) stream2 = StringIO() Y.dump(ge2, stream2) tmp2 = stream2.getvalue() if tmp1 == tmp2: print("Round-trip test passed.") else: print("TMP1: \n" + tmp1) print("TMP2: \n" + tmp2) return tmp1 == tmp2
def execute(self, cluster_runnable, extensions: Dict[str, str], new_secrets: str, force: bool) -> None: """Execute a ClusterRunnable in the cluster. It will first upload the runnable file + extensions to the orchestrator (under $HOME/flambe.yaml) and then it will execute it based on the provided secrets Parameters ---------- cluster_runnable: ClusterRunnable The ClusterRunnable to run in the cluster extensions: Dict[str, str] The extensions for the ClusterRunnable new_secrets: str The path (relative to the orchestrator) where the secrets are located. IMPORTANT: previous to calling this method, the secrets should have been uploaded to the orchestrator force: bool The force parameter provided when running flambe locally """ if not self.orchestrator: raise man_errors.ClusterError( "Orchestrator instance was not loaded.") orch_exp = (f"{self.orchestrator.get_home_path()}/flambe.yaml") with tempfile.NamedTemporaryFile("w") as t: with StringIO() as s: yaml.dump_all([extensions, cluster_runnable], s) t.write(s.getvalue()) t.flush() self.orchestrator.send_rsync(t.name, orch_exp) logger.info(cl.BL("Remote runnable file sent to orchestrator")) self.orchestrator.launch_flambe(orch_exp, new_secrets, force)
def test_unicode_transfer(unicode_filename, verbose=False): with open(unicode_filename, 'rb') as fp: data = fp.read().decode('utf-8') for encoding in [None, 'utf-8', 'utf-16-be', 'utf-16-le']: input = data if PY3: if encoding is not None: input = ('\ufeff' + input).encode(encoding) output1 = yaml.emit(yaml.parse(input), allow_unicode=True) if encoding is None: stream = StringIO() else: stream = BytesIO() yaml.emit(yaml.parse(input), stream, allow_unicode=True) output2 = stream.getvalue() assert isinstance(output1, str), (type(output1), encoding) if encoding is None: assert isinstance(output2, str), (type(output1), encoding) else: assert isinstance(output2, bytes), (type(output1), encoding) output2.decode(encoding) else: if encoding is not None: input = (u'\ufeff' + input).encode(encoding) output1 = yaml.emit(yaml.parse(input), allow_unicode=True) stream = StringIO() yaml.emit(yaml.parse(input), _unicode_open(stream, 'utf-8'), allow_unicode=True) output2 = stream.getvalue() if encoding is None: assert isinstance(output1, unicode), (type(output1), encoding) # NOQA else: assert isinstance(output1, str), (type(output1), encoding) output1.decode(encoding) assert isinstance(output2, str), (type(output2), encoding) output2.decode('utf-8')
def discover_files(walk_dir): print('walk_dir = ' + walk_dir) print('walk_dir (absolute) = ' + os.path.abspath(walk_dir)) for root, subdirs, files in os.walk(walk_dir): if (root.startswith(walk_dir + 'images') or root.startswith(walk_dir + '_site') or root.startswith(walk_dir + 'downloads') or root.startswith(walk_dir + '.grunt') or root.startswith(walk_dir + 'scripts') or root.startswith(walk_dir + '_sass') or root.startswith(walk_dir + '_plugins') or root.startswith(walk_dir + 'bower_components') or root.startswith(walk_dir + 'node_modules') or root.startswith(walk_dir + '.github')): continue for filename in files: _, extension = os.path.splitext(filename) if extension is None: continue if extension.lower() not in ['.html', '.md']: continue if filename.lower() in ['readme.md', 'yaml']: continue file_path = os.path.join(root, filename) # print('\t- file %s (full path: %s)' % (filename, file_path)) with open(file_path, 'r') as original, open(file_path + '.bak', 'w') as updated: try: front_matter = yaml.load(pluck_yaml(original)) except Exception as e: print(e) os.remove(file_path + '.bak') continue if front_matter is None: print('skipping {}'.format(original)) # os.rename(file_path + '.bak', file_path) os.remove(file_path + '.bak') continue front_matter = prune_keys(front_matter) tags = prune_tags(front_matter) if tags is not None: front_matter['tags'] = tags categories = prune_categories(front_matter) if categories is not None: front_matter['categories'] = categories stream = StringIO() yaml.dump((front_matter), stream) front_matter = stream.getvalue() content = pluck_content(original) updated.write('---\n') updated.write(front_matter) updated.write('---\n') updated.write(content) os.rename(file_path + '.bak', file_path)
def dump(obj: dict, default_flow_style=False) -> str: yaml = YAML() yaml.default_flow_style = default_flow_style stream = StringIO() yaml.dump(obj, stream) return stream.getvalue()
def get_yaml(self): yaml_dump = YAML() yaml_dump.indent(mapping=2, sequence=4, offset=2) stream = StringIO() yaml_dump.dump(self.get_dict(), stream) return stream.getValue()
def to_yaml(obj): stream = StringIO() yaml_parser.dump(cattr_converter.unstructure(obj), stream) return stream.getvalue()
def to_string(self, data): stream = StringIO() RuamelYaml.dump(self, data, stream) return stream.getvalue()
def dump_all(documents, stream=None, Dumper=Dumper, default_style=None, default_flow_style=None, canonical=None, indent=None, width=None, allow_unicode=None, line_break=None, encoding=enc, explicit_start=None, explicit_end=None, version=None, tags=None, block_seq_indent=None, top_level_colon_align=None, prefix_colon=None): # type: (Any, StreamType, Any, Any, Any, bool, Union[None, int], Union[None, int], bool, Any, Any, Union[None, bool], Union[None, bool], Any, Any, Any, Any, Any) -> Union[None, str] # NOQA """ Serialize a sequence of Python objects into a YAML stream. If stream is None, return the produced string instead. """ getvalue = None if top_level_colon_align is True: top_level_colon_align = max([len(str(x)) for x in documents[0]]) if stream is None: if encoding is None: stream = StringIO() else: stream = BytesIO() getvalue = stream.getvalue dumper = Dumper( stream, default_style=default_style, default_flow_style=default_flow_style, canonical=canonical, indent=indent, width=width, allow_unicode=allow_unicode, line_break=line_break, encoding=encoding, explicit_start=explicit_start, explicit_end=explicit_end, version=version, tags=tags, block_seq_indent=block_seq_indent, top_level_colon_align=top_level_colon_align, prefix_colon=prefix_colon, ) try: dumper._serializer.open() for data in documents: try: dumper._representer.represent(data) except AttributeError: # print(dir(dumper._representer)) raise dumper._serializer.close() finally: dumper._emitter.dispose() if getvalue is not None: return getvalue() return None
def export(self, metadata, **kwargs): stream = StringIO() self.myyaml.dump(data=metadata, stream=stream) metadata = stream.getvalue() metadata = metadata[:-1] return u(metadata)
def get_batch_data_and_markers( self, batch_spec: BatchSpec) -> Tuple[Any, BatchMarkers]: # batch_data # We need to build a batch_markers to be used in the dataframe batch_markers: BatchMarkers = BatchMarkers({ "ge_load_time": datetime.datetime.now( datetime.timezone.utc).strftime("%Y%m%dT%H%M%S.%fZ") }) batch_data: PandasBatchData if isinstance(batch_spec, RuntimeDataBatchSpec): # batch_data != None is already checked when RuntimeDataBatchSpec is instantiated if isinstance(batch_spec.batch_data, pd.DataFrame): df = batch_spec.batch_data elif isinstance(batch_spec.batch_data, PandasBatchData): df = batch_spec.batch_data.dataframe else: raise ValueError( "RuntimeDataBatchSpec must provide a Pandas DataFrame or PandasBatchData object." ) batch_spec.batch_data = "PandasDataFrame" elif isinstance(batch_spec, S3BatchSpec): if self._s3 is None: raise ge_exceptions.ExecutionEngineError( f"""PandasExecutionEngine has been passed a S3BatchSpec, but the ExecutionEngine does not have a boto3 client configured. Please check your config.""" ) s3_engine = self._s3 s3_url = S3Url(batch_spec.path) reader_method: str = batch_spec.reader_method reader_options: dict = batch_spec.reader_options or {} s3_object = s3_engine.get_object(Bucket=s3_url.bucket, Key=s3_url.key) logger.debug("Fetching s3 object. Bucket: {} Key: {}".format( s3_url.bucket, s3_url.key)) reader_fn = self._get_reader_fn(reader_method, s3_url.key) df = reader_fn( StringIO(s3_object["Body"].read().decode( s3_object.get("ContentEncoding", "utf-8"))), **reader_options, ) elif isinstance(batch_spec, PathBatchSpec): reader_method: str = batch_spec.reader_method reader_options: dict = batch_spec.reader_options path: str = batch_spec.path reader_fn: Callable = self._get_reader_fn(reader_method, path) df = reader_fn(path, **reader_options) else: raise BatchSpecError( f"batch_spec must be of type RuntimeDataBatchSpec, PathBatchSpec, or S3BatchSpec, not {batch_spec.__class__.__name__}" ) df = self._apply_splitting_and_sampling_methods(batch_spec, df) if df.memory_usage().sum() < HASH_THRESHOLD: batch_markers["pandas_data_fingerprint"] = hash_pandas_dataframe( df) typed_batch_data = PandasBatchData(execution_engine=self, dataframe=df) return typed_batch_data, batch_markers
def test_component_dumping_factory(make_instances): a = make_instances() config = "!A {}\n" with StringIO() as stream: yaml.dump(a, stream) assert config == stream.getvalue()
def build_toc(content_folder, filename_split_char='_'): """Auto-generate a Table of Contents from files/folders. Parameters ---------- content_folder : str Path to the folder where content exists. The TOC will be generated according to the alphanumeric sort of these files/folders. filename_split_char : str The character used in inferring spaces in page names from filenames. """ if not op.isdir(content_folder): raise ValueError(f"Could not find the provided content folder\n{content_folder}") # Generate YAML from the directory structure out = [YAML_TOP, YAML_WARN] toc_pages = [] for ii, (ifolder, folders, ifiles) in enumerate(sorted(os.walk(content_folder))): if ".ipynb_checkpoints" in ifolder: continue path_rel_to_content = ifolder.replace(content_folder, '') if ii == 0: # Create a dictionary of top-level folders we'll append to top_level_dict = {folder: [] for folder in folders if len(folder) > 0} # Append files for the top-most folder for ifile in ifiles: if any(ifile.endswith(ii) for ii in [".ipynb", ".md"]): if ifile == "LICENSE.md": continue suff = os.path.splitext(ifile)[-1] i_title = _filename_to_title(ifile, filename_split_char) i_url = os.path.join(path_rel_to_content, os.path.basename(ifile)).replace(suff, '') toc_pages.append({'title': i_title, 'url': i_url}) else: # Grab the top-most folder to choose which list we'll append to folder = path_rel_to_content.lstrip('/').split(os.sep)[0] # If the file ends in ipynb or md, add it to this section for ifile in ifiles: suff = os.path.splitext(ifile)[-1] if not any(ii == suff for ii in [".ipynb", ".md"]): continue # Convert to Jupyter-book ready names i_title = _filename_to_title(ifile, filename_split_char) i_url = os.path.join(path_rel_to_content, os.path.basename(ifile)).replace(suff, '') top_level_dict[folder].append({'title': i_title, 'url': i_url}) # Iterate through our top level dict and convert to yaml-style dict top_level_dict = {key: val for key, val in top_level_dict.items() if len(val) > 0} out_children = [] for folder, subsections in top_level_dict.items(): name = _filename_to_title(folder) out_children.append("## REPLACE ##") out_children.append({'header': name}) out_children += subsections toc_pages += out_children # Convert the dictionary into YAML and append it to our output yaml = YAML() string = StringIO() yaml.dump(toc_pages, string) out.append(string.getvalue().replace("- '## REPLACE ##'", '')) return '\n'.join(out)
def object_to_yaml_str(obj): output_str: str with StringIO() as string_stream: yaml.dump(obj, string_stream) output_str = string_stream.getvalue() return output_str
def stringify_dict(as_dict: dict) -> str: as_dict = format_node(as_dict, node_path=[]) stream = StringIO() yaml.dump(as_dict, stream) return stream.getvalue()
def assert_graphql_resp_expected(resp_orig, exp_response_orig, query, resp_hdrs={}, skip_if_err_msg=False, skip_assertion=False, exp_resp_hdrs={}): print('Reponse Headers: ', resp_hdrs) print(exp_resp_hdrs) # Prepare actual and expected responses so comparison takes into # consideration only the ordering that we care about: resp = collapse_order_not_selset(resp_orig, query) exp_response = collapse_order_not_selset(exp_response_orig, query) matched = equal_CommentedMap( resp, exp_response) and (exp_resp_hdrs or {}).items() <= resp_hdrs.items() if PytestConf.config.getoption("--accept"): print('skipping assertion since we chose to --accept new output') else: yml = yaml.YAML() # https://yaml.readthedocs.io/en/latest/example.html#output-of-dump-as-a-string : dump_str = StringIO() test_output = { # Keep strict received order when displaying errors: 'response': resp_orig, 'expected': exp_response_orig, 'diff': (lambda diff: "(results differ only in their order of keys)" if diff == {} else diff)(stringify_keys( jsondiff.diff(exp_response, resp))), 'query': query } if 'x-request-id' in resp_hdrs: test_output['request id'] = resp_hdrs['x-request-id'] if exp_resp_hdrs: diff_hdrs = { key: val for key, val in resp_hdrs.items() if key in exp_resp_hdrs } test_output['headers'] = { 'actual': dict(resp_hdrs), 'expected': exp_resp_hdrs, 'diff': (stringify_keys(jsondiff.diff(exp_resp_hdrs, diff_hdrs))) } yml.dump(test_output, stream=dump_str) if not skip_if_err_msg: if skip_assertion: return resp, matched else: assert matched, '\n' + dump_str.getvalue() elif matched: return resp, matched else: def is_err_msg(msg): return any(msg.get(x) for x in ['error', 'errors']) def as_list(x): return x if isinstance(x, list) else [x] # If it is a batch GraphQL query, compare each individual response separately for (exp, out) in zip(as_list(exp_response), as_list(resp)): matched_ = equal_CommentedMap(exp, out) if is_err_msg(exp) and is_err_msg(out): if not matched_: warnings.warn( "Response does not have the expected error message\n" + dump_str.getvalue()) return resp, matched else: if skip_assertion: return resp, matched_ else: assert matched_, '\n' + dump_str.getvalue() return resp, matched # matched always True unless --accept
def getConfigHash(self): s = StringIO() yaml.dump(self.get_config(), s) return hashlib.md5(s.getvalue().encode('utf-8')).hexdigest()
def yaml_to_string(v: dict): s = StringIO() fast_load.dump(v, s) return s.getvalue()