Exemplo n.º 1
0
def _clean_non_serializables(data: Dict) -> Dict:
    copy = {}
    for key, value in data.items():
        if callable(value):
            continue

        if hasattr(value, "to_dict"):
            value = value.to_dict()

        if isinstance(value, dict):
            value = _clean_non_serializables(value)

        if isinstance(value, bytes):
            value = to_hex(value)

        if isinstance(value, tuple):
            value = list(value)

        if isinstance(key, Enum):
            key = key.name

        if isinstance(value, Enum):
            value = value.value

        if value and not isinstance(value, builtin_types):
            try:
                pytoml.dumps({key: value})
            except RuntimeError:
                continue

        copy[key] = value
    return copy
Exemplo n.º 2
0
def _clean_non_serializables(data):
    copy = {}
    for key, value in data.items():
        if callable(value):
            continue

        if hasattr(value, 'to_dict'):
            value = value.to_dict()

        if isinstance(value, dict):
            value = _clean_non_serializables(value)

        if isinstance(value, bytes):
            value = serialize_bytes(value)

        if isinstance(value, tuple):
            value = list(value)

        if isinstance(key, Enum):
            key = key.name

        if isinstance(value, Enum):
            value = value.value

        if value and not isinstance(value, builtin_types):
            try:
                pytoml.dumps({key: value})
            except RuntimeError:
                continue

        copy[key] = value
    return copy
Exemplo n.º 3
0
def update_config(cfg):
    deps = get_dependencies(cfg)
    snakeye = snakeye_metadata(cfg)
    flit = flit_metadata(cfg)
    return TEMPLATE.format(metadata=toml.dumps(flit),
                           snakeye_metadata=toml.dumps(snakeye),
                           dependencies=toml.dumps(deps))
Exemplo n.º 4
0
def main(filesystem, locator, link_dir, handle_error):
    with filesystem.open(locator.root.descendant("virtualenvs.toml")) as venvs:
        contents = pytoml.load(venvs)

    for name, config in contents["virtualenv"].items():
        config.setdefault("sys.version", sys.version)

        virtualenv = locator.for_name(name=name)
        existing_config_path = virtualenv.path.descendant("installed.toml")

        try:
            with filesystem.open(existing_config_path) as existing_config:
                if pytoml.loads(existing_config.read()) == config:
                    continue
        except FileNotFound:
            virtualenv.create()
        else:
            virtualenv.recreate_on(filesystem=filesystem)

        packages, requirements = _to_install(config=config)
        try:
            virtualenv.install(packages=packages, requirements=requirements)
        except Exception:
            handle_error(virtualenv)
            continue

        for link in config.get("link", []):
            _link(
                source=virtualenv.binary(name=link),
                to=link_dir.descendant(link),
                filesystem=filesystem,
            )

        with filesystem.open(existing_config_path, "w") as existing_config:
            existing_config.write(pytoml.dumps(config).encode("utf-8"))
Exemplo n.º 5
0
    def dumps(self, data, sort=False):  # type: (...) -> str
        if not isinstance(data, TOMLFile):
            data = toml.dumps(data, sort_keys=sort)
        else:
            data = dumps(data)

        return data
Exemplo n.º 6
0
 def toml_dump(data):
     dump = pytoml.dumps(data)
     if isinstance(data, dict):
         for k, v in data.items():
             if all(isinstance(v2, dict) for v2 in v.values()):
                 dump = dump.replace('[%s]' % k, '')
     return dump.strip()
Exemplo n.º 7
0
 def serialise(self, fields=("name", "summary"), recurse=True, format=None):
     if format == "pinned":
         # user-specified fields are ignored/invalid in this case
         fields = ("pinned",)
     data = [OrderedDict([(f, getattr(self, f, None)) for f in fields])]
     if format == "human":
         table = gen_table(self, extra_cols=fields)
         tabulate.PRESERVE_WHITESPACE = True
         return tabulate.tabulate(table, headers="keys")
     if recurse and self.requires:
         deps = flatten_deps(self)
         next(deps)  # skip over root
         data += [d for dep in deps for d in dep.serialise(fields=fields, recurse=False)]
     if format is None or format == "python":
         result = data
     elif format == "json":
         result = json.dumps(data, indent=2, default=str, separators=(",", ": "))
     elif format == "yaml":
         result = oyaml.dump(data)
     elif format == "toml":
         result = "\n".join([pytoml.dumps(d) for d in data])
     elif format == "pinned":
         result = "\n".join([d["pinned"] for d in data])
     else:
         raise Exception("Unsupported format")
     return result
Exemplo n.º 8
0
def _migrate(registries_conf_file, output_file):
    toml = pytoml.dumps(loadYAML.get_registries(registries_conf_file))
    if output_file:
        with open(output_file,"w") as f:
            f.write(toml)
    else:
        print(toml)
Exemplo n.º 9
0
 def dumps(self, obj):
     try:
         return pytoml.dumps(obj, sort_keys=True)
     except Exception as e:
         raise j.exceptions.Value("Toml serialization failed",
                                  data=obj,
                                  exception=e)
Exemplo n.º 10
0
def fill_volumes_with_model(model_file,
                            volumes,
                            filename,
                            resume_filename=None,
                            partition=False,
                            viewer=False,
                            **kwargs):
    if '{volume}' not in filename:
        raise ValueError(
            'HDF5 filename must contain "{volume}" for volume name replacement.'
        )
    if resume_filename is not None and '{volume}' not in resume_filename:
        raise ValueError(
            'TOML resume filename must contain "{volume}" for volume name replacement.'
        )

    if partition:
        _, volumes = partition_volumes(volumes)

    for volume_name, volume in six.iteritems(volumes):
        logging.info('Filling volume %s...', volume_name)
        volume = volume.downsample(CONFIG.volume.resolution)
        if resume_filename is not None:
            resume_volume_filename = resume_filename.format(volume=volume_name)
            resume_volume = six.next(
                six.itervalues(HDF5Volume.from_toml(resume_volume_filename)))
            resume_prediction = resume_volume.to_memory_volume().label_data
        else:
            resume_prediction = None

        volume_filename = filename.format(volume=volume_name)
        checkpoint_filename = volume_filename + '_checkpoint'
        prediction, conflict_count = fill_volume_with_model(
            model_file,
            volume,
            resume_prediction=resume_prediction,
            checkpoint_filename=checkpoint_filename,
            **kwargs)

        config = HDF5Volume.write_file(volume_filename + '.hdf5',
                                       CONFIG.volume.resolution,
                                       label_data=prediction)
        config['name'] = volume_name + ' segmentation'
        with open(volume_filename + '.toml', 'wb') as tomlfile:
            tomlfile.write('# Filling model: {}\n'.format(model_file))
            tomlfile.write('# Filling kwargs: {}\n'.format(str(kwargs)))
            tomlfile.write(str(toml.dumps({'dataset': [config]})))

        if viewer:
            viewer = WrappedViewer(
                voxel_size=list(np.flipud(CONFIG.volume.resolution)))
            subvolume = volume.get_subvolume(
                SubvolumeBounds(start=np.zeros(3, dtype=np.int64),
                                stop=volume.shape))
            viewer.add(subvolume.image, name='Image')
            viewer.add(prediction, name='Labels')
            viewer.add(conflict_count, name='Conflicts')

            viewer.print_view_prompt()
Exemplo n.º 11
0
 def __str__(self):
     sanitized = {}
     for k, v in six.iteritems(self.__dict__):
         if isinstance(v, np.ndarray):
             sanitized[k] = v.tolist()
         else:
             sanitized[k] = v
     return toml.dumps(sanitized)
Exemplo n.º 12
0
def _main():
    ap = argparse.ArgumentParser()
    ap.add_argument('-d', '--dir', action='append')
    ap.add_argument('testcase', nargs='*')
    args = ap.parse_args()

    if not args.dir:
        args.dir = [os.path.join(os.path.split(__file__)[0], 'toml-test/tests')]

    succeeded = []
    failed = []

    for path in args.dir:
        if not os.path.isdir(path):
            print('error: not a dir: {0}'.format(path))
            return 2
        for top, dirnames, fnames in os.walk(path):
            for fname in fnames:
                if not fname.endswith('.toml'):
                    continue

                if args.testcase and not any(arg in fname for arg in args.testcase):
                    continue

                parse_error = None
                try:
                    with open(os.path.join(top, fname), 'rb') as fin:
                        parsed = toml.load(fin)
                except toml.TomlError:
                    parsed = None
                    parse_error = sys.exc_info()
                else:
                    dumped = toml.dumps(parsed)
                    parsed2 = toml.loads(dumped)
                    if parsed != parsed2:
                        failed.append((fname, None))
                        continue

                    with open(os.path.join(top, fname), 'rb') as fin:
                        parsed = toml.load(fin, translate=_testbench_literal)

                try:
                    with io.open(os.path.join(top, fname[:-5] + '.json'), 'rt', encoding='utf-8') as fin:
                        bench = json.load(fin)
                except IOError:
                    bench = None

                if parsed != adjust_bench(bench):
                    failed.append((fname, parsed, bench, parse_error))
                else:
                    succeeded.append(fname)

    for f, parsed, bench, e in failed:
        print('failed: {}\n{}\n{}'.format(f, json.dumps(parsed, indent=4), json.dumps(bench, indent=4)))
        if e:
            traceback.print_exception(*e)
    print('succeeded: {0}'.format(len(succeeded)))
    return 1 if failed or not succeeded else 0
Exemplo n.º 13
0
def _main():
    ap = argparse.ArgumentParser()
    ap.add_argument('-d', '--dir', action='append')
    ap.add_argument('testcase', nargs='*')
    args = ap.parse_args()

    if not args.dir:
        args.dir = [os.path.join(os.path.split(__file__)[0], 'toml-test/tests')]

    succeeded = []
    failed = []

    for path in args.dir:
        if not os.path.isdir(path):
            print('error: not a dir: {}'.format(path))
            return 2
        for top, dirnames, fnames in os.walk(path):
            for fname in fnames:
                if not fname.endswith('.toml'):
                    continue

                if args.testcase and not any(arg in fname for arg in args.testcase):
                    continue

                parse_error = None
                try:
                    with open(os.path.join(top, fname), 'rb') as fin:
                        parsed = toml.load(fin)
                except toml.TomlError:
                    parsed = None
                    parse_error = sys.exc_info()
                else:
                    dumped = toml.dumps(parsed)
                    parsed2 = toml.loads(dumped)
                    if parsed != parsed2:
                        failed.append((fname, None))
                        continue

                    with open(os.path.join(top, fname), 'rb') as fin:
                        parsed = toml.load(fin, translate=_testbench_literal)

                try:
                    with io.open(os.path.join(top, fname[:-5] + '.json'), 'rt', encoding='utf-8') as fin:
                        bench = json.load(fin)
                except IOError:
                    bench = None

                if parsed != adjust_bench(bench):
                    failed.append((fname, parsed, bench, parse_error))
                else:
                    succeeded.append(fname)

    for f, parsed, bench, e in failed:
        print('failed: {}\n{}\n{}'.format(f, json.dumps(parsed, indent=4), json.dumps(bench, indent=4)))
        if e:
            traceback.print_exception(*e)
    print('succeeded: {}'.format(len(succeeded)))
    return 1 if failed or not succeeded else 0
Exemplo n.º 14
0
def render_toml(obj):
    """
    Serializes the specified object to TOML.

    :param obj: the object to serialize
    :returns: str
    """

    return pytoml.dumps(obj)
Exemplo n.º 15
0
def save_config(config, path):
    config['revision'] = get_revision()
    lines = []
    for line in toml.dumps(config).splitlines():
        if line.startswith('['):
            lines.append('')
        lines.append(line)
    with open(path, 'w') as f:
        f.write('\n'.join(lines))
Exemplo n.º 16
0
 def configSave(self, config=None, path=""):
     """
     """
     if self.executor.state_disabled:
         return
     if self.readonly:
         raise j.exceptions.Input(
             message="cannot write config to '%s', because is readonly" %
             self, level=1, source="", tags="", msgpub="")
     if config and path:
         data = pytoml.dumps(config)
         self.executor.file_write(path, data)
         return
     data = pytoml.dumps(self._configJS)
     self.executor.file_write(self.configJSPath, data, sudo=True)
     data = pytoml.dumps(self._configState)
     self.executor.file_write(self.configStatePath, data, sudo=True)
     self.executor.reset()  # make sure all caching is reset
Exemplo n.º 17
0
def create_hugo_post(content, dest_dir):
    """Create a hugo post from the given content."""

    text = content.pop("content")
    post = "+++\n{}+++\n\n{}\n".format(pytoml.dumps(content), text.strip())
    issue_dir = join(dest_dir, "issue-{}".format(content["issue"]))
    makedirs(issue_dir, exist_ok=True)
    post_path = join(issue_dir, "{}.md".format(slugify(content["title"])))
    with open(post_path, "w") as f:
        f.write(post)
Exemplo n.º 18
0
def write_out_file(front_data, body_text, out_file_path):
    out_lines = ['+++']
    front_string = toml.dumps(front_data)
    out_lines.extend(front_string.splitlines())
    out_lines.append('+++')
    out_lines.extend(body_text.splitlines())

    with open(out_file_path, 'w') as f:

        f.write('\n'.join(out_lines))
Exemplo n.º 19
0
def dump_module(header, module):
    attribs = dict()
    for name, value in inspect.getmembers(module):
        if name.isupper():
            attribs[name] = value

    attribs = _clean_non_serializables(attribs)

    print(pytoml.dumps({header: attribs}))
    print()
Exemplo n.º 20
0
def dump_module(header: str, module: Any) -> None:
    attribs = dict()
    for name, value in inspect.getmembers(module):
        if name.isupper():
            attribs[name] = value

    attribs = _clean_non_serializables(attribs)

    print(pytoml.dumps({header: attribs}))
    print()
Exemplo n.º 21
0
def dumps(d):
    """dump dict object into toml stream 
    
    Arguments:
        d (dict) : the dict which will be dumped     
    
    Returns:
        string : the toml stream
    """
    s = pytoml.dumps(d)
    return s
Exemplo n.º 22
0
def main():
    parser = argparse.ArgumentParser(
        description=f'SSH bastion server {__version__}')
    parser.add_argument('-c', '--config', default='~/.sshgateway/config.toml')
    parser.add_argument('--show-config',
                        dest='show',
                        action='store_true',
                        default=False)
    parser.add_argument('-v', dest='verbose', action='count', default=1)
    args = parser.parse_args()
    config['verbose'] = args.verbose
    try:
        with open(expanduser(args.config)) as configfile:
            config.update(pytoml.loads(configfile.read()))
    except FileNotFoundError:
        print('Can not find configuration file. use default config')
    except PermissionError:
        print('Can not read configuration file.', file=sys.stderr)
        sys.exit(1)
    except pytoml.TomlError as e:
        print(f'Invalid configuration file: {e}', file=sys.stderr)

    if args.show:
        print(pytoml.dumps(config))
        sys.exit(0)

    for host in config['hosts']:
        host.setdefault('proxy', None)

    try:
        config['hosts'] = [Host(**host) for host in config['hosts']]
        for host in config['hosts']:
            host_dict[host.name] = host

        config['groups'] = [Group(**group) for group in config['groups']]
        config['permissions'] = [
            Permission(**perm) for perm in config['permissions']
        ]
    except Exception as e:
        print(f'Invalid configuration file: {e}', file=sys.stderr)
        sys.exit(1)

    loop = asyncio.get_event_loop()
    try:
        loop.run_until_complete(start_server())
    except (OSError, asyncssh.Error) as exc:
        sys.exit('Error starting server: ' + str(exc))
    try:
        loop.run_forever()
    except KeyboardInterrupt:
        pass
    finally:
        loop.run_until_complete(loop.shutdown_asyncgens())
        loop.close()
Exemplo n.º 23
0
def add(index_path, key, value):
    index = pytoml.loads(subprocess.check_output(decrypt + [index_path]))
    if key not in index:
        index[key] = value
        cmd = encrypt + ['--yes', '--output', index_path]
        with subprocess.Popen(cmd,
                              stdin=subprocess.PIPE, stderr=subprocess.PIPE,
                              universal_newlines=True) as gpg:
            err = gpg.communicate(pytoml.dumps(index))[1].encode()
            if gpg.wait() != 0:
                raise subprocess.CalledProcessError(gpg.poll(), cmd, err)
Exemplo n.º 24
0
def dumps(data, meta_format="yaml"):
    body = data.pop("body", "")
    file_string = "---" + meta_format + new_line
    if meta_format == "yaml":
        file_string += yaml.dump(data, default_flow_style=False)
    elif meta_format == "json":
        file_string += json.dumps(data)
    elif meta_format == "toml":
        file_string += toml.dumps(data)
    file_string = new_line + "---" + new_line + body
    return file_string
Exemplo n.º 25
0
    def initialise(self):
        if (self.directory / 'pyproject.toml').exists():
            resp = input("pyproject.toml exists - overwrite it? [y/N]: ")
            if (not resp) or resp[0].lower() != 'y':
                return

        module = self.prompt_text('Module name', self.guess_module_name(),
                                  str.isidentifier)
        author = self.prompt_text('Author', self.defaults.get('author'),
                                  lambda s: s != '')
        author_email = self.prompt_text('Author email',
                                        self.defaults.get('author_email'),
                                        self.validate_email)
        if 'home_page_template' in self.defaults:
            home_page_default = self.defaults['home_page_template'].replace(
                '{modulename}', module)
        else:
            home_page_default = None
        home_page = self.prompt_text(
            'Home page',
            home_page_default,
            self.validate_homepage,
            retry_msg="Should start with http:// or https:// - try again.")
        license = self.prompt_options(
            'Choose a license (see http://choosealicense.com/ for more info)',
            license_choices, self.defaults.get('license'))

        self.update_defaults(author=author,
                             author_email=author_email,
                             home_page=home_page,
                             module=module,
                             license=license)

        metadata = OrderedDict([
            ('module', module),
            ('author', author),
        ])
        if author_email:
            metadata['author-email'] = author_email
        if home_page:
            metadata['home-page'] = home_page
        if license != 'skip':
            metadata['classifiers'] = [license_names_to_classifiers[license]]
            self.write_license(license, author)

        with (self.directory / 'pyproject.toml').open('w',
                                                      encoding='utf-8') as f:
            f.write(TEMPLATE.format(metadata=toml.dumps(metadata)))

        print()
        print(
            "Written pyproject.toml; edit that file to add optional extra info."
        )
Exemplo n.º 26
0
 def __str__(self):
     sanitized = {}
     for n, c in six.iteritems(self.__dict__):
         if not isinstance(c, BaseConfig):
             sanitized[n] = c
             continue
         sanitized[n] = {}
         for k, v in six.iteritems(c.__dict__):
             if isinstance(v, np.ndarray):
                 sanitized[n][k] = v.tolist()
             else:
                 sanitized[n][k] = v
     return toml.dumps(sanitized)
Exemplo n.º 27
0
def write_toml(path,
               contents,
               auto_generated_from=None,
               keys_to_convert_to_list=None,
               types_to_str=(enum.Enum, )):
    convert_dict_item_type(contents, _type=types_to_str, func=str)
    if keys_to_convert_to_list:
        convert_dict_keys(contents, keys=keys_to_convert_to_list, func=list)
    dump = pytoml.dumps(contents)
    if auto_generated_from:
        dump = f"# NOTE: THIS FILE IS AUTO GENERATED BASED ON THE ANALYSIS OF {auto_generated_from}.\n# DO NOT MODIFY THIS FILE DIRECTLY.\n{dump}"
    with open(path, 'w') as the_file:
        the_file.write(dump)
    return dump
Exemplo n.º 28
0
def new(title):
    article_info = {
        "title": title,
        "create_time": datetime.datetime.now().isoformat("T")
    }
    article_default_text = "%s+++++++\n\n##1" % pytoml.dumps(article_info)

    if os.path.isfile("article/%s.md" % title):
        print("Error: Article '%s' is existed." % title)
        return

    file = open("article/%s.md" % title, "wt")
    file.write(article_default_text)
    file.close()
Exemplo n.º 29
0
def claim(simulation):
    """Clearly not thread safe, but it'll do the trick."""
    s3 = boto3.resource('s3')
    key = os.path.join(os.path.basename(simulation['directory']),
                       'simulation.toml')
    try:
        remote = toml.loads(
            s3.Object('xcite-simulations', key).get()['Body'].read())
    except BotoClientError:
        body = io.BytesIO(toml.dumps(simulation).encode('utf-8'))
        s3.Object('xcite-simulations', key).put(Body=body)
        return True
    else:
        return simulation['server'] == remote['server']
Exemplo n.º 30
0
def new(title):
    article_info = {
        "title": title,
        "create_time": datetime.datetime.now().isoformat("T")
    }
    article_default_text = "%s+++++++\n\n##1" % pytoml.dumps(article_info)

    if os.path.isfile("article/%s.md" % title):
        print("Error: Article '%s' is existed." % title)
        return

    file = open("article/%s.md" % title, "wt")
    file.write(article_default_text)
    file.close()
Exemplo n.º 31
0
def serialize(obj, **options):
    """
    Serialize Python data to TOML.

    :param obj: the data structure to serialize.
    :param options: options given to lower pytoml module.
    """

    try:
        if "file_out" in options:
            return toml.dump(obj, options["file_out"], **options)
        else:
            return toml.dumps(obj, **options)
    except Exception as error:  # pylint: disable=broad-except
        raise SerializationError(error)
Exemplo n.º 32
0
def serialize(obj, **options):
    '''
    Serialize Python data to TOML.

    :param obj: the data structure to serialize.
    :param options: options given to lower pytoml module.
    '''

    try:
        if 'file_out' in options:
            return toml.dump(obj, options['file_out'], **options)
        else:
            return toml.dumps(obj, **options)
    except Exception as error:
        raise SerializationError(error)
Exemplo n.º 33
0
def encode_toml(data, ordered):
    try:
        return pytoml.dumps(data, sort_keys=not ordered)
    except AttributeError as e:
        if str(e) == "'list' object has no attribute 'keys'":
            raise ValueError('Cannot convert non-dictionary data to '
                             'TOML; use "wrap" to wrap it in a '
                             'dictionary')
        else:
            raise e
    except TypeError as e:
        if str(e) == "'in <string>' requires string as left operand, not int":
            raise ValueError('Cannot convert binary to TOML')
        else:
            raise ValueError('Cannot convert data to TOML ({0})'.format(e))
Exemplo n.º 34
0
def convert(path):
    cp = configparser.ConfigParser()
    with path.open(encoding='utf-8') as f:
        cp.read_file(f)

    ep_file = Path('entry_points.txt')
    metadata = OrderedDict()
    for name, value in cp['metadata'].items():
        if name in metadata_list_fields:
            metadata[name] = [l for l in value.splitlines() if l.strip()]
        elif name == 'entry-points-file':
            ep_file = Path(value)
        else:
            metadata[name] = value

    if 'scripts' in cp:
        scripts = OrderedDict(cp['scripts'])
    else:
        scripts = {}

    entrypoints = CaseSensitiveConfigParser()
    if ep_file.is_file():
        with ep_file.open(encoding='utf-8') as f:
            entrypoints.read_file(f)

    written_entrypoints = False
    with Path('pyproject.toml').open('w', encoding='utf-8') as f:
        f.write(TEMPLATE.format(metadata=pytoml.dumps(metadata)))

        if scripts:
            f.write('\n[tool.flit.scripts]\n')
            pytoml.dump(scripts, f)

        for groupname, group in entrypoints.items():
            if not dict(group):
                continue

            if '.' in groupname:
                groupname = '"{}"'.format(groupname)
            f.write('\n[tool.flit.entrypoints.{}]\n'.format(groupname))
            pytoml.dump(OrderedDict(group), f)
            written_entrypoints = True

    print("Written 'pyproject.toml'")
    files = str(path)
    if written_entrypoints:
        files += ' and ' + str(ep_file)
    print("Please check the new file, then remove", files)
Exemplo n.º 35
0
def _update_entry_to_local_data(entry, local_files_data):

    add_to_end = True

    if entry["name"] in local_files_data.items():
        local_files_data["version"] = entry["version"]
        local_files_data["checksum"] = entry["checksum"]
        add_to_end = False

    if add_to_end:
        local_files_data[entry["name"]] = entry

    f = open(path_finder.get_local_version_data_path(),'w')
    f.write(toml.dumps(local_files_data))
    f.flush()
    f.close()
Exemplo n.º 36
0
def writeVSphereSettings(knobName=DEFAULT_VSPHERE_KNOB,
                         knobDirectory=DEFAULT_KNOB_DIRECTORY,
                         hostname=None,
                         username=None,
                         password=None):
    '''
  Write the vsphere host information to a knob file so we don't have to determine it
  every single time we read a tag.

  :param str knobName: What file name to write the vsphere data to
  :param str knobDirectory: What directory to write the knob file in
  :param str hostname: hostname of the vSphere host
  :param str user: username to connect to vSphere
  :param str password: password to connect to vSphere
  '''
    assert isinstance(hostname,
                      basestring), ("hostname must be a string but is %r" %
                                    hostname)
    assert isinstance(
        knobDirectory,
        basestring), ("knobDirectory must be a string but is %r" %
                      knobDirectory)
    assert isinstance(knobName,
                      basestring), ("knobName must be a string but is %r" %
                                    knobName)
    assert isinstance(password,
                      basestring), ("password must be a string but is %r" %
                                    password)
    assert isinstance(username,
                      basestring), ("username must be a string but is %r" %
                                    username)

    loadSharedLogger()
    knobPath = "%s/%s" % (knobDirectory, knobName)
    if not os.path.isdir(knobDirectory):
        this.logger.info(
            'writeVsphereSettings: directory %s does not exist, creating it',
            knobDirectory)
        systemCall('mkdir -p %s' % knobDirectory)

    with open(knobPath, 'w') as knobFile:
        this.logger.info('Writing vSphere connection info to %s', knobPath)
        settings = {}
        settings['hostname'] = hostname
        settings['username'] = username
        settings['password'] = password
        knobFile.write(toml.dumps(settings))
Exemplo n.º 37
0
def main(filesystem, locator, link_dir, handle_error):
    with filesystem.open(locator.root.descendant("virtualenvs.toml")) as venvs:
        contents = pytoml.load(
            venvs,
            object_pairs_hook=collections.OrderedDict,
        )

    progress = tqdm(contents["virtualenv"].items())
    for name, config in progress:
        progress.set_description(name)

        python = config.pop("python", sys.executable)
        config.setdefault(
            "sys.version", subprocess.check_output(
                [python, "--version"],
                stderr=subprocess.STDOUT,
            ).decode('ascii'),
        )

        virtualenv = locator.for_name(name=name)
        existing_config_path = virtualenv.path.descendant("installed.toml")

        try:
            with filesystem.open(existing_config_path) as existing_config:
                if pytoml.loads(existing_config.read()) == config:
                    continue
        except FileNotFound:
            virtualenv.create(python=python)
        else:
            virtualenv.recreate_on(filesystem=filesystem, python=python)

        packages, requirements = _to_install(config=config)
        try:
            virtualenv.install(packages=packages, requirements=requirements)
        except Exception:
            handle_error(virtualenv)
            continue

        for link in config.get("link", []):
            _link(
                source=virtualenv.binary(name=link),
                to=link_dir.descendant(link),
                filesystem=filesystem,
            )

        with filesystem.open(existing_config_path, "wt") as existing_config:
            existing_config.write(pytoml.dumps(config))
Exemplo n.º 38
0
Arquivo: test.py Projeto: ExpHP/pytoml
def _main():
    succeeded = []
    failed = []

    for top, dirnames, fnames in os.walk('.'):
        for fname in fnames:
            if not fname.endswith('.toml'):
                continue

            if sys.argv[1:] and not any(arg in fname for arg in sys.argv[1:]):
                continue

            parse_error = None
            try:
                with open(os.path.join(top, fname), 'rb') as fin:
                    parsed = toml.load(fin)
            except toml.TomlError:
                parsed = None
                parse_error = sys.exc_info()
            else:
                dumped = toml.dumps(parsed)
                parsed2 = toml.loads(dumped)
                if parsed != parsed2:
                    failed.append((fname, None))
                    continue

                with open(os.path.join(top, fname), 'rb') as fin:
                    parsed = toml.load(fin, translate=_testbench_literal)

            try:
                with io.open(os.path.join(top, fname[:-5] + '.json'), 'rt', encoding='utf-8') as fin:
                    bench = json.load(fin)
            except IOError:
                bench = None

            if parsed != bench:
                failed.append((fname, parsed, bench, parse_error))
            else:
                succeeded.append(fname)

    for f, parsed, bench, e in failed:
        print('failed: {}\n{}\n{}'.format(f, json.dumps(parsed, indent=4), json.dumps(bench, indent=4)))
        if e:
            traceback.print_exception(*e)
    print('succeeded: {}'.format(len(succeeded)))
    return 1 if failed else 0
Exemplo n.º 39
0
def serialize_to_main_conf(keys : list, values : list):
    config = None
    print("Setting keys: ", keys, " to values: ", values)
    with open(os.path.join(config_path, "main_config.toml")) as conffile:
        config = toml.load(conffile)

    for i in range(0, len(keys)):
        config[keys[i]] = values[i]

    global main_conf_values
    main_conf_values = config

    if config is not None:
        f = open(os.path.join(config_path, "main_config.toml"),'w')
        f.write(toml.dumps(config))
        f.flush()
        f.close()
Exemplo n.º 40
0
def _main():
    succeeded = []
    failed = []

    for top, dirnames, fnames in os.walk('.'):
        for fname in fnames:
            if not fname.endswith('.toml'):
                continue

            try:
                with open(os.path.join(top, fname), 'rb') as fin:
                    parsed = toml.load(fin)
            except toml.TomlError:
                parsed = None
            else:
                dumped = toml.dumps(parsed)
                parsed2 = toml.loads(dumped)
                if parsed != parsed2:
                    failed.append(fname)
                    continue

                with open(os.path.join(top, fname), 'rb') as fin:
                    parsed = toml.load(fin, _testbench_literal, _testbench_array)

            try:
                with io.open(os.path.join(top, fname[:-5] + '.json'), 'rt', encoding='utf-8') as fin:
                    bench = json.load(fin)
            except IOError:
                bench = None

            if parsed != bench:
                failed.append(fname)
            else:
                succeeded.append(fname)

    for f in failed:
        print('failed: {}'.format(f))
    print('succeeded: {}'.format(len(succeeded)))
    return 1 if failed else 0
Exemplo n.º 41
0
    def configure(self, service_obj):
        import pytoml
        """
        this gets executed when files are installed
        this step is used to do configuration steps to the platform
        after this step the system will try to start the ays if anything needs to be started
        """

        # for backwards compatibility
        base = '/opt/jumpscale7/apps/agentcontroller2'

        toml = '/opt/jumpscale7/apps/agentcontroller2/agentcontroller2.toml'
        cfg = pytoml.load(open(toml))
        redis = service_obj.hrd.get('instance.param.redis.host')
        cfg['main']['redis_host'] = redis
        cfg['main']['redis_password'] = service_obj.hrd.get('instance.param.redis.password')

        # configure env var for events handlers
        redis_host, _, redis_port = redis.partition(':')
        cfg['events']['settings']['redis_address'] = redis_host
        cfg['events']['settings']['redis_port'] = redis_port
        cfg['events']['settings']['redis_password'] = service_obj.hrd.get('instance.param.redis.password')

        syncthing = j.atyourservice.get(name='syncthing')
        cfg['events']['settings']['syncthing_url'] = 'http://localhost:%s/' % syncthing.hrd.get('instance.param.port')

        content = pytoml.dumps(cfg)
        j.system.fs.writeFile(filename=toml, contents=content)

        # Start script syncing (syncthing)
        jumpscripts = j.system.fs.joinPaths(base, 'jumpscripts')

        j.system.fs.createDir(jumpscripts)

        syncthing_id = syncthing.actions.get_syncthing_id(syncthing)
        folderid = 'jumpscripts-%s' % hashlib.md5(syncthing_id).hexdigest()
        syncthing.actions.add_folder(syncthing, folderid, jumpscripts)
Exemplo n.º 42
0
def _update_checksum_entry(entry, local_checksum_data):
    
    file_path = os.path.join(path_finder.get_nwn_path(), entry["target_dir"])
    file_path = os.path.join(file_path, entry["name"])

    print("Doing checksum for: ", entry["name"])
    # Skip generating a checksum for portraits, since there's so bloody many of them!
    checksum = "portrait"
    if entry["target_dir"] != "portraits":
        checksum = _generate_file_md5(file_path)
    else:
        return

    print ("Checksum: ", checksum)

    modified_at = time.ctime(os.path.getmtime(file_path))
    print ("Modified at: ", modified_at)

    checksum_entry = {}
    checksum_entry["name"] = entry["name"]
    checksum_entry["checksum"] = checksum
    checksum_entry["modified"] = modified_at

    found_entry = False
    if checksum_entry["name"] in local_checksum_data:
        found_entry = True
        local_checksum_data[checksum_entry["name"]]["checksum"] = checksum_entry["checksum"]

    if found_entry == False:
        local_checksum_data[checksum_entry["name"]] = checksum_entry

    f = open(path_finder.get_local_checksums_path(),'w')
    f.write(toml.dumps(local_checksum_data))
    f.flush()
    f.close()

    return checksum_entry
Exemplo n.º 43
0
 def get_dot_rmotr_as_toml(self):
     return toml.dumps({
         'uuid': self.uuid,
         'name': self.name
     })
  frontmatter, article_contents = file_data.split('---')
  metadata = yaml.load(frontmatter)
  _, filename = os.path.split(arguments['<in_path>'])

  # Generate the output file.
  created_date = time.strptime(metadata['created'], '%B %d, %Y')
  formatted_date = time.strftime('%Y-%m-%d', created_date)
  output_metadata = {
    'date': '%sT13:00:00-08:00' % formatted_date,
    'location': '',
    'tags': metadata['tags'],
    'thumbnail': '',
    'title': metadata['title'],
  }

  # Make sure a 'drafts' tag is present so we can do some QC later.
  if 'drafts' not in output_metadata['tags']:
    output_metadata['tags'].append('drafts')

  # Build up the file.
  output_contents = ''.join((
    '+++\n',
    toml.dumps(output_metadata),
    '\n+++',
    article_contents))

  # Save the output to the default location.
  out_path = os.path.join('content/notes', filename)
  with open(out_path, 'w') as out_file:
    out_file.write(output_contents)
Exemplo n.º 45
0
def remarshal(input, output, input_format, output_format, wrap=None,
              unwrap=None, indent_json=None, yaml_options={}):
    try:
        if input == '-':
            input_file = getattr(sys.stdin, 'buffer', sys.stdin)
        else:
            input_file = open(input, 'rb')

        if output == '-':
            output_file = getattr(sys.stdout, 'buffer', sys.stdout)
        else:
            try:
                output_file = open(output, 'wb')
            except FileNotFoundError as e:
                # There should never be another reason for a FileNotFoundError
                # here than a missing parent directory.
                raise NotADirectoryError("Not a directory: '{0}'"
                                         .format(os.path.dirname(output)))

        input_data = input_file.read()

        if input_format == 'json':
            try:
                parsed = json.loads(input_data.decode('utf-8'))
            except JSONDecodeError as e:
                raise ValueError('Cannot parse as JSON ({0})'.format(e))
        elif input_format == 'toml':
            try:
                parsed = pytoml.loads(input_data)
            except pytoml.core.TomlError as e:
                raise ValueError('Cannot parse as TOML ({0})'.format(e))
        elif input_format == 'yaml':
            try:
                parsed = yaml.load(input_data)
            except (yaml.scanner.ScannerError, yaml.parser.ParserError) as e:
                raise ValueError('Cannot parse as YAML ({0})'.format(e))
        else:
            raise ValueError('Unknown input format: {0}'.format(input_format))

        if unwrap is not None:
            parsed = parsed[unwrap]
        if wrap is not None:
            temp = {}
            temp[wrap] = parsed
            parsed = temp

        if output_format == 'json':
            if indent_json is True:
                indent_json = 2
            if indent_json:
                separators = (',', ': ')
            else:
                separators = (',', ':')
            output_data = json.dumps(parsed, default=json_serialize,
                                     ensure_ascii=False, indent=indent_json,
                                     separators=separators,
                                     sort_keys=True) + "\n"
        elif output_format == 'toml':
            try:
                output_data = pytoml.dumps(parsed, sort_keys=True)
            except AttributeError as e:
                if str(e) == "'list' object has no attribute 'keys'":
                    raise ValueError('Cannot convert non-dictionary data to '
                                     'TOML; use "wrap" to wrap it in a '
                                     'dictionary')
                else:
                    raise e
        elif output_format == 'yaml':
            output_data = yaml.safe_dump(parsed, allow_unicode=True,
                                         default_flow_style=False,
                                         encoding=None, **yaml_options)
        else:
            raise ValueError('Unknown output format: {0}'.
                             format(output_format))
        output_file.write(output_data.encode('utf-8'))
        output_file.close()
    finally:
        if 'input_file' in locals():
            input_file.close()
        if 'output_file' in locals():
            output_file.close()
Exemplo n.º 46
0
def to_toml(a, *args, **kw):
    '''Make verbose, human readable toml'''
    transformed = pytoml.dumps(a, **kw)
    return to_text(transformed)
Exemplo n.º 47
0
def main():
    # config
    default_conf = agilebot.AgileBot.default_conf()
    conf = copy(default_conf)

    # config file
    if os.path.isfile(CONFIG_PATH):
        with open(CONFIG_PATH, 'r') as f:
            toml_config = toml.load(f)
            conf = util.left_merge(default_conf, toml_config)

    # logging conf
    log_level = os.environ.get('AGILEBOT_LOG_LEVEL') or conf['logging']['level']
    logger.setLevel(log_level)
    logger.info('log level: {}'.format(log_level))

    # library logging config
    lib_log_level = logging.CRITICAL
    logging.getLogger('oauthlib').setLevel(lib_log_level)
    logging.getLogger('requests').setLevel(lib_log_level)
    logging.getLogger('requests_oauthlib').setLevel(lib_log_level)

    # command line args
    parser = argparse.ArgumentParser(
        description='Automate functions for Agile development sprints.')
    subparsers = parser.add_subparsers(help='sub-command help', dest='subparser_0')
    parser.add_argument('--conf', action='store_true', default=False, help='print current configuration')

    # boards sub-command
    sub_commands = {
        'slack': cmd_slack.sub_command(subparsers),
        'sprint': cmd_sprint.sub_command(subparsers),
        'trello': cmd_trello.sub_command(subparsers)
    }

    # set defaults, ENV var first, then config file, then command line args
    # don't set func or func_help due to a bug in argparse that isn't fixed in python 3.4 yet
    # http://bugs.python.org/issue9351
    parser.set_defaults(
        # func=None,
        # func_help=parser.print_help,
        agile_sprint_lists=get_first_value(
            os.environ.get('AGILE_SPRINT_LISTS'),
            conf['agile']['sprint_lists']
        ),
    )

    # parse the arguments
    args = parser.parse_args()
    sc_0 = getattr(args, 'subparser_0', '')
    sc_1 = getattr(args, 'subparser_1', '')
    logger.debug('subparser_0: {}'.format(sc_0))
    logger.debug('subparser_1: {}'.format(sc_1))

    # agile
    conf['agile']['sprint_lists'] = args.agile_sprint_lists

    if not len(sys.argv) > 1:
        # no arguments given, show help
        logger.debug('sys.argv: {}'.format(len(sys.argv)))
        parser.print_help()
    elif args.conf:
        # show current config
        logger.debug('printing current configuration')
        print(toml.dumps(conf, sort_keys=True))
    elif not getattr(args, 'func', None):
        # if the sub-command function is not set, show help
        logger.debug('sub-command function not found for: {}'.format(str(sys.argv)))
        if hasattr(args, 'func_help'):
            logger.debug('show sub-command specific help')
            func_help = args.func_help
        else:
            logger.debug('show general help')
            func_help = parser.print_help
        func_help()
        logger.debug('args namespace: {}'.format(pformat(args)))
        logger.debug('main parser: {}'.format(pformat(parser)))
        sys.exit(1)
    else:
        # run the sub-command
        logger.debug('executing: agilebot {} {}'.format(sc_0, sc_1))
        args.func(args, conf)
Exemplo n.º 48
0
def dump_cmd_options(options):
    print(pytoml.dumps({
        'options': _clean_non_serializables(options),
    }))
    print()
Exemplo n.º 49
0
def dump_config(config):
    print(pytoml.dumps({'configs': _clean_non_serializables(config)}))
    print()
Exemplo n.º 50
0
 def dump_str(self, data):
     return toml.dumps(data)
Exemplo n.º 51
0
def remarshal(input,
              output,
              input_format,
              output_format,
              wrap=None,
              unwrap=None,
              indent_json=None,
              yaml_options={},
              ordered=False):
    try:
        if input == '-':
            input_file = getattr(sys.stdin, 'buffer', sys.stdin)
        else:
            input_file = open(input, 'rb')

        if output == '-':
            output_file = getattr(sys.stdout, 'buffer', sys.stdout)
        else:
            output_file = open(output, 'wb')

        input_data = input_file.read()

        if input_format == 'json':
            try:
                pairs_hook = OrderedDict if ordered else dict
                parsed = json.loads(input_data.decode('utf-8'),
                                    object_pairs_hook=pairs_hook)
            except JSONDecodeError as e:
                raise ValueError('Cannot parse as JSON ({0})'.format(e))
        elif input_format == 'toml':
            try:
                pairs_hook = OrderedDict if ordered else dict
                parsed = pytoml.loads(input_data,
                                      object_pairs_hook=pairs_hook)
            except pytoml.core.TomlError as e:
                raise ValueError('Cannot parse as TOML ({0})'.format(e))
        elif input_format == 'yaml':
            try:
                loader = OrderedLoader if ordered else TimezoneLoader
                parsed = yaml.load(input_data,
                                   loader)
            except (yaml.scanner.ScannerError, yaml.parser.ParserError) as e:
                raise ValueError('Cannot parse as YAML ({0})'.format(e))
        else:
            raise ValueError('Unknown input format: {0}'.format(input_format))

        if unwrap is not None:
            parsed = parsed[unwrap]
        if wrap is not None:
            temp = {}
            temp[wrap] = parsed
            parsed = temp

        if output_format == 'json':
            if indent_json is True:
                indent_json = 2
            if indent_json:
                separators = (',', ': ')
            else:
                separators = (',', ':')
            output_data = json.dumps(parsed,
                                     default=json_serialize,
                                     ensure_ascii=False,
                                     indent=indent_json,
                                     separators=separators,
                                     sort_keys=not ordered) + "\n"
        elif output_format == 'toml':
            try:
                output_data = pytoml.dumps(parsed,
                                           sort_keys=not ordered)
            except AttributeError as e:
                if str(e) == "'list' object has no attribute 'keys'":
                    raise ValueError('Cannot convert non-dictionary data to '
                                     'TOML; use "wrap" to wrap it in a '
                                     'dictionary')
                else:
                    raise e
        elif output_format == 'yaml':
            dumper = OrderedDumper if ordered else yaml.SafeDumper
            output_data = yaml.dump(parsed,
                                    None,
                                    dumper,
                                    allow_unicode=True,
                                    default_flow_style=False,
                                    encoding=None,
                                    **yaml_options)
        else:
            raise ValueError('Unknown output format: {0}'.
                             format(output_format))
        output_file.write(output_data.encode('utf-8'))
        output_file.close()
    finally:
        if 'input_file' in locals():
            input_file.close()
        if 'output_file' in locals():
            output_file.close()
Exemplo n.º 52
0
def generate_lesson_dot_rmotr_file(name, _type, uuid=None):
    return toml.dumps({
        'uuid': str(uuid or uuid_module.uuid4()),
        'name': name,
        'type': _type
    })
Exemplo n.º 53
0
def generate_unit_dot_rmotr_file(name, uuid=None):
    return toml.dumps({
        'uuid': str(uuid or uuid_module.uuid4()),
        'name': name
    })