Ejemplo n.º 1
0
    def to_json(self):
        """Get JSON for a kapsel.yml env spec section."""
        # Note that we use _conda_packages (only the packages we
        # introduce ourselves) rather than conda_packages
        # (includes inherited packages).
        packages = list(self._conda_packages)
        pip_packages = list(self._pip_packages)
        if pip_packages:
            packages.append(dict(pip=pip_packages))
        channels = list(self._channels)

        # this is a gross, roundabout hack to get ryaml dicts that
        # have ordering... OrderedDict doesn't work because the
        # yaml saver saves them as some "!!omap" nonsense. Other
        # than ordering, the formatting isn't even preserved here.
        template_json = ryaml.load("something:\n    packages: []\n" + "    channels: []\n",
                                   Loader=ryaml.RoundTripLoader)

        template_json['something']['packages'] = packages
        template_json['something']['channels'] = channels

        if len(self.inherit_from_names) > 0:
            if len(self.inherit_from_names) == 1:
                names = self.inherit_from_names[0]
            else:
                names = list(self.inherit_from_names)
            template_json['something']['inherit_from'] = names

        return template_json['something']
Ejemplo n.º 2
0
def test_pypi_section_order_preserved(testing_workdir):
    """
    Test whether sections have been written in the correct order.
    """
    from conda_build.render import FIELDS
    from conda_build.skeletons.pypi import (ABOUT_ORDER,
                                            REQUIREMENTS_ORDER,
                                            PYPI_META_STATIC)

    api.skeletonize(packages='sympy', repo='pypi')
    # Since we want to check the order of items in the recipe (not whether
    # the metadata values themselves are sensible), read the file as (ordered)
    # yaml, and check the order.
    with open('sympy/meta.yaml', 'r') as file:
        lines = [l for l in file.readlines() if not l.startswith("{%")]

    # The loader below preserves the order of entries...
    recipe = ruamel_yaml.load('\n'.join(lines), Loader=ruamel_yaml.RoundTripLoader)

    major_sections = list(recipe.keys())
    # Blank fields are omitted when skeletonizing, so prune any missing ones
    # before comparing.
    pruned_fields = [f for f in FIELDS if f in major_sections]
    assert major_sections == pruned_fields
    assert list(recipe['about']) == ABOUT_ORDER
    assert list(recipe['requirements']) == REQUIREMENTS_ORDER
    for k, v in PYPI_META_STATIC.items():
        assert list(v.keys()) == list(recipe[k])
Ejemplo n.º 3
0
def prepare_input(argv=None):
    """
    Get, parse and prepare input file.
    """
    p = ArgumentParser(description='insiliChem.bio OpenMM launcher: '
                       'easy to deploy MD protocols for OpenMM')
    p.add_argument('input', metavar='INPUT FILE', type=str,
                   help='YAML input file')
    p.add_argument('--version', action='version', version='%(prog)s v{}'.format(__version__))
    args = p.parse_args(argv if argv else sys.argv[1:])

    # Load config file
    with open(args.input) as f:
        cfg = yaml.load(f, Loader=YamlLoader)
    # Paths and dirs
    cfg['_path'] = os.path.abspath(args.input)
    cfg['system_options'] = prepare_system_options(cfg)
    cfg['outputpath'] = sanitize_path_for_file(cfg.get('outputpath', '.'), args.input)
    try: 
        os.makedirs(cfg['outputpath'])
    except OSError:
        pass

    handler = prepare_handler(cfg)

    return handler, cfg
Ejemplo n.º 4
0
def _remote_or_local(fn, branch='master', remote=False):
    """
    Downloads a temp file directly from the specified github branch or
    the current one on disk.
    """
    if remote:
        url = (
            'https://raw.githubusercontent.com/bioconda/bioconda-recipes/'
            '{branch}/{path}'.format(branch=branch, path=fn)
        )
        print('Using config file {}'.format(url))
        with conda.fetch.TmpDownload(url) as f:
            cfg = yaml.load(open(f))
    else:
        cfg = yaml.load(open(os.path.join(HERE, fn)))
    return cfg
Ejemplo n.º 5
0
    def test_create_advanced_pip(self):
        with make_temp_envs_dir() as envs_dir:
            with env_vars({
                'CONDA_ENVS_DIRS': envs_dir,
                'CONDA_PIP_INTEROP_ENABLED': 'true',
            }, reset_context):
                env_name = str(uuid4())[:8]
                prefix = join(envs_dir, env_name)
                python_path = join(prefix, PYTHON_BINARY)

                run_command(Commands.CREATE, env_name,
                            support_file('pip_argh.yml'))
                out_file = join(envs_dir, 'test_env.yaml')

            # make sure that the export reconsiders the presence of pip interop being enabled
            PrefixData._cache_.clear()

            with env_vars({
                'CONDA_ENVS_DIRS': envs_dir,
            }, reset_context):
                # note: out of scope of pip interop var.  Should be enabling conda pip interop itself.
                run_command(Commands.EXPORT, env_name, out_file)
                with open(out_file) as f:
                    d = ruamel_yaml.load(f)
                assert {'pip': ['argh==0.26.2']} in d['dependencies']
Ejemplo n.º 6
0
def yaml_load_standard(string):
    """Uses the default (unsafe) loader.

    Examples:
        >>> yaml_load_standard("prefix: !!python/unicode '/Users/darwin/test'")
        {'prefix': '/Users/darwin/test'}
    """
    return yaml.load(string, Loader=yaml.Loader, version="1.2")
Ejemplo n.º 7
0
def yaml_load_safe(string):
    """
    Examples:
        >>> yaml_load_safe("key: value")
        {'key': 'value'}

    """
    return yaml.load(string, Loader=yaml.SafeLoader, version="1.2")
Ejemplo n.º 8
0
def _save_file(yaml, filename):
    contents = ryaml.dump(yaml, Dumper=ryaml.RoundTripDumper)

    try:
        # This is to ensure we don't corrupt the file, even if ruamel.yaml is broken
        ryaml.load(contents, Loader=ryaml.RoundTripLoader)
    except YAMLError as e:  # pragma: no cover (should not happen)
        print("ruamel.yaml bug; it failed to parse a file that it generated.", file=sys.stderr)
        print("  the parse error was: " + str(e), file=sys.stderr)
        print("Generated file was:", file=sys.stderr)
        print(contents, file=sys.stderr)
        raise RuntimeError("Bug in ruamel.yaml library; failed to parse a file that it generated: " + str(e))

    if not os.path.isfile(filename):
        # might have to make the directory
        dirname = os.path.dirname(filename)
        makedirs_ok_if_exists(dirname)
    _atomic_replace(filename, contents)
Ejemplo n.º 9
0
    def construct_include(self, node):
        """Include file referenced at node."""

        filename = os.path.join(self._root, self.construct_scalar(node))
        filename = os.path.abspath(filename)
        extension = os.path.splitext(filename)[1].lstrip('.')

        with open(filename, 'r') as f:
            if extension in ('yaml', 'yml'):
                return yaml.load(f, Loader=self)
            else:
                return ''.join(f.readlines())
Ejemplo n.º 10
0
    def save_environment_yml(self, filename):
        """Save as an environment.yml file."""
        # here we want to flatten the env spec to include all inherited stuff
        packages = list(self.conda_packages)
        pip_packages = list(self.pip_packages)
        if pip_packages:
            packages.append(dict(pip=pip_packages))
        channels = list(self.channels)

        yaml = ryaml.load("name: " "\ndependencies: []\nchannels: []\n", Loader=ryaml.RoundTripLoader)

        assert self.name is not None  # the global anonymous spec can't be saved
        yaml['name'] = self.name
        yaml['dependencies'] = packages
        yaml['channels'] = channels

        _save_file(yaml, filename)
Ejemplo n.º 11
0
def yaml_load(string):
    return yaml.load(string, Loader=yaml.RoundTripLoader, version="1.2")
Ejemplo n.º 12
0
"""
import boto3
from awacs.aws import Allow, Statement, Policy, Action
from troposphere import Ref, Template, GetAtt, Output, iam
from utils import create_or_update_stack, stack_info
import ruamel_yaml as yaml
from troposphere.firehose import (
    KinesisStreamSourceConfiguration,
    DeliveryStream,
    BufferingHints,
    S3DestinationConfiguration
)

with open('../config/firehose_config.yml') as f:
    cfg = yaml.load(f)

STACK_NAME = cfg['firehose']['stack_name']
ACCOUNT_ID = cfg['firehose']['account_id']
BUCKET_NAME = cfg['firehose']['bucket_name']
REGION = cfg['firehose']['region']
STREAM_NAME = stack_info(stack_name='ScraperStreamStack')['StreamName']
STREAM_ARN = stack_info(stack_name='ScraperStreamStack')['StreamARN']

t = Template()
description = 'Stack for kinesis firehose stream to deliver to s3 from kinesis'
t.add_description(description)
t.add_version('2010-09-09')

firehose_policy_doc = Policy(
    Statement=[
Ejemplo n.º 13
0
args = parser.parse_args()

if pygments:
    if args.color:
        formatter = Terminal256Formatter
    else:
        formatter = NullFormatter

try: # ... to read json
    i = args.infile.read()
    d = json.loads( i )
    if args.alwaysjson:
        if pygments:
            i = highlight( out, JsonLexer(), formatter() )
        print( i )
    else:
        out = yaml.safe_dump(d, indent=args.indent, allow_unicode=True )
        if pygments:
            out = highlight( out, YamlLexer(), formatter() )
        print( out )
except:
    try: # ... to read yaml
        d = yaml.load( i )
        out = json.dumps(d, indent=args.indent)
        if pygments:
            out = highlight(out, JsonLexer(), formatter() )
        print(out)
    except:
        print("input error: invalid json or yaml format")

Ejemplo n.º 14
0
    def _default_content(self):
        header = (
            "This is an Anaconda project file.\n" + "\n" + "Here you can describe your project and how to run it.\n" +
            "Use `conda-kapsel run` to run the project.\n" +
            "The file is in YAML format, please see http://www.yaml.org/start.html for more.\n")
        sections = OrderedDict()

        sections['name'] = ("Set the 'name' key to name your project\n")

        sections['icon'] = ("Set the 'icon' key to give your project an icon\n")

        sections['commands'] = ("In the commands section, list your runnable scripts, notebooks, and other code.\n" +
                                "Use `conda-kapsel add-command` to add commands.\n")

        sections['variables'] = ("In the variables section, list any environment variables your code depends on.\n"
                                 "Use `conda-kapsel add-variable` to add variables.\n")

        sections['services'] = (
            "In the services section, list any services that should be\n" + "available before your code runs.\n" +
            "Use `conda-kapsel add-service` to add services.\n")

        sections['downloads'] = ("In the downloads section, list any URLs to download to local files\n" +
                                 "before your code runs.\n" + "Use `conda-kapsel add-download` to add downloads.\n")

        sections['packages'] = ("In the packages section, list any packages that must be installed\n" +
                                "before your code runs.\n" + "Use `conda-kapsel add-packages` to add packages.\n")

        sections['channels'] = (
            "In the channels section, list any Conda channel URLs to be searched\n" + "for packages.\n" + "\n" +
            "For example,\n" + "\n" + "channels:\n" + "   - https://conda.anaconda.org/asmeurer\n")

        sections['env_specs'] = (
            "You can define multiple, named environment specs.\n" + "Each inherits any global packages or channels,\n" +
            "but can have its own unique ones also.\n" + "Use `conda-kapsel add-env-spec` to add environment specs.\n")

        assert self._default_env_specs_func is not None
        default_env_specs = self._default_env_specs_func()
        assert default_env_specs is not None

        # we make a big string and then parse it because I can't figure out the
        # ruamel.yaml API to insert comments in front of map keys.
        def comment_out(comment):
            return ("# " + "\n# ".join(comment.split("\n")) + "\n").replace("# \n", "#\n")

        to_parse = comment_out(header)
        for section_name, comment in sections.items():
            # future: this is if/else is silly, we should be
            # assigning these bodies up above when we assign the
            # comments.
            if section_name in ('name', 'icon'):
                section_body = ""
            elif section_name in ('channels', 'packages'):
                section_body = "  []"
            else:
                section_body = "  {}"
            to_parse = to_parse + "\n#\n" + comment_out(comment) + section_name + ":\n" + section_body + "\n\n\n"

        as_json = ryaml.load(to_parse, Loader=ryaml.RoundTripLoader)

        for env_spec in default_env_specs:
            as_json['env_specs'][env_spec.name] = env_spec.to_json()

        return as_json
Ejemplo n.º 15
0
def yaml_load(filehandle):
    yaml = get_yaml()
    return yaml.load(filehandle, Loader=yaml.RoundTripLoader, version="1.1")
Ejemplo n.º 16
0
def yaml_load(filehandle):
    yaml = get_yaml()
    try:
        return yaml.load(filehandle, Loader=yaml.RoundTripLoader, version="1.2")
    except AttributeError:
        return yaml.load(filehandle)
Ejemplo n.º 17
0
def _load_string(contents):
    # using RoundTripLoader incorporates safe_load
    # (we don't load code)
    assert issubclass(ryaml.RoundTripLoader, ryaml.constructor.SafeConstructor)
    return ryaml.load(contents, Loader=ryaml.RoundTripLoader)