def test_saneyaml_load_report_error_for_invalid_field_name(self):
test = get_test_content('test_model/parse/invalid_names.about')
try:
saneyaml.load(test)
self.fail('Exception not raised')
except Exception:
pass
def test_load_does_not_raise_exception_on_dupe_by_default(self):
test = '''
a: 12
b: 23
a: 45
'''
saneyaml.load(test)
def test_load_yaml_about_file_with_no_dupe(self):
test = '''
name: test
license_expression: mit
notes: dup key here
'''
saneyaml.load(test, allow_duplicate_keys=False)
def test_load_optionally_raise_exception_on_dupe(self):
test = '''
a: 12
b: 23
a: 45
'''
try:
saneyaml.load(test, allow_duplicate_keys=False)
self.fail('Exception not raised')
except saneyaml.UnsupportedYamlFeatureError as e:
assert 'Duplicate key in YAML source: a' == str(e)
def test_saneyaml_loads_blank_lines_and_lines_without_no_colon(self):
test = '''
name: no colon test
test
version: 0.7.0
about_resource: about.py
test with no colon
'''
try:
saneyaml.load(test)
self.fail('Exception not raised')
except Exception:
pass
def test_load_optionally_raise_exception_on_dupe_in_nested_mappings(self):
test = '''
2:
3: 4
4: 5
5:
6: 9
8: 9
6: 8
'''
try:
saneyaml.load(test, allow_duplicate_keys=False)
self.fail('Exception not raised')
except saneyaml.UnsupportedYamlFeatureError:
pass
def test_load_yaml_about_file_raise_exception_on__duplicate(self):
test = '''
name: test
notes: some notes
notes: dup key here
notes: dup key here
license_expression: mit
notes: dup key here
'''
try:
saneyaml.load(test, allow_duplicate_keys=False)
self.fail('Exception not raised')
except saneyaml.UnsupportedYamlFeatureError as e :
assert 'Duplicate key in YAML source: notes' == str(e)
def parse(cls, location):
metadata = extract_gem_metadata(location)
metadata = saneyaml.load(metadata)
yield build_rubygem_package_data(
gem_data=metadata,
datasource_id=cls.datasource_id,
)
def recognize(cls, location):
# an unextracted .gen archive
if location.endswith('.gem'):
return get_gem_package(location)
# an extractcode-extracted .gen archive
if location.endswith('metadata.gz-extract'):
with open(location, 'rb') as met:
metadata = met.read()
metadata = saneyaml.load(metadata)
return build_rubygem_package(metadata)
if location.endswith('.gemspec'):
# TODO implement me
return
if location.endswith('Gemfile'):
# TODO implement me
return
if location.endswith('Gemfile.lock'):
# TODO implement me
return
return
def get_meta_yaml_data(location):
"""
Return a mapping of conda metadata loaded from a meta.yaml files. The format
support Jinja-based templating and we try a crude resolution of variables
before loading the data as YAML.
"""
# FIXME: use Jinja to process these
variables = get_variables(location)
yaml_lines = []
with io.open(location, encoding='utf-8') as metayaml:
for line in metayaml:
if not line:
continue
pure_line = line.strip()
if (pure_line.startswith('{%') and pure_line.endswith('%}')
and '=' in pure_line):
continue
# Replace the variable with the value
if '{{' in line and '}}' in line:
for variable, value in variables.items():
line = line.replace('{{ ' + variable + ' }}', value)
yaml_lines.append(line)
return saneyaml.load('\n'.join(yaml_lines))
def parse(cls, location):
with open(location) as inp:
pubspec_data = saneyaml.load(inp.read())
package_data = build_package(pubspec_data)
if package_data:
yield package_data
def test_load_yaml_about_file_raise_exception_on_invalid_yaml_ignore_non_key_line(self):
test = '''
name: test
- notes: some notes
- notes: dup key here
# some
notes: dup key here
license_expression: mit
notes dup key here
'''
try:
saneyaml.load(test, allow_duplicate_keys=False)
self.fail('Exception not raised')
except Exception:
pass
def parse_pkg_info(location):
"""
Return a PythonPackage from a a 'PKG-INFO' file at 'location' or None.
"""
if not location:
return
if not location.endswith('PKG-INFO'):
return
with io.open(location, encoding='utf-8') as loc:
infos = saneyaml.load(loc.read())
logger.error(logger)
if not infos.get('Name'):
return
parties = []
author = infos.get('Author')
if author:
parties.append(
models.Party(type=models.party_person, name=author, role=''))
package = PythonPackage(
name=infos.get('Name'),
version=infos.get('Version'),
description=infos.get('Summary') or infos.get('Description'),
homepage_url=infos.get('Home-page') or None,
# FIXME: this is NOT correct as classifiers can be used for this too
declared_license=infos.get('License') or None,
# FIXME: what about email?
# FIXME: what about maintainers?
parties=parties,
)
return package
def test_About_hydrate_normalize_field_names_to_lowercase(self):
test_content = get_test_content(
'test_gen/parser_tests/upper_field_names.ABOUT')
fields = saneyaml.load(test_content).items()
a = model.About()
for _ in range(3):
self.check_About_hydrate(a, fields)
def test_saneyaml_dangling_text_is_not_an_invalid_continuation(self):
test = get_test_content('test_model/parse/invalid_continuation.about')
result = saneyaml.load(test)
expected = [(u'single_line', u'optional'), (u'other_field', u'value'),
(u'multi_line',
u'some value and more\ninvalid continuation2')]
assert expected == list(result.items())
def test_scan_output_for_timestamp():
test_dir = test_env.get_test_loc('yaml/simple')
result_file = test_env.get_temp_file('yaml')
run_scan_click(['-clip', test_dir, '--yaml', result_file])
result_yaml = saneyaml.load(open(result_file).read())
header = result_yaml['headers'][0]
assert 'start_timestamp' in header
assert 'end_timestamp' in header
def parse(cls, location):
with open(location, 'rb') as met:
metadata = met.read()
metadata = saneyaml.load(metadata)
yield build_rubygem_package_data(
gem_data=metadata,
datasource_id=cls.datasource_id,
)
def test_saneyaml_load_does_not_convert_to_crlf(self):
test = get_test_content('test_model/crlf/about.ABOUT')
result = saneyaml.load(test)
expected = [(u'about_resource', u'.'), (u'name', u'pytest'),
(u'description', u'first line\nsecond line\nthird line\n'),
(u'copyright', u'copyright')]
assert expected == list(result.items())
def test_saneyaml_load_can_parse_continuations(self):
test = get_test_content('test_model/parse/continuation.about')
result = saneyaml.load(test)
expected = [('single_line', 'optional'), ('other_field', 'value'),
(u'multi_line', u'some value and more and yet more')]
assert expected == list(result.items())
def test_saneyaml_load_accepts_unicode_keys_and_values(self):
test = get_test_content(
'test_model/parse/non_ascii_field_name_value.about')
result = saneyaml.load(test)
expected = [('name', 'name'), ('about_resource', '.'),
('owner', 'Matías Aguirre'),
(u'Matías', u'unicode field name')]
assert expected == list(result.items())
def read_podfile_lock(location):
"""
Reads from podfile.lock file at location as YML.
"""
with open(location, 'r') as file:
data = saneyaml.load(file)
return data
def from_file(cls, data_file):
with open(data_file) as df:
data = saneyaml.load(df.read())
data['data_file'] = data_file
alptest = cls(**data)
alptest.license_expression = cls.licensing.parse(
alptest.license_expression).render()
return alptest
def recognize(cls, location):
"""
Yield one or more Package manifest objects given a file ``location`` pointing to a
package archive, manifest or similar.
"""
with open(location) as inp:
locks_data = saneyaml.load(inp.read())
yield cls(dependencies=list(collect_locks(locks_data)))
def recognize(cls, location):
"""
Yield one or more Package manifest objects given a file ``location`` pointing to a
package archive, manifest or similar.
"""
with open(location, 'rb') as met:
metadata = met.read()
metadata = saneyaml.load(metadata)
yield build_rubygem_package(cls, metadata)
def test_load_yaml_about_file_with_multiline(self):
test = '''
name: test
owner: test
notes: |
String block here
license_expression: mit
owner: test1
notes: continuation
line
description: sample
'''
try:
saneyaml.load(test, allow_duplicate_keys=False)
self.fail('Exception not raised')
except saneyaml.UnsupportedYamlFeatureError as e :
# notes: exceptio is rasied only for the first dupe
assert 'Duplicate key in YAML source: owner' == str(e)
def parse(cls, location):
"""
Yield one or more Package manifest objects given a file ``location`` pointing to a
package archive, manifest or similar.
"""
with io.open(location, encoding='utf-8') as loc:
freebsd_manifest = saneyaml.load(loc)
package_data = models.PackageData(
datasource_id=cls.datasource_id,
type=cls.default_package_type,
qualifiers=dict(
arch=freebsd_manifest.get('arch'),
origin=freebsd_manifest.get('origin'),
))
# mapping of top level manifest items to the PackageData object field name
plain_fields = [
('name', 'name'),
('version', 'version'),
('www', 'homepage_url'),
('desc', 'description'),
('categories', 'keywords'),
]
for source, target in plain_fields:
value = freebsd_manifest.get(source)
if value:
if isinstance(value, str):
value = value.strip()
if value:
setattr(package_data, target, value)
# mapping of top level +COMPACT_MANIFEST items to a function accepting as
# arguments the package.json element value and returning an iterable of key,
# values Package Object to update
field_mappers = [
('maintainer', maintainer_mapper),
('origin', origin_mapper),
('arch', arch_mapper),
]
for source, func in field_mappers:
logger.debug('parse: %(source)r, %(func)r' % locals())
value = freebsd_manifest.get(source) or None
if value:
func(value, package_data)
# license_mapper needs multiple fields
license_mapper(freebsd_manifest, package_data)
if package_data.declared_license:
package_data.license_expression = cls.compute_normalized_license(
package_data)
yield package_data
def get_package_root(cls, manifest_resource, codebase):
with io.open(manifest_resource.location, encoding='utf-8') as loc:
package_data = saneyaml.load(loc.read())
about_resource = package_data.get('about_resource')
if about_resource:
manifest_resource_parent = manifest_resource.parent(codebase)
for child in manifest_resource_parent.children(codebase):
if child.name == about_resource:
return child
return manifest_resource
def get_gem_package(location, download_url=None, purl=None):
"""
Return a RubyGem Package built from the .gem file at `location` or None.
"""
if not location.endswith('.gem'):
return
metadata = get_gem_metadata(location)
metadata = saneyaml.load(metadata)
return build_rubygem_package(metadata, download_url, purl)
def test_saneyaml_load_can_parse_verbatim_text_unstripped(self):
test = get_test_content('test_model/parse/continuation_verbatim.about')
result = saneyaml.load(test)
expected = [(u'single_line', u'optional'), (u'other_field', u'value'),
(u'multi_line',
u'some value \n and more \n and yet more \n \n')
]
assert expected == list(result.items())
def parse(cls, location):
with open(location) as inp:
locks_data = saneyaml.load(inp.read())
dependencies = list(collect_locks(locks_data))
yield models.PackageData(datasource_id=cls.datasource_id,
type=cls.default_package_type,
primary_language=cls.default_primary_language,
dependencies=dependencies)