def test_multiple_lines_separated_in_project(self):
global_content = textwrap.dedent("""\
foo<2;python_version=='2.7'
foo>1;python_version!='2.7'
""")
project_content = textwrap.dedent("""\
foo<1.8;python_version=='2.7'
# mumbo gumbo
foo>0.9;python_version!='2.7'
""")
global_reqs = requirement.parse(global_content)
project_reqs = list(requirement.to_reqs(project_content))
actions, reqs = update._sync_requirements_file(global_reqs,
project_reqs, 'f',
False, False, False)
self.assertEqual(
requirement.Requirements([
requirement.Requirement('foo', '', '<2',
"python_version=='2.7'", ''),
requirement.Requirement('foo', '', '>1',
"python_version!='2.7'", ''),
requirement.Requirement('', '', '', '', "# mumbo gumbo")
]), reqs)
self.assertEqual(
project.StdOut(" foo<1.8;python_version=='2.7' -> "
"foo<2;python_version=='2.7'\n"), actions[2])
self.assertEqual(
project.StdOut(" foo>0.9;python_version!='2.7' -> "
"foo>1;python_version!='2.7'\n"), actions[3])
self.assertThat(actions, matchers.HasLength(4))
def test_replace_non_canonical(self):
new_req = '-e file:///path#egg=foo_baz'
reqs = requirement.parse("foo-baz===1.0.2\n")
res = edit.edit(reqs, 'foo_baz', new_req)
self.assertEqual(
res,
requirement.Requirements(
[requirement.Requirement('', '', '', '', new_req)]))
def test_location(self):
reqs = requirement.to_content(requirement.Requirements(
[requirement.Requirement(
'foo', 'file://foo', '', "python_version=='2.7'", '# BSD')]))
self.assertEqual(
''.join(requirement._REQS_HEADER
+ ["file://foo#egg=foo;python_version=='2.7' # BSD\n"]),
reqs)
def test_unparseable_line(self):
global_content = textwrap.dedent("""\
foo
""")
project_content = textwrap.dedent("""\
foo
-e https://git.openstack.org/openstack/neutron.git#egg=neutron
""")
global_reqs = requirement.parse(global_content)
project_reqs = list(requirement.to_reqs(project_content))
actions, reqs = update._sync_requirements_file(
global_reqs, project_reqs, 'f', False, False, False)
n = '-e https://git.openstack.org/openstack/neutron.git#egg=neutron'
self.assertEqual(requirement.Requirements([
requirement.Requirement('foo', '', '', '', ''),
requirement.Requirement('', '', '', '', n)]),
reqs)
def test_smoke(self):
reqs = requirement.to_content(requirement.Requirements([
requirement.Requirement('foo', '', '<=1', "python_version=='2.7'",
'# BSD')
]),
marker_sep='!')
self.assertEqual(
''.join(requirement._REQS_HEADER +
["foo<=1!python_version=='2.7' # BSD\n"]), reqs)
def test_merge_extras(self):
old_content = textwrap.dedent(u"""
[metadata]
name = fred
[extras]
# Comment
a =
b
# comment
c =
d
[entry_points]
console_scripts =
foo = bar:quux
""")
blank = requirement.Requirement('', '', '', '', '')
r1 = requirement.Requirement(
'b', '', '>=1', "python_version=='2.7'", '')
r2 = requirement.Requirement('d', '', '', '', '# BSD')
reqs = {
'a': requirement.Requirements([blank, r1]),
'c': requirement.Requirements([blank, r2])}
merged = project.merge_setup_cfg(old_content, reqs)
expected = textwrap.dedent(u"""
[metadata]
name = fred
[extras]
# Comment
a =
b>=1:python_version=='2.7'
# comment
c =
d # BSD
[entry_points]
console_scripts =
foo = bar:quux
""")
self.assertEqual(expected, merged)
def edit(reqs, name, replacement):
if not replacement:
reqs.pop(name, None)
else:
reqs[name] = [(requirement.Requirement('', '', '', '',
replacement), '')]
result = []
for entries in reqs.values():
for entry, _ in entries:
result.append(entry)
return requirement.Requirements(sorted(result))
def test_multiple_lines_nochange(self):
global_content = textwrap.dedent("""\
foo<2;python_version=='2.7'
foo>1;python_version!='2.7'
""")
project_content = textwrap.dedent("""\
foo<2;python_version=='2.7'
foo>1;python_version!='2.7'
""")
global_reqs = requirement.parse(global_content)
project_reqs = list(requirement.to_reqs(project_content))
actions, reqs = update._sync_requirements_file(
global_reqs, project_reqs, 'f', False, False, False)
self.assertEqual(requirement.Requirements([
requirement.Requirement(
'foo', '', '<2', "python_version=='2.7'", ''),
requirement.Requirement(
'foo', '', '>1', "python_version!='2.7'", '')]),
reqs)
self.assertThat(actions, matchers.HasLength(0))
def edit(reqs, name, replacement):
key = requirement.canonical_name(name)
if not replacement:
reqs.pop(key, None)
else:
reqs[key] = [(requirement.Requirement('', '', '', '',
replacement), '')]
result = []
for entries in reqs.values():
for entry, _ in entries:
result.append(entry)
return requirement.Requirements(sorted(result))
def test_extras_kept(self):
global_content = textwrap.dedent("""\
oslo.db>1.4.1
""")
project_content = textwrap.dedent("""\
oslo.db[fixture,mysql]>1.3
""")
global_reqs = requirement.parse(global_content)
project_reqs = list(requirement.to_reqs(project_content))
actions, reqs = update._sync_requirements_file(
global_reqs, project_reqs, 'f', False, False, False)
self.assertEqual(requirement.Requirements([
requirement.Requirement(
'oslo.db', '', '>1.4.1', '', '', ['fixture', 'mysql'])]),
reqs)
self.assertThat(actions, matchers.HasLength(3))
self.assertEqual(project.StdOut(
" oslo.db[fixture,mysql]>1.3 -> "
"oslo.db[fixture,mysql]>1.4.1\n"), actions[2])
def test_single_global_multiple_in_project(self):
global_content = textwrap.dedent("""\
foo>1
""")
project_content = textwrap.dedent("""\
foo<2;python_version=='2.7'
foo>1;python_version!='2.7'
""")
global_reqs = requirement.parse(global_content)
project_reqs = list(requirement.to_reqs(project_content))
actions, reqs = update._sync_requirements_file(
global_reqs, project_reqs, 'f', False, False, False)
self.assertEqual(requirement.Requirements([
requirement.Requirement('foo', '', '>1', "", '')]),
reqs)
self.assertEqual(project.StdOut(
" foo<2;python_version=='2.7' -> foo>1\n"), actions[2])
self.assertEqual(project.StdOut(
" foo>1;python_version!='2.7' -> \n"), actions[3])
self.assertThat(actions, matchers.HasLength(4))
def test_replace_many(self):
reqs = requirement.parse('foo==1.2;p\nfoo==1.3;q')
res = edit.edit(reqs, 'foo', 'foo==1.3')
self.assertEqual(
requirement.Requirements(
[requirement.Requirement('', '', '', '', 'foo==1.3')]), res)
def test_add(self):
reqs = {}
res = edit.edit(reqs, 'foo', 'foo==1.2')
self.assertEqual(
requirement.Requirements(
[requirement.Requirement('', '', '', '', 'foo==1.2')]), res)
class TestParseRequirement(testtools.TestCase):
dist_scenarios = [
('package',
dict(line='swift',
req=requirement.Requirement('swift', '', '', '', ''))),
('specifier',
dict(line='alembic>=0.4.1',
req=requirement.Requirement('alembic', '', '>=0.4.1', '', ''))),
('specifiers',
dict(line='alembic>=0.4.1,!=1.1.8',
req=requirement.Requirement('alembic', '', '!=1.1.8,>=0.4.1', '',
''))),
('comment-only',
dict(line='# foo',
req=requirement.Requirement('', '', '', '', '# foo'))),
('comment',
dict(line='Pint>=0.5 # BSD',
req=requirement.Requirement('Pint', '', '>=0.5', '', '# BSD'))),
('comment-with-semicolon',
dict(line='Pint>=0.5 # BSD;fred',
req=requirement.Requirement('Pint', '', '>=0.5', '',
'# BSD;fred'))),
('case',
dict(line='Babel>=1.3',
req=requirement.Requirement('Babel', '', '>=1.3', '', ''))),
('markers',
dict(line="pywin32;sys_platform=='win32'",
req=requirement.Requirement('pywin32', '', '',
"sys_platform=='win32'", ''))),
('markers-with-comment',
dict(line="Sphinx<=1.2; python_version=='2.7'# Sadface",
req=requirement.Requirement('Sphinx', '', '<=1.2',
"python_version=='2.7'",
'# Sadface')))
]
url_scenarios = [
('url',
dict(line='file:///path/to/thing#egg=thing',
req=requirement.Requirement('thing', 'file:///path/to/thing', '',
'', ''),
permit_urls=True)),
('oslo-url',
dict(line='file:///path/to/oslo.thing#egg=oslo.thing',
req=requirement.Requirement('oslo.thing',
'file:///path/to/oslo.thing', '', '',
''),
permit_urls=True)),
('url-comment',
dict(line='file:///path/to/thing#egg=thing # http://altpath#egg=boo',
req=requirement.Requirement('thing', 'file:///path/to/thing', '',
'', '# http://altpath#egg=boo'),
permit_urls=True)),
('editable',
dict(line='-e file:///path/to/bar#egg=bar',
req=requirement.Requirement('bar', '-e file:///path/to/bar', '',
'', ''),
permit_urls=True))
]
scenarios = dist_scenarios + url_scenarios
def test_parse(self):
parsed = requirement.parse_line(self.line,
permit_urls=getattr(
self, 'permit_urls', False))
self.assertEqual(self.req, parsed)
def test_canonicalises(self):
req = requirement.Requirement('Foo_bar', '', '', '', '')
self.assertEqual({'foo-bar': [(req, '')]},
requirement.to_dict([(req, '')]))
def test_urls(self):
line = 'file:///foo#egg=foo'
reqs = list(requirement.to_reqs(line, permit_urls=True))
req = requirement.Requirement('foo', 'file:///foo', '', '', '')
self.assertEqual(reqs, [(req, line)])
def _sync_requirements_file(source_reqs, dest_sequence, dest_label, softupdate,
hacking, non_std_reqs):
actions = []
dest_reqs = requirement.to_dict(dest_sequence)
changes = []
output_requirements = []
processed_packages = set()
for req, req_line in dest_sequence:
# Skip the instructions header
if req_line in requirement._REQS_HEADER:
continue
elif req is None:
# Unparsable lines.
output_requirements.append(
requirement.Requirement('', '', '', '', req_line.rstrip()))
continue
elif not req.package:
# Comment-only lines
output_requirements.append(req)
continue
elif req.package.lower() in processed_packages:
continue
processed_packages.add(req.package.lower())
# Special cases:
# projects need to align hacking version on their own time
if req.package == "hacking" and not hacking:
output_requirements.append(req)
continue
reference = source_reqs.get(req.package.lower())
if reference:
actual = dest_reqs.get(req.package.lower())
for req, ref in six.moves.zip_longest(actual, reference):
if not req:
# More in globals
changes.append(Change(ref[0].package, '', ref[1]))
elif not ref:
# less in globals
changes.append(Change(req[0].package, req[1], ''))
elif req[0] != ref[0]:
# NOTE(jamielennox): extras are allowed to be specified in
# a project's requirements and the version be updated and
# extras maintained. Create a new ref object the same as
# the original but with the req's extras.
merged_ref = requirement.Requirement(
ref[0].package, ref[0].location, ref[0].specifiers,
ref[0].markers, ref[0].comment, req[0].extras)
ref = (merged_ref, merged_ref.to_line())
if req[0] != ref[0]:
# A change on this entry
changes.append(Change(req[0].package, req[1], ref[1]))
if ref:
output_requirements.append(ref[0])
elif softupdate:
# under softupdate we pass through anything unknown packages,
# this is intended for ecosystem projects that want to stay in
# sync with existing requirements, but also add their own above
# and beyond.
output_requirements.append(req)
else:
# What do we do if we find something unexpected?
#
# In the default cause we should die horribly, because
# the point of global requirements was a single lever
# to control all the pip installs in the gate.
#
# However, we do have other projects using
# devstack jobs that might have legitimate reasons to
# override. For those we support NON_STANDARD_REQS=1
# environment variable to turn this into a warning only.
# However this drops the unknown requirement.
actions.append(
project.Error("'%s' is not in global-requirements.txt" %
req.package))
# always print out what we did if we did a thing
if changes:
actions.append(
project.StdOut("Version change for: %s\n" %
", ".join([x.name for x in changes])))
actions.append(project.StdOut("Updated %s:\n" % dest_label))
for change in changes:
actions.append(project.StdOut(" %s\n" % change))
return actions, requirement.Requirements(output_requirements)
