def _do_main(root, source, suffix, softupdate, hacking, stdout, verbose,
non_std_reqs):
"""No options or environment variable access from here on in."""
proj = project.read(root)
global_req_content = open(os.path.join(source, 'global-requirements.txt'),
'rt').read()
global_reqs = requirement.parse(global_req_content)
actions = _process_project(proj, global_reqs, suffix, softupdate, hacking,
non_std_reqs)
project.write(proj, actions, stdout=stdout, verbose=verbose)
def test_pbr(self):
root = self.useFixture(common.pbr_fixture).root
proj = project.read(root)
self.expectThat(proj['root'], matchers.Equals(root))
setup_py = open(root + '/setup.py', 'rt').read()
self.expectThat(proj['setup.py'], matchers.Equals(setup_py))
setup_cfg = open(root + '/setup.cfg', 'rt').read()
self.expectThat(proj['setup.cfg'], matchers.Equals(setup_cfg))
self.expectThat(
proj['requirements'],
matchers.KeysEqual('requirements.txt', 'test-requirements.txt'))
def _do_main(
root, source, suffix, softupdate, hacking, stdout, verbose,
non_std_reqs):
"""No options or environment variable access from here on in."""
proj = project.read(root)
global_req_content = open(
os.path.join(source, 'global-requirements.txt'), 'rt').read()
global_reqs = requirement.parse(global_req_content)
actions = _process_project(
proj, global_reqs, suffix, softupdate, hacking, non_std_reqs)
project.write(proj, actions, stdout=stdout, verbose=verbose)
def test_project_req_bigger_then_uc(self):
self.useFixture(common.project_fixture)
# lets change the six requirement not include the u-c version
proj_read = project.read(common.project_fixture.root)
proj_read['requirements']['requirements.txt'] = \
proj_read['requirements']['requirements.txt'][:-1] + '>1.10.0\n'
expected_out = ('six must be <= 1.10.0 from upper-constraints and '
'include the upper-constraints version')
# Start capturing some output
mock_stdout = io.StringIO()
with mock.patch('openstack_requirements.project.read',
return_value=proj_read), \
mock.patch('sys.stdout', mock_stdout):
ret = check_exists.main([common.project_fixture.root])
self.assertEqual(ret, 1)
self.assertIn(expected_out, mock_stdout.getvalue())
def test_project_req_bigger_then_uc(self):
self.useFixture(common.project_fixture)
# lets change the six requirement not include the u-c version
proj_read = project.read(common.project_fixture.root)
proj_read['requirements']['requirements.txt'] = \
proj_read['requirements']['requirements.txt'][:-1] + '>1.10.0\n'
expected_out = ('six must be <= 1.10.0 from upper-constraints and '
'include the upper-constraints version')
# Start capturing some output
mock_stdout = io.StringIO()
with mock.patch('openstack_requirements.project.read',
return_value=proj_read), \
mock.patch('sys.stdout', mock_stdout):
ret = check_exists.main([common.project_fixture.root])
self.assertEqual(ret, 1)
self.assertIn(expected_out, mock_stdout.getvalue())
def test_project_multiple_missing_from_uc_and_gr(self):
self.useFixture(common.project_fixture)
orig_mocked_read_req = check_exists.read_requirements_file
read_req_path = ('openstack_requirements.cmds.check_exists.'
'read_requirements_file')
def remove_req_read_reqs_file(filename):
if filename == 'upper-constraints.txt':
upper_cons = common.upper_constraints.copy()
upper_cons.pop('lxml')
return upper_cons
return orig_mocked_read_req(filename)
new_reqs = '>1.10.0\nsomerandommodule\n'
# lets change the six requirement not include the u-c version
proj_read = project.read(common.project_fixture.root)
proj_read['requirements']['requirements.txt'] = \
proj_read['requirements']['requirements.txt'][:-1] + new_reqs
proj_read['requirements']['test-requirements.txt'] = \
proj_read['requirements']['test-requirements.txt'] + \
'anotherrandommodule\n'
expected_outs = [
'lxml from requirements.txt not found in upper-constraints',
'somerandommodule from requirements.txt not found in '
'global-requirements',
'anotherrandommodule from test-requirements.txt not found in '
'global-requirements',
'six must be <= 1.10.0 from upper-constraints and include the '
'upper-constraints version'
]
# Start capturing some output
mock_stdout = io.StringIO()
with mock.patch('openstack_requirements.project.read',
return_value=proj_read), \
mock.patch('sys.stdout', mock_stdout), \
mock.patch(read_req_path, remove_req_read_reqs_file):
ret = check_exists.main([common.project_fixture.root])
self.assertEqual(ret, 1)
for expected in expected_outs:
self.assertIn(expected, mock_stdout.getvalue())
def test_project_multiple_missing_from_uc_and_gr(self):
self.useFixture(common.project_fixture)
orig_mocked_read_req = check_exists.read_requirements_file
read_req_path = ('openstack_requirements.cmds.check_exists.'
'read_requirements_file')
def remove_req_read_reqs_file(filename):
if filename == 'upper-constraints.txt':
upper_cons = common.upper_constraints.copy()
upper_cons.pop('lxml')
return upper_cons
return orig_mocked_read_req(filename)
new_reqs = '>1.10.0\nsomerandommodule\n'
# lets change the six requirement not include the u-c version
proj_read = project.read(common.project_fixture.root)
proj_read['requirements']['requirements.txt'] = \
proj_read['requirements']['requirements.txt'][:-1] + new_reqs
proj_read['requirements']['test-requirements.txt'] = \
proj_read['requirements']['test-requirements.txt'] + \
'anotherrandommodule\n'
expected_outs = [
'lxml from requirements.txt not found in upper-constraints',
'somerandommodule from requirements.txt not found in '
'global-requirements',
'anotherrandommodule from test-requirements.txt not found in '
'global-requirements',
'six must be <= 1.10.0 from upper-constraints and include the '
'upper-constraints version']
# Start capturing some output
mock_stdout = io.StringIO()
with mock.patch('openstack_requirements.project.read',
return_value=proj_read), \
mock.patch('sys.stdout', mock_stdout), \
mock.patch(read_req_path, remove_req_read_reqs_file):
ret = check_exists.main([common.project_fixture.root])
self.assertEqual(ret, 1)
for expected in expected_outs:
self.assertIn(expected, mock_stdout.getvalue())
def test_project_missing_from_gr(self):
self.useFixture(common.project_fixture)
# Add some random package that wont exist in G-R
with open(common.project_fixture.req_file, 'a') as req_file:
req_file.write(u'SomeRandomModule #Some random module\n')
req_file.flush()
expected_out = ('somerandommodule from requirements.txt not found in'
' global-requirements')
# Start capturing some output
mock_stdout = io.StringIO()
proj_read = project.read(common.project_fixture.root)
with mock.patch('openstack_requirements.project.read',
return_value=proj_read), \
mock.patch('sys.stdout', mock_stdout):
ret = check_exists.main([common.project_fixture.root])
self.assertEqual(ret, 1)
self.assertIn(expected_out, mock_stdout.getvalue())
def test_project_missing_from_gr(self):
self.useFixture(common.project_fixture)
# Add some random package that wont exist in G-R
with open(common.project_fixture.req_file, 'a') as req_file:
req_file.write(u'SomeRandomModule #Some random module\n')
req_file.flush()
expected_out = ('somerandommodule from requirements.txt not found in'
' global-requirements')
# Start capturing some output
mock_stdout = io.StringIO()
proj_read = project.read(common.project_fixture.root)
with mock.patch('openstack_requirements.project.read',
return_value=proj_read), \
mock.patch('sys.stdout', mock_stdout):
ret = check_exists.main([common.project_fixture.root])
self.assertEqual(ret, 1)
self.assertIn(expected_out, mock_stdout.getvalue())
def make_project(fixture):
with fixture:
return project.read(fixture.root)
def main(args=None):
parser = argparse.ArgumentParser()
parser.add_argument('project',
default='',
help='path to the project source root folder.')
parser.add_argument('-u',
'--upper-constraints',
default='upper-constraints.txt',
help='path to the upper-constraints.txt file')
parser.add_argument('-g',
'--global-requirements',
default='global-requirements.txt',
help='Path to the global-requirements.txt file')
parser.add_argument('-b',
'--blacklist',
default='blacklist.txt',
help='Path to the blacklist.txt file')
parser.add_argument('-G',
'--gr-check',
action='store_true',
help='Do a specifier check of global-requirements')
args = parser.parse_args(args)
upper_constraints = read_requirements_file(args.upper_constraints)
global_requirements = read_requirements_file(args.global_requirements)
blacklist = read_requirements_file(args.blacklist)
project_data = project.read(args.project)
error_count = 0
for require_file, data in project_data.get('requirements', {}).items():
print(
u'\nComparing %s with global-requirements and upper-constraints' %
require_file)
requirements = requirement.parse(data)
for name, spec_list in requirements.items():
if not name or name in blacklist:
continue
if name not in global_requirements:
print(u'%s from %s not found in global-requirements' %
(name, require_file))
error_count += 1
continue
if name not in upper_constraints:
print(u'%s from %s not found in upper-constraints' %
(name, require_file))
error_count += 1
continue
elif spec_list:
uc = upper_constraints[name][0][0]
gr = global_requirements[name][0][0]
spec_gr = SpecifierSet(gr.specifiers)
for req, _ in spec_list:
specs = SpecifierSet(req.specifiers)
# This assumes uc will only have == specifiers
for uc_spec in SpecifierSet(uc.specifiers):
# if the uc version isn't in the lower specifier
# then something is wrong.
if Version(uc_spec.version) not in specs:
print(
u'%s must be <= %s from upper-constraints and '
'include the upper-constraints version' %
(name, uc_spec.version))
error_count += 1
continue
if args.gr_check:
for spec in specs:
# g-r will mostly define blocked versions. And a
# local project may define there own, so there is
# no point checking a != specifier
if spec.operator == '!=':
continue
if spec.version not in spec_gr:
print(
u'Specifier %s from %s is failing check '
'from global-requirements specifiers %s' %
(spec.version, name, str(spec_gr)))
error_count += 1
continue
return 1 if error_count else 0
def make_project(fixture):
with fixture:
return project.read(fixture.root)
def main():
args = grab_args()
branch = args.branch
os.chdir(args.src_dir)
reqdir = args.reqs
print(sys.version_info)
if reqdir is None:
if args.local:
print('selecting default requirements directory for local mode')
reqdir = os.path.dirname(
os.path.dirname(os.path.dirname(sys.argv[0])))
else:
print('selecting default requirements directory for normal mode')
reqdir = _DEFAULT_REQS_DIR
print('Branch: {}'.format(branch))
print('Source: {}'.format(args.src_dir))
print('Requirements: {}'.format(reqdir))
sha, _ = run_command('git log -n 1 --format=%H')
print('Patch under test: {}'.format(sha))
# build a list of requirements from the global list in the
# openstack/requirements project so we can match them to the changes
with tempdir() as reqroot:
install_and_load_requirements(reqroot, reqdir)
with open(reqdir + '/global-requirements.txt', 'rt') as f:
global_reqs = check.get_global_reqs(f.read())
blacklist = requirement.parse(
open(reqdir + '/blacklist.txt', 'rt').read())
cwd = os.getcwd()
# build a list of requirements in the proposed change,
# and check them for style violations while doing so
head_proj = project.read(cwd)
head_reqs = check.RequirementsList(sha, head_proj)
# Don't apply strict parsing rules to stable branches.
# Reasoning is:
# - devstack etc protect us from functional issues
# - we're backporting to stable, so guarding against
# aesthetics and DRY concerns is not our business anymore
# - if in future we have other not-functional linty style
# things to add, we don't want them to affect stable
# either.
head_strict = not branch.startswith('stable/')
head_reqs.process(strict=head_strict)
failed = check.validate(head_reqs, blacklist, global_reqs)
failed = (check.validate_lower_constraints(
head_reqs,
head_proj['lower-constraints.txt'],
blacklist,
) or failed)
# report the results
if failed or head_reqs.failed:
print("*** Incompatible requirement found!")
print("*** See http://docs.openstack.org/developer/requirements")
sys.exit(1)
print("Updated requirements match openstack/requirements.")
def test_no_setup_py(self):
root = self.useFixture(fixtures.TempDir()).path
proj = project.read(root)
self.expectThat(
proj, matchers.Equals({'root': root, 'requirements': {}}))
def main():
args = grab_args()
branch = args.branch
os.chdir(args.src_dir)
reqdir = args.reqs
if reqdir is None:
if args.local:
reqdir = os.path.dirname(
os.path.dirname(
os.path.dirname(sys.argv[0])))
else:
reqdir = _DEFAULT_REQS_DIR
# build a list of requirements from the global list in the
# openstack/requirements project so we can match them to the changes
with tempdir() as reqroot:
install_and_load_requirements(reqroot, reqdir)
with open(reqdir + '/global-requirements.txt', 'rt') as f:
global_reqs = check.get_global_reqs(f.read())
blacklist = requirement.parse(
open(reqdir + '/blacklist.txt', 'rt').read())
cwd = os.getcwd()
# build a list of requirements in the proposed change,
# and check them for style violations while doing so
head_proj = project.read(cwd)
head_reqs = check.RequirementsList('HEAD', head_proj)
# Don't apply strict parsing rules to stable branches.
# Reasoning is:
# - devstack etc protect us from functional issues
# - we're backporting to stable, so guarding against
# aesthetics and DRY concerns is not our business anymore
# - if in future we have other not-functional linty style
# things to add, we don't want them to affect stable
# either.
head_strict = not branch.startswith('stable/')
head_reqs.process(strict=head_strict)
if not args.local:
# build a list of requirements already in the target branch,
# so that we can create a diff and identify what's being changed
run_command("git checkout HEAD^1")
branch_proj = project.read(cwd)
# switch back to the proposed change now
run_command("git checkout %s" % branch)
else:
branch_proj = {'root': cwd}
branch_reqs = check.RequirementsList(branch, branch_proj)
# Don't error on the target branch being broken.
branch_reqs.process(strict=False)
failed = check.validate(head_reqs, branch_reqs, blacklist, global_reqs)
failed = (
check.validate_lower_constraints(
head_reqs,
head_proj['lower-constraints.txt'],
blacklist,
)
or failed
)
# report the results
if failed or head_reqs.failed or branch_reqs.failed:
print("*** Incompatible requirement found!")
print("*** See http://docs.openstack.org/developer/requirements")
sys.exit(1)
print("Updated requirements match openstack/requirements.")
def main():
args = grab_args()
branch = args.branch
failed = False
# build a list of requirements from the global list in the
# openstack/requirements project so we can match them to the changes
with tempdir() as reqroot:
# Only clone requirements repo if no local repo is specified
# on the command line.
if args.reqs is None:
reqdir = os.path.join(reqroot, "openstack/requirements")
if args.zc is not None:
zc = args.zc
else:
zc = '/usr/zuul-env/bin/zuul-cloner'
out, err = run_command("%(zc)s "
"--cache-dir /opt/git "
"--workspace %(root)s "
"git://git.openstack.org "
"openstack/requirements" %
dict(zc=zc, root=reqroot))
print out
print err
else:
reqdir = args.reqs
install_and_load_requirements(reqroot, reqdir)
global_reqs = requirement.parse(
open(reqdir + '/global-requirements.txt', 'rt').read())
for k, entries in global_reqs.items():
# Discard the lines: we don't need them.
global_reqs[k] = set(r for (r, line) in entries)
blacklist = requirement.parse(
open(reqdir + '/blacklist.txt', 'rt').read())
cwd = os.getcwd()
# build a list of requirements in the proposed change,
# and check them for style violations while doing so
head = run_command("git rev-parse HEAD")[0]
head_proj = project.read(cwd)
head_reqs = RequirementsList('HEAD', head_proj)
# Don't apply strict parsing rules to stable branches.
# Reasoning is:
# - devstack etc protect us from functional issues
# - we're backporting to stable, so guarding against
# aesthetics and DRY concerns is not our business anymore
# - if in future we have other not-functional linty style
# things to add, we don't want them to affect stable
# either.
head_strict = not branch.startswith('stable/')
head_reqs.process(strict=head_strict)
if not args.local:
# build a list of requirements already in the target branch,
# so that we can create a diff and identify what's being changed
run_command("git remote update")
run_command("git checkout remotes/origin/%s" % branch)
branch_proj = project.read(cwd)
# switch back to the proposed change now
run_command("git checkout %s" % head)
else:
branch_proj = {'root': cwd}
branch_reqs = RequirementsList(branch, branch_proj)
# Don't error on the target branch being broken.
branch_reqs.process(strict=False)
# iterate through the changing entries and see if they match the global
# equivalents we want enforced
for fname, freqs in head_reqs.reqs_by_file.items():
print("Validating %(fname)s" % {'fname': fname})
for name, reqs in freqs.items():
counts = {}
if (name in branch_reqs.reqs
and reqs == branch_reqs.reqs[name]):
# Unchanged [or a change that preserves a current value]
continue
if name in blacklist:
# Blacklisted items are not synced and are managed
# by project teams as they see fit, so no further
# testing is needed.
continue
if name not in global_reqs:
failed = True
print("Requirement %s not in openstack/requirements" %
str(reqs))
continue
if reqs == global_reqs[name]:
continue
for req in reqs:
if req.extras:
for extra in req.extras:
counts[extra] = counts.get(extra, 0) + 1
else:
counts[''] = counts.get('', 0) + 1
if not _is_requirement_in_global_reqs(
req, global_reqs[name]):
failed = True
print(
"Requirement for package %s : %s does "
"not match openstack/requirements value : %s" %
(name, str(req), str(global_reqs[name])))
for extra, count in counts.items():
if count != len(global_reqs[name]):
failed = True
print(
"Package %s%s requirement does not match "
"number of lines (%d) in "
"openstack/requirements" %
(name, ('[%s]' % extra) if extra else '',
len(global_reqs[name])))
# report the results
if failed or head_reqs.failed or branch_reqs.failed:
print("*** Incompatible requirement found!")
print("*** See http://docs.openstack.org/developer/requirements")
sys.exit(1)
print("Updated requirements match openstack/requirements.")
def main():
args = grab_args()
branch = args.branch
# build a list of requirements from the global list in the
# openstack/requirements project so we can match them to the changes
with tempdir() as reqroot:
# Only clone requirements repo if no local repo is specified
# on the command line.
if args.reqs is None:
reqdir = os.path.join(reqroot, "openstack/requirements")
if args.zc is not None:
zc = args.zc
else:
zc = '/usr/zuul-env/bin/zuul-cloner'
out, err = run_command("%(zc)s "
"--cache-dir /opt/git "
"--workspace %(root)s "
"git://git.openstack.org "
"openstack/requirements"
% dict(zc=zc, root=reqroot))
print out
print err
else:
reqdir = args.reqs
install_and_load_requirements(reqroot, reqdir)
global_reqs = requirement.parse(
open(reqdir + '/global-requirements.txt', 'rt').read())
for k, entries in global_reqs.items():
# Discard the lines: we don't need them.
global_reqs[k] = set(r for (r, line) in entries)
cwd = os.getcwd()
# build a list of requirements in the proposed change,
# and check them for style violations while doing so
head = run_command("git rev-parse HEAD")[0]
head_proj = project.read(cwd)
head_reqs = RequirementsList('HEAD', head_proj)
# Don't apply strict parsing rules to stable branches.
# Reasoning is:
# - devstack etc protect us from functional issues
# - we're backporting to stable, so guarding against
# aesthetics and DRY concerns is not our business anymore
# - if in future we have other not-functional linty style
# things to add, we don't want them to affect stable
# either.
head_strict = not branch.startswith('stable/')
head_reqs.process(strict=head_strict)
if not args.local:
# build a list of requirements already in the target branch,
# so that we can create a diff and identify what's being changed
run_command("git remote update")
run_command("git checkout remotes/origin/%s" % branch)
branch_proj = project.read(cwd)
# switch back to the proposed change now
run_command("git checkout %s" % head)
else:
branch_proj = {'root': cwd}
branch_reqs = RequirementsList(branch, branch_proj)
# Don't error on the target branch being broken.
branch_reqs.process(strict=False)
# iterate through the changing entries and see if they match the global
# equivalents we want enforced
failed = False
for name, reqs in head_reqs.reqs.items():
if name in branch_reqs.reqs and reqs == branch_reqs.reqs[name]:
# Unchanged [or a change that preserves a current value]
continue
if name not in global_reqs:
print(
"Requirement %s not in openstack/requirements" % str(reqs))
failed = True
continue
if reqs != global_reqs[name]:
print("Requirement %s does not match openstack/requirements "
"value %s" % (str(reqs), str(global_reqs[name])))
failed = True
# report the results
if failed or head_reqs.failed or branch_reqs.failed:
sys.exit(1)
print("Updated requirements match openstack/requirements.")
def main(args=None):
parser = argparse.ArgumentParser()
parser.add_argument(
'project',
default='',
help='path to the project source root folder.')
parser.add_argument(
'-u', '--upper-constraints',
default='upper-constraints.txt',
help='path to the upper-constraints.txt file')
parser.add_argument(
'-g', '--global-requirements',
default='global-requirements.txt',
help='Path to the global-requirements.txt file')
parser.add_argument(
'-b', '--blacklist',
default='blacklist.txt',
help='Path to the blacklist.txt file')
parser.add_argument(
'-G', '--gr-check', action='store_true',
help='Do a specifier check of global-requirements')
args = parser.parse_args(args)
upper_constraints = read_requirements_file(args.upper_constraints)
global_requirements = read_requirements_file(args.global_requirements)
blacklist = read_requirements_file(args.blacklist)
project_data = project.read(args.project)
error_count = 0
for require_file, data in project_data.get('requirements', {}).items():
print(u'\nComparing %s with global-requirements and upper-constraints'
% require_file)
requirements = requirement.parse(data)
for name, spec_list in requirements.items():
if not name or name in blacklist:
continue
if name not in global_requirements:
print(u'%s from %s not found in global-requirements' % (
name, require_file))
error_count += 1
continue
if name not in upper_constraints:
print(u'%s from %s not found in upper-constraints' % (
name, require_file))
error_count += 1
continue
elif spec_list:
uc = upper_constraints[name][0][0]
gr = global_requirements[name][0][0]
spec_gr = SpecifierSet(gr.specifiers)
for req, _ in spec_list:
specs = SpecifierSet(req.specifiers)
# This assumes uc will only have == specifiers
for uc_spec in SpecifierSet(uc.specifiers):
# if the uc version isn't in the lower specifier
# then something is wrong.
if Version(uc_spec.version) not in specs:
print(
u'%s must be <= %s from upper-constraints and '
'include the upper-constraints version' %
(name, uc_spec.version))
error_count += 1
continue
if args.gr_check:
for spec in specs:
# g-r will mostly define blocked versions. And a
# local project may define there own, so there is
# no point checking a != specifier
if spec.operator == '!=':
continue
if spec.version not in spec_gr:
print(
u'Specifier %s from %s is failing check '
'from global-requirements specifiers %s' %
(spec.version, name, str(spec_gr)))
error_count += 1
continue
return 1 if error_count else 0
