def _do_main(root, source, suffix, softupdate, hacking, stdout, verbose, non_std_reqs): """No options or environment variable access from here on in.""" proj = project.read(root) global_req_content = open(os.path.join(source, 'global-requirements.txt'), 'rt').read() global_reqs = requirement.parse(global_req_content) actions = _process_project(proj, global_reqs, suffix, softupdate, hacking, non_std_reqs) project.write(proj, actions, stdout=stdout, verbose=verbose)
def test_pbr(self): root = self.useFixture(common.pbr_fixture).root proj = project.read(root) self.expectThat(proj['root'], matchers.Equals(root)) setup_py = open(root + '/setup.py', 'rt').read() self.expectThat(proj['setup.py'], matchers.Equals(setup_py)) setup_cfg = open(root + '/setup.cfg', 'rt').read() self.expectThat(proj['setup.cfg'], matchers.Equals(setup_cfg)) self.expectThat( proj['requirements'], matchers.KeysEqual('requirements.txt', 'test-requirements.txt'))
def _do_main( root, source, suffix, softupdate, hacking, stdout, verbose, non_std_reqs): """No options or environment variable access from here on in.""" proj = project.read(root) global_req_content = open( os.path.join(source, 'global-requirements.txt'), 'rt').read() global_reqs = requirement.parse(global_req_content) actions = _process_project( proj, global_reqs, suffix, softupdate, hacking, non_std_reqs) project.write(proj, actions, stdout=stdout, verbose=verbose)
def test_project_req_bigger_then_uc(self): self.useFixture(common.project_fixture) # lets change the six requirement not include the u-c version proj_read = project.read(common.project_fixture.root) proj_read['requirements']['requirements.txt'] = \ proj_read['requirements']['requirements.txt'][:-1] + '>1.10.0\n' expected_out = ('six must be <= 1.10.0 from upper-constraints and ' 'include the upper-constraints version') # Start capturing some output mock_stdout = io.StringIO() with mock.patch('openstack_requirements.project.read', return_value=proj_read), \ mock.patch('sys.stdout', mock_stdout): ret = check_exists.main([common.project_fixture.root]) self.assertEqual(ret, 1) self.assertIn(expected_out, mock_stdout.getvalue())
def test_project_req_bigger_then_uc(self): self.useFixture(common.project_fixture) # lets change the six requirement not include the u-c version proj_read = project.read(common.project_fixture.root) proj_read['requirements']['requirements.txt'] = \ proj_read['requirements']['requirements.txt'][:-1] + '>1.10.0\n' expected_out = ('six must be <= 1.10.0 from upper-constraints and ' 'include the upper-constraints version') # Start capturing some output mock_stdout = io.StringIO() with mock.patch('openstack_requirements.project.read', return_value=proj_read), \ mock.patch('sys.stdout', mock_stdout): ret = check_exists.main([common.project_fixture.root]) self.assertEqual(ret, 1) self.assertIn(expected_out, mock_stdout.getvalue())
def test_project_multiple_missing_from_uc_and_gr(self): self.useFixture(common.project_fixture) orig_mocked_read_req = check_exists.read_requirements_file read_req_path = ('openstack_requirements.cmds.check_exists.' 'read_requirements_file') def remove_req_read_reqs_file(filename): if filename == 'upper-constraints.txt': upper_cons = common.upper_constraints.copy() upper_cons.pop('lxml') return upper_cons return orig_mocked_read_req(filename) new_reqs = '>1.10.0\nsomerandommodule\n' # lets change the six requirement not include the u-c version proj_read = project.read(common.project_fixture.root) proj_read['requirements']['requirements.txt'] = \ proj_read['requirements']['requirements.txt'][:-1] + new_reqs proj_read['requirements']['test-requirements.txt'] = \ proj_read['requirements']['test-requirements.txt'] + \ 'anotherrandommodule\n' expected_outs = [ 'lxml from requirements.txt not found in upper-constraints', 'somerandommodule from requirements.txt not found in ' 'global-requirements', 'anotherrandommodule from test-requirements.txt not found in ' 'global-requirements', 'six must be <= 1.10.0 from upper-constraints and include the ' 'upper-constraints version' ] # Start capturing some output mock_stdout = io.StringIO() with mock.patch('openstack_requirements.project.read', return_value=proj_read), \ mock.patch('sys.stdout', mock_stdout), \ mock.patch(read_req_path, remove_req_read_reqs_file): ret = check_exists.main([common.project_fixture.root]) self.assertEqual(ret, 1) for expected in expected_outs: self.assertIn(expected, mock_stdout.getvalue())
def test_project_multiple_missing_from_uc_and_gr(self): self.useFixture(common.project_fixture) orig_mocked_read_req = check_exists.read_requirements_file read_req_path = ('openstack_requirements.cmds.check_exists.' 'read_requirements_file') def remove_req_read_reqs_file(filename): if filename == 'upper-constraints.txt': upper_cons = common.upper_constraints.copy() upper_cons.pop('lxml') return upper_cons return orig_mocked_read_req(filename) new_reqs = '>1.10.0\nsomerandommodule\n' # lets change the six requirement not include the u-c version proj_read = project.read(common.project_fixture.root) proj_read['requirements']['requirements.txt'] = \ proj_read['requirements']['requirements.txt'][:-1] + new_reqs proj_read['requirements']['test-requirements.txt'] = \ proj_read['requirements']['test-requirements.txt'] + \ 'anotherrandommodule\n' expected_outs = [ 'lxml from requirements.txt not found in upper-constraints', 'somerandommodule from requirements.txt not found in ' 'global-requirements', 'anotherrandommodule from test-requirements.txt not found in ' 'global-requirements', 'six must be <= 1.10.0 from upper-constraints and include the ' 'upper-constraints version'] # Start capturing some output mock_stdout = io.StringIO() with mock.patch('openstack_requirements.project.read', return_value=proj_read), \ mock.patch('sys.stdout', mock_stdout), \ mock.patch(read_req_path, remove_req_read_reqs_file): ret = check_exists.main([common.project_fixture.root]) self.assertEqual(ret, 1) for expected in expected_outs: self.assertIn(expected, mock_stdout.getvalue())
def test_project_missing_from_gr(self): self.useFixture(common.project_fixture) # Add some random package that wont exist in G-R with open(common.project_fixture.req_file, 'a') as req_file: req_file.write(u'SomeRandomModule #Some random module\n') req_file.flush() expected_out = ('somerandommodule from requirements.txt not found in' ' global-requirements') # Start capturing some output mock_stdout = io.StringIO() proj_read = project.read(common.project_fixture.root) with mock.patch('openstack_requirements.project.read', return_value=proj_read), \ mock.patch('sys.stdout', mock_stdout): ret = check_exists.main([common.project_fixture.root]) self.assertEqual(ret, 1) self.assertIn(expected_out, mock_stdout.getvalue())
def test_project_missing_from_gr(self): self.useFixture(common.project_fixture) # Add some random package that wont exist in G-R with open(common.project_fixture.req_file, 'a') as req_file: req_file.write(u'SomeRandomModule #Some random module\n') req_file.flush() expected_out = ('somerandommodule from requirements.txt not found in' ' global-requirements') # Start capturing some output mock_stdout = io.StringIO() proj_read = project.read(common.project_fixture.root) with mock.patch('openstack_requirements.project.read', return_value=proj_read), \ mock.patch('sys.stdout', mock_stdout): ret = check_exists.main([common.project_fixture.root]) self.assertEqual(ret, 1) self.assertIn(expected_out, mock_stdout.getvalue())
def make_project(fixture): with fixture: return project.read(fixture.root)
def main(args=None): parser = argparse.ArgumentParser() parser.add_argument('project', default='', help='path to the project source root folder.') parser.add_argument('-u', '--upper-constraints', default='upper-constraints.txt', help='path to the upper-constraints.txt file') parser.add_argument('-g', '--global-requirements', default='global-requirements.txt', help='Path to the global-requirements.txt file') parser.add_argument('-b', '--blacklist', default='blacklist.txt', help='Path to the blacklist.txt file') parser.add_argument('-G', '--gr-check', action='store_true', help='Do a specifier check of global-requirements') args = parser.parse_args(args) upper_constraints = read_requirements_file(args.upper_constraints) global_requirements = read_requirements_file(args.global_requirements) blacklist = read_requirements_file(args.blacklist) project_data = project.read(args.project) error_count = 0 for require_file, data in project_data.get('requirements', {}).items(): print( u'\nComparing %s with global-requirements and upper-constraints' % require_file) requirements = requirement.parse(data) for name, spec_list in requirements.items(): if not name or name in blacklist: continue if name not in global_requirements: print(u'%s from %s not found in global-requirements' % (name, require_file)) error_count += 1 continue if name not in upper_constraints: print(u'%s from %s not found in upper-constraints' % (name, require_file)) error_count += 1 continue elif spec_list: uc = upper_constraints[name][0][0] gr = global_requirements[name][0][0] spec_gr = SpecifierSet(gr.specifiers) for req, _ in spec_list: specs = SpecifierSet(req.specifiers) # This assumes uc will only have == specifiers for uc_spec in SpecifierSet(uc.specifiers): # if the uc version isn't in the lower specifier # then something is wrong. if Version(uc_spec.version) not in specs: print( u'%s must be <= %s from upper-constraints and ' 'include the upper-constraints version' % (name, uc_spec.version)) error_count += 1 continue if args.gr_check: for spec in specs: # g-r will mostly define blocked versions. And a # local project may define there own, so there is # no point checking a != specifier if spec.operator == '!=': continue if spec.version not in spec_gr: print( u'Specifier %s from %s is failing check ' 'from global-requirements specifiers %s' % (spec.version, name, str(spec_gr))) error_count += 1 continue return 1 if error_count else 0
def make_project(fixture): with fixture: return project.read(fixture.root)
def main(): args = grab_args() branch = args.branch os.chdir(args.src_dir) reqdir = args.reqs print(sys.version_info) if reqdir is None: if args.local: print('selecting default requirements directory for local mode') reqdir = os.path.dirname( os.path.dirname(os.path.dirname(sys.argv[0]))) else: print('selecting default requirements directory for normal mode') reqdir = _DEFAULT_REQS_DIR print('Branch: {}'.format(branch)) print('Source: {}'.format(args.src_dir)) print('Requirements: {}'.format(reqdir)) sha, _ = run_command('git log -n 1 --format=%H') print('Patch under test: {}'.format(sha)) # build a list of requirements from the global list in the # openstack/requirements project so we can match them to the changes with tempdir() as reqroot: install_and_load_requirements(reqroot, reqdir) with open(reqdir + '/global-requirements.txt', 'rt') as f: global_reqs = check.get_global_reqs(f.read()) blacklist = requirement.parse( open(reqdir + '/blacklist.txt', 'rt').read()) cwd = os.getcwd() # build a list of requirements in the proposed change, # and check them for style violations while doing so head_proj = project.read(cwd) head_reqs = check.RequirementsList(sha, head_proj) # Don't apply strict parsing rules to stable branches. # Reasoning is: # - devstack etc protect us from functional issues # - we're backporting to stable, so guarding against # aesthetics and DRY concerns is not our business anymore # - if in future we have other not-functional linty style # things to add, we don't want them to affect stable # either. head_strict = not branch.startswith('stable/') head_reqs.process(strict=head_strict) failed = check.validate(head_reqs, blacklist, global_reqs) failed = (check.validate_lower_constraints( head_reqs, head_proj['lower-constraints.txt'], blacklist, ) or failed) # report the results if failed or head_reqs.failed: print("*** Incompatible requirement found!") print("*** See http://docs.openstack.org/developer/requirements") sys.exit(1) print("Updated requirements match openstack/requirements.")
def test_no_setup_py(self): root = self.useFixture(fixtures.TempDir()).path proj = project.read(root) self.expectThat( proj, matchers.Equals({'root': root, 'requirements': {}}))
def main(): args = grab_args() branch = args.branch os.chdir(args.src_dir) reqdir = args.reqs if reqdir is None: if args.local: reqdir = os.path.dirname( os.path.dirname( os.path.dirname(sys.argv[0]))) else: reqdir = _DEFAULT_REQS_DIR # build a list of requirements from the global list in the # openstack/requirements project so we can match them to the changes with tempdir() as reqroot: install_and_load_requirements(reqroot, reqdir) with open(reqdir + '/global-requirements.txt', 'rt') as f: global_reqs = check.get_global_reqs(f.read()) blacklist = requirement.parse( open(reqdir + '/blacklist.txt', 'rt').read()) cwd = os.getcwd() # build a list of requirements in the proposed change, # and check them for style violations while doing so head_proj = project.read(cwd) head_reqs = check.RequirementsList('HEAD', head_proj) # Don't apply strict parsing rules to stable branches. # Reasoning is: # - devstack etc protect us from functional issues # - we're backporting to stable, so guarding against # aesthetics and DRY concerns is not our business anymore # - if in future we have other not-functional linty style # things to add, we don't want them to affect stable # either. head_strict = not branch.startswith('stable/') head_reqs.process(strict=head_strict) if not args.local: # build a list of requirements already in the target branch, # so that we can create a diff and identify what's being changed run_command("git checkout HEAD^1") branch_proj = project.read(cwd) # switch back to the proposed change now run_command("git checkout %s" % branch) else: branch_proj = {'root': cwd} branch_reqs = check.RequirementsList(branch, branch_proj) # Don't error on the target branch being broken. branch_reqs.process(strict=False) failed = check.validate(head_reqs, branch_reqs, blacklist, global_reqs) failed = ( check.validate_lower_constraints( head_reqs, head_proj['lower-constraints.txt'], blacklist, ) or failed ) # report the results if failed or head_reqs.failed or branch_reqs.failed: print("*** Incompatible requirement found!") print("*** See http://docs.openstack.org/developer/requirements") sys.exit(1) print("Updated requirements match openstack/requirements.")
def main(): args = grab_args() branch = args.branch failed = False # build a list of requirements from the global list in the # openstack/requirements project so we can match them to the changes with tempdir() as reqroot: # Only clone requirements repo if no local repo is specified # on the command line. if args.reqs is None: reqdir = os.path.join(reqroot, "openstack/requirements") if args.zc is not None: zc = args.zc else: zc = '/usr/zuul-env/bin/zuul-cloner' out, err = run_command("%(zc)s " "--cache-dir /opt/git " "--workspace %(root)s " "git://git.openstack.org " "openstack/requirements" % dict(zc=zc, root=reqroot)) print out print err else: reqdir = args.reqs install_and_load_requirements(reqroot, reqdir) global_reqs = requirement.parse( open(reqdir + '/global-requirements.txt', 'rt').read()) for k, entries in global_reqs.items(): # Discard the lines: we don't need them. global_reqs[k] = set(r for (r, line) in entries) blacklist = requirement.parse( open(reqdir + '/blacklist.txt', 'rt').read()) cwd = os.getcwd() # build a list of requirements in the proposed change, # and check them for style violations while doing so head = run_command("git rev-parse HEAD")[0] head_proj = project.read(cwd) head_reqs = RequirementsList('HEAD', head_proj) # Don't apply strict parsing rules to stable branches. # Reasoning is: # - devstack etc protect us from functional issues # - we're backporting to stable, so guarding against # aesthetics and DRY concerns is not our business anymore # - if in future we have other not-functional linty style # things to add, we don't want them to affect stable # either. head_strict = not branch.startswith('stable/') head_reqs.process(strict=head_strict) if not args.local: # build a list of requirements already in the target branch, # so that we can create a diff and identify what's being changed run_command("git remote update") run_command("git checkout remotes/origin/%s" % branch) branch_proj = project.read(cwd) # switch back to the proposed change now run_command("git checkout %s" % head) else: branch_proj = {'root': cwd} branch_reqs = RequirementsList(branch, branch_proj) # Don't error on the target branch being broken. branch_reqs.process(strict=False) # iterate through the changing entries and see if they match the global # equivalents we want enforced for fname, freqs in head_reqs.reqs_by_file.items(): print("Validating %(fname)s" % {'fname': fname}) for name, reqs in freqs.items(): counts = {} if (name in branch_reqs.reqs and reqs == branch_reqs.reqs[name]): # Unchanged [or a change that preserves a current value] continue if name in blacklist: # Blacklisted items are not synced and are managed # by project teams as they see fit, so no further # testing is needed. continue if name not in global_reqs: failed = True print("Requirement %s not in openstack/requirements" % str(reqs)) continue if reqs == global_reqs[name]: continue for req in reqs: if req.extras: for extra in req.extras: counts[extra] = counts.get(extra, 0) + 1 else: counts[''] = counts.get('', 0) + 1 if not _is_requirement_in_global_reqs( req, global_reqs[name]): failed = True print( "Requirement for package %s : %s does " "not match openstack/requirements value : %s" % (name, str(req), str(global_reqs[name]))) for extra, count in counts.items(): if count != len(global_reqs[name]): failed = True print( "Package %s%s requirement does not match " "number of lines (%d) in " "openstack/requirements" % (name, ('[%s]' % extra) if extra else '', len(global_reqs[name]))) # report the results if failed or head_reqs.failed or branch_reqs.failed: print("*** Incompatible requirement found!") print("*** See http://docs.openstack.org/developer/requirements") sys.exit(1) print("Updated requirements match openstack/requirements.")
def main(): args = grab_args() branch = args.branch # build a list of requirements from the global list in the # openstack/requirements project so we can match them to the changes with tempdir() as reqroot: # Only clone requirements repo if no local repo is specified # on the command line. if args.reqs is None: reqdir = os.path.join(reqroot, "openstack/requirements") if args.zc is not None: zc = args.zc else: zc = '/usr/zuul-env/bin/zuul-cloner' out, err = run_command("%(zc)s " "--cache-dir /opt/git " "--workspace %(root)s " "git://git.openstack.org " "openstack/requirements" % dict(zc=zc, root=reqroot)) print out print err else: reqdir = args.reqs install_and_load_requirements(reqroot, reqdir) global_reqs = requirement.parse( open(reqdir + '/global-requirements.txt', 'rt').read()) for k, entries in global_reqs.items(): # Discard the lines: we don't need them. global_reqs[k] = set(r for (r, line) in entries) cwd = os.getcwd() # build a list of requirements in the proposed change, # and check them for style violations while doing so head = run_command("git rev-parse HEAD")[0] head_proj = project.read(cwd) head_reqs = RequirementsList('HEAD', head_proj) # Don't apply strict parsing rules to stable branches. # Reasoning is: # - devstack etc protect us from functional issues # - we're backporting to stable, so guarding against # aesthetics and DRY concerns is not our business anymore # - if in future we have other not-functional linty style # things to add, we don't want them to affect stable # either. head_strict = not branch.startswith('stable/') head_reqs.process(strict=head_strict) if not args.local: # build a list of requirements already in the target branch, # so that we can create a diff and identify what's being changed run_command("git remote update") run_command("git checkout remotes/origin/%s" % branch) branch_proj = project.read(cwd) # switch back to the proposed change now run_command("git checkout %s" % head) else: branch_proj = {'root': cwd} branch_reqs = RequirementsList(branch, branch_proj) # Don't error on the target branch being broken. branch_reqs.process(strict=False) # iterate through the changing entries and see if they match the global # equivalents we want enforced failed = False for name, reqs in head_reqs.reqs.items(): if name in branch_reqs.reqs and reqs == branch_reqs.reqs[name]: # Unchanged [or a change that preserves a current value] continue if name not in global_reqs: print( "Requirement %s not in openstack/requirements" % str(reqs)) failed = True continue if reqs != global_reqs[name]: print("Requirement %s does not match openstack/requirements " "value %s" % (str(reqs), str(global_reqs[name]))) failed = True # report the results if failed or head_reqs.failed or branch_reqs.failed: sys.exit(1) print("Updated requirements match openstack/requirements.")
def main(args=None): parser = argparse.ArgumentParser() parser.add_argument( 'project', default='', help='path to the project source root folder.') parser.add_argument( '-u', '--upper-constraints', default='upper-constraints.txt', help='path to the upper-constraints.txt file') parser.add_argument( '-g', '--global-requirements', default='global-requirements.txt', help='Path to the global-requirements.txt file') parser.add_argument( '-b', '--blacklist', default='blacklist.txt', help='Path to the blacklist.txt file') parser.add_argument( '-G', '--gr-check', action='store_true', help='Do a specifier check of global-requirements') args = parser.parse_args(args) upper_constraints = read_requirements_file(args.upper_constraints) global_requirements = read_requirements_file(args.global_requirements) blacklist = read_requirements_file(args.blacklist) project_data = project.read(args.project) error_count = 0 for require_file, data in project_data.get('requirements', {}).items(): print(u'\nComparing %s with global-requirements and upper-constraints' % require_file) requirements = requirement.parse(data) for name, spec_list in requirements.items(): if not name or name in blacklist: continue if name not in global_requirements: print(u'%s from %s not found in global-requirements' % ( name, require_file)) error_count += 1 continue if name not in upper_constraints: print(u'%s from %s not found in upper-constraints' % ( name, require_file)) error_count += 1 continue elif spec_list: uc = upper_constraints[name][0][0] gr = global_requirements[name][0][0] spec_gr = SpecifierSet(gr.specifiers) for req, _ in spec_list: specs = SpecifierSet(req.specifiers) # This assumes uc will only have == specifiers for uc_spec in SpecifierSet(uc.specifiers): # if the uc version isn't in the lower specifier # then something is wrong. if Version(uc_spec.version) not in specs: print( u'%s must be <= %s from upper-constraints and ' 'include the upper-constraints version' % (name, uc_spec.version)) error_count += 1 continue if args.gr_check: for spec in specs: # g-r will mostly define blocked versions. And a # local project may define there own, so there is # no point checking a != specifier if spec.operator == '!=': continue if spec.version not in spec_gr: print( u'Specifier %s from %s is failing check ' 'from global-requirements specifiers %s' % (spec.version, name, str(spec_gr))) error_count += 1 continue return 1 if error_count else 0