def test_load_inventory_with_errors(self): location = get_test_loc('test_gen/inv4.csv') base_dir = get_temp_dir() errors, abouts = gen.load_inventory(location, base_dir) expected_errors = [ Error( CRITICAL, "Field name: 'confirmed copyright' contains illegal name characters: 0 to 9, a to z, A to Z and _." ), Error(INFO, 'Field resource is a custom field.'), Error(INFO, 'Field test is a custom field.'), Error(INFO, 'Field about_resource: Path') ] # assert [] == errors for exp, err in zip(expected_errors, errors): assert exp.severity == err.severity assert err.message.startswith(exp.message) expected = ( 'about_resource: .\n' 'name: AboutCode\n' 'version: 0.11.0\n' 'description: |\n' ' multi\n' ' line\n' # 'confirmed copyright: Copyright (c) nexB, Inc.\n' 'resource: this.ABOUT\n' 'test: This is a test\n') result = [a.dumps() for a in abouts] assert expected == result[0]
def test_load_inventory(self): location = get_test_loc('test_gen/inv.csv') base_dir = get_temp_dir() errors, abouts = gen.load_inventory(location, base_dir) expected_errors = [ Error(INFO, 'Field custom1 is a custom field.'), Error(INFO, 'Field about_resource: Path') ] for exp, err in zip(expected_errors, errors): assert exp.severity == err.severity assert err.message.startswith(exp.message) expected = ('''about_resource: . name: AboutCode version: 0.11.0 description: | multi line custom1: | multi line ''') result = [a.dumps() for a in abouts] assert expected == result[0]
def test_filter_errors_no_errors(self): errors = [ Error(INFO, 'msg3'), Error(DEBUG, 'msg4'), Error(NOTSET, 'msg4'), ] assert [] == cmd.filter_errors(errors)
def test_load_inventory_with_mapping(self): location = get_test_loc('gen/inv4.csv') base_dir = get_test_loc('inv') license_notice_text_location = None use_mapping = True errors, abouts = gen.load_inventory(location, base_dir, license_notice_text_location, use_mapping) expected_errors = [ Error( INFO, 'Field test is not a supported field and is not defined in the mapping file. This field is ignored.' ), Error(INFO, 'Field resource is a custom field') ] assert sorted(expected_errors) == sorted(errors) expected = [ u'about_resource: .\n' u'name: AboutCode\n' u'version: 0.11.0\n' u'description: |\n' u' multi\n' u' line\n' u'copyright: Copyright (c) nexB, Inc.\n' u'resource: this.ABOUT\n' ] result = [a.dumps(with_absent=False, with_empty=False) for a in abouts] assert expected == result
def generate(abouts, license_dict, min_license_score, template=None, variables=None): """ Generate an attribution text from an `abouts` list of About objects, a `template` template text and a `variables` optional dict of extra variables. Return a tuple of (error, attribution text) where error is an Error object or None and attribution text is the generated text or None. """ rendered = None error = None template_error = check_template(template) if template_error: lineno, message = template_error error = Error( CRITICAL, 'Template validation error at line: {lineno}: "{message}"'.format( **locals())) return error, None template = jinja2.Template(template) # Get the current UTC time utcnow = datetime.datetime.utcnow() try: # Convert the field object to dictionary as it's needed for the # groupby in JINJA2 template about_dict_list = [] for about in abouts: about_dict = convert_object_to_dict(about) about_dict_list.append(about_dict) rendered = template.render(abouts=about_dict_list, license_dict=license_dict, min_license_score=min_license_score, utcnow=utcnow, tkversion=__version__, variables=variables) except Exception as e: lineno = getattr(e, 'lineno', '') or '' if lineno: lineno = ' at line: {}'.format(lineno) err = getattr(e, 'message', '') or '' error = Error( CRITICAL, 'Template processing error {lineno}: {err}'.format(**locals()), ) error = Error( CRITICAL, 'Template processing error:' + str(e), ) return error, rendered
def test_About_has_errors_when_required_fields_are_empty(self): test_file = get_test_loc('test_model/parse/empty_required.ABOUT') a = model.About(test_file) expected = [ Error(CRITICAL, 'Field about_resource is required and empty'), Error(CRITICAL, 'Field name is required and empty'), ] result = a.errors assert expected == result
def test_ignore_about_resource_path_not_exist_error(self): input_err = [ Error(ERROR, 'Field about_resource_path: test.tar.gz does not exist'), Error(ERROR, 'Field about_resource_path: test.tar.gz does not exist') ] expected_err = [] assert util.ignore_about_resource_path_not_exist_error( input_err) == expected_err
def request_license_data(api_url, api_key, license_key): """ Return a tuple of (dictionary of license data, list of errors) given a `license_key`. Send a request to `api_url` authenticating with `api_key`. """ headers = { 'Authorization': 'Token %s' % api_key, } payload = {'api_key': api_key, 'key': license_key, 'format': 'json'} api_url = api_url.rstrip('/') payload = urlencode(payload) full_url = '%(api_url)s/?%(payload)s' % locals() # handle special characters in URL such as space etc. quoted_url = quote(full_url, safe="%/:=&?~#+!$,;'@()*[]") license_data = {} errors = [] try: request = Request(quoted_url, headers=headers) response = urlopen(request) response_content = response.read().decode('utf-8') # FIXME: this should be an ordered dict license_data = json.loads(response_content) if not license_data['results']: msg = u"Invalid 'license': %s" % license_key errors.append(Error(ERROR, msg)) except HTTPError as http_e: # some auth problem if http_e.code == 403: msg = (u"Authorization denied. Invalid '--api_key'. " u"License generation is skipped.") errors.append(Error(ERROR, msg)) else: # Since no api_url/api_key/network status have # problem detected, it yields 'license' is the cause of # this exception. msg = u"Invalid 'license': %s" % license_key errors.append(Error(ERROR, msg)) except Exception as e: errors.append(Error(ERROR, str(e))) finally: if license_data.get('count') == 1: license_data = license_data.get('results')[0] else: license_data = {} return license_data, errors
def test_About_has_errors_for_illegal_custom_field_name(self): test_file = get_test_loc('test_model/parse/illegal_custom_field.about') a = model.About(test_file) expected_errors = [ Error(INFO, 'Field hydrate is a custom field.'), Error( CRITICAL, "Internal error with custom field: 'hydrate': 'illegal name'.") ] assert expected_errors == a.errors assert not hasattr(getattr(a, 'hydrate'), 'value') field = list(a.custom_fields.values())[0] assert 'hydrate' == field.name assert 'illegal name' == field.value
def transform_csv_to_csv(location, output, transformer): """ Read a CSV file at `location` and write a new CSV file at `output`. Apply transformations using the `transformer` Transformer. Return a list of Error objects. """ if not transformer: raise ValueError('Cannot transform without Transformer') rows = read_csv_rows(location) errors = [] data = iter(rows) field_names = next(rows) dupes = check_duplicate_fields(field_names) if dupes: msg = u'Duplicated field name: %(name)s' for name in dupes: errors.append(Error(CRITICAL, msg % locals())) return field_names, [], errors # Convert to dicts new_data = [OrderedDict(zip_longest(field_names, item)) for item in data] field_names, updated_data, errors = transform_data(new_data, transformer) if errors: return errors else: write_csv(output, updated_data, field_names) return []
def check_duplicated_columns(location): """ Return a list of errors for duplicated column names in a CSV file at location. """ location = add_unc(location) with codecs.open(location, 'rb', encoding='utf-8-sig', errors='replace') as csvfile: reader = csv.reader(csvfile) columns = next(reader) columns = [col for col in columns] seen = set() dupes = OrderedDict() for col in columns: c = col.lower() if c in seen: if c in dupes: dupes[c].append(col) else: dupes[c] = [col] seen.add(c.lower()) errors = [] if dupes: dup_msg = [] for name, names in dupes.items(): names = u', '.join(names) msg = '%(name)s with %(names)s' % locals() dup_msg.append(msg) dup_msg = u', '.join(dup_msg) msg = ('Duplicated column name(s): %(dup_msg)s\n' % locals() + 'Please correct the input and re-run.') errors.append(Error(ERROR, msg)) return unique(errors)
def generate_and_save(abouts, output_location, template_loc=None, variables=None): """ Generate an attribution text from an `abouts` list of About objects, a `template_loc` template file location and a `variables` optional dict of extra variables. Save the generated attribution text in the `output_location` file. Return a list of Error objects if any. """ errors = [] # Parse license_expression and save to the license list for about in abouts: if not about.license_expression.value: continue special_char_in_expression, lic_list = parse_license_expression(about.license_expression.value) if special_char_in_expression: msg = (u"The following character(s) cannot be in the license_expression: " + str(special_char_in_expression)) errors.append(Error(ERROR, msg)) rendering_error, rendered = generate_from_file( abouts, template_loc=template_loc, variables=variables ) if rendering_error: errors.append(rendering_error) if rendered: output_location = add_unc(output_location) with io.open(output_location, 'w', encoding='utf-8') as of: of.write(rendered) return errors
def test_check_file_names_with_invalid_chars_return_errors(self): paths = [ 'locations/file', 'locations/file with space', 'locations/dir1/dir2/file1', 'locations/dir2/file1', 'Accessibilité/ périmètre' ] import sys if sys.version_info[0] < 3: # python2 expected = [Error(CRITICAL, b"Invalid characters '\xe9\xe8' in file name at: 'Accessibilit\xe9/ p\xe9rim\xe8tre'")] else: expected = [Error(CRITICAL, "Invalid characters 'éè' in file name at: 'Accessibilité/ périmètre'")] result = util.check_file_names(paths) assert expected[0].message == result[0].message assert expected == result
def test_api_request_license_data_without_result(self, mock_data): response_content = b'{"count":0,"results":[]}' mock_data.return_value = FakeResponse(response_content) license_data = api.request_license_data(api_url='http://fake.url/', api_key='api_key', license_key='apache-2.0') expected = ({}, [Error(ERROR, "Invalid 'license': apache-2.0")]) assert expected == license_data
def test_check_file_names_with_dupes_return_errors(self): paths = ['some/path', 'some/PAth'] result = util.check_file_names(paths) expected = [ Error( CRITICAL, "Duplicate files: 'some/PAth' and 'some/path' have the same case-insensitive file name") ] assert expected == result
def test_filter_errors_default(self): errors = [ Error(CRITICAL, 'msg1'), Error(ERROR, 'msg2'), Error(INFO, 'msg3'), Error(WARNING, 'msg4'), Error(DEBUG, 'msg4'), Error(NOTSET, 'msg4'), ] expected = [ Error(CRITICAL, 'msg1'), Error(ERROR, 'msg2'), Error(WARNING, 'msg4'), ] assert expected == cmd.filter_errors(errors)
def test_check_duplicated_columns_handles_lower_upper_case(self): test_file = get_test_loc('test_util/dup_keys_with_diff_case.csv') expected = [ Error( ERROR, 'Duplicated column name(s): copyright with Copyright\nPlease correct the input and re-run.' ) ] result = util.check_duplicated_columns(test_file) assert expected == result
def test_check_duplicated_columns(self): test_file = get_test_loc('test_gen/dup_keys.csv') expected = [ Error( ERROR, 'Duplicated column name(s): copyright with copyright\nPlease correct the input and re-run.' ) ] result = gen.check_duplicated_columns(test_file) assert expected == result
def test_SingleLineField_has_errors_if_multiline(self): value = '''line1 line2''' field_class = model.SingleLineField expected = value expected_errors = [ Error(ERROR, 'Field s: Cannot span multiple lines: line1\n line2') ] self.check_validate(field_class, value, expected, expected_errors)
def check_About_hydrate(self, about, fields): expected = set([ 'name', 'homepage_url', 'download_url', 'version', 'copyright', 'date', 'license_spdx', 'license_text_file', 'notice_file', 'about_resource' ]) expected_errors = [ Error(INFO, 'Field date is a custom field.'), Error(INFO, 'Field license_spdx is a custom field.'), Error(INFO, 'Field license_text_file is a custom field.') ] errors = about.hydrate(fields) assert expected_errors == errors result = set([f.name for f in about.all_fields() if f.present]) assert expected == result
def test_About_dumps_all_non_empty_fields(self): test_file = get_test_loc('test_model/parse/complete2/about.ABOUT') a = model.About(test_file) expected_error = [ Error(INFO, 'Field custom1 is a custom field.'), Error(INFO, 'Field custom2 is a custom field.'), Error(INFO, 'Field custom2 is present but empty.') ] assert sorted(expected_error) == sorted(a.errors) expected = '''about_resource: . name: AboutCode version: 0.11.0 custom1: | multi line ''' result = a.dumps() assert expected == result
def test_About_has_errors_when_about_resource_does_not_exist(self): test_file = get_test_loc( 'test_gen/parser_tests/missing_about_ref.ABOUT') file_path = posixpath.join(posixpath.dirname(test_file), 'about_file_missing.c') a = model.About(test_file) err_msg = 'Field about_resource: Path %s not found' % file_path expected = [Error(INFO, err_msg)] result = a.errors assert expected == result
def test_collect_inventory_does_not_raise_error_and_maintains_order_on_custom_fields( self): test_loc = get_test_loc('test_model/inventory/custom_fields2.ABOUT') errors, abouts = model.collect_inventory(test_loc) expected_errors = [ Error( INFO, 'inventory/custom_fields2.ABOUT: Field resource is a custom field.' ), Error( INFO, 'inventory/custom_fields2.ABOUT: Field custom_mapping is a custom field.' ) ] assert expected_errors == errors expected = [ u'about_resource: .\nname: test\nresource: .\ncustom_mapping: test\n' ] assert expected == [a.dumps() for a in abouts]
def test_have_problematic_error(): have_problematic_errors = [ Error(CRITICAL, 'msg1'), Error(ERROR, 'msg2'), Error(INFO, 'msg3'), Error(WARNING, 'msg4'), Error(DEBUG, 'msg4'), Error(NOTSET, 'msg4'), ] no_problematic_errors = [ Error(INFO, 'msg3'), Error(DEBUG, 'msg4'), Error(NOTSET, 'msg4'), ] assert cmd.have_problematic_error(have_problematic_errors) assert cmd.have_problematic_error(no_problematic_errors) == False
def check_file_names(paths): """ Given a sequence of file paths, check that file names are valid and that there are no case-insensitive duplicates in any given directories. Return a list of errors. From spec : A file name can contain only these US-ASCII characters: - digits from 0 to 9 - uppercase and lowercase letters from A to Z - the _ underscore, - dash and . period signs. From spec: The case of a file name is not significant. On case-sensitive file systems (such as Linux), a tool must raise an error if two ABOUT files stored in the same directory have the same lowercase file name. """ # FIXME: this should be a defaultdicts that accumulates all duplicated paths seen = {} errors = [] for orig_path in paths: path = orig_path invalid = invalid_chars(path) if invalid: invalid = ''.join(invalid) msg = ('Invalid characters %(invalid)r in file name at: ' '%(path)r' % locals()) errors.append(Error(CRITICAL, msg)) path = to_posix(orig_path) name = resource_name(path).lower() parent = posixpath.dirname(path) path = posixpath.join(parent, name) path = posixpath.normpath(path) path = posixpath.abspath(path) existing = seen.get(path) if existing: msg = ('Duplicate files: %(orig_path)r and %(existing)r ' 'have the same case-insensitive file name' % locals()) errors.append(Error(CRITICAL, msg)) else: seen[path] = orig_path return errors
def test_collect_inventory_return_errors(self): test_loc = get_test_loc('test_model/collect_inventory_errors') errors, _abouts = model.collect_inventory(test_loc) file_path1 = posixpath.join(test_loc, 'distribute_setup.py') file_path2 = posixpath.join(test_loc, 'date_test.py') err_msg1 = 'non-supported_date_format.ABOUT: Field about_resource: Path %s not found' % file_path1 err_msg2 = 'supported_date_format.ABOUT: Field about_resource: Path %s not found' % file_path2 expected_errors = [ Error( INFO, 'non-supported_date_format.ABOUT: Field date is a custom field.' ), Error( INFO, 'supported_date_format.ABOUT: Field date is a custom field.'), Error(INFO, err_msg1), Error(INFO, err_msg2) ] assert sorted(expected_errors) == sorted(errors)
def test_About_rejects_non_ascii_names_and_accepts_unicode_values(self): test_file = get_test_loc( 'test_model/parse/non_ascii_field_name_value.about') a = model.About(test_file) expected = [ Error( CRITICAL, "Field name: 'mat\xedas' contains illegal name characters: 0 to 9, a to z, A to Z and _. (or empty spaces)" ) ] assert expected == a.errors
def test_About_duplicate_field_names_are_detected_with_different_case( self): # This test is failing because the YAML does not keep the order when # loads the test files. For instance, it treat the 'About_Resource' as the # first element and therefore the dup key is 'about_resource'. test_file = get_test_loc('test_model/parse/dupe_field_name.ABOUT') a = model.About(test_file) expected = [ Error( WARNING, 'Field About_Resource is a duplicate. Original value: "." replaced with: "new value"' ), Error( WARNING, 'Field Name is a duplicate. Original value: "old" replaced with: "new"' ) ] result = a.errors assert sorted(expected) == sorted(result)
def test_About_file_fields_are_empty_if_present_and_path_missing(self): test_file = get_test_loc( 'test_model/parse/missing_notice_license_files.ABOUT') a = model.About(test_file) file_path1 = posixpath.join(posixpath.dirname(test_file), 'test.LICENSE') file_path2 = posixpath.join(posixpath.dirname(test_file), 'test.NOTICE') err_msg1 = Error(CRITICAL, 'Field license_file: Path %s not found' % file_path1) err_msg2 = Error(CRITICAL, 'Field notice_file: Path %s not found' % file_path2) expected_errors = [err_msg1, err_msg2] assert expected_errors == a.errors assert {'test.LICENSE': None} == a.license_file.value assert {'test.NOTICE': None} == a.notice_file.value
def test_PathField_contains_dict_after_validate(self): value = 'string' field_class = model.PathField expected = OrderedDict([('string', None)]) expected_errors = [ Error( ERROR, 'Field s: Unable to verify path: string: No base directory provided' ) ] self.check_validate(field_class, value, expected, expected_errors)