def test_collect_inventory_populate_about_file_path(self): test_loc = get_test_loc('test_model/inventory/complete') errors, abouts = model.collect_inventory(test_loc) assert [] == errors expected = 'about.ABOUT' result = abouts[0].about_file_path assert expected == result
def inventory(location, output, mapping, quiet, format, show_all): """ Collect a JSON or CSV inventory of components from .ABOUT files. LOCATION: Path to an .ABOUT file or a directory with .ABOUT files. OUTPUT: Path to the JSON or CSV inventory file to create. """ print_version() if not exists(os.path.dirname(output)): # FIXME: there is likely a better way to return an error click.echo('ERROR: <OUTPUT> path does not exists.') # FIXME: return error code? return click.echo( 'Collecting inventory from: %(location)r and writing output to: %(output)r' % locals()) # FIXME: do we really want to continue support zip as an input? if location.lower().endswith('.zip'): # accept zipped ABOUT files as input location = extract_zip(location) errors, abouts = model.collect_inventory(location, use_mapping=mapping) write_errors = model.write_output(abouts, output, format) for err in write_errors: errors.append(err) finalized_errors = ignore_about_resource_path_not_exist_error(errors) log_errors(finalized_errors, quiet, show_all, os.path.dirname(output)) sys.exit(0)
def test_collect_inventory_return_no_warnings_and_model_can_uuse_relative_paths( self): test_loc = get_test_loc('test_model/rel/allAboutInOneDir') errors, _abouts = model.collect_inventory(test_loc) expected_errors = [] result = [(level, e) for level, e in errors if level > INFO] assert expected_errors == result
def inventory(location, output, format, quiet, verbose): # NOQA """ Collect the inventory of .ABOUT file data as CSV or JSON. LOCATION: Path to an .ABOUT file or a directory with .ABOUT files. OUTPUT: Path to the JSON or CSV inventory file to create. """ if not quiet: print_version() click.echo('Collecting inventory from ABOUT files...') if location.lower().endswith('.zip'): # accept zipped ABOUT files as input location = extract_zip(location) errors, abouts = collect_inventory(location) write_errors = write_output(abouts=abouts, location=output, format=format) errors.extend(write_errors) errors_count = report_errors(errors, quiet, verbose, log_file_loc=output + '-error.log') if not quiet: msg = 'Inventory collected in {output}.'.format(**locals()) click.echo(msg) sys.exit(errors_count)
def attrib(location, output, template, vartext, quiet, verbose): """ Generate an attribution document at OUTPUT using .ABOUT files at LOCATION. LOCATION: Path to a file, directory or .zip archive containing .ABOUT files. OUTPUT: Path where to write the attribution document. """ if not quiet: print_version() click.echo('Generating attribution...') # accept zipped ABOUT files as input if location.lower().endswith('.zip'): location = extract_zip(location) errors, abouts = collect_inventory(location) attrib_errors = generate_attribution_doc( abouts=abouts, output_location=output, template_loc=template, variables=vartext, ) errors.extend(attrib_errors) errors_count = report_errors(errors, quiet, verbose, log_file_loc=output + '-error.log') if not quiet: msg = 'Attribution generated in: {output}'.format(**locals()) click.echo(msg) sys.exit(errors_count)
def check(location, show_all): """ Check and validate .ABOUT file(s) at LOCATION for errors and print error messages on the terminal. LOCATION: Path to a .ABOUT file or a directory containing .ABOUT files. """ click.echo('Running aboutcode-toolkit version ' + __version__) click.echo('Checking ABOUT files...') errors, abouts = model.collect_inventory(location) msg_format = '%(sever)s: %(message)s' print_errors = [] for severity, message in errors: sever = severities[severity] if show_all: print_errors.append(msg_format % locals()) elif sever in problematic_errors: print_errors.append(msg_format % locals()) number_of_errors = len(print_errors) for err in print_errors: print(err) if print_errors: click.echo('Found {} errors.'.format(number_of_errors)) # FIXME: not sure this is the right way to exit with a retrun code sys.exit(1) else: click.echo('No error found.') sys.exit(0)
def test_collect_inventory_always_collects_custom_fieldsg(self): test_loc = get_test_loc('test_model/inventory/custom_fields.ABOUT') errors, abouts = model.collect_inventory(test_loc) expected_msg1 = 'Field resource is a custom field' assert len(errors) == 2 assert expected_msg1 in errors[0].message # The not supported 'resource' value is collected assert abouts[0].resource.value
def test_collect_inventory_can_collect_a_single_file(self): test_loc = get_test_loc( 'test_model/single_file/django_snippets_2413.ABOUT') _errors, abouts = model.collect_inventory(test_loc) assert 1 == len(abouts) expected = ['single_file/django_snippets_2413.ABOUT'] result = [a.about_file_path for a in abouts] assert expected == result
def test_collect_inventory_with_license_expression(self): test_loc = get_test_loc( 'test_model/parse/multi_line_license_expresion.ABOUT') errors, abouts = model.collect_inventory(test_loc) assert [] == errors expected_lic = 'mit or apache-2.0' returned_lic = abouts[0].license_expression.value assert expected_lic == returned_lic
def test_generate(self): expected = (u'Apache HTTP Server: 2.4.3\n' u'resource: httpd-2.4.3.tar.gz\n') test_file = get_test_loc('attrib_gen/attrib.ABOUT') with open(get_test_loc('attrib_gen/test.template')) as tmpl: template = tmpl.read() _errors, abouts = model.collect_inventory(test_file) result = attrib.generate(abouts, template) self.assertEqual(expected, result)
def test_generate_from_file_with_default_template(self): test_file = get_test_loc('attrib_gen/attrib.ABOUT') _errors, abouts = model.collect_inventory(test_file) result = attrib.generate_from_file(abouts) with open(get_test_loc( 'attrib_gen/expected_default_attrib.html')) as exp: expected = exp.read() self.assertEqual([x.rstrip() for x in expected.splitlines()], [x.rstrip() for x in result.splitlines()])
def test_generate_from_file_with_default_template(self): test_file = get_test_loc('attrib_gen/attrib.ABOUT') _errors, abouts = model.collect_inventory(test_file) result = attrib.generate_from_file(abouts) with open(get_test_loc('attrib_gen/expected_default_attrib.html')) as exp: expected = exp.read() # strip the timestamp: the timestamp is wrapped in italic block self.assertEqual([x.rstrip() for x in expected.splitlines()], [x.rstrip() for x in result.splitlines() if not '<i>' in x])
def test_generate(self): expected = (u'Apache HTTP Server: 2.4.3\n' u'resource: httpd-2.4.3.tar.gz\n') test_file = get_test_loc('attrib_gen/attrib.ABOUT') with open(get_test_loc('attrib_gen/test.template')) as tmpl: template = tmpl.read() _errors, abouts = model.collect_inventory(test_file) result = attrib.generate(abouts, template) self.assertEqual(expected, result)
def test_collect_inventory_works_with_relative_paths(self): # FIXME: This test need to be run under src/attributecode/ # or otherwise it will fail as the test depends on the launching # location test_loc = get_test_loc('test_model/inventory/relative') # Use '.' as the indication of the current directory test_loc1 = test_loc + '/./' # Use '..' to go back to the parent directory test_loc2 = test_loc + '/../relative' errors1, abouts1 = model.collect_inventory(test_loc1) errors2, abouts2 = model.collect_inventory(test_loc2) assert [] == errors1 assert [] == errors2 expected = 'about.ABOUT' result1 = abouts1[0].about_file_path result2 = abouts2[0].about_file_path assert expected == result1 assert expected == result2
def test_inventory_filter(self): test_loc = get_test_loc('basic') _errors, abouts = model.collect_inventory(test_loc) filter_dict = {'name': ['simple']} # The test loc has 2 .about files, only the simple.about is taken after # the filtering updated_abouts = util.inventory_filter(abouts, filter_dict) for about in updated_abouts: assert about.name.value == 'simple'
def test_inventory_filter(self): test_loc = get_test_loc('basic') _errors, abouts = model.collect_inventory(test_loc) filter_dict = {'name': ['simple']} # The test loc has 2 .about files, only the simple.about is taken after # the filtering updated_abouts = util.inventory_filter(abouts, filter_dict) for about in updated_abouts: assert about.name.value == 'simple'
def test_collect_inventory_with_multi_line(self): test_loc = get_test_loc( 'test_model/parse/multi_line_license_expresion.ABOUT') errors, abouts = model.collect_inventory(test_loc) assert [] == errors expected_lic_url = [ 'https://enterprise.dejacode.com/urn/?urn=urn:dje:license:mit', 'https://enterprise.dejacode.com/urn/?urn=urn:dje:license:apache-2.0' ] returned_lic_url = abouts[0].license_url.value assert expected_lic_url == returned_lic_url
def test_collect_inventory_does_not_convert_lf_to_crlf_from_directory( self): location = get_test_loc('test_model/crlf/about.ABOUT') result = get_temp_file() errors, abouts = model.collect_inventory(location) errors2 = model.write_output(abouts, result, format='csv') errors.extend(errors2) assert all(e.severity == INFO for e in errors) expected = get_test_loc('test_model/crlf/expected.csv') check_csv(expected, result, fix_cell_linesep=True, regen=False)
def test_collect_inventory_complex_from_directory(self): location = get_test_loc('test_model/inventory/complex') result = get_temp_file() errors, abouts = model.collect_inventory(location) model.write_output(abouts, result, format='csv') assert all(e.severity == INFO for e in errors) expected = get_test_loc('test_model/inventory/complex/expected.csv') check_csv(expected, result, fix_cell_linesep=True, regen=False)
def check(location, verbose): """ Check .ABOUT file(s) at LOCATION for validity and print error messages. LOCATION: Path to a file or directory containing .ABOUT files. """ print_version() click.echo('Checking ABOUT files...') errors, _abouts = collect_inventory(location) severe_errors_count = report_errors(errors, quiet=False, verbose=verbose) sys.exit(severe_errors_count)
def test_collect_inventory_basic_from_directory(self): location = get_test_loc('test_model/inventory/basic') result = get_temp_file() errors, abouts = model.collect_inventory(location) model.write_output(abouts, result, format='csv') expected_errors = [] assert expected_errors == errors expected = get_test_loc('test_model/inventory/basic/expected.csv') check_csv(expected, result)
def test_collect_inventory_with_no_about_resource_from_directory(self): location = get_test_loc('test_model/inventory/no_about_resource_key') result = get_temp_file() errors, abouts = model.collect_inventory(location) model.write_output(abouts, result, format='csv') expected_errors = [ Error(CRITICAL, 'about/about.ABOUT: Field about_resource is required') ] assert expected_errors == errors
def attrib(location, output, template, mapping, mapping_file, inventory, vartext, quiet, verbose): """ Generate an attribution document at OUTPUT using .ABOUT files at LOCATION. LOCATION: Path to an .ABOUT file, a directory containing .ABOUT files or a .zip archive containing .ABOUT files. OUTPUT: Path to output file to write the attribution to. """ print_version() click.echo('Generating attribution...') # accept zipped ABOUT files as input if location.lower().endswith('.zip'): location = extract_zip(location) inv_errors, abouts = model.collect_inventory(location, use_mapping=mapping, mapping_file=mapping_file) no_match_errors = attrib_generate_and_save(abouts=abouts, output_location=output, use_mapping=mapping, mapping_file=mapping_file, template_loc=template, inventory_location=inventory, vartext=vartext) if not no_match_errors: # Check for template error with open(output, 'r') as output_file: first_line = output_file.readline() if first_line.startswith('Template'): click.echo(first_line) sys.exit(errno.ENOEXEC) for no_match_error in no_match_errors: inv_errors.append(no_match_error) error_count = 0 for e in inv_errors: # Only count as warning/error if CRITICAL, ERROR and WARNING if e.severity > 20: error_count = error_count + 1 log_errors(inv_errors, error_count, quiet, verbose, os.path.dirname(output)) click.echo(' %(error_count)d errors or warnings detected.' % locals()) click.echo('Finished.') sys.exit(0)
def test_generate_from_collected_inventory_wih_custom_temaplte(self): test_file = get_test_loc('test_attrib/gen_simple/attrib.ABOUT') errors, abouts = model.collect_inventory(test_file) assert not errors test_template = get_test_loc('test_attrib/gen_simple/test.template') with open(test_template) as tmpl: template = tmpl.read() expected = ('Apache HTTP Server: 2.4.3\n' 'resource: httpd-2.4.3.tar.gz\n') error, result = attrib.generate(abouts, template) assert expected == result assert not error
def test_lic_key_name_sync(self): test_file = get_test_loc( 'test_attrib/gen_license_key_name_check/test.ABOUT') expected = get_test_loc( 'test_attrib/gen_license_key_name_check/expected/expected.html') template_loc = get_test_loc( 'test_attrib/gen_license_key_name_check/custom.template') output_file = get_temp_file() errors, abouts = model.collect_inventory(test_file) attrib.generate_and_save(abouts, output_file, template_loc) with open(output_file) as of: f1 = '\n'.join(of.readlines(False)) with open(expected) as ef: f2 = '\n'.join(ef.readlines(False)) assert f1 == f2
def test_generate_with_default_template(self): test_file = get_test_loc( 'test_attrib/gen_default_template/attrib.ABOUT') errors, abouts = model.collect_inventory(test_file) assert not errors error, result = attrib.generate_from_file(abouts) assert not error expected_file = get_test_loc( 'test_attrib/gen_default_template/expected_default_attrib.html') with open(expected_file) as exp: expected = exp.read() # strip the timestamp: the timestamp is wrapped in italic block result = remove_timestamp(result) expected = remove_timestamp(expected) assert expected == result
def attrib(location, output, template, mapping, mapping_file, inventory, vartext, quiet, verbose): """ Generate an attribution document at OUTPUT using .ABOUT files at LOCATION. LOCATION: Path to an .ABOUT file, a directory containing .ABOUT files or a .zip archive containing .ABOUT files. OUTPUT: Path to output file to write the attribution to. """ print_version() click.echo('Generating attribution...') # accept zipped ABOUT files as input if location.lower().endswith('.zip'): location = extract_zip(location) inv_errors, abouts = model.collect_inventory(location, use_mapping=mapping, mapping_file=mapping_file) no_match_errors = attrib_generate_and_save( abouts=abouts, output_location=output, use_mapping=mapping, mapping_file=mapping_file, template_loc=template, inventory_location=inventory, vartext=vartext) if not no_match_errors: # Check for template error with open(output, 'r') as output_file: first_line = output_file.readline() if first_line.startswith('Template'): click.echo(first_line) sys.exit(errno.ENOEXEC) for no_match_error in no_match_errors: inv_errors.append(no_match_error) error_count = 0 for e in inv_errors: # Only count as warning/error if CRITICAL, ERROR and WARNING if e.severity > 20: error_count = error_count + 1 log_errors(inv_errors, error_count, quiet, verbose, os.path.dirname(output)) click.echo(' %(error_count)d errors or warnings detected.' % locals()) click.echo('Finished.') sys.exit(0)
def test_collect_inventory_does_not_raise_error_and_maintains_order_on_custom_fields( self): test_loc = get_test_loc('test_model/inventory/custom_fields2.ABOUT') errors, abouts = model.collect_inventory(test_loc) expected_errors = [ Error( INFO, 'inventory/custom_fields2.ABOUT: Field resource is a custom field.' ), Error( INFO, 'inventory/custom_fields2.ABOUT: Field custom_mapping is a custom field.' ) ] assert expected_errors == errors expected = [ u'about_resource: .\nname: test\nresource: .\ncustom_mapping: test\n' ] assert expected == [a.dumps() for a in abouts]
def test_collect_inventory_return_errors(self): test_loc = get_test_loc('test_model/collect_inventory_errors') errors, _abouts = model.collect_inventory(test_loc) file_path1 = posixpath.join(test_loc, 'distribute_setup.py') file_path2 = posixpath.join(test_loc, 'date_test.py') err_msg1 = 'non-supported_date_format.ABOUT: Field about_resource: Path %s not found' % file_path1 err_msg2 = 'supported_date_format.ABOUT: Field about_resource: Path %s not found' % file_path2 expected_errors = [ Error( INFO, 'non-supported_date_format.ABOUT: Field date is a custom field.' ), Error( INFO, 'supported_date_format.ABOUT: Field date is a custom field.'), Error(INFO, err_msg1), Error(INFO, err_msg2) ] assert sorted(expected_errors) == sorted(errors)
def test_collect_inventory_with_long_path(self): test_loc = extract_test_loc('test_model/longpath.zip') _errors, abouts = model.collect_inventory(test_loc) assert 2 == len(abouts) expected_paths = ( 'longpath/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1' '/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1' '/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1' '/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1' '/longpath1/non-supported_date_format.ABOUT', 'longpath/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1' '/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1' '/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1' '/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1' '/longpath1/supported_date_format.ABOUT') results = [a.about_file_path for a in abouts] assert all(r.endswith(expected_paths) for r in results) expected_name = ['distribute', 'date_test'] result_name = [a.name.value for a in abouts] assert sorted(expected_name) == sorted(result_name)
def attrib(location, output, template, mapping, inventory, quiet, show_all): """ Generate an attribution document at OUTPUT using .ABOUT files at LOCATION. LOCATION: Path to an .ABOUT file, a directory containing .ABOUT files or a .zip archive containing .ABOUT files. OUTPUT: Path to output file to write the attribution to. """ print_version() click.echo('Generating attribution...') # accept zipped ABOUT files as input if location.lower().endswith('.zip'): location = extract_zip(location) inv_errors, abouts = model.collect_inventory(location, use_mapping=mapping) no_match_errors = attrib_generate_and_save(abouts=abouts, output_location=output, use_mapping=mapping, template_loc=template, inventory_location=inventory) if not no_match_errors: # Check for template error with open(output, 'r') as output_file: first_line = output_file.readline() if first_line.startswith('Template'): click.echo(first_line) sys.exit(errno.ENOEXEC) for no_match_error in no_match_errors: inv_errors.append(no_match_error) finalized_errors = ignore_about_resource_path_not_exist_error(inv_errors) log_errors(finalized_errors, quiet, show_all, os.path.dirname(output)) click.echo('Finished.') sys.exit(0)
def check(location, verbose): """ Check and validate .ABOUT file(s) at LOCATION for errors and print error messages on the terminal. LOCATION: Path to a .ABOUT file or a directory containing .ABOUT files. """ click.echo('Running aboutcode-toolkit version ' + __version__) click.echo('Checking ABOUT files...') errors, abouts = model.collect_inventory(location) msg_format = '%(sever)s: %(message)s' print_errors = [] number_of_errors = 0 for severity, message in errors: sever = severities[severity] # Only problematic_errors should be counted. # Others such as INFO should not be counted as error. if sever in problematic_errors: number_of_errors = number_of_errors + 1 if verbose: print_errors.append(msg_format % locals()) elif sever in problematic_errors: print_errors.append(msg_format % locals()) for err in print_errors: print(err) if print_errors: click.echo('Found {} errors.'.format(number_of_errors)) # FIXME: not sure this is the right way to exit with a return code sys.exit(1) else: click.echo('No error found.') sys.exit(0)
def inventory(location, output, mapping, mapping_file, mapping_output, filter, quiet, format, verbose): # NOQA """ Collect a JSON or CSV inventory of components from .ABOUT files. LOCATION: Path to an .ABOUT file or a directory with .ABOUT files. OUTPUT: Path to the JSON or CSV inventory file to create. """ print_version() if not exists(os.path.dirname(output)): # FIXME: there is likely a better way to return an error click.echo('ERROR: <OUTPUT> path does not exists.') # FIXME: return error code? return click.echo('Collecting inventory from: %(location)s and writing output to: %(output)s' % locals()) # FIXME: do we really want to continue support zip as an input? if location.lower().endswith('.zip'): # accept zipped ABOUT files as input location = extract_zip(location) errors, abouts = model.collect_inventory(location, use_mapping=mapping, mapping_file=mapping_file) updated_abouts = [] if filter: filter_dict = {} # Parse the filter and save to the filter dictionary with a list of value for element in filter: key = element.partition('=')[0] value = element.partition('=')[2] if key in filter_dict: filter_dict[key].append(value) else: value_list = [value] filter_dict[key] = value_list updated_abouts = inventory_filter(abouts, filter_dict) else: updated_abouts = abouts # Do not write the output if one of the ABOUT files has duplicated key names dup_error_msg = u'Duplicated key name(s)' halt_output = False for err in errors: if dup_error_msg in err.message: halt_output = True break if not halt_output: write_errors = model.write_output(updated_abouts, output, format, mapping_output) for err in write_errors: errors.append(err) else: msg = u'Duplicated key names are not supported.\n' + \ 'Please correct and re-run.' print(msg) error_count = 0 for e in errors: # Only count as warning/error if CRITICAL, ERROR and WARNING if e.severity > 20: error_count = error_count + 1 log_errors(errors, error_count, quiet, verbose, os.path.dirname(output)) click.echo(' %(error_count)d errors or warnings detected.' % locals()) sys.exit(0)
def inventory(location, output, mapping, mapping_file, mapping_output, filter, quiet, format, verbose): # NOQA """ Collect a JSON or CSV inventory of components from .ABOUT files. LOCATION: Path to an .ABOUT file or a directory with .ABOUT files. OUTPUT: Path to the JSON or CSV inventory file to create. """ print_version() if not exists(os.path.dirname(output)): # FIXME: there is likely a better way to return an error click.echo('ERROR: <OUTPUT> path does not exists.') # FIXME: return error code? return click.echo( 'Collecting inventory from: %(location)s and writing output to: %(output)s' % locals()) # FIXME: do we really want to continue support zip as an input? if location.lower().endswith('.zip'): # accept zipped ABOUT files as input location = extract_zip(location) errors, abouts = model.collect_inventory(location, use_mapping=mapping, mapping_file=mapping_file) updated_abouts = [] if filter: filter_dict = {} # Parse the filter and save to the filter dictionary with a list of value for element in filter: key = element.partition('=')[0] value = element.partition('=')[2] if key in filter_dict: filter_dict[key].append(value) else: value_list = [value] filter_dict[key] = value_list updated_abouts = inventory_filter(abouts, filter_dict) else: updated_abouts = abouts # Do not write the output if one of the ABOUT files has duplicated key names dup_error_msg = u'Duplicated key name(s)' halt_output = False for err in errors: if dup_error_msg in err.message: halt_output = True break if not halt_output: write_errors = model.write_output(updated_abouts, output, format, mapping_output) for err in write_errors: errors.append(err) else: msg = u'Duplicated key names are not supported.\n' + \ 'Please correct and re-run.' print(msg) error_count = 0 for e in errors: # Only count as warning/error if CRITICAL, ERROR and WARNING if e.severity > 20: error_count = error_count + 1 log_errors(errors, error_count, quiet, verbose, os.path.dirname(output)) click.echo(' %(error_count)d errors or warnings detected.' % locals()) sys.exit(0)