def check_csv(expected, result, regen=False, fix_cell_linesep=False): """ Assert that the contents of two CSV files locations `expected` and `result` are equal. """ if regen: shutil.copyfile(result, expected) expected = sorted([sorted(d.items()) for d in load_csv(expected)]) result = [d.items() for d in load_csv(result)] if fix_cell_linesep: result = [list(fix_crlf(items)) for items in result] result = sorted(sorted(items) for items in result) assert expected == result
def test_load_csv_utf_8(self): test_file = get_test_loc('test_util/csv/test_utf8.csv') expected = [ OrderedDict([(u'about_resource', u'/myFile'), (u'name', u'\u540d')]) ] result = util.load_csv(test_file) assert expected == result
def test_load_csv_does_convert_column_names_to_lowercase(self): test_file = get_test_loc('test_util/csv/about_key_with_upper_case.csv') expected = [ OrderedDict([('about_file', 'about.ABOUT'), ('about_resource', '.'), ('name', 'ABOUT tool'), ('version', '0.8.1')]) ] result = util.load_csv(test_file) assert expected == result
def test_load_csv_load_rows(self): test_file = get_test_loc('test_util/csv/about.csv') expected = [ OrderedDict([('about_file', 'about.ABOUT'), ('about_resource', '.'), ('name', 'ABOUT tool'), ('version', '0.8.1')]) ] result = util.load_csv(test_file) assert expected == result
def test_load_csv_does_not_convert_column_names_to_lowercase(self): test_file = get_test_loc('util/about_key_with_upper_case.csv') expected = [OrderedDict( [('about_file', 'about.ABOUT'), ('about_resource', '.'), ('nAme', 'ABOUT tool'), ('Version', '0.8.1')]) ] result = util.load_csv(test_file) assert expected == result
def test_load_csv_without_mapping(self): test_file = get_test_loc('util/about.csv') expected = [OrderedDict([ ('about_file', 'about.ABOUT'), ('about_resource', '.'), ('name', 'ABOUT tool'), ('version', '0.8.1')]) ] result = util.load_csv(test_file) assert expected == result
def test_load_csv_with_conf(self): location = get_test_loc('test_util/load/simple_sample.csv') base_dir = get_temp_dir() configuration = get_test_loc('test_util/load/key.config') inventory = util.load_csv(location, configuration) expected = [ OrderedDict([('name', 'cryptohash-sha256'), ('version', 'v 0.11.100.1'), ('license_expression', 'bsd-new and mit'), ('resource', '/project/cryptohash-sha256')]), OrderedDict([('name', 'some_component'), ('version', 'v 0.0.1'), ('license_expression', 'mit'), ('resource', '/project/some_component')]) ] assert inventory == expected
def load_inventory(location, base_dir, reference_dir=None): """ Load the inventory file at `location` for ABOUT and LICENSE files stored in the `base_dir`. Return a list of errors and a list of About objects validated against the `base_dir`. Optionally use `reference_dir` as the directory location of extra reference license and notice files to reuse. """ errors = [] abouts = [] base_dir = util.to_posix(base_dir) # FIXME: do not mix up CSV and JSON if location.endswith('.csv'): # FIXME: this should not be done here. dup_cols_err = check_duplicated_columns(location) if dup_cols_err: errors.extend(dup_cols_err) return errors, abouts inventory = util.load_csv(location) else: inventory = util.load_json(location) try: # FIXME: this should not be done here. dup_about_resource_err = check_duplicated_about_resource(inventory) if dup_about_resource_err: errors.extend(dup_about_resource_err) return errors, abouts newline_in_file = check_newline_in_file_field(inventory) if newline_in_file: errors.extend(newline_in_file) return errors, abouts except Exception as e: # TODO: why catch ALL Exception msg = "The essential field 'about_resource' is not found in the <input>" errors.append(Error(CRITICAL, msg)) return errors, abouts for i, fields in enumerate(inventory): # check does the input contains the required fields required_fields = model.About.required_fields for f in required_fields: if f not in fields: msg = "Required field: %(f)r not found in the <input>" % locals( ) errors.append(Error(ERROR, msg)) return errors, abouts afp = fields.get(model.About.ABOUT_RESOURCE_ATTR) # FIXME: this should not be a failure condition if not afp or not afp.strip(): msg = 'Empty column: %(afp)r. Cannot generate .ABOUT file.' % locals( ) errors.append(Error(ERROR, msg)) continue else: afp = util.to_posix(afp) loc = join(base_dir, afp) about = model.About(about_file_path=afp) about.location = loc # Update value for 'about_resource' # keep only the filename or '.' if it's a directory if 'about_resource' in fields: updated_resource_value = u'' resource_path = fields['about_resource'] if resource_path.endswith(u'/'): updated_resource_value = u'.' else: updated_resource_value = basename(resource_path) fields['about_resource'] = updated_resource_value ld_errors = about.load_dict( fields, base_dir, running_inventory=False, reference_dir=reference_dir, ) """ # 'about_resource' field will be generated during the process. # No error need to be raise for the missing 'about_resource'. for e in ld_errors: if e.message == 'Field about_resource is required': ld_errors.remove(e) """ for e in ld_errors: if not e in errors: errors.extend(ld_errors) abouts.append(about) return unique(errors), abouts
def load_inventory(location, base_dir, license_notice_text_location=None, use_mapping=False, mapping_file=None): """ Load the inventory file at `location` for ABOUT and LICENSE files stored in the `base_dir`. Return a list of errors and a list of About objects validated against the base_dir. Optionally use `license_notice_text_location` as the location of license and notice texts. Optionally use mappings for field names if `use_mapping` is True or a custom mapping_file if provided. """ errors = [] abouts = [] base_dir = util.to_posix(base_dir) if location.endswith('.csv'): dup_cols_err = check_duplicated_columns(location) if dup_cols_err: errors.extend(dup_cols_err) return errors, abouts inventory = util.load_csv(location, use_mapping, mapping_file) else: inventory = util.load_json(location, use_mapping, mapping_file) try: dup_about_paths_err = check_duplicated_about_file_path(inventory) if dup_about_paths_err: errors.extend(dup_about_paths_err) return errors, abouts except: msg = ( "The essential field 'about_file_path' is not found.\n" "Use the --mapping or --mapping-file option to map the " "input keys and verify the mapping information are correct.\n" "OR correct the column names in the <input>" ) errors.append(Error(CRITICAL, msg)) return errors, abouts for i, fields in enumerate(inventory): # check does the input contains the required fields required_fields = model.About.required_fields for f in required_fields: if f not in fields: msg = ( "Required column: %(f)r not found.\n" "Use the --mapping or --mapping-file option to map the " "input keys and verify the mapping information are correct.\n" "OR correct the column names in the <input>" ) % locals() errors.append(Error(ERROR, msg)) return errors, abouts afp = fields.get(model.About.about_file_path_attr) if not afp or not afp.strip(): msg = 'Empty column: %(afp)r. Cannot generate .ABOUT file.' % locals() errors.append(Error(ERROR, msg)) continue else: afp = util.to_posix(afp) loc = join(base_dir, afp) about = model.About(about_file_path=afp) about.location = loc running_inventory = False ld_errors = about.load_dict(fields, base_dir, running_inventory, use_mapping, mapping_file, license_notice_text_location, with_empty=False) # 'about_resource' field will be generated during the process. # No error need to be raise for the missing 'about_resource'. for e in ld_errors: if e.message == 'Field about_resource is required': ld_errors.remove(e) for e in ld_errors: if not e in errors: errors.extend(ld_errors) abouts.append(about) return errors, abouts