def generate_and_save(abouts, output_location, template_loc=None, variables=None):
"""
Generate an attribution text from an `abouts` list of About objects, a
`template_loc` template file location and a `variables` optional
dict of extra variables. Save the generated attribution text in the
`output_location` file.
Return a list of Error objects if any.
"""
errors = []
# Parse license_expression and save to the license list
for about in abouts:
if not about.license_expression.value:
continue
special_char_in_expression, lic_list = parse_license_expression(about.license_expression.value)
if special_char_in_expression:
msg = (u"The following character(s) cannot be in the license_expression: " +
str(special_char_in_expression))
errors.append(Error(ERROR, msg))
rendering_error, rendered = generate_from_file(
abouts,
template_loc=template_loc,
variables=variables
)
if rendering_error:
errors.append(rendering_error)
if rendered:
output_location = add_unc(output_location)
with io.open(output_location, 'w', encoding='utf-8') as of:
of.write(rendered)
return errors
def generate_and_save(abouts,
license_dict,
output_location,
min_license_score=0,
template_loc=None,
variables=None):
"""
Generate an attribution text from an `abouts` list of About objects, a
`template_loc` template file location and a `variables` optional
dict of extra variables. Save the generated attribution text in the
`output_location` file.
Return a list of Error objects if any.
"""
errors = []
rendering_error, rendered = generate_from_file(
abouts,
license_dict,
min_license_score=min_license_score,
template_loc=template_loc,
variables=variables)
if rendering_error:
errors.append(rendering_error)
if rendered:
output_location = add_unc(output_location)
with io.open(output_location, 'w', encoding='utf-8') as of:
of.write(rendered)
return errors, rendered
def check_duplicated_columns(location):
"""
Return a list of errors for duplicated column names in a CSV file
at location.
"""
location = add_unc(location)
with codecs.open(location, 'rb', encoding='utf-8-sig',
errors='replace') as csvfile:
reader = csv.reader(csvfile)
columns = next(reader)
columns = [col for col in columns]
seen = set()
dupes = OrderedDict()
for col in columns:
c = col.lower()
if c in seen:
if c in dupes:
dupes[c].append(col)
else:
dupes[c] = [col]
seen.add(c.lower())
errors = []
if dupes:
dup_msg = []
for name, names in dupes.items():
names = u', '.join(names)
msg = '%(name)s with %(names)s' % locals()
dup_msg.append(msg)
dup_msg = u', '.join(dup_msg)
msg = ('Duplicated column name(s): %(dup_msg)s\n' % locals() +
'Please correct the input and re-run.')
errors.append(Error(ERROR, msg))
return unique(errors)
def check_duplicated_columns(location):
"""
Return a list of errors for duplicated column names in a CSV file
at location.
"""
location = add_unc(location)
# FIXME: why ignore errors?
with codecs.open(location, 'rb', encoding='utf-8', errors='ignore') as csvfile:
reader = csv.reader(csvfile)
columns = next(reader)
columns = [col for col in columns]
seen = set()
dupes = OrderedDict()
for col in columns:
c = col.lower()
if c in seen:
if c in dupes:
dupes[c].append(col)
else:
dupes[c] = [col]
seen.add(c.lower())
errors = []
if dupes:
dup_msg = []
for name, names in dupes.items():
names = u', '.join(names)
msg = '%(name)s with %(names)s' % locals()
dup_msg.append(msg)
dup_msg = u', '.join(dup_msg)
msg = ('Duplicated column name(s): %(dup_msg)s\n' % locals() +
'Please correct the input and re-run.')
errors.append(Error(ERROR, msg))
return errors
def generate_from_file(abouts, template_loc=None):
"""
Generate and return attribution string from a list of About objects and a
template location.
"""
if not template_loc:
template_loc = default_template
template_loc = add_unc(template_loc)
with codecs.open(template_loc, 'rb', encoding='utf-8') as tplf:
tpls = tplf.read()
return generate(abouts, template_string=tpls)
def generate_from_file(abouts, template_loc=None, vartext_dict=None):
"""
Generate and return attribution string from a list of About objects and a
template location.
"""
if not template_loc:
template_loc = default_template
template_loc = add_unc(template_loc)
with codecs.open(template_loc, 'rb', encoding='utf-8') as tplf:
tpls = tplf.read()
return generate(abouts, template_string=tpls, vartext_dict=vartext_dict)
def test_PathField_check_location(self):
test_file = 'license.LICENSE'
field = model.PathField(name='f', value=test_file, present=True)
base_dir = get_test_loc('test_model/base_dir')
errors = field.validate(base_dir=base_dir)
expected_errrors = []
assert expected_errrors == errors
result = field.value[test_file]
expected = add_unc(posixpath.join(to_posix(base_dir), test_file))
assert expected == result
def extract_test_loc(path, extract_func=extract_zip):
"""
Given an archive file identified by a path relative
to a test files directory, return a new temp directory where the
archive file has been extracted using extract_func.
"""
archive = get_test_loc(path)
if on_windows:
target_dir = add_unc(get_temp_dir())
else:
target_dir = get_temp_dir()
extract_func(archive, target_dir)
return target_dir
def extract_test_loc(path, extract_func=extract_zip):
"""
Given an archive file identified by a path relative
to a test files directory, return a new temp directory where the
archive file has been extracted using extract_func.
"""
archive = get_test_loc(path)
if on_windows:
target_dir = add_unc(get_temp_dir())
else:
target_dir = get_temp_dir()
extract_func(archive, target_dir)
return target_dir
def generate_from_file(abouts, template_loc=DEFAULT_TEMPLATE_FILE, variables=None):
"""
Generate an attribution text from an `abouts` list of About objects, a
`template_loc` template file location and a `variables` optional
dict of extra variables.
Return a tuple of (error, attribution text) where error is an Error object
or None and attribution text is the generated text or None.
"""
template_loc = add_unc(template_loc)
with io.open(template_loc, encoding='utf-8') as tplf:
tpls = tplf.read()
return generate(abouts, template=tpls, variables=variables)
def generate_and_save(abouts,
output_location,
use_mapping=False,
template_loc=None,
inventory_location=None):
"""
Generate attribution file using the `abouts` list of About object
at `output_location`.
Use the optional `template_loc` custom temaplte or a default template.
Optionally filter `abouts` object based on the inventory JSON or
CSV at `inventory_location`.
Optionally use the mapping.config file is `use_mapping` is True
"""
updated_abouts = []
lstrip_afp = []
afp_list = []
not_match_path = []
errors = []
if not inventory_location:
updated_abouts = abouts
# Do the following if an filter list (inventory_location) is provided
else:
if not exists(inventory_location):
# FIXME: this message does not make sense
msg = (u'"INVENTORY_LOCATION" does not exist. Generation halted.')
errors.append(Error(ERROR, msg))
return errors
if inventory_location.endswith('.csv') or inventory_location.endswith(
'.json'):
# FIXME: we should use the same inventory lodaing that we use everywhere!!!!
try:
# Return a list which contains only the about file path
about_list = attributecode.util.get_about_file_path(
inventory_location, use_mapping=use_mapping)
# FIXME: why catching all exceptions?
except Exception:
# 'about_file_path' key/column doesn't exist
msg = u"The required key: 'about_file_path' does not exist. Generation halted."
errors.append(Error(ERROR, msg))
return errors
else:
# FIXME: this message does not make sense
msg = u'Only .csv and .json are supported for the "INVENTORY_LOCATION". Generation halted.'
errors.append(Error(ERROR, msg))
return errors
for afp in about_list:
lstrip_afp.append(afp.lstrip('/'))
# return a list of paths that point all to .ABOUT files
about_files_list = as_about_paths(lstrip_afp)
# Collect all the about_file_path
for about in abouts:
afp_list.append(about.about_file_path)
# Get the not matching list if any
for fp in about_files_list:
if not fp in afp_list:
not_match_path.append(fp)
if not_match_path:
if len(not_match_path) == len(about_files_list):
msg = "None of the paths in the provided 'inventory_location' match with the 'LOCATION'."
errors.append(Error(ERROR, msg))
return errors
else:
for path in not_match_path:
msg = 'Path: ' + path + ' cannot be found.'
errors.append(Error(ERROR, msg))
for about in abouts:
for fp in about_files_list:
if about.about_file_path == fp:
updated_abouts.append(about)
# Parse license_expression and save to the license list
for about in updated_abouts:
if about.license_expression.value:
special_char_in_expression, lic_list = parse_license_expression(
about.license_expression.value)
if special_char_in_expression:
msg = (
u"The following character(s) cannot be in the licesne_expression: "
+ str(special_char_in_expression))
errors.append(Error(ERROR, msg))
else:
about.license.value = lic_list
rendered = generate_from_file(updated_abouts, template_loc=template_loc)
if rendered:
output_location = add_unc(output_location)
with codecs.open(output_location, 'wb', encoding='utf-8') as of:
of.write(rendered)
return errors
def generate_and_save(abouts, output_location, use_mapping=False, mapping_file=None,
template_loc=None, inventory_location=None, vartext=None):
"""
Generate attribution file using the `abouts` list of About object
at `output_location`.
Optionally use the mapping.config file if `use_mapping` is True.
Optionally use the custom mapping file if mapping_file is set.
Use the optional `template_loc` custom temaplte or a default template.
Optionally filter `abouts` object based on the inventory JSON or
CSV at `inventory_location`.
"""
updated_abouts = []
lstrip_afp = []
afp_list = []
not_match_path = []
errors = []
vartext_dict = {}
if not inventory_location:
updated_abouts = abouts
# Do the following if an filter list (inventory_location) is provided
else:
if not exists(inventory_location):
# FIXME: this message does not make sense
msg = (u'"INVENTORY_LOCATION" does not exist. Generation halted.')
errors.append(Error(ERROR, msg))
return errors
if inventory_location.endswith('.csv') or inventory_location.endswith('.json'):
# FIXME: we should use the same inventory lodaing that we use everywhere!!!!
try:
# Return a list which contains only the about file path
about_list = attributecode.util.get_about_file_path(
inventory_location, use_mapping=use_mapping, mapping_file=mapping_file)
# FIXME: why catching all exceptions?
except Exception:
# 'about_file_path' key/column doesn't exist
msg = u"The required key: 'about_file_path' does not exist. Generation halted."
errors.append(Error(ERROR, msg))
return errors
else:
# FIXME: this message does not make sense
msg = u'Only .csv and .json are supported for the "INVENTORY_LOCATION". Generation halted.'
errors.append(Error(ERROR, msg))
return errors
for afp in about_list:
lstrip_afp.append(afp.lstrip('/'))
# return a list of paths that point all to .ABOUT files
about_files_list = as_about_paths(lstrip_afp)
# Collect all the about_file_path
for about in abouts:
afp_list.append(about.about_file_path)
# Get the not matching list if any
for fp in about_files_list:
if not fp in afp_list:
not_match_path.append(fp)
if not_match_path:
if len(not_match_path) == len(about_files_list):
msg = "None of the paths in the provided 'inventory_location' match with the 'LOCATION'."
errors.append(Error(ERROR, msg))
return errors
else:
for path in not_match_path:
msg = 'Path: ' + path + ' cannot be found.'
errors.append(Error(ERROR, msg))
for about in abouts:
for fp in about_files_list:
if about.about_file_path == fp:
updated_abouts.append(about)
# Parse license_expression and save to the license list
for about in updated_abouts:
if about.license_expression.value:
special_char_in_expression, lic_list = parse_license_expression(about.license_expression.value)
if special_char_in_expression:
msg = (u"The following character(s) cannot be in the licesne_expression: " +
str(special_char_in_expression))
errors.append(Error(ERROR, msg))
else:
about.license_key.value = lic_list
# Parse the vartext and save to the vartext dictionary
if vartext:
for var in vartext:
key = var.partition('=')[0]
value = var.partition('=')[2]
vartext_dict[key] = value
rendered = generate_from_file(updated_abouts, template_loc=template_loc, vartext_dict=vartext_dict)
if rendered:
output_location = add_unc(output_location)
with codecs.open(output_location, 'wb', encoding='utf-8') as of:
of.write(rendered)
return errors