def get_test_loc(path):
"""
Return the location of a test file or directory given a path relative to
the testdata directory.
"""
base = to_posix(TESTDATA_DIR)
path = to_posix(path)
path = posixpath.join(base, path)
# path = to_native(path)
return path
def get_test_loc(path):
"""
Return the location of a test file or directory given a path relative to
the testdata directory.
"""
base = to_posix(TESTDATA_DIR)
path = to_posix(path)
path = posixpath.join(base, path)
# path = to_native(path)
return path
def get_test_loc(path, must_exists=True):
"""
Return the location of a test file or directory given a path relative to
the testdata directory.
"""
base = to_posix(TESTDATA_DIR)
path = to_posix(path)
path = posixpath.join(base, path)
if must_exists:
assert os.path.exists(path)
return path
def log_errors(errors, quiet, show_all, base_dir=False):
"""
Iterate of sequence of Error objects and print and log errors with
a severity superior or equal to level.
"""
logger = logging.getLogger(__name__)
handler = logging.StreamHandler()
handler.setLevel(logging.CRITICAL)
handler.setFormatter(logging.Formatter('%(levelname)s: %(message)s'))
logger.addHandler(handler)
file_logger = logging.getLogger(__name__ + '_file')
msg_format = '%(sever)s: %(message)s'
# Create error.log if problematic_error detected
if base_dir and have_problematic_error(errors):
bdir = to_posix(base_dir)
LOG_FILENAME = 'error.log'
log_path = join(bdir, LOG_FILENAME)
if exists(log_path):
os.remove(log_path)
file_handler = logging.FileHandler(log_path)
file_logger.addHandler(file_handler)
for severity, message in errors:
sever = severities[severity]
if not quiet:
if show_all:
print(msg_format % locals())
elif sever in problematic_errors:
print(msg_format % locals())
if base_dir:
# The logger will only log error if severity >= 30
file_logger.log(severity, msg_format % locals())
def test_PathField_check_location(self):
test_file = 'license.LICENSE'
field = model.PathField(name='f', value=test_file, present=True)
base_dir = get_test_loc('test_model/base_dir')
errors = field.validate(base_dir=base_dir)
expected_errrors = []
assert expected_errrors == errors
result = field.value[test_file]
expected = add_unc(posixpath.join(to_posix(base_dir), test_file))
assert expected == result
def log_errors(errors, err_count, quiet, verbose, base_dir=False):
"""
Iterate of sequence of Error objects and print and log errors with
a severity superior or equal to level.
"""
logger = logging.getLogger(__name__)
handler = logging.StreamHandler()
handler.setLevel(logging.CRITICAL)
handler.setFormatter(logging.Formatter('%(levelname)s: %(message)s'))
logger.addHandler(handler)
file_logger = logging.getLogger(__name__ + '_file')
msg_format = '%(sever)s: %(message)s'
# Create error.log if problematic_error detected
if base_dir and have_problematic_error(errors):
bdir = to_posix(base_dir)
LOG_FILENAME = 'error.log'
log_path = join(bdir, LOG_FILENAME)
if exists(log_path):
os.remove(log_path)
f = open(log_path, "a")
error_msg = str(err_count) + u" errors or warnings detected."
f.write(error_msg)
file_handler = logging.FileHandler(log_path)
file_logger.addHandler(file_handler)
for severity, message in errors:
sever = severities[severity]
if not quiet:
if verbose:
print(msg_format % locals())
elif sever in problematic_errors:
print(msg_format % locals())
if base_dir:
# The logger will only log error for severity >= 30
file_logger.log(severity, msg_format % locals())
def generate(location,
base_dir,
android=None,
reference_dir=None,
fetch_license=False):
"""
Load ABOUT data from a CSV inventory at `location`. Write ABOUT files to
base_dir. Return errors and about objects.
"""
not_exist_errors = []
notice_dict = {}
api_url = ''
api_key = ''
gen_license = False
# FIXME: use two different arguments: key and url
# Check if the fetch_license contains valid argument
if fetch_license:
# Strip the ' and " for api_url, and api_key from input
api_url = fetch_license[0].strip("'").strip('"')
api_key = fetch_license[1].strip("'").strip('"')
gen_license = True
# TODO: WHY use posix??
bdir = to_posix(base_dir)
errors, abouts = load_inventory(location=location,
base_dir=bdir,
reference_dir=reference_dir)
if gen_license:
license_dict, err = model.pre_process_and_fetch_license_dict(
abouts, api_url, api_key)
if err:
for e in err:
# Avoid having same error multiple times
if not e in errors:
errors.append(e)
for about in abouts:
if about.about_file_path.startswith('/'):
about.about_file_path = about.about_file_path.lstrip('/')
dump_loc = join(bdir, about.about_file_path.lstrip('/'))
# The following code is to check if there is any directory ends with spaces
split_path = about.about_file_path.split('/')
dir_endswith_space = False
for segment in split_path:
if segment.endswith(' '):
msg = (
u'File path : '
u'%(dump_loc)s '
u'contains directory name ends with spaces which is not '
u'allowed. Generation skipped.' % locals())
errors.append(Error(ERROR, msg))
dir_endswith_space = True
break
if dir_endswith_space:
# Continue to work on the next about object
continue
try:
# Generate value for 'about_resource' if it does not exist
if not about.about_resource.value:
about.about_resource.value = OrderedDict()
about_resource_value = ''
if about.about_file_path.endswith('/'):
about_resource_value = u'.'
else:
about_resource_value = basename(about.about_file_path)
about.about_resource.value[about_resource_value] = None
about.about_resource.present = True
# Check for the existence of the 'about_resource'
# If the input already have the 'about_resource' field, it will
# be validated when creating the about object
loc = util.to_posix(dump_loc)
about_file_loc = loc
path = join(dirname(util.to_posix(about_file_loc)),
about_resource_value)
if not exists(path):
path = util.to_posix(path.strip(UNC_PREFIX_POSIX))
path = normpath(path)
msg = (u'Field about_resource: '
u'%(path)s '
u'does not exist' % locals())
not_exist_errors.append(msg)
if gen_license:
# Write generated LICENSE file
license_key_name_context_url_list = about.dump_lic(
dump_loc, license_dict)
if license_key_name_context_url_list:
# use value not "presence"
if not about.license_file.present:
about.license_file.value = OrderedDict()
for lic_key, lic_name, lic_context, lic_url in license_key_name_context_url_list:
gen_license_name = lic_key + u'.LICENSE'
about.license_file.value[
gen_license_name] = lic_context
about.license_file.present = True
if not about.license_name.present:
about.license_name.value.append(lic_name)
if not about.license_url.present:
about.license_url.value.append(lic_url)
if about.license_url.value:
about.license_url.present = True
if about.license_name.value:
about.license_name.present = True
about.dump(dump_loc)
if android:
"""
Create MODULE_LICENSE_XXX and get context to create NOTICE file
follow the standard from Android Open Source Project
"""
import os
parent_path = os.path.dirname(util.to_posix(dump_loc))
about.android_module_license(parent_path)
notice_path, notice_context = about.android_notice(parent_path)
if notice_path in notice_dict.keys():
notice_dict[notice_path] += '\n\n' + notice_context
else:
notice_dict[notice_path] = notice_context
for e in not_exist_errors:
errors.append(Error(INFO, e))
except Exception as e:
# only keep the first 100 char of the exception
# TODO: truncated errors are likely making diagnotics harder
emsg = repr(e)[:100]
msg = (u'Failed to write .ABOUT file at : '
u'%(dump_loc)s '
u'with error: %(emsg)s' % locals())
errors.append(Error(ERROR, msg))
if android:
# Check if there is already a NOTICE file present
for path in notice_dict.keys():
if os.path.exists(path):
msg = (u'NOTICE file already exist at: %s' % path)
errors.append(Error(ERROR, msg))
else:
about.dump_android_notice(path, notice_dict[path])
return unique(errors), abouts
def load_inventory(location, base_dir, reference_dir=None):
"""
Load the inventory file at `location` for ABOUT and LICENSE files stored in
the `base_dir`. Return a list of errors and a list of About objects
validated against the `base_dir`.
Optionally use `reference_dir` as the directory location of extra reference
license and notice files to reuse.
"""
errors = []
abouts = []
base_dir = util.to_posix(base_dir)
# FIXME: do not mix up CSV and JSON
if location.endswith('.csv'):
# FIXME: this should not be done here.
dup_cols_err = check_duplicated_columns(location)
if dup_cols_err:
errors.extend(dup_cols_err)
return errors, abouts
inventory = util.load_csv(location)
else:
inventory = util.load_json(location)
try:
# FIXME: this should not be done here.
dup_about_resource_err = check_duplicated_about_resource(inventory)
if dup_about_resource_err:
errors.extend(dup_about_resource_err)
return errors, abouts
newline_in_file = check_newline_in_file_field(inventory)
if newline_in_file:
errors.extend(newline_in_file)
return errors, abouts
except Exception as e:
# TODO: why catch ALL Exception
msg = "The essential field 'about_resource' is not found in the <input>"
errors.append(Error(CRITICAL, msg))
return errors, abouts
for i, fields in enumerate(inventory):
# check does the input contains the required fields
required_fields = model.About.required_fields
for f in required_fields:
if f not in fields:
msg = "Required field: %(f)r not found in the <input>" % locals(
)
errors.append(Error(ERROR, msg))
return errors, abouts
afp = fields.get(model.About.ABOUT_RESOURCE_ATTR)
# FIXME: this should not be a failure condition
if not afp or not afp.strip():
msg = 'Empty column: %(afp)r. Cannot generate .ABOUT file.' % locals(
)
errors.append(Error(ERROR, msg))
continue
else:
afp = util.to_posix(afp)
loc = join(base_dir, afp)
about = model.About(about_file_path=afp)
about.location = loc
# Update value for 'about_resource'
# keep only the filename or '.' if it's a directory
if 'about_resource' in fields:
updated_resource_value = u''
resource_path = fields['about_resource']
if resource_path.endswith(u'/'):
updated_resource_value = u'.'
else:
updated_resource_value = basename(resource_path)
fields['about_resource'] = updated_resource_value
ld_errors = about.load_dict(
fields,
base_dir,
running_inventory=False,
reference_dir=reference_dir,
)
"""
# 'about_resource' field will be generated during the process.
# No error need to be raise for the missing 'about_resource'.
for e in ld_errors:
if e.message == 'Field about_resource is required':
ld_errors.remove(e)
"""
for e in ld_errors:
if not e in errors:
errors.extend(ld_errors)
abouts.append(about)
return unique(errors), abouts
def test_to_posix_from_posix(self):
test = r'/this/that'
expected = '/this/that'
result = util.to_posix(test)
assert expected == result
def test_to_posix_from_win(self):
test = r'c:\this\that'
expected = 'c:/this/that'
result = util.to_posix(test)
assert expected == result
def test_to_posix_from_mixed(self):
test = r'/this/that\this'
expected = '/this/that/this'
result = util.to_posix(test)
assert expected == result
def generate(location, base_dir, license_notice_text_location=None,
fetch_license=False, policy=None, conf_location=None,
with_empty=False, with_absent=False, use_mapping=False, mapping_file=None):
"""
Load ABOUT data from a CSV inventory at `location`. Write ABOUT files to
base_dir using policy flags and configuration file at conf_location.
Policy defines which action to take for merging or overwriting fields and
files. Return errors and about objects.
"""
not_exist_errors = []
api_url = ''
api_key = ''
gen_license = False
# Check if the fetch_license contains valid argument
if fetch_license:
# Strip the ' and " for api_url, and api_key from input
api_url = fetch_license[0].strip("'").strip('"')
api_key = fetch_license[1].strip("'").strip('"')
gen_license = True
bdir = to_posix(base_dir)
errors, abouts = load_inventory(
location=location,
base_dir=bdir,
license_notice_text_location=license_notice_text_location,
use_mapping=use_mapping,
mapping_file=mapping_file)
if gen_license:
license_dict, err = model.pre_process_and_fetch_license_dict(abouts, api_url, api_key)
if err:
for e in err:
# Avoid having same error multiple times
if not e in errors:
errors.append(e)
for about in abouts:
if about.about_file_path.startswith('/'):
about.about_file_path = about.about_file_path.lstrip('/')
dump_loc = join(bdir, about.about_file_path.lstrip('/'))
# The following code is to check if there is any directory ends with spaces
split_path = about.about_file_path.split('/')
dir_endswith_space = False
for segment in split_path:
if segment.endswith(' '):
msg = (u'File path : '
u'%(dump_loc)s '
u'contains directory name ends with spaces which is not '
u'allowed. Generation skipped.' % locals())
errors.append(Error(ERROR, msg))
dir_endswith_space = True
break
if dir_endswith_space:
# Continue to work on the next about object
continue
try:
# Generate value for 'about_resource' if it does not exist
if not about.about_resource.value:
about.about_resource.value = OrderedDict()
about_resource_value = ''
if about.about_file_path.endswith('/'):
about_resource_value = u'.'
else:
about_resource_value = basename(about.about_file_path)
about.about_resource.value[about_resource_value] = None
about.about_resource.present = True
# Check for the existence of the 'about_resource'
# If the input already have the 'about_resource' field, it will
# be validated when creating the about object
loc = util.to_posix(dump_loc)
about_file_loc = loc
path = join(dirname(util.to_posix(about_file_loc)), about_resource_value)
if not exists(path):
path = util.to_posix(path.strip(UNC_PREFIX_POSIX))
path = normpath(path)
msg = (u'Field about_resource: '
u'%(path)s '
u'does not exist' % locals())
not_exist_errors.append(msg)
if gen_license:
# Write generated LICENSE file
license_key_name_context_url_list = about.dump_lic(dump_loc, license_dict)
if license_key_name_context_url_list:
# Do not help user to fill in the license name
# if not about.license_name.present:
# about.license_name.value = lic_name
# about.license_name.present = True
if not about.license_file.present:
about.license_file.value = OrderedDict()
for lic_key, lic_name, lic_context, lic_url in license_key_name_context_url_list:
gen_license_name = lic_key + u'.LICENSE'
about.license_file.value[gen_license_name] = lic_context
about.license_file.present = True
if not about.license_name.present:
about.license_name.value.append(lic_name)
if not about.license_url.present:
about.license_url.value.append(lic_url)
if about.license_url.value:
about.license_url.present = True
if about.license_name.value:
about.license_name.present = True
# Write the ABOUT files
about.dump(dump_loc, use_mapping=use_mapping, mapping_file=mapping_file, with_empty=with_empty, with_absent=with_absent)
for e in not_exist_errors:
errors.append(Error(INFO, e))
except Exception as e:
# only keep the first 100 char of the exception
emsg = repr(e)[:100]
msg = (u'Failed to write .ABOUT file at : '
u'%(dump_loc)s '
u'with error: %(emsg)s' % locals())
errors.append(Error(ERROR, msg))
dedup_errors = deduplicate(errors)
return dedup_errors, abouts
def load_inventory(location, base_dir, license_notice_text_location=None,
use_mapping=False, mapping_file=None):
"""
Load the inventory file at `location` for ABOUT and LICENSE files
stored in the `base_dir`. Return a list of errors and a list of
About objects validated against the base_dir.
Optionally use `license_notice_text_location` as the location of
license and notice texts.
Optionally use mappings for field names if `use_mapping` is True
or a custom mapping_file if provided.
"""
errors = []
abouts = []
base_dir = util.to_posix(base_dir)
if location.endswith('.csv'):
dup_cols_err = check_duplicated_columns(location)
if dup_cols_err:
errors.extend(dup_cols_err)
return errors, abouts
inventory = util.load_csv(location, use_mapping, mapping_file)
else:
inventory = util.load_json(location, use_mapping, mapping_file)
try:
dup_about_paths_err = check_duplicated_about_file_path(inventory)
if dup_about_paths_err:
errors.extend(dup_about_paths_err)
return errors, abouts
except:
msg = (
"The essential field 'about_file_path' is not found.\n"
"Use the --mapping or --mapping-file option to map the "
"input keys and verify the mapping information are correct.\n"
"OR correct the column names in the <input>"
)
errors.append(Error(CRITICAL, msg))
return errors, abouts
for i, fields in enumerate(inventory):
# check does the input contains the required fields
required_fields = model.About.required_fields
for f in required_fields:
if f not in fields:
msg = (
"Required column: %(f)r not found.\n"
"Use the --mapping or --mapping-file option to map the "
"input keys and verify the mapping information are correct.\n"
"OR correct the column names in the <input>"
) % locals()
errors.append(Error(ERROR, msg))
return errors, abouts
afp = fields.get(model.About.about_file_path_attr)
if not afp or not afp.strip():
msg = 'Empty column: %(afp)r. Cannot generate .ABOUT file.' % locals()
errors.append(Error(ERROR, msg))
continue
else:
afp = util.to_posix(afp)
loc = join(base_dir, afp)
about = model.About(about_file_path=afp)
about.location = loc
running_inventory = False
ld_errors = about.load_dict(fields, base_dir, running_inventory,
use_mapping, mapping_file, license_notice_text_location,
with_empty=False)
# 'about_resource' field will be generated during the process.
# No error need to be raise for the missing 'about_resource'.
for e in ld_errors:
if e.message == 'Field about_resource is required':
ld_errors.remove(e)
for e in ld_errors:
if not e in errors:
errors.extend(ld_errors)
abouts.append(about)
return errors, abouts
def test_to_posix_from_posix(self):
test = r'/this/that'
expected = '/this/that'
result = util.to_posix(test)
assert expected == result
def test_to_posix_from_win(self):
test = r'c:\this\that'
expected = 'c:/this/that'
result = util.to_posix(test)
assert expected == result
def test_to_posix_from_mixed(self):
test = r'/this/that\this'
expected = '/this/that/this'
result = util.to_posix(test)
assert expected == result
