def get_test_loc(path): """ Return the location of a test file or directory given a path relative to the testdata directory. """ base = to_posix(TESTDATA_DIR) path = to_posix(path) path = posixpath.join(base, path) # path = to_native(path) return path
def get_test_loc(path, must_exists=True): """ Return the location of a test file or directory given a path relative to the testdata directory. """ base = to_posix(TESTDATA_DIR) path = to_posix(path) path = posixpath.join(base, path) if must_exists: assert os.path.exists(path) return path
def log_errors(errors, quiet, show_all, base_dir=False): """ Iterate of sequence of Error objects and print and log errors with a severity superior or equal to level. """ logger = logging.getLogger(__name__) handler = logging.StreamHandler() handler.setLevel(logging.CRITICAL) handler.setFormatter(logging.Formatter('%(levelname)s: %(message)s')) logger.addHandler(handler) file_logger = logging.getLogger(__name__ + '_file') msg_format = '%(sever)s: %(message)s' # Create error.log if problematic_error detected if base_dir and have_problematic_error(errors): bdir = to_posix(base_dir) LOG_FILENAME = 'error.log' log_path = join(bdir, LOG_FILENAME) if exists(log_path): os.remove(log_path) file_handler = logging.FileHandler(log_path) file_logger.addHandler(file_handler) for severity, message in errors: sever = severities[severity] if not quiet: if show_all: print(msg_format % locals()) elif sever in problematic_errors: print(msg_format % locals()) if base_dir: # The logger will only log error if severity >= 30 file_logger.log(severity, msg_format % locals())
def test_PathField_check_location(self): test_file = 'license.LICENSE' field = model.PathField(name='f', value=test_file, present=True) base_dir = get_test_loc('test_model/base_dir') errors = field.validate(base_dir=base_dir) expected_errrors = [] assert expected_errrors == errors result = field.value[test_file] expected = add_unc(posixpath.join(to_posix(base_dir), test_file)) assert expected == result
def log_errors(errors, err_count, quiet, verbose, base_dir=False): """ Iterate of sequence of Error objects and print and log errors with a severity superior or equal to level. """ logger = logging.getLogger(__name__) handler = logging.StreamHandler() handler.setLevel(logging.CRITICAL) handler.setFormatter(logging.Formatter('%(levelname)s: %(message)s')) logger.addHandler(handler) file_logger = logging.getLogger(__name__ + '_file') msg_format = '%(sever)s: %(message)s' # Create error.log if problematic_error detected if base_dir and have_problematic_error(errors): bdir = to_posix(base_dir) LOG_FILENAME = 'error.log' log_path = join(bdir, LOG_FILENAME) if exists(log_path): os.remove(log_path) f = open(log_path, "a") error_msg = str(err_count) + u" errors or warnings detected." f.write(error_msg) file_handler = logging.FileHandler(log_path) file_logger.addHandler(file_handler) for severity, message in errors: sever = severities[severity] if not quiet: if verbose: print(msg_format % locals()) elif sever in problematic_errors: print(msg_format % locals()) if base_dir: # The logger will only log error for severity >= 30 file_logger.log(severity, msg_format % locals())
def generate(location, base_dir, android=None, reference_dir=None, fetch_license=False): """ Load ABOUT data from a CSV inventory at `location`. Write ABOUT files to base_dir. Return errors and about objects. """ not_exist_errors = [] notice_dict = {} api_url = '' api_key = '' gen_license = False # FIXME: use two different arguments: key and url # Check if the fetch_license contains valid argument if fetch_license: # Strip the ' and " for api_url, and api_key from input api_url = fetch_license[0].strip("'").strip('"') api_key = fetch_license[1].strip("'").strip('"') gen_license = True # TODO: WHY use posix?? bdir = to_posix(base_dir) errors, abouts = load_inventory(location=location, base_dir=bdir, reference_dir=reference_dir) if gen_license: license_dict, err = model.pre_process_and_fetch_license_dict( abouts, api_url, api_key) if err: for e in err: # Avoid having same error multiple times if not e in errors: errors.append(e) for about in abouts: if about.about_file_path.startswith('/'): about.about_file_path = about.about_file_path.lstrip('/') dump_loc = join(bdir, about.about_file_path.lstrip('/')) # The following code is to check if there is any directory ends with spaces split_path = about.about_file_path.split('/') dir_endswith_space = False for segment in split_path: if segment.endswith(' '): msg = ( u'File path : ' u'%(dump_loc)s ' u'contains directory name ends with spaces which is not ' u'allowed. Generation skipped.' % locals()) errors.append(Error(ERROR, msg)) dir_endswith_space = True break if dir_endswith_space: # Continue to work on the next about object continue try: # Generate value for 'about_resource' if it does not exist if not about.about_resource.value: about.about_resource.value = OrderedDict() about_resource_value = '' if about.about_file_path.endswith('/'): about_resource_value = u'.' else: about_resource_value = basename(about.about_file_path) about.about_resource.value[about_resource_value] = None about.about_resource.present = True # Check for the existence of the 'about_resource' # If the input already have the 'about_resource' field, it will # be validated when creating the about object loc = util.to_posix(dump_loc) about_file_loc = loc path = join(dirname(util.to_posix(about_file_loc)), about_resource_value) if not exists(path): path = util.to_posix(path.strip(UNC_PREFIX_POSIX)) path = normpath(path) msg = (u'Field about_resource: ' u'%(path)s ' u'does not exist' % locals()) not_exist_errors.append(msg) if gen_license: # Write generated LICENSE file license_key_name_context_url_list = about.dump_lic( dump_loc, license_dict) if license_key_name_context_url_list: # use value not "presence" if not about.license_file.present: about.license_file.value = OrderedDict() for lic_key, lic_name, lic_context, lic_url in license_key_name_context_url_list: gen_license_name = lic_key + u'.LICENSE' about.license_file.value[ gen_license_name] = lic_context about.license_file.present = True if not about.license_name.present: about.license_name.value.append(lic_name) if not about.license_url.present: about.license_url.value.append(lic_url) if about.license_url.value: about.license_url.present = True if about.license_name.value: about.license_name.present = True about.dump(dump_loc) if android: """ Create MODULE_LICENSE_XXX and get context to create NOTICE file follow the standard from Android Open Source Project """ import os parent_path = os.path.dirname(util.to_posix(dump_loc)) about.android_module_license(parent_path) notice_path, notice_context = about.android_notice(parent_path) if notice_path in notice_dict.keys(): notice_dict[notice_path] += '\n\n' + notice_context else: notice_dict[notice_path] = notice_context for e in not_exist_errors: errors.append(Error(INFO, e)) except Exception as e: # only keep the first 100 char of the exception # TODO: truncated errors are likely making diagnotics harder emsg = repr(e)[:100] msg = (u'Failed to write .ABOUT file at : ' u'%(dump_loc)s ' u'with error: %(emsg)s' % locals()) errors.append(Error(ERROR, msg)) if android: # Check if there is already a NOTICE file present for path in notice_dict.keys(): if os.path.exists(path): msg = (u'NOTICE file already exist at: %s' % path) errors.append(Error(ERROR, msg)) else: about.dump_android_notice(path, notice_dict[path]) return unique(errors), abouts
def load_inventory(location, base_dir, reference_dir=None): """ Load the inventory file at `location` for ABOUT and LICENSE files stored in the `base_dir`. Return a list of errors and a list of About objects validated against the `base_dir`. Optionally use `reference_dir` as the directory location of extra reference license and notice files to reuse. """ errors = [] abouts = [] base_dir = util.to_posix(base_dir) # FIXME: do not mix up CSV and JSON if location.endswith('.csv'): # FIXME: this should not be done here. dup_cols_err = check_duplicated_columns(location) if dup_cols_err: errors.extend(dup_cols_err) return errors, abouts inventory = util.load_csv(location) else: inventory = util.load_json(location) try: # FIXME: this should not be done here. dup_about_resource_err = check_duplicated_about_resource(inventory) if dup_about_resource_err: errors.extend(dup_about_resource_err) return errors, abouts newline_in_file = check_newline_in_file_field(inventory) if newline_in_file: errors.extend(newline_in_file) return errors, abouts except Exception as e: # TODO: why catch ALL Exception msg = "The essential field 'about_resource' is not found in the <input>" errors.append(Error(CRITICAL, msg)) return errors, abouts for i, fields in enumerate(inventory): # check does the input contains the required fields required_fields = model.About.required_fields for f in required_fields: if f not in fields: msg = "Required field: %(f)r not found in the <input>" % locals( ) errors.append(Error(ERROR, msg)) return errors, abouts afp = fields.get(model.About.ABOUT_RESOURCE_ATTR) # FIXME: this should not be a failure condition if not afp or not afp.strip(): msg = 'Empty column: %(afp)r. Cannot generate .ABOUT file.' % locals( ) errors.append(Error(ERROR, msg)) continue else: afp = util.to_posix(afp) loc = join(base_dir, afp) about = model.About(about_file_path=afp) about.location = loc # Update value for 'about_resource' # keep only the filename or '.' if it's a directory if 'about_resource' in fields: updated_resource_value = u'' resource_path = fields['about_resource'] if resource_path.endswith(u'/'): updated_resource_value = u'.' else: updated_resource_value = basename(resource_path) fields['about_resource'] = updated_resource_value ld_errors = about.load_dict( fields, base_dir, running_inventory=False, reference_dir=reference_dir, ) """ # 'about_resource' field will be generated during the process. # No error need to be raise for the missing 'about_resource'. for e in ld_errors: if e.message == 'Field about_resource is required': ld_errors.remove(e) """ for e in ld_errors: if not e in errors: errors.extend(ld_errors) abouts.append(about) return unique(errors), abouts
def test_to_posix_from_posix(self): test = r'/this/that' expected = '/this/that' result = util.to_posix(test) assert expected == result
def test_to_posix_from_win(self): test = r'c:\this\that' expected = 'c:/this/that' result = util.to_posix(test) assert expected == result
def test_to_posix_from_mixed(self): test = r'/this/that\this' expected = '/this/that/this' result = util.to_posix(test) assert expected == result
def generate(location, base_dir, license_notice_text_location=None, fetch_license=False, policy=None, conf_location=None, with_empty=False, with_absent=False, use_mapping=False, mapping_file=None): """ Load ABOUT data from a CSV inventory at `location`. Write ABOUT files to base_dir using policy flags and configuration file at conf_location. Policy defines which action to take for merging or overwriting fields and files. Return errors and about objects. """ not_exist_errors = [] api_url = '' api_key = '' gen_license = False # Check if the fetch_license contains valid argument if fetch_license: # Strip the ' and " for api_url, and api_key from input api_url = fetch_license[0].strip("'").strip('"') api_key = fetch_license[1].strip("'").strip('"') gen_license = True bdir = to_posix(base_dir) errors, abouts = load_inventory( location=location, base_dir=bdir, license_notice_text_location=license_notice_text_location, use_mapping=use_mapping, mapping_file=mapping_file) if gen_license: license_dict, err = model.pre_process_and_fetch_license_dict(abouts, api_url, api_key) if err: for e in err: # Avoid having same error multiple times if not e in errors: errors.append(e) for about in abouts: if about.about_file_path.startswith('/'): about.about_file_path = about.about_file_path.lstrip('/') dump_loc = join(bdir, about.about_file_path.lstrip('/')) # The following code is to check if there is any directory ends with spaces split_path = about.about_file_path.split('/') dir_endswith_space = False for segment in split_path: if segment.endswith(' '): msg = (u'File path : ' u'%(dump_loc)s ' u'contains directory name ends with spaces which is not ' u'allowed. Generation skipped.' % locals()) errors.append(Error(ERROR, msg)) dir_endswith_space = True break if dir_endswith_space: # Continue to work on the next about object continue try: # Generate value for 'about_resource' if it does not exist if not about.about_resource.value: about.about_resource.value = OrderedDict() about_resource_value = '' if about.about_file_path.endswith('/'): about_resource_value = u'.' else: about_resource_value = basename(about.about_file_path) about.about_resource.value[about_resource_value] = None about.about_resource.present = True # Check for the existence of the 'about_resource' # If the input already have the 'about_resource' field, it will # be validated when creating the about object loc = util.to_posix(dump_loc) about_file_loc = loc path = join(dirname(util.to_posix(about_file_loc)), about_resource_value) if not exists(path): path = util.to_posix(path.strip(UNC_PREFIX_POSIX)) path = normpath(path) msg = (u'Field about_resource: ' u'%(path)s ' u'does not exist' % locals()) not_exist_errors.append(msg) if gen_license: # Write generated LICENSE file license_key_name_context_url_list = about.dump_lic(dump_loc, license_dict) if license_key_name_context_url_list: # Do not help user to fill in the license name # if not about.license_name.present: # about.license_name.value = lic_name # about.license_name.present = True if not about.license_file.present: about.license_file.value = OrderedDict() for lic_key, lic_name, lic_context, lic_url in license_key_name_context_url_list: gen_license_name = lic_key + u'.LICENSE' about.license_file.value[gen_license_name] = lic_context about.license_file.present = True if not about.license_name.present: about.license_name.value.append(lic_name) if not about.license_url.present: about.license_url.value.append(lic_url) if about.license_url.value: about.license_url.present = True if about.license_name.value: about.license_name.present = True # Write the ABOUT files about.dump(dump_loc, use_mapping=use_mapping, mapping_file=mapping_file, with_empty=with_empty, with_absent=with_absent) for e in not_exist_errors: errors.append(Error(INFO, e)) except Exception as e: # only keep the first 100 char of the exception emsg = repr(e)[:100] msg = (u'Failed to write .ABOUT file at : ' u'%(dump_loc)s ' u'with error: %(emsg)s' % locals()) errors.append(Error(ERROR, msg)) dedup_errors = deduplicate(errors) return dedup_errors, abouts
def load_inventory(location, base_dir, license_notice_text_location=None, use_mapping=False, mapping_file=None): """ Load the inventory file at `location` for ABOUT and LICENSE files stored in the `base_dir`. Return a list of errors and a list of About objects validated against the base_dir. Optionally use `license_notice_text_location` as the location of license and notice texts. Optionally use mappings for field names if `use_mapping` is True or a custom mapping_file if provided. """ errors = [] abouts = [] base_dir = util.to_posix(base_dir) if location.endswith('.csv'): dup_cols_err = check_duplicated_columns(location) if dup_cols_err: errors.extend(dup_cols_err) return errors, abouts inventory = util.load_csv(location, use_mapping, mapping_file) else: inventory = util.load_json(location, use_mapping, mapping_file) try: dup_about_paths_err = check_duplicated_about_file_path(inventory) if dup_about_paths_err: errors.extend(dup_about_paths_err) return errors, abouts except: msg = ( "The essential field 'about_file_path' is not found.\n" "Use the --mapping or --mapping-file option to map the " "input keys and verify the mapping information are correct.\n" "OR correct the column names in the <input>" ) errors.append(Error(CRITICAL, msg)) return errors, abouts for i, fields in enumerate(inventory): # check does the input contains the required fields required_fields = model.About.required_fields for f in required_fields: if f not in fields: msg = ( "Required column: %(f)r not found.\n" "Use the --mapping or --mapping-file option to map the " "input keys and verify the mapping information are correct.\n" "OR correct the column names in the <input>" ) % locals() errors.append(Error(ERROR, msg)) return errors, abouts afp = fields.get(model.About.about_file_path_attr) if not afp or not afp.strip(): msg = 'Empty column: %(afp)r. Cannot generate .ABOUT file.' % locals() errors.append(Error(ERROR, msg)) continue else: afp = util.to_posix(afp) loc = join(base_dir, afp) about = model.About(about_file_path=afp) about.location = loc running_inventory = False ld_errors = about.load_dict(fields, base_dir, running_inventory, use_mapping, mapping_file, license_notice_text_location, with_empty=False) # 'about_resource' field will be generated during the process. # No error need to be raise for the missing 'about_resource'. for e in ld_errors: if e.message == 'Field about_resource is required': ld_errors.remove(e) for e in ld_errors: if not e in errors: errors.extend(ld_errors) abouts.append(about) return errors, abouts