def generate_and_save(abouts, output_location, template_loc=None, variables=None): """ Generate an attribution text from an `abouts` list of About objects, a `template_loc` template file location and a `variables` optional dict of extra variables. Save the generated attribution text in the `output_location` file. Return a list of Error objects if any. """ errors = [] # Parse license_expression and save to the license list for about in abouts: if not about.license_expression.value: continue special_char_in_expression, lic_list = parse_license_expression(about.license_expression.value) if special_char_in_expression: msg = (u"The following character(s) cannot be in the license_expression: " + str(special_char_in_expression)) errors.append(Error(ERROR, msg)) rendering_error, rendered = generate_from_file( abouts, template_loc=template_loc, variables=variables ) if rendering_error: errors.append(rendering_error) if rendered: output_location = add_unc(output_location) with io.open(output_location, 'w', encoding='utf-8') as of: of.write(rendered) return errors
def generate_and_save(abouts, license_dict, output_location, min_license_score=0, template_loc=None, variables=None): """ Generate an attribution text from an `abouts` list of About objects, a `template_loc` template file location and a `variables` optional dict of extra variables. Save the generated attribution text in the `output_location` file. Return a list of Error objects if any. """ errors = [] rendering_error, rendered = generate_from_file( abouts, license_dict, min_license_score=min_license_score, template_loc=template_loc, variables=variables) if rendering_error: errors.append(rendering_error) if rendered: output_location = add_unc(output_location) with io.open(output_location, 'w', encoding='utf-8') as of: of.write(rendered) return errors, rendered
def check_duplicated_columns(location): """ Return a list of errors for duplicated column names in a CSV file at location. """ location = add_unc(location) with codecs.open(location, 'rb', encoding='utf-8-sig', errors='replace') as csvfile: reader = csv.reader(csvfile) columns = next(reader) columns = [col for col in columns] seen = set() dupes = OrderedDict() for col in columns: c = col.lower() if c in seen: if c in dupes: dupes[c].append(col) else: dupes[c] = [col] seen.add(c.lower()) errors = [] if dupes: dup_msg = [] for name, names in dupes.items(): names = u', '.join(names) msg = '%(name)s with %(names)s' % locals() dup_msg.append(msg) dup_msg = u', '.join(dup_msg) msg = ('Duplicated column name(s): %(dup_msg)s\n' % locals() + 'Please correct the input and re-run.') errors.append(Error(ERROR, msg)) return unique(errors)
def check_duplicated_columns(location): """ Return a list of errors for duplicated column names in a CSV file at location. """ location = add_unc(location) # FIXME: why ignore errors? with codecs.open(location, 'rb', encoding='utf-8', errors='ignore') as csvfile: reader = csv.reader(csvfile) columns = next(reader) columns = [col for col in columns] seen = set() dupes = OrderedDict() for col in columns: c = col.lower() if c in seen: if c in dupes: dupes[c].append(col) else: dupes[c] = [col] seen.add(c.lower()) errors = [] if dupes: dup_msg = [] for name, names in dupes.items(): names = u', '.join(names) msg = '%(name)s with %(names)s' % locals() dup_msg.append(msg) dup_msg = u', '.join(dup_msg) msg = ('Duplicated column name(s): %(dup_msg)s\n' % locals() + 'Please correct the input and re-run.') errors.append(Error(ERROR, msg)) return errors
def generate_from_file(abouts, template_loc=None): """ Generate and return attribution string from a list of About objects and a template location. """ if not template_loc: template_loc = default_template template_loc = add_unc(template_loc) with codecs.open(template_loc, 'rb', encoding='utf-8') as tplf: tpls = tplf.read() return generate(abouts, template_string=tpls)
def generate_from_file(abouts, template_loc=None, vartext_dict=None): """ Generate and return attribution string from a list of About objects and a template location. """ if not template_loc: template_loc = default_template template_loc = add_unc(template_loc) with codecs.open(template_loc, 'rb', encoding='utf-8') as tplf: tpls = tplf.read() return generate(abouts, template_string=tpls, vartext_dict=vartext_dict)
def test_PathField_check_location(self): test_file = 'license.LICENSE' field = model.PathField(name='f', value=test_file, present=True) base_dir = get_test_loc('test_model/base_dir') errors = field.validate(base_dir=base_dir) expected_errrors = [] assert expected_errrors == errors result = field.value[test_file] expected = add_unc(posixpath.join(to_posix(base_dir), test_file)) assert expected == result
def extract_test_loc(path, extract_func=extract_zip): """ Given an archive file identified by a path relative to a test files directory, return a new temp directory where the archive file has been extracted using extract_func. """ archive = get_test_loc(path) if on_windows: target_dir = add_unc(get_temp_dir()) else: target_dir = get_temp_dir() extract_func(archive, target_dir) return target_dir
def extract_test_loc(path, extract_func=extract_zip): """ Given an archive file identified by a path relative to a test files directory, return a new temp directory where the archive file has been extracted using extract_func. """ archive = get_test_loc(path) if on_windows: target_dir = add_unc(get_temp_dir()) else: target_dir = get_temp_dir() extract_func(archive, target_dir) return target_dir
def generate_from_file(abouts, template_loc=DEFAULT_TEMPLATE_FILE, variables=None): """ Generate an attribution text from an `abouts` list of About objects, a `template_loc` template file location and a `variables` optional dict of extra variables. Return a tuple of (error, attribution text) where error is an Error object or None and attribution text is the generated text or None. """ template_loc = add_unc(template_loc) with io.open(template_loc, encoding='utf-8') as tplf: tpls = tplf.read() return generate(abouts, template=tpls, variables=variables)
def generate_and_save(abouts, output_location, use_mapping=False, template_loc=None, inventory_location=None): """ Generate attribution file using the `abouts` list of About object at `output_location`. Use the optional `template_loc` custom temaplte or a default template. Optionally filter `abouts` object based on the inventory JSON or CSV at `inventory_location`. Optionally use the mapping.config file is `use_mapping` is True """ updated_abouts = [] lstrip_afp = [] afp_list = [] not_match_path = [] errors = [] if not inventory_location: updated_abouts = abouts # Do the following if an filter list (inventory_location) is provided else: if not exists(inventory_location): # FIXME: this message does not make sense msg = (u'"INVENTORY_LOCATION" does not exist. Generation halted.') errors.append(Error(ERROR, msg)) return errors if inventory_location.endswith('.csv') or inventory_location.endswith( '.json'): # FIXME: we should use the same inventory lodaing that we use everywhere!!!! try: # Return a list which contains only the about file path about_list = attributecode.util.get_about_file_path( inventory_location, use_mapping=use_mapping) # FIXME: why catching all exceptions? except Exception: # 'about_file_path' key/column doesn't exist msg = u"The required key: 'about_file_path' does not exist. Generation halted." errors.append(Error(ERROR, msg)) return errors else: # FIXME: this message does not make sense msg = u'Only .csv and .json are supported for the "INVENTORY_LOCATION". Generation halted.' errors.append(Error(ERROR, msg)) return errors for afp in about_list: lstrip_afp.append(afp.lstrip('/')) # return a list of paths that point all to .ABOUT files about_files_list = as_about_paths(lstrip_afp) # Collect all the about_file_path for about in abouts: afp_list.append(about.about_file_path) # Get the not matching list if any for fp in about_files_list: if not fp in afp_list: not_match_path.append(fp) if not_match_path: if len(not_match_path) == len(about_files_list): msg = "None of the paths in the provided 'inventory_location' match with the 'LOCATION'." errors.append(Error(ERROR, msg)) return errors else: for path in not_match_path: msg = 'Path: ' + path + ' cannot be found.' errors.append(Error(ERROR, msg)) for about in abouts: for fp in about_files_list: if about.about_file_path == fp: updated_abouts.append(about) # Parse license_expression and save to the license list for about in updated_abouts: if about.license_expression.value: special_char_in_expression, lic_list = parse_license_expression( about.license_expression.value) if special_char_in_expression: msg = ( u"The following character(s) cannot be in the licesne_expression: " + str(special_char_in_expression)) errors.append(Error(ERROR, msg)) else: about.license.value = lic_list rendered = generate_from_file(updated_abouts, template_loc=template_loc) if rendered: output_location = add_unc(output_location) with codecs.open(output_location, 'wb', encoding='utf-8') as of: of.write(rendered) return errors
def generate_and_save(abouts, output_location, use_mapping=False, mapping_file=None, template_loc=None, inventory_location=None, vartext=None): """ Generate attribution file using the `abouts` list of About object at `output_location`. Optionally use the mapping.config file if `use_mapping` is True. Optionally use the custom mapping file if mapping_file is set. Use the optional `template_loc` custom temaplte or a default template. Optionally filter `abouts` object based on the inventory JSON or CSV at `inventory_location`. """ updated_abouts = [] lstrip_afp = [] afp_list = [] not_match_path = [] errors = [] vartext_dict = {} if not inventory_location: updated_abouts = abouts # Do the following if an filter list (inventory_location) is provided else: if not exists(inventory_location): # FIXME: this message does not make sense msg = (u'"INVENTORY_LOCATION" does not exist. Generation halted.') errors.append(Error(ERROR, msg)) return errors if inventory_location.endswith('.csv') or inventory_location.endswith('.json'): # FIXME: we should use the same inventory lodaing that we use everywhere!!!! try: # Return a list which contains only the about file path about_list = attributecode.util.get_about_file_path( inventory_location, use_mapping=use_mapping, mapping_file=mapping_file) # FIXME: why catching all exceptions? except Exception: # 'about_file_path' key/column doesn't exist msg = u"The required key: 'about_file_path' does not exist. Generation halted." errors.append(Error(ERROR, msg)) return errors else: # FIXME: this message does not make sense msg = u'Only .csv and .json are supported for the "INVENTORY_LOCATION". Generation halted.' errors.append(Error(ERROR, msg)) return errors for afp in about_list: lstrip_afp.append(afp.lstrip('/')) # return a list of paths that point all to .ABOUT files about_files_list = as_about_paths(lstrip_afp) # Collect all the about_file_path for about in abouts: afp_list.append(about.about_file_path) # Get the not matching list if any for fp in about_files_list: if not fp in afp_list: not_match_path.append(fp) if not_match_path: if len(not_match_path) == len(about_files_list): msg = "None of the paths in the provided 'inventory_location' match with the 'LOCATION'." errors.append(Error(ERROR, msg)) return errors else: for path in not_match_path: msg = 'Path: ' + path + ' cannot be found.' errors.append(Error(ERROR, msg)) for about in abouts: for fp in about_files_list: if about.about_file_path == fp: updated_abouts.append(about) # Parse license_expression and save to the license list for about in updated_abouts: if about.license_expression.value: special_char_in_expression, lic_list = parse_license_expression(about.license_expression.value) if special_char_in_expression: msg = (u"The following character(s) cannot be in the licesne_expression: " + str(special_char_in_expression)) errors.append(Error(ERROR, msg)) else: about.license_key.value = lic_list # Parse the vartext and save to the vartext dictionary if vartext: for var in vartext: key = var.partition('=')[0] value = var.partition('=')[2] vartext_dict[key] = value rendered = generate_from_file(updated_abouts, template_loc=template_loc, vartext_dict=vartext_dict) if rendered: output_location = add_unc(output_location) with codecs.open(output_location, 'wb', encoding='utf-8') as of: of.write(rendered) return errors