def main():
"""
Configuration root flattener.
"""
parser = get_parser()
options, _ = parser.parse_args()
cone_common.handle_common_options(options)
action = configroot2flat.ConeConfigroot2FlatAction(
project=options.project,
configs=options.configs,
config_wildcards=options.config_wildcards,
config_regexes=options.config_regexes)
try:
status = action.run()
if status:
action.save()
action.close()
except configroot2flat.Configroot2FlatFailed, e:
parser.error(str(e))
def main():
""" Export configurations. """
parser = OptionParser(version="%%prog %s" % VERSION)
parser.add_options(cone_common.COMMON_OPTIONS)
parser.add_option("-c", "--configuration",
dest="configs",
action="append",
help="Defines the name of the configuration for the action, can be "\
"specified multiple times to include multiple configurations.",
metavar="CONFIG",
default=[])
parser.add_option("--config-wildcard",
action="append",
dest="config_wildcards",
help="Wildcard pattern for including configurations, e.g. "\
"product_langpack_*_root.confml",
metavar="WILDCARD",
default=[])
parser.add_option("--config-regex",
action="append",
dest="config_regexes",
help="Regular expression for including configurations, e.g. "\
"product_langpack_\\d{2}_root.confml",
metavar="REGEX",
default=[])
parser.add_option("-p", "--project",
dest="project",
help="defines the location of current project. Default is the "\
"current working directory.",\
default=".",
metavar="STORAGE")
group = OptionGroup(parser, 'Export options',
'The export action is intended for exporting configurations '\
'from one project (storage) to another. A project can be a '\
'folder, a CPF or ZIP file, or a Carbon web storage URL. '\
# An ugly way to make newlines, someone should look into
# sub-classing optparse.HelpFormatter...
' '\
'Two different ways of exporting are supported: '\
' '\
'1. Exporting multiple configurations into one new project using --remote '\
' '\
'2. Exporting configurations into a number of new projects using --export-dir')
group.add_option("-r", "--remote",
dest="remote",
help="Defines the location of remote storage. All configurations included using "\
"--configuration, --config-wildcard and --config-regex are exported into "\
"the storage. If the remote storage location is not given, the default "\
"location is determined based on the first included source configuration name. "\
"E.g. 'example.confml' would be exported into 'example.cpf'",
metavar="STORAGE")
group.add_option("--export-dir",
help="Defines the directory where each included configuration is exported "\
"as a new project.",
default=None)
group.add_option("--export-format",
help="Defines the format into which projects are exported when using "\
"--export-dir. Possible values are 'cpf' (the default) and 'dir'.",
default=None)
group.add_option("-a","--add",
dest="added",
action="append",
type="string",
help="Adds a configuration layer to the given configuration as last element. "\
"The add operation can be used several times in a single command and it "\
"can create even an empty layer. "\
"Example --add foo/root.confml --add bar/root-confml.",
metavar="CONFIG",
default=None)
group.add_option("--run-action",
dest="action",
action="append",
type="string",
help="Adds a execution of extra step that can manipulate the configuration before it is exported to external storage. "\
"The --run-action operation can be used several times in a single command and it "\
"will execute the actions in given order."\
"Example --run-action=fix, which would execute fix action during export.",
metavar="PLUGIN",
default=None)
group.add_option("--exclude-folders",
dest="exclude_empty_folders",
action="store_true",
help="Excludes empty folders from export",
default=False)
group.add_option("--exclude-content-filter",
dest="exclude_content_filter",
help="Filters out files and folders from content folder which matches with "\
"the given regular expression. Matching is case-insensitive. "\
"Example --exclude-content-filter=\".*\.sisx|.*\.sis|.*\.wgz|.*wgt\" "\
"Filters out all files with extension sis, sisx, wgz or wgt.",
default=None)
group.add_option("--include-content-filter",
dest="include_content_filter",
help="Filters out files and folders from content folder which don't match with "\
"the given regular expression. Matching is case-insensitive. "\
"Example --include-content-filter=\".*(manual|configurator).*\" "\
"Filters out all content files which don't have manual or configurator in their path.",
default=None)
parser.add_option_group(group)
(options, args) = parser.parse_args()
cone_common.handle_common_options(options)
# Check options
if options.export_format and options.export_dir is None:
parser.error(
"--export-format can only be used in conjunction with --export-dir"
)
if options.export_dir and options.remote:
parser.error(
"--export-dir and --remote cannot be used at the same time")
if options.export_format and options.export_format.lower() not in ('dir',
'cpf'):
parser.error("Invalid export format '%s'" % options.export_format)
if options.export_dir and not (options.configs or options.config_wildcards
or options.config_regexes):
parser.error(
"Use of --export-dir requires at least one configuration to be specified"
)
if options.export_dir and os.path.isfile(options.export_dir):
parser.error("Given export directory '%s' is a file")
try:
run_export(options)
except ExportFailedException, e:
parser.error(str(e))
def main():
""" Create report of existing report data. """
parser = OptionParser(version="%%prog %s" % VERSION)
parser.add_options(cone_common.COMMON_OPTIONS)
group = OptionGroup(parser, 'Report options',
'The report function generates a report using previously generated '\
'intermediary report data as input.')
group.add_option("-i", "--input-data",\
action="append",\
help="Defines an input file for report generation. "\
"If specified more than once, the data of all specified "\
"report data files is merged.",
metavar="FILE",
default=[])
group.add_option("-d", "--input-data-dir",\
help="Defines a directory containing the data files to use for "\
"generating the report. This is an alternative to specifying "\
"a number of --input-data files. The order of the data files "\
"is determined by the generation time stamps found in the data "\
"files.",
metavar="DIR",
default=None)
group.add_option("-r", "--report",\
dest="report",\
action="store",
type="string",
help="Specifies the report file to create."\
"Example -r report.html.",
metavar="FILE",\
default="report.html")
group.add_option("--report-option",\
action="append",
help="Specifies the report verbose options, that defines "\
"what data is included to the report. The option can be "\
"used multiple times."\
"choises=[default|all]"\
"Example --report-option=all",
metavar="OPTION",\
default=[])
group.add_option("-t", "--template",\
dest="template",\
action="store",
type="string",
help="Template used in report generation."\
"Example -t report_template.html.",
metavar="FILE",\
default=None)
parser.add_option_group(group)
(options, args) = parser.parse_args()
cone_common.handle_common_options(options)
if len(options.input_data) == 0 and not options.input_data_dir:
parser.error("Input data must be specified with either --input-data or --input-data-dir")
if len(options.input_data) > 0 and options.input_data_dir:
parser.error("Both --input-data and --input-data-dir specified, use one or the other.")
if options.input_data_dir and not os.path.isdir(options.input_data_dir):
parser.error('Given --input-data-dir does not exist or is not a directory.')
if options.input_data:
files = options.input_data
else:
files = get_input_data_files(options.input_data_dir)
if len(files) == 0:
parser.error("At least one input data file must be specified.")
# Load all data files
data_reports = []
for data_file in files:
print "Loading data file '%s'" % data_file
data = generation_report.load_report_data(data_file)
data_reports.append(data)
# Sort by time stamp
data_reports.sort(key=lambda entry: entry.generation_timestamp)
# Generate the report
print "Generating report to '%s'" % options.report
generation_report.generate_report(data_reports, options.report, options.template, [ROOT_PATH], options.report_option)
print "Done!'"
def main(argv=sys.argv):
""" Merge a configuration/layer to the project. """
parser = OptionParser(version="%%prog %s" % VERSION)
parser.add_options(cone_common.COMMON_OPTIONS)
parser.add_option("-c", "--configuration",\
dest="configuration",\
help="defines the name of the target configuration for the action",\
metavar="CONFIG")
parser.add_option("-p", "--project",\
dest="project",\
help="defines the location of current project. Default is the current working directory.",\
default=".",\
metavar="STORAGE")
group = OptionGroup(parser, 'Merge options',
'The merge functionality is meant to merge configurations/layers '
'from a remote project (defined with -r) to the current project (defined with -p). '
'Default value for the current project is the currently working directory. '
'A project can be either a folder or a cpf/zip file. There are two ways to '
'use merge: merge configuration roots (multiple layers), or specific layers. '
'See the ConE documentation for details and examples.')
group.add_option("-r", "--remote",\
dest="remote",\
help="defines the location of remote storage",\
metavar="STORAGE")
group.add_option("-s", "--sourceconfiguration",\
dest="sourceconfiguration",\
help="defines the name of the remote configuration inside the remote storage for the merge action. "\
"Default is the active root of the remote project.",\
metavar="CONFIG")
group.add_option("--sourcelayer",
help="Defines a specific layer to use as the layer to merge "\
"from the remote project. Must be the layer root (ConfML file)."\
"For example: --sourcelayer assets/somelayer/root.confml",
metavar="LAYER_ROOT",
default=None)
group.add_option("--targetlayer",
help="Defines a specific layer (root) to use as the layer to merge "\
"into the target project. Must be the layer root (ConfML file)."\
"For example: --targetlayer assets/somelayer/root.confml",
metavar="LAYER_ROOT",
default=None)
group.add_option("--rename",\
action="store_true",
dest="rename",\
help="defines that the merged layers need to be renamed",
default=False)
group.add_option("--all",\
action="store_true",
dest="all",\
help="Defines that the entire configuration (all layers) needs to be merged. "\
"This has no effect when merging layers directly using --sourcelayer and --targetlayer.",
default=False)
group.add_option("-l", "--layer",\
dest="layers",\
type="int",
action="append",
help="Define the layers of the source configuration that are included to merge action. "\
"The layer operation can be used several times in a single command. "\
"Note that this can only be used when merging configuration roots, not "\
"specific layers using --sourcelayer and --targetlayer. "\
"Example -l -1 --layer=-2, which would append a layers -1 and -2 to the layers => layers = -1,-2",
metavar="LAYERS",\
default=None)
group.add_option("--merge-policy",
help="Specifies the merge policy to use when merging layers. "\
"Possible values: "\
"replace-add - Add/replace files from source layer, but leave other files in the target as they are. "\
" "\
"overwrite-layer - Overwrite the entire layer (remove all previous content).",
default=MergePolicy.REPLACE_ADD)
parser.add_option_group(group)
(options, _) = parser.parse_args(argv)
cone_common.handle_common_options(options)
# Check the passed options
if not MergePolicy.is_valid(options.merge_policy):
parser.error("Invalid merge policy: %s\nMust be one of %s" % (options.merge_policy, '\n'.join(MergePolicy.ALL)))
if not options.remote: parser.error("Remote project must be given")
if options.layers and (options.sourcelayer or options.targetlayer):
parser.error("Specifying layer indices using --layer is not supported when using --sourcelayer or --targetlayer!")
if options.sourcelayer and not options.targetlayer:
parser.error("Merging a layer into a configuration is not supported at the moment!")
if options.sourcelayer and not options.sourcelayer.lower().endswith('.confml'):
parser.error("Source layer root should be a .confml file")
if options.targetlayer and not options.targetlayer.lower().endswith('.confml'):
parser.error("Target layer root should be a .confml file")
if not options.sourcelayer and options.targetlayer:
parser.error("Cannot merge a configuration into a layer!")
# If layers for configuration root merging are not specifically given,
# the default is the last layer
if options.layers is None:
options.layers = [-1]
target_project = api.Project(api.Storage.open(options.project,"a", username=options.username, password=options.password))
source_project = api.Project(api.Storage.open(options.remote,"r", username=options.username, password=options.password))
print "Target project: %s" % options.project
print "Source project: %s" % options.remote
target_config = None
try:
if options.sourcelayer and options.targetlayer:
print "Target layer: %s" % options.targetlayer
print "Source layer: %s" % options.sourcelayer
try:
source_config = source_project.get_configuration(options.sourcelayer)
except exceptions.NotFound:
raise MergeFailedException("Layer root '%s' not found in source project" % options.sourcelayer)
try:
target_config = target_project.get_configuration(options.targetlayer)
except exceptions.NotFound:
logger.info('Creating new layer %s' % (options.targetlayer))
target_config = target_project.create_configuration(options.targetlayer)
print "Merging layers..."
merge_configuration_layer(source_config, target_config, options.merge_policy)
else:
# Merging a configuration root into a configuration root
if options.all: layer_indices = None
else: layer_indices = utils.distinct_array(options.layers)
def find_layers(source_config, target_config):
return find_layers_to_merge(
layer_indices = layer_indices,
rename = options.rename,
sourceconfig = source_config,
targetconfig = target_config)
merge_config_root_to_config_root(
source_project = source_project,
target_project = target_project,
source_config = options.sourceconfiguration,
target_config = options.configuration,
layer_finder_func = find_layers,
merge_policy = options.merge_policy)
except MergeFailedException, e:
print "Could not merge: %s" % e
sys.exit(2)
def main():
""" Update/set values to features in configuration(s). """
parser = OptionParser(version="%%prog %s" % VERSION)
parser.add_options(cone_common.COMMON_OPTIONS)
parser.add_option("-c", "--configuration",
dest="configs",
action="append",
help="Defines the name of the configuration for the action, can be "\
"specified multiple times to include multiple configurations.",
metavar="CONFIG",
default=[])
parser.add_option("--config-wildcard",
action="append",
dest="config_wildcards",
help="Wildcard pattern for including configurations, e.g. "\
"product_langpack_*_root.confml",
metavar="WILDCARD",
default=[])
parser.add_option("--config-regex",
action="append",
dest="config_regexes",
help="Regular expression for including configurations, e.g. "\
"product_langpack_\\d{2}_root.confml",
metavar="REGEX",
default=[])
parser.add_option("-p", "--project",\
dest="project",\
help="defines the location of current project. Default is the current working directory.",\
default=".",\
metavar="STORAGE")
group = OptionGroup(
parser, 'Update options',
'The update functionality is meant for ConfML manipulation '
'in current project (defined with -p). '
'Default value for the current project is the currently working directory. '
'A project can be either a folder or a cpf/zip file.')
group.add_option("-m","--add-meta",\
dest="meta",\
action="append",
type="string",
help="Add given metadata to defined configuration."\
"Example --add-meta \"owner=John Cone\" -m product=E75",
default=None)
group.add_option("--add-cpf-meta",\
dest="cpfmeta",\
action="append",
type="string",
help="Add given CPF identification metadata to defined configuration."\
"Example --add-cpf-meta \"coreplat_name=Platform1\"",
default=None)
group.add_option("-d","--add-desc",\
dest="desc",\
type="string",\
help="Add given description to defined configuration."\
"Example --add-desc \"Customer one CPF\" -d Description1",
default=None)
group.add_option("--remove-meta",\
dest="remove_meta",\
action="append",
type="string",
help="Removes given metadata from defined configuration."\
"Example --remove-meta owner --remove-meta coreplat_name",
metavar="META",\
default=None)
group.add_option("--remove-desc",\
dest="remove_desc",\
action="store_true",\
help="Removes description from defined configuration."\
"Example --remove-desc",
default=False)
group.add_option("--add-data",\
dest="data",\
action="append",
type="string",
help="Add given ConfML data to defined configuration."\
"Example --add-data \"KCRUidAvkon.KAknDefaultAppOrientation=1\"",
default=None)
group.add_option("--add-configuration",\
dest="configurations",\
action="append",
type="string",
help="Include given configuration inside the updated configuration."\
"Example --add-configuration \"test_configuration.confml\"",
default=None)
group.add_option("--add-policy",\
dest="policy",\
type="string",
help="Define the data update policy, which can be append|prepend|replace."\
"Example --add-configuration \"test_configuration.confml\" --add-policy=prepend"\
"This would add the configuration as the first include to the target configuration.",
default='append')
parser.add_option_group(group)
(options, args) = parser.parse_args()
cone_common.handle_common_options(options)
# Open the project and find out the active configuration
project = api.Project(
api.Storage.open(options.project,
"a",
username=options.username,
password=options.password))
try:
active_root = project.get_storage().get_active_configuration()
except AttributeError:
active_root = None
# Collect the list of configurations specified from the command line
config_list = []
if options.configs or options.config_wildcards or options.config_regexes:
try:
config_list = cone_common.get_config_list_from_project(
project=project,
configs=options.configs,
config_wildcards=options.config_wildcards,
config_regexes=options.config_regexes)
except cone_common.ConfigurationNotFoundError, e:
parser.error(str(e))
def main():
""" Validate a configuration, or individual confml/implml files. """
shortcut_container = report_util.ReportShortcutContainer(
REPORT_SHORTCUTS, 'html')
parser = OptionParser(version="ConE validate %s" % VERSION)
parser.add_options(cone_common.COMMON_OPTIONS)
parser.add_option(
"-c",
"--configuration",
dest="configuration",
help="Defines the name of the configuration for the action",
metavar="CONFIG")
parser.add_option("-p", "--project",
dest="project",
default=".",
help="defines the location of current project. Default is the "\
"current working directory.",
metavar="STORAGE")
group = OptionGroup(parser, 'Validate options',
'The validate action is intended for performing validation on a '\
'configuration or individual files. ')
group.add_option('--confml-file',
action="append",
help='Validate only the given single ConfML file.',
metavar="FILE",
default=None)
group.add_option('--implml-file',
action="append",
help='Validate only the given single ImplML file.',
metavar="FILE",
default=None)
group.add_option("--template",
help="Template used in report generation. "\
"Example: --template=report_template.html.",
metavar="FILE",
default=None)
group.add_option("--report-type",
help="The type of the report to generate. This is a convenience "\
"switch for setting the used template. If --template is given, this option has no effect. "\
"Possible values: "\
+ shortcut_container.get_shortcut_help_text(),
metavar="TYPE",\
default=None)
group.add_option("--report",
help="Specifies the location of the validation report. "\
"Example --report=report.html.",
metavar="FILE",
default=None)
group.add_option(
"--dump-schema-files",
help=
"Dump the XML schema files used for validation into the specified directory.",
metavar="DIR",
default=None)
group.add_option("--exclude-filter",
action="append",
help="Exclude validation problems by given filter. "\
"Examples: --exclude-filter=schema, --exclude-filter=schema.implml, --exclude-filter=schema.confml, --exclude-filter=schema.implml.ruleml",
default=None)
group.add_option("--include-filter",
action="append",
help="Include validation problems by given filter."\
"Examples: --include-filter=schema.implml, --include-filter=schema.implml.ruleml",
default=None)
parser.add_option_group(group)
(options, _) = parser.parse_args()
cone_common.handle_common_options(options)
if not shortcut_container.is_valid_shortcut(options.report_type):
parser.error("Invalid report type: %s" % options.report_type)
if options.dump_schema_files:
dump_dir = options.dump_schema_files
print "Dumping XML schema files to '%s'" % dump_dir
cone.validation.schemavalidation.dump_schema_files(dump_dir)
return
pt_filter = ProblemTypeFilter(includes=options.include_filter or [],
excludes=options.exclude_filter or [])
problems = []
if options.confml_file or options.implml_file:
if options.confml_file:
func = cone.validation.schemavalidation.validate_confml_data
for file in options.confml_file:
problems.extend(validate_file(file, func))
if options.implml_file:
func = cone.validation.schemavalidation.validate_implml_data
for file in options.implml_file:
problems.extend(validate_file(file, func))
else:
problems = validate_configuration(options.project,
options.configuration, pt_filter)
if problems is not None:
filters = {'filter_by_severity': filter_by_severity}
problems = pt_filter.filter(problems)
print "Total %d problem(s) after filtering" % len(problems)
print "Generating report..."
problems.sort(key=lambda p: (p.severity, p.file, p.line))
template, report = shortcut_container.determine_template_and_report(
options.report_type, options.template, options.report,
'validation_report')
report_util.generate_report(template,
report, {'problems': problems},
extra_filters=filters)
def main(argv=sys.argv):
""" Pack (zip) the variant layers of a configuration. """
parser = OptionParser(version="%%prog %s" % VERSION)
parser.add_options(cone_common.COMMON_OPTIONS)
parser.add_option(
"-p",
"--project",
dest="project",
help=
"Defines the location of current project. Default is the current working directory.",
default=".",
metavar="STORAGE")
group = OptionGroup(
parser, "Packvariant options",
"The packvariant action is intended for packing variant to a zip-file for integration purposes."
)
group.add_option("-c",
"--configuration",
dest="configuration",
help="Name of the configuration wanted to be packed.",
metavar="CONFIG")
group.add_option(
"-r",
"--remote",
dest="remote",
help="Defines a location and a name of remote storage (ZIP)",
metavar="STORAGE")
group.add_option(
"-l",
"--convert-location",
dest="convertlocation",
help="Defines a location of convertprojectml file."
"Default location is <PROJECT>/convertpluginlayer/implml/",
default="/convertpluginlayer/implml/")
parser.add_option_group(group)
(options, _) = parser.parse_args(argv)
cone_common.handle_common_options(options)
# Check the passed options
if not options.remote:
parser.error("Target where variant package is placed must be given")
if not options.configuration:
parser.error("Configuration root to be packed must be given")
try:
target_storage = api.Storage.open(options.remote,
'w',
username=options.username,
password=options.password)
target_project = api.Project(target_storage)
source_storage = api.Storage.open(options.project,
'r',
username=options.username,
password=options.password)
if not os.path.isdir(source_storage.get_path()):
print "ERROR: --Project must be a directory. Terminating the program."
sys.exit(1)
source_project = api.Project(source_storage)
source_config = get_active_root_if_necessary(source_project,
options.configuration,
'source')
source_config = source_project.get_configuration(source_config)
fname, _ = os.path.splitext(options.configuration)
conv_project_filename = fname + ".convertprojectml"
print "Packing configuration: %s" % options.configuration
print "Source project: %s" % options.project
print "Target project: %s" % options.remote
# Adding all files in layers
layer_list = find_variant_layers(source_config)
for add_layer in layer_list:
layer_config = source_project.get_configuration(add_layer)
layer = layer_config.get_layer()
path_part = layer.path + "/"
target_project.import_configuration(layer_config)
resource_list = layer_config.layer.list_all_resources(recurse=True)
for single_resource in resource_list:
parsed_path = path_part + single_resource
if source_storage.is_resource(parsed_path):
logger.info("Adding file: %s" % parsed_path)
target_storage.import_resources([parsed_path],
source_storage)
if source_storage.is_folder(path_part + single_resource):
logger.info("Adding folder: %s" % parsed_path)
target_storage.create_folder(parsed_path)
layer_list = get_layerlist_as_convertprojectml(source_config)
metadata_list = get_metadatas_as_convertprojectml(source_config)
if not source_storage.is_folder(
os.path.normpath(options.convertlocation + "/")):
source_storage.create_folder(
os.path.normpath(options.convertlocation + "/"))
create_convertprojectml_file(
options.configuration, conv_project_filename,
options.project + "/" + options.convertlocation + "/", layer_list,
metadata_list, source_config.get_name())
target_storage.import_resources([
os.path.normpath(options.convertlocation + "/" +
conv_project_filename)
], source_storage)
except Exception, e:
print "Could not create Zip archive: %s" % e
sys.exit(2)
def main():
""" Run automatic fixes for configurations. """
parser = OptionParser(version="ConE validate %s" % VERSION)
parser.add_options(cone_common.COMMON_OPTIONS)
parser.add_option(
"-c",
"--configuration",
dest="configuration",
help="Defines the name of the configuration for the action",
metavar="CONFIG")
parser.add_option("-p", "--project",
dest="project",
default=".",
help="defines the location of current project. Default is the "\
"current working directory.",
metavar="STORAGE")
group = OptionGroup(parser, 'Fix options',
'The fix action is intended for performing fixes on a '\
'configuration. ')
group.add_option("--print-available-fixes",
action="store_true",
help="Print all configuration fixer objects available.",
default=False)
group.add_option("--exclude-filter",
action="append",
help="Exclude problems by given filter. "\
"Examples: --exclude-filter=schema, --exclude-filter=schema.implml, --exclude-filter=schema.confml, --exclude-filter=schema.implml.ruleml",
default=None)
group.add_option("--include-filter",
action="append",
help="Include problems by given filter."\
"Examples: --include-filter=schema.implml, --include-filter=schema.implml.ruleml",
default=None)
parser.add_option_group(group)
(options, _) = parser.parse_args()
cone_common.handle_common_options(options)
if options.print_available_fixes:
print "Available fixers:"
for fix_class in confmlvalidation.get_fixer_classes():
print "%s: %s" % (fix_class, fix_class.__doc__)
return 0
else:
if not options.configuration:
parser.error(
"A configuration must be given! Use -c / --configuration option."
)
action = fix.ConeFixAction(include_filter=options.include_filter or [],
exclude_filter=options.exclude_filter or [],
username=options.username,
password=options.password,
project_name=options.project,
configuration_name=options.configuration)
status = action.run()
if status:
action.save()
action.close()
return status
def main():
""" Get information about project / configurations. """
shortcut_container = report_util.ReportShortcutContainer(REPORT_SHORTCUTS,
None)
parser = OptionParser(version="%%prog %s" % VERSION)
parser.add_options(cone_common.COMMON_OPTIONS)
parser.add_option("-c", "--configuration",
action="append",
dest="configs",
help="Defines a configuration to use in report generation or info printout. "\
"May be defined more than once, but multiple configuration will only "\
"do anything if the report type supports them. If multiple configurations "\
"are not supported, the first one will be used",
metavar="CONFIG",
default=[])
parser.add_option("--config-wildcard",\
action="append",
dest="config_wildcards",
help="Wildcard pattern for including configurations, e.g. product_langpack_*_root.confml",
metavar="WILDCARD",
default=[])
parser.add_option("--config-regex",\
action="append",
dest="config_regexes",
help="Regular expression for including configurations, e.g. product_langpack_\\d{2}_root.confml",
metavar="REGEX",
default=[])
parser.add_option("-p", "--project",\
dest="project",\
help="defines the location of current project. Default is the current working directory.",\
default=".",\
metavar="STORAGE")
info_group = OptionGroup(parser, 'Info options',
'The info functionality is meant for printing information about the '\
'contents of a cpf/zip file or Configuration Project (folder). Two '\
'separate use cases are currently supported: '\
'1. Printing basic information about a project or configuration to '\
' stdout. '\
'2. Creating a report of the contents of a configuration or configurations.')
info_group.add_option("--report-type",
help="The type of the report to generate. This is a convenience "\
"switch for setting the used template. "\
"Possible values: "\
+ shortcut_container.get_shortcut_help_text(),
metavar="TYPE",\
default=None)
info_group.add_option("--report",
help="The file where the configuration info report is written."\
"By default this value is determined by the used "\
"report type. Example: --report report.html.",
metavar="FILE",\
default=None)
info_group.add_option("--template",
help="Template used in a report generation. By default "\
"this value is determined by the used report type. "\
"Example: --template report_template.html.",
metavar="FILE",\
default=None)
info_group.add_option("--impl-filter",
help="The pattern used for filtering implementations for the "\
"report. See the switch --impl in action generate for "\
"more info. ",
metavar="PATTERN",
default='.*')
info_group.add_option("--view-file",
help="External ConfML file containing the view used for filtering "\
"the features listed in the setting value report. The first view "\
"defined in the file will be used.",
metavar="FILE",
default=None)
info_group.add_option("--print-active-root",
action="store_true",
help="Print active root in the current project and exit.")
parser.add_option_group(info_group)
(options, args) = parser.parse_args()
cone_common.handle_common_options(options)
if options.print_active_root:
print_active_root(options)
sys.exit(0)
if not shortcut_container.is_valid_shortcut(options.report_type):
parser.error("Invalid report type: %s" % options.report_type)
if (options.report_type or options.template) and \
(not options.configs and not options.config_wildcards and not options.config_regexes):
parser.error("Report type or template specified but configuration(s) not given!")
if options.view_file and not os.path.isfile(options.view_file):
parser.error("No such file: %s" % options.view_file)
# Load view from the specified file if necessary
view = None
if options.view_file:
try:
view = _load_view_from_file(options.view_file)
print "Loaded view '%s' from '%s'" % (view.get_name(), options.view_file)
except ViewLoadError, e:
print e
sys.exit(1)
def main():
""" Generate a configuration. """
parser = OptionParser(version="%%prog %s" % VERSION)
parser.add_options(cone_common.COMMON_OPTIONS)
parser.add_option("-c", "--configuration",\
dest="configuration",\
help="defines the name of the configuration for the action",\
metavar="CONFIG")
parser.add_option("-p", "--project",\
dest="project",\
help="defines the location of current project. Default is the current working directory.",\
default=".",\
metavar="STORAGE")
gen_group = OptionGroup(parser, 'Generate options',
'The generate function will create target files from a specific configuration.'\
'The generate will always work with read-only mode of the project, so no changes are saved to project')
gen_group.add_option("-o", "--output",\
dest="output",\
help="defines the target folder where the files are is generated or copied",\
metavar="FOLDER",\
default="output")
gen_group.add_option("-i", "--impl",\
dest="impls",\
action="append",
help=\
"""Define a Python regular expression filter for actual ImplML plugin(s) that needs to be executed. The whole path to ImplML filename is used in the regexp matching.
The impl operation can be used several times in a single command.
Example1 --impl crml => matches for any ImplML file that has a CrML string in the path.
Example2 --impl makeml$ => matches for ImplML file that has ends with MakeML string.
""",
metavar="IMPLS",\
default=None)
gen_group.add_option("--impl-tag",\
dest="tags",\
type="string",
action="append",
help="define a tag for the implementations that are included to the output. "\
"A tag is name value pair and has the following format: name:value, e.g. target:rofs3."\
"Example --impl-tag=target:uda --impl-tag=target:content, which would include impls include both tags.",
metavar="TAG",\
default=None)
gen_group.add_option("--impl-tag-policy",\
dest="tags_policy",\
type="string",
action="append",
help="Policy for implementation tags. May have one of the following values: --impl-tag-policy=AND, --impl-tag-policy=OR. "\
"Default is OR.",
metavar="TAGS_POLICY",\
default=None)
gen_group.add_option("-s", "--set",\
dest="overrides",\
action="append",
type="string",
help="Override a ConfML reference in the execution."\
"The set operation can be used several times in a single command."\
"Example -s foo.bar=10 -s foo.fea='test'.",
metavar="SET",\
default=None)
gen_group.add_option("--add",\
dest="added",\
action="append",
type="string",
help="Add a given configuration to the given configuration as last element."\
"The add operation can be used several times in a single command."\
"Example --add foo/root.confml --add bar/root-confml.",
metavar="CONF",\
default=None)
gen_group.add_option("-r", "--report",\
dest="report",\
action="store",
type="string",
help="Generates a report about settings that are properly generated."\
"Example -r report.html.",
metavar="FILE",\
default=None)
gen_group.add_option("--report-option",\
action="append",
help="Specifies the report verbose options, that defines "\
"what data is included to the report. The option can be "\
"used multiple times."\
"choises=[default|all]"\
"Example --report-option=all",
metavar="OPTION",\
default=[])
gen_group.add_option("-t", "--template",\
dest="template",\
action="store",
type="string",
help="Template used in report generation."\
"Example -t report_template.html.",
metavar="FILE",\
default=None)
gen_group.add_option("--report-data-output",\
type="string",
help="Specifies a file where intermediary report data is generated.",
metavar="FILE",\
default=None)
gen_group.add_option("-n", "--dryrun",\
dest="dryrun",\
action="store_true",
help="Executes generation without generation output.",
default=False)
gen_group.add_option("--add-setting-file",\
dest="settings",\
action="append",
type="string",
help="Generate specific settings in ini format."\
"Example -o my_generate_settings.cfg.",
metavar="FILE",\
default=None)
gen_group.add_option("--dump-autodata",\
dest="dump_autodata",\
action="store",
type="string",
metavar="FILE",
help="Specifies a confml file for storing autodata.confml permanently.",
default=None)
gen_group.add_option("-w", "--what",\
dest="what",\
action="store",
type="string",
metavar="FILE",
help="List output files to a txt file",
default=None)
lf_group = OptionGroup(parser, 'Layer filtering options',
'Layer filtering options define configuration layers to be used for filtering '\
'the implementations that are used to generate output. Filtering by a layer means that '\
'only implementations that generate their output based on settings changed on that layer '\
'are included in the generation.')
lf_group.add_option("-l", "--layer",\
dest="layers",\
type="int",
action="append",
help="Define a layer by giving its index in the root configuration. "\
"0 is first, 1 the second, -1 the last, -2 the second to last and so on. "\
"The layer operation can be used several times in a single command. "\
"Example -l -1 --layer=-2, which would append a layers -1 and -2 to the layers => layers = -1,-2",
metavar="LAYER",\
default=None)
lf_group.add_option("--layer-regex",
dest="layer_regexes",
action="append",
help="Define a regular expression for including layers into the generation process, "\
"e.g. --layer-regex layer[0-9]/root.confml. The pattern is matched against the layer root "\
"path, which could be e.g. 'assets/layer1/root.confml'.",
metavar="REGEX",)
lf_group.add_option("--layer-wildcard",
dest="layer_wildcards",
action="append",
help="Define a wildcard for including layers into the generation process, e.g "\
"--layer-wildcard layer*",
metavar="WILDCARD",)
lf_group.add_option("--all-layers",
dest="all_layers",
action="store_true",
help="Include all layers in generation. This switch overrides all other layer "\
"configurations (iMaker API and using the --layer, --layer-regex and --layer-wildcard parameters)",
default=False)
start_time = time.time()
parser.add_option_group(gen_group)
parser.add_option_group(lf_group)
(options, _) = parser.parse_args()
settinglist = [os.path.join(ROOT_PATH,'conesub_generate.cfg')]
if options.settings:
for setting_file in options.settings:
settinglist.append(os.path.normpath(os.path.join(ROOT_PATH, setting_file)))
gset = cone_common.get_settings(settinglist)
cone_common.handle_common_options(options, settings=gset)
current = api.Project(api.Storage.open(options.project,"r"))
active_root = current.get_storage().get_active_configuration()
if not options.configuration:
if active_root == "":
parser.error("configuration must be given")
else:
logging.getLogger('cone').info('No configuration given! Using active root configuration %s' % active_root)
options.configuration = active_root
try:
config = current.get_configuration(options.configuration)
except exceptions.NotFound:
parser.error("No such configuration: %s" % options.configuration)
reffilters = None
implfilters = None
impltags = None
# Include possible additional configurations
if options.added:
for configname in options.added:
logging.getLogger('cone').info('Adding configuration %s' % configname)
config.include_configuration(utils.resourceref.norm(configname))
# Get implementation filters from configuration
try:
implfilters = (config.get_default_view().get_feature('imakerapi.cone_impls').get_value() or '').split(',')
except exceptions.NotFound:
implfilters = []
# Get filters from command line if they exist => cmd overrides configuration
if options.impls:
implfilters = options.impls
if options.tags and len(options.tags) > 0:
impltags = {}
for tag in options.tags:
(name,value) = tag.split(':',2)
existingvalue = impltags.get(name,[])
existingvalue.append(value)
impltags[name] = existingvalue
logging.getLogger('cone').info('Tag filter %s' % impltags)
else:
impltags = None
tags_policy = 'OR'
if options.tags_policy:
tags_policy = options.tags_policy[0]
layerdefs = _get_included_layers(config, options, parser)
filter_by_refs = _filter_by_refs(config, options, parser)
if layerdefs:
logging.getLogger('cone').info('Included layers:\n%s' % '\n'.join(layerdefs))
else:
logging.getLogger('cone').info('Including all layers')
dview = config.get_default_view()
# Add data references if included layers are defined
if len(layerdefs) > 0:
# get the data references from given layers
logging.getLogger('cone').info('Getting layer specific data reference from %s' % layerdefs)
reffilters = []
for layer_path in utils.distinct_array(layerdefs):
logging.getLogger('cone').info('Searching layer %s' % layer_path)
layer = config.get_configuration(layer_path)
refs = _get_new_refs(reffilters, layer.list_leaf_datas())
# reduce the refs of sequences to single reference of the sequence feature
layerrefs = set()
for fea in dview.get_features(refs):
layerrefs.add(fea.fqr)
if fea.is_sequence():
layerrefs.add(fea.get_sequence_parent().fqr)
refs = sorted(list(layerrefs))
#logging.getLogger('cone').info("Refs from layer '%s'\n%s" % (layer.get_path(), '\n'.join(refs)))
reffilters += refs
# Make sure that the output folder exists
if not os.path.exists(options.output):
os.makedirs(options.output)
impls = plugin.filtered_impl_set(config,implfilters)
impls.output = options.output
log.info("Parsed %s implementation(s)" % len(impls))
logging.getLogger('cone').info("Supported implementation file extensions: %r" % plugin.get_supported_file_extensions())
# logging.getLogger('cone').debug('Loaded implementations:')
# for impl in impls:
# msg = "File '%s', impl. type '%s', class '%s', phase '%s'" % \
# (impl.ref, impl.IMPL_TYPE_ID, type(impl).__name__, impl.invocation_phase())
# logging.getLogger('cone').debug(msg)
# Create temporary variables
temp_feature_refs = impls.create_temp_features(config)
if reffilters is not None:
reffilters.extend(temp_feature_refs)
logging.getLogger('cone').info('Refs from temporary variables:\n%s' % '\n'.join(temp_feature_refs))
# Set overrides only after temp variables are created, so that
# they can also be modified from the command line
if options.overrides:
# Make sure that the last layer is the autodata layer
plugin.get_autoconfig(config)
for override in options.overrides:
(ref,value) = override.split('=',1)
config.get_default_view().get_feature(ref).set_value(value)
# ---------------
# Generate output
# ---------------
context = plugin.GenerationContext(configuration = config,
tags = impltags or {},
tags_policy = tags_policy,
output = options.output,
impl_set = impls,
temp_features = temp_feature_refs,
filter_by_refs = filter_by_refs)
context.changed_refs = reffilters
context.output = options.output
impls.output = options.output
for phase in impls.INVOCATION_PHASES:
log.info("Generating phase '%s'" % phase)
context.phase = phase
impls.generate(context)
impls.post_generate(context)
if options.what:
log.info("Write output files to '%s'" % options.what)
output_files = []
for op in context.get_output():
# Only append once
if op.type == 'file' and output_files.count(op.abspath) < 1:
output_files.append(op.abspath)
try:
mkpath(os.path.dirname(os.path.abspath(options.what)))
what_fh = open(os.path.abspath(options.what), 'w')
try:
[what_fh.write('%s\n' % ofile) for ofile in output_files]
print "Wrote output file list to '%s'" % options.what
finally:
what_fh.close()
except Exception:
log.info("Could not create directory for '%s'" % options.what)
print "Generated %s to %s!" % (options.configuration, impls.output)
# Store temporary rule execution outputs to a new configuration
if options.dump_autodata:
# Make sure autodata layer is the one we're dealing with
plugin.get_autoconfig(config)
lastconfig = config.get_last_configuration()
lastconfig.set_name(utils.resourceref.to_objref(utils.resourceref.get_filename(utils.resourceref.norm(options.dump_autodata))))
data = persistentconfml.dumps(lastconfig)
try:
mkpath(utils.resourceref.get_path(utils.resourceref.norm(options.dump_autodata)))
fh = open(options.dump_autodata, 'w')
try: fh.write(data)
finally: fh.close()
print 'Saved autodata to %s' % options.dump_autodata
except DistutilsFileError:
log.info('Unable to dump autodata')
# ---------------
# Generate report
# ---------------
# If reporting is enabled collect data for report
if options.report != None or options.report_data_output != None:
logging.getLogger('cone').info('Collecting data for report.')
rep_data = generation_report.ReportData()
rep_data.context = context
rep_data.context.log_file = os.path.abspath(options.log_file)
rep_data.context.log = _read_log(options.log_file)
rep_data.project_dir = options.project
logging.getLogger('cone').info('Collecting data found rep_data %s' % rep_data)
duration = str("%.3f" % (time.time() - start_time) )
rep_data.set_duration( duration )
rep_data.options = options
# Save intermediary report data file if necessary
if options.report_data_output != None:
logging.getLogger('cone').info('Dumping report data to %s' % options.report_data_output)
print "Dumping report data to '%s'" % options.report_data_output
generation_report.save_report_data(rep_data, options.report_data_output)
# Generate the report if necessary
if options.report != None:
generation_report.generate_report([rep_data], options.report, options.template, [ROOT_PATH], options.report_option)
print_summary(rep_data)
if current: current.close()
def main(argv=sys.argv):
""" Initialize a variant from a cpf. """
parser = OptionParser(version="%%prog %s" % VERSION)
parser.add_options(cone_common.COMMON_OPTIONS)
parser.add_option("-p", "--project",
dest="project",
help="Defines the location of current project. Default is the current working directory.",
default=".",
metavar="STORAGE")
group = OptionGroup(parser, 'Initvariant options',
'The initvariant action is intended for merging a variant CPF back into the '
'configuration project, or creating a new empty variant based on an existing '
'configuration. It merges all customer variant layers (layers with '
'custvariant* in their path name) and renames them based on the variant ID '
'and variant name ("custvariant_<id>_<name>").')
group.add_option("-c", "--configuration",
dest="configuration",
help="Defines the name of the target configuration. By default the "
"configuration file name is composed of product name, variant ID "
"and variant name like this: <product>_custvariant_<id>_<name>_root.confml",
metavar="CONFIG")
group.add_option("-r", "--remote",
dest="remote",
help="Defines the location of remote storage (CPF)",
metavar="STORAGE")
group.add_option("-s", "--sourceconfiguration",
dest="sourceconfiguration",
help="Defines the name of the remote configuration inside the remote storage. "
"Default is the active root of the remote project.",
metavar="CONFIG")
group.add_option("--variant-id", help="Variant ID, mandatory.")
group.add_option("--variant-name", help="Variant name, optional.")
group.add_option("--product-name",
help="Product name, taken from the configuration data by default "
"(i.e. defaults to '${imakerapi.productname}')",
default="${imakerapi.productname}")
group.add_option("--set-active-root",
action="store_true",
help="Set the newly created (or update) configuration root as the "
"project's active root after the merge is done.")
group.add_option("-b","--based-on-configuration",
dest="boconfig",
help="Defines the configuration root which is used as a base "
"configuration for the new empty variant.")
group.add_option("--find-layer-regexp",
dest="find_pattern",
default='.*/manual/.*|.*/configurator/.*',
help="Defines the pattern which is used to find the layers "
"from source configuration that will be merged"
"Default: '.*/manual/.*|.*/configurator/.*' " )
parser.add_option_group(group)
(options, _) = parser.parse_args(argv)
cone_common.handle_common_options(options)
# Check the passed options
if not options.remote and not options.boconfig:
parser.error("Remote project or based-on-configuration must be given")
if options.remote and options.boconfig:
parser.error("Only either remote project or based-on-configuration can be given, but not both")
if not options.variant_id: parser.error("Variant ID must be given")
temp_cpf_folder = None
if options.boconfig:
class ExportOptions(object):
pass
path = ''
coreplat_name = ''
product = ''
project = api.Project(api.Storage.open(options.project,"a", username=options.username, password=options.password))
config = project.get_configuration(options.boconfig)
meta = config.meta
if meta:
for prop in meta.array:
if 'name' in prop.attrs and 'value' in prop.attrs:
name = prop.attrs['name']
if name == 'coreplat_name':
coreplat_name = prop.attrs['value']
if name == 'product_name':
product = prop.attrs['value']
if not coreplat_name or not product:
print >>sys.stderr, "Could not find coreplat_name or product_name from meta data."
print >>sys.stderr, "Are you sure the given based-on-configuration is valid?"
sys.exit(2)
path = coreplat_name + '/' + product
# the new way (product)
if (os.path.exists(os.path.join(options.project, product, "root.confml"))):
path = product
# the old way (coreplat/product)
elif (os.path.exists(os.path.join(options.project, coreplat_name, product, "root.confml"))):
path = coreplat_name + '/' + product
# any other way, product root somewhere else (?/?/product/root.confml)
else:
for root, dirs, files in os.walk(os.path.abspath(options.project)):
if os.path.exists(os.path.join(root, product, "root.confml")):
fullpath = os.path.abspath(os.path.join(root, product, "root.confml"))
m = re.search(r'%s[\\/](.*)[\\/]root.confml' % re.escape(os.path.abspath(options.project)), fullpath)
if m:
path = m.group(1)
path = re.sub(r'\\','/', path)
break
temp_cpf_folder = tempfile.mkdtemp()
export_options = ExportOptions()
export_options.project = options.project
export_options.remote = os.path.join(temp_cpf_folder, 'temp.cpf')
export_options.configs = [options.boconfig]
export_options.username = options.username
export_options.password = options.password
export_options.config_wildcards = None
export_options.config_regexes = None
export_options.export_dir = None
export_options.exclude_content_filter = None
export_options.include_content_filter = None
export_options.added = [path + '/customer/custvariant/manual/root.confml',
path + '/customer/custvariant/configurator/root.confml']
export_options.exclude_empty_folders = False
export_options.action = None
options.remote = export_options.remote
print "Exporting variant CPF to a temporary directory"
run_export(export_options)
target_project = api.Project(api.Storage.open(options.project,"a", username=options.username, password=options.password))
source_project = api.Project(api.Storage.open(options.remote,"r", username=options.username, password=options.password))
print "Target project: %s" % options.project
print "Source project: %s" % options.remote
replace_dict = {"VAR_ID": options.variant_id, "VAR_NAME": options.variant_name}
try:
# Open the source configuration
source_config = get_active_root_if_necessary(source_project, options.sourceconfiguration, 'source')
source_config = source_project.get_configuration(source_config)
# Determine the new name of the layer part (replaces 'custvariant[^/]*')
if options.variant_name:
new_name = "custvariant_%s_%s" % (options.variant_id, options.variant_name)
else:
new_name = "custvariant_%s" % options.variant_id
# Determine the target configuration
if options.configuration:
target_config = options.configuration
else:
# Target configuration not given explicitly, automatically
# determine the name based on the product name and the new
# layer name
try:
product_name = utils.expand_refs_by_default_view(
options.product_name,
source_config.get_default_view(),
catch_not_found = False)
except Exception, e:
print "Could not determine product name: %s" % e
sys.exit(2)
print "Product name: %s" % product_name
target_config = "%s_%s_root.confml" % (product_name, new_name)
def find_layers(source_config, target_config):
ret_list = []
layers = find_variant_layers_to_merge(source_config,
target_config,
options.find_pattern)
p = re.compile(r'custvariant[^/]*')
for layer in layers:
tgt_layer = p.sub(new_name, layer)
ret_list.append((layer, tgt_layer))
return ret_list
# Perform the merge
merge_config_root_to_config_root(
source_project = source_project,
target_project = target_project,
source_config = options.sourceconfiguration,
target_config = target_config,
layer_finder_func = find_layers,
merge_policy = MergePolicy.OVERWRITE_LAYER,
find_pattern = options.find_pattern)
if options.set_active_root:
target_project.get_storage().set_active_configuration(target_config)