def _run_task(self):
self.logger.info("Querying Salesforce for changed source members")
changes = self._get_changes()
self._filtered = self._filter_changes(changes)
if not self._filtered:
self.logger.info("No changes to retrieve")
return
super(RetrieveChanges, self)._run_task()
# update package.xml
package_xml_opts = {
"directory": self.options["path"],
"api_version": self.options["api_version"],
}
if self.options["path"] == "src":
package_xml_opts[
"package_name"
] = self.project_config.project__package__name
package_xml = PackageXmlGenerator(**package_xml_opts)()
with open(os.path.join(self.options["path"], "package.xml"), "w") as f:
f.write(package_xml)
self._store_snapshot()
def _get_destructive_changes(self, path=None):
if not path:
path = self.options["path"]
generator = PackageXmlGenerator(
directory=path,
api_version=self.project_config.project__package__api_version,
delete=True,
)
return generator()
def test_render_xml__managed(self):
with temporary_dir() as path:
generator = PackageXmlGenerator(
path,
"43.0",
"Test Package",
managed=True,
install_class="Install",
uninstall_class="Uninstall",
)
result = generator()
self.assertEqual(EXPECTED_MANAGED, result)
def test_namespaced_report_folder(self):
api_version = "36.0"
package_name = "Test Package"
test_dir = "namespaced_report_folder"
path = os.path.join(__location__, "package_metadata", test_dir)
generator = PackageXmlGenerator(path, api_version, package_name)
with open(os.path.join(path, "package.xml"), "r") as f:
expected_package_xml = f.read().strip()
package_xml = generator()
self.assertEqual(package_xml, expected_package_xml)
def _write_manifest(changes, path, api_version):
"""Write a package.xml for the specified changes and API version."""
type_members = defaultdict(list)
for change in changes:
type_members[change["MemberType"]].append(change["MemberName"])
generator = PackageXmlGenerator(
".",
api_version,
types=[MetadataType(name, members) for name, members in type_members.items()],
)
package_xml = generator()
with open(os.path.join(path, "package.xml"), "w") as f:
f.write(package_xml)
def test_package_name_urlencoding(self):
api_version = "36.0"
package_name = "Test & Package"
expected = '<?xml version="1.0" encoding="UTF-8"?>\n'
expected += '<Package xmlns="http://soap.sforce.com/2006/04/metadata">\n'
expected += " <fullName>Test %26 Package</fullName>\n"
expected += " <version>{}</version>\n".format(api_version)
expected += "</Package>"
with temporary_dir() as path:
generator = PackageXmlGenerator(path, api_version, package_name)
package_xml = generator()
self.assertEqual(package_xml, expected)
def test_package_name_urlencoding(self):
api_version = '36.0'
package_name = 'Test & Package'
path = tempfile.mkdtemp()
expected = '<?xml version="1.0" encoding="UTF-8"?>\n'
expected += '<Package xmlns="http://soap.sforce.com/2006/04/metadata">\n'
expected += ' <fullName>Test %26 Package</fullName>\n'
expected += ' <version>{}</version>\n'.format(api_version)
expected += '</Package>'
generator = PackageXmlGenerator(path, api_version, package_name)
package_xml = generator()
self.assertEquals(package_xml, expected)
def test_namespaced_report_folder(self):
api_version = '36.0'
package_name = 'Test Package'
test_dir = 'namespaced_report_folder'
path = os.path.join(
__location__,
'package_metadata',
test_dir,
)
generator = PackageXmlGenerator(path, api_version, package_name)
with open(os.path.join(path, 'package.xml'), 'r') as f:
expected_package_xml = f.read().strip()
package_xml = generator()
self.assertEquals(package_xml, expected_package_xml)
def _get_destructive_changes(self, path=None):
if not path:
path = self.options['path']
generator = PackageXmlGenerator(
directory=path,
api_version=self.project_config.project__package__api_version,
delete=True,
)
namespace = ''
if self.options['managed'] in [True, 'True', 'true']:
if self.options['namespace']:
namespace = self.options['namespace'] + '__'
destructive_changes = generator()
destructive_changes = destructive_changes.replace(
self.options['filename_token'], namespace)
return destructive_changes
def _get_destructive_changes(self, path=None):
if not path:
path = self.options["path"]
generator = PackageXmlGenerator(
directory=path,
api_version=self.project_config.project__package__api_version,
delete=True,
)
namespace = ""
if self.options["managed"]:
if self.options["namespace"]:
namespace = self.options["namespace"] + "__"
destructive_changes = generator()
destructive_changes = destructive_changes.replace(
self.options["filename_token"], namespace)
return destructive_changes
def _write_manifest(changes, path, api_version):
"""Write a package.xml for the specified changes and API version."""
type_members = defaultdict(list)
for change in changes:
mdtype = change["MemberType"]
# folders are retrieved along with their contained type
if mdtype.endswith("Folder"):
mdtype = mdtype[:-len("Folder")]
type_members[mdtype].append(change["MemberName"])
generator = PackageXmlGenerator(
".",
api_version,
types=[
MetadataType(name, members)
for name, members in type_members.items()
],
)
package_xml = generator()
with open(os.path.join(path, "package.xml"), "w") as f:
f.write(package_xml)
def _get_api(self):
type_members = defaultdict(list)
for change in self._filtered:
type_members[change["MemberType"]].append(change["MemberName"])
self.logger.info("{MemberType}: {MemberName}".format(**change))
package_xml_path = os.path.join(self.options["path"], "package.xml")
if os.path.isfile(package_xml_path):
with open(package_xml_path, "rb") as f:
current_package_xml = xmltodict.parse(f)
else:
current_package_xml = {"Package": {}}
merged_type_members = {}
mdtypes = current_package_xml["Package"].get("types", [])
mdtypes = mdtypes if isinstance(mdtypes, list) else [mdtypes]
for mdtype in mdtypes:
members = mdtype.get("members", [])
members = members if isinstance(members, list) else [members]
if members:
type_name = mdtype["name"]
merged_type_members[type_name] = members
for name, members in type_members.items():
if name in merged_type_members:
merged_type_members[name].extend(members)
else:
merged_type_members[name] = members
types = []
for name, members in merged_type_members.items():
types.append(MetadataType(name, members))
package_xml = PackageXmlGenerator(".",
self.options["api_version"],
types=types)()
return self.api_class(self, package_xml,
self.options.get("api_version"))
def _transform(self):
# call _transform_entity once per retrieved entity
# if the entity is an XML file, provide a parsed version
# and write the returned metadata into the deploy directory
parser = PackageXmlGenerator(
None, self.api_version) # We'll use it for its metadata_map
entity_configurations = [
entry for entry in parser.metadata_map if any([
subentry["type"] == self.entity
for subentry in parser.metadata_map[entry]
])
]
if not entity_configurations:
raise CumulusCIException(
f"Unable to locate configuration for entity {self.entity}")
configuration = parser.metadata_map[entity_configurations[0]][0]
if configuration["class"] not in [
"MetadataFilenameParser",
"CustomObjectParser",
]:
raise CumulusCIException(
f"MetadataSingleEntityTransformTask only supports manipulating complete, file-based XML entities (not {self.entity})"
)
extension = configuration["extension"]
directory = entity_configurations[0]
source_metadata_dir = self.retrieve_dir / directory
if "*" in self.api_names:
# Walk the retrieved directory to get the actual suite
# of API names retrieved and rebuild our api_names list.
self.api_names.remove("*")
self.api_names = self.api_names.union(
metadata_file.stem
for metadata_file in source_metadata_dir.iterdir()
if metadata_file.suffix == f".{extension}")
removed_api_names = set()
for api_name in self.api_names:
# Page Layout names can contain spaces, but parentheses and other
# characters like ' and < are quoted.
# We quote user-specified API names so we can locate the corresponding
# metadata files, but present them un-quoted in messages to the user.
unquoted_api_name = unquote(api_name)
path = source_metadata_dir / f"{api_name}.{extension}"
if not path.exists():
raise CumulusCIException(f"Cannot find metadata file {path}")
try:
tree = metadata_tree.parse(str(path))
except SyntaxError as err:
err.filename = path
raise err
transformed_xml = self._transform_entity(tree, unquoted_api_name)
if transformed_xml:
parent_dir = self.deploy_dir / directory
if not parent_dir.exists():
parent_dir.mkdir()
destination_path = parent_dir / f"{api_name}.{extension}"
with destination_path.open(mode="w", encoding="utf-8") as f:
f.write(transformed_xml.tostring(xml_declaration=True))
else:
# Make sure to remove from our package.xml
removed_api_names.add(api_name)
self.api_names = self.api_names - removed_api_names
def _generate_package_xml(self, deploy):
"""Synthesize a package.xml for generated metadata."""
generator = PackageXmlGenerator(str(self.deploy_dir), self.api_version)
return generator()
def test_parse_types_unknown_md_type(self):
with temporary_dir() as path:
os.mkdir(os.path.join(path, "bogus"))
generator = PackageXmlGenerator(path, "43.0", "Test Package")
with self.assertRaises(MetadataParserMissingError):
generator.parse_types()
def test_parse_types_unknown_md_type(self):
with temporary_dir() as path:
os.mkdir(os.path.join(path, "bogus"))
generator = PackageXmlGenerator(path, "43.0", "Test Package")
with self.assertRaises(MetadataParserMissingError):
generator.parse_types()
def retrieve_components(
components,
org_config,
target: str,
md_format: bool,
extra_package_xml_opts: dict,
namespace_tokenize: str,
api_version: str,
):
"""Retrieve specified components from an org into a target folder.
Retrieval is done using the sfdx force:source:retrieve command.
Set `md_format` to True if retrieving into a folder with a package
in metadata format. In this case the folder will be temporarily
converted to dx format for the retrieval and then converted back.
Retrievals to metadata format can also set `namespace_tokenize`
to a namespace prefix to replace it with a `%%%NAMESPACE%%%` token.
"""
with contextlib.ExitStack() as stack:
if md_format:
# Create target if it doesn't exist
if not os.path.exists(target):
os.mkdir(target)
touch(os.path.join(target, "package.xml"))
# Inject namespace
if namespace_tokenize:
process_text_in_directory(
target,
functools.partial(inject_namespace,
namespace=namespace_tokenize,
managed=True),
)
# Temporarily convert metadata format to DX format
stack.enter_context(temporary_dir())
os.mkdir("target")
# We need to create sfdx-project.json
# so that sfdx will recognize force-app as a package directory.
with open("sfdx-project.json", "w") as f:
json.dump(
{
"packageDirectories": [{
"path": "force-app",
"default": True
}]
}, f)
sfdx(
"force:mdapi:convert",
log_note="Converting to DX format",
args=["-r", target, "-d", "force-app"],
check_return=True,
)
# Construct package.xml with components to retrieve, in its own tempdir
package_xml_path = stack.enter_context(temporary_dir(chdir=False))
_write_manifest(components, package_xml_path, api_version)
# Retrieve specified components in DX format
sfdx(
"force:source:retrieve",
access_token=org_config.access_token,
log_note="Retrieving components",
args=[
"-a",
str(api_version),
"-x",
os.path.join(package_xml_path, "package.xml"),
"-w",
"5",
],
capture_output=False,
check_return=True,
env={"SFDX_INSTANCE_URL": org_config.instance_url},
)
if md_format:
# Convert back to metadata format
sfdx(
"force:source:convert",
log_note="Converting back to metadata format",
args=["-r", "force-app", "-d", target],
capture_output=False,
check_return=True,
)
# Reinject namespace tokens
if namespace_tokenize:
process_text_in_directory(
target,
functools.partial(tokenize_namespace,
namespace=namespace_tokenize),
)
# Regenerate package.xml,
# to avoid reformatting or losing package name/scripts
package_xml_opts = {
"directory": target,
"api_version": api_version,
**extra_package_xml_opts,
}
package_xml = PackageXmlGenerator(**package_xml_opts)()
with open(os.path.join(target, "package.xml"), "w") as f:
f.write(package_xml)
