def find_and_replace(build, *files, **kwargs):
'''replace one string with another in a set of files
:param kwargs: must contain ``find`` and ``replace`` keys,
representing the string to look for, and string to replace
with, respectively.
:param kwargs: can also contain the ``template`` boolean
argument, which determines if we will run the ``replace``
argument through genshi templating first (defaults to True).
:param files: array of glob patterns to select files
:param kwargs: must contain ``find`` and ``replace`` keys
'''
if "in" in kwargs:
files = kwargs['in']
if "find" not in kwargs:
raise ConfigurationError("Find not passed in to find_and_replace")
if "replace" not in kwargs:
raise ConfigurationError("Replace not passed in to find_and_replace")
template = kwargs.get('template', True)
find = kwargs["find"]
replace = kwargs['replace']
if template:
replace = utils.render_string(build.config, replace)
replace_summary = replace[:60]+'...' if len(replace) > 60 else replace
build.log.debug("replacing %s with %s" % (find, repr(replace_summary)))
for glob_str in files:
found_files = glob.glob(utils.render_string(build.config, glob_str))
if len(found_files) == 0:
build.log.warning('No files were found to match pattern "%s"' % glob_str)
for _file in found_files:
_replace_in_file(build, _file, find, replace)
def _rename_or_copy_files(build, frm, to, rename=True, ignore_patterns=None):
if ignore_patterns is None:
ignore_patterns = []
from_, to = utils.render_string(build.config, frm), utils.render_string(
build.config, to)
if path.isdir(from_):
ignore_func = git_ignore(from_, ignore_patterns)
else:
ignore_func = None
if rename:
build.log.debug('renaming {from_} to {to}'.format(**locals()))
shutil.move(from_, to)
else:
if '*' in to:
# looks like a glob - last directory in path might not exist.
tos = glob.glob(path.dirname(to))
tos = [path.join(t, path.basename(to)) for t in tos]
else:
# don't glob in case the to path doesn't exist yet
tos = [to]
for found_to in tos:
build.log.debug('copying {from_} to {found_to}'.format(**locals()))
if path.isdir(from_):
shutil.copytree(from_, found_to, ignore=ignore_func)
else:
shutil.copy(from_, found_to)
def _rename_or_copy_files(build, frm, to, rename=True, ignore_patterns=None):
if ignore_patterns is None:
ignore_patterns = []
from_, to = utils.render_string(build.config, frm), utils.render_string(build.config, to)
if path.isdir(from_):
ignore_func = git_ignore(from_, ignore_patterns)
else:
ignore_func = None
if rename:
build.log.debug('renaming {from_} to {to}'.format(**locals()))
shutil.move(from_, to)
else:
if '*' in to:
# looks like a glob - last directory in path might not exist.
tos = glob.glob(path.dirname(to))
tos = [path.join(t,path.basename(to)) for t in tos]
else:
# don't glob in case the to path doesn't exist yet
tos = [to]
for found_to in tos:
build.log.debug('copying {from_} to {found_to}'.format(**locals()))
if path.isdir(from_):
shutil.copytree(from_, found_to, ignore=ignore_func)
else:
shutil.copy(from_, found_to)
def set_attribute_value_xml(build, file, value, attribute, element=None):
'''set contents of an XML element's attribute
:param build: the current build.Build
:param file: filename or file object
:param value: the new attribute value (will be templated)
:param attribute: attribute name
:param element: tag name or path to change (defaults to root node)
'''
xml = ElementTree.ElementTree()
xml.parse(file)
if element is None:
el = xml.getroot()
else:
el = xml.find(element, dict((v,k) for k,v in ElementTree._namespace_map.items()))
# set is not aware of namespaces, so we have to replace "namespace" with "{schema}"
namespaces = dict((v,k) for k,v in ElementTree._namespace_map.items())
if ":" in attribute:
parts = attribute.split(":")
attribute = "{%s}%s" % (namespaces[parts[0]], parts[1])
el.set(attribute, utils.render_string(build.config, value))
xml.write(file)
def find_and_replace_in_dir(build,
root_dir,
find,
replace,
file_suffixes=("html", ),
template=False,
**kw):
'For all files ending with one of the suffixes, under the root_dir, replace ``find`` with ``replace``'
if template:
replace = utils.render_string(build.config, replace)
build.log.debug("replacing {find} with {replace} in {files}".format(
find=find,
replace=replace,
files="{0}/**/*.{1}".format(root_dir, file_suffixes)))
found_roots = glob.glob(root_dir)
if len(found_roots) == 0:
build.log.warning('No files were found to match pattern "%s"' %
root_dir)
for found_root in found_roots:
for root, _, files, depth in walk_with_depth(found_root):
for file_ in files:
if file_.rpartition('.')[2] in file_suffixes:
find_with_fixed_path = find.replace(
"%{back_to_parent}%", "../" * (depth + 1))
replace_with_fixed_path = replace.replace(
"%{back_to_parent}%", "../" * (depth + 1))
_replace_in_file(build, path.join(root, file_),
find_with_fixed_path,
replace_with_fixed_path)
def set_attribute_value_xml(build, file, value, attribute, element=None):
'''set contents of an XML element's attribute
:param build: the current build.Build
:param file: filename or file object
:param value: the new attribute value (will be templated)
:param attribute: attribute name
:param element: tag name or path to change (defaults to root node)
'''
xml = ElementTree.ElementTree()
xml.parse(file)
if element is None:
el = xml.getroot()
else:
el = xml.find(
element, dict(
(v, k) for k, v in ElementTree._namespace_map.items()))
# set is not aware of namespaces, so we have to replace "namespace" with "{schema}"
namespaces = dict((v, k) for k, v in ElementTree._namespace_map.items())
if ":" in attribute:
parts = attribute.split(":")
attribute = "{%s}%s" % (namespaces[parts[0]], parts[1])
el.set(attribute, utils.render_string(build.config, value))
xml.write(file)
def remove_files(build, *removes):
build.log.info('deleting %d files' % len(removes))
for rem in removes:
real_rem = utils.render_string(build.config, rem)
build.log.debug('deleting %s' % real_rem)
if path.isfile(real_rem):
os.remove(real_rem)
else:
shutil.rmtree(real_rem, ignore_errors=True)
def remove_files(build, *removes):
build.log.info('deleting %d files' % len(removes))
for rem in removes:
real_rem = utils.render_string(build.config, rem)
build.log.debug('deleting %s' % real_rem)
if path.isfile(real_rem):
os.remove(real_rem)
else:
shutil.rmtree(real_rem, ignore_errors=True)
def write_config(build, filename, content):
# We hang various things we shouldn't off config, this is pretty horrible
clean_config = copy(build.config)
if 'json' in clean_config:
clean_config.pop('json')
if 'android_sdk_dir' in clean_config:
clean_config.pop('android_sdk_dir')
content = utils.render_string({'config': json.dumps(clean_config, indent=4, sort_keys=True)}, content)
with open(filename, 'w') as fileobj:
fileobj.write(content)
def regex_replace_in_file(build, filename, find, replace, template=False):
build.log.debug("regex replace in {filename}".format(**locals()))
if template:
replace = utils.render_string(build.config, replace)
tmp_file = uuid.uuid4().hex
in_file_contents = read_file_as_str(filename)
in_file_contents = re.sub(find, replace, in_file_contents)
with codecs.open(tmp_file, 'w', encoding='utf8') as out_file:
out_file.write(in_file_contents)
os.remove(filename)
shutil.move(tmp_file, filename)
def write_config(build, filename, content):
# We hang various things we shouldn't off config, this is pretty horrible
clean_config = copy(build.config)
if 'json' in clean_config:
clean_config.pop('json')
if 'android_sdk_dir' in clean_config:
clean_config.pop('android_sdk_dir')
content = utils.render_string(
{'config': json.dumps(clean_config, indent=4, sort_keys=True)},
content)
with open(filename, 'w') as fileobj:
fileobj.write(content)
def regex_replace_in_file(build, filename, find, replace, template=False):
build.log.debug("regex replace in {filename}".format(**locals()))
if template:
replace = utils.render_string(build.config, replace)
tmp_file = uuid.uuid4().hex
in_file_contents = read_file_as_str(filename)
in_file_contents = re.sub(find, replace, in_file_contents)
with codecs.open(tmp_file, 'w', encoding='utf8') as out_file:
out_file.write(in_file_contents)
os.remove(filename)
shutil.move(tmp_file, filename)
def find_and_replace(build, *files, **kwargs):
'''replace one string with another in a set of files
:param kwargs: must contain ``find`` and ``replace`` keys,
representing the string to look for, and string to replace
with, respectively.
:param kwargs: can also contain the ``template`` boolean
argument, which determines if we will run the ``replace``
argument through genshi templating first (defaults to True).
:param files: array of glob patterns to select files
:param kwargs: must contain ``find`` and ``replace`` keys
'''
if "in" in kwargs:
files = kwargs['in']
if "find" not in kwargs:
raise ConfigurationError("Find not passed in to find_and_replace")
if "replace" not in kwargs:
raise ConfigurationError("Replace not passed in to find_and_replace")
template = kwargs.get('template', True)
find = kwargs["find"]
replace = kwargs['replace']
if template:
replace = utils.render_string(build.config, replace)
replace_summary = replace[:60] + '...' if len(replace) > 60 else replace
build.log.debug("replacing %s with %s" % (find, repr(replace_summary)))
for glob_str in files:
found_files = glob.glob(utils.render_string(build.config, glob_str))
if len(found_files) == 0:
build.log.warning('No files were found to match pattern "%s"' %
glob_str)
for _file in found_files:
_replace_in_file(build, _file, find, replace)
def set_in_config(build, key, value):
if isinstance(value, str):
value = utils.render_string(build.config, value)
build.log.debug("Setting {key} to {value} in app_config.json".format(key=key, value=value))
key = key.split(".")
last = key.pop()
at = build.config
for part in key:
if not part in at or not isinstance(at[part], dict):
at[part] = {}
at = at[part]
at[last] = value
def set_in_config(build, key, value):
if isinstance(value, str):
value = utils.render_string(build.config, value)
build.log.debug("Setting {key} to {value} in app_config.json".format(
key=key, value=value))
key = key.split(".")
last = key.pop()
at = build.config
for part in key:
if not part in at or not isinstance(at[part], dict):
at[part] = {}
at = at[part]
at[last] = value
def set_element_value_xml(build, file, value, element=None):
'''set text contents of an XML element
:param build: the current build.Build
:param file: filename or file object
:param value: the new element contents (will be templated)
:param element: tag name or path to change (defaults to root node)
'''
xml = ElementTree.ElementTree()
xml.parse(file)
if element is None:
el = xml.getroot()
else:
el = xml.find(element, dict((v,k) for k,v in ElementTree._namespace_map.items()))
el.text = utils.render_string(build.config, value).decode('utf8', errors='replace')
xml.write(file)
def set_in_biplist(build, filename, key, value):
# biplist import must be done here, as in the server context, biplist doesn't exist
import biplist
if isinstance(value, str):
value = utils.render_string(build.config, value)
build.log.debug(u"setting {key} to {value} in {files}".format(
key=key, value=value, files=filename))
found_files = glob.glob(filename)
if len(found_files) == 0:
build.log.warning('No files were found to match pattern "%s"' %
filename)
for found_file in found_files:
plist = biplist.readPlist(found_file)
plist = utils.transform(plist, key, lambda _: value, allow_set=True)
biplist.writePlist(plist, found_file)
def set_in_biplist(build, filename, key, value):
# biplist import must be done here, as in the server context, biplist doesn't exist
import biplist
if isinstance(value, str):
value = utils.render_string(build.config, value)
build.log.debug(u"setting {key} to {value} in {files}".format(
key=key, value=value, files=filename
))
found_files = glob.glob(filename)
if len(found_files) == 0:
build.log.warning('No files were found to match pattern "%s"' % filename)
for found_file in found_files:
plist = biplist.readPlist(found_file)
plist = utils.transform(plist, key, lambda _: value, allow_set=True)
biplist.writePlist(plist, found_file)
def add_to_json_array(build, filename, key, value):
if isinstance(value, str):
value = utils.render_string(build.config, value)
build.log.debug("adding '{value}' to '{key}' in {files}".format(
key=key, value=value, files=filename))
found_files = glob.glob(filename)
if len(found_files) == 0:
build.log.warning('No files were found to match pattern "%s"' %
filename)
for found_file in found_files:
file_json = {}
with open(found_file, "r") as opened_file:
file_json = json.load(opened_file)
# TODO: . separated keys?
file_json[key].append(value)
with open(found_file, "w") as opened_file:
json.dump(file_json, opened_file, indent=2, sort_keys=True)
def find_and_replace_in_dir(build, root_dir, find, replace, file_suffixes=("html",), template=False, **kw):
'For all files ending with one of the suffixes, under the root_dir, replace ``find`` with ``replace``'
if template:
replace = utils.render_string(build.config, replace)
build.log.debug("replacing {find} with {replace} in {files}".format(
find=find, replace=replace, files="{0}/**/*.{1}".format(root_dir, file_suffixes)
))
found_roots = glob.glob(root_dir)
if len(found_roots) == 0:
build.log.warning('No files were found to match pattern "%s"' % root_dir)
for found_root in found_roots:
for root, _, files, depth in walk_with_depth(found_root):
for file_ in files:
if file_.rpartition('.')[2] in file_suffixes:
find_with_fixed_path = find.replace("%{back_to_parent}%", "../" * (depth+1))
replace_with_fixed_path = replace.replace("%{back_to_parent}%", "../" * (depth+1))
_replace_in_file(build, path.join(root, file_), find_with_fixed_path, replace_with_fixed_path)
def set_element_value_xml(build, file, value, element=None):
'''set text contents of an XML element
:param build: the current build.Build
:param file: filename or file object
:param value: the new element contents (will be templated)
:param element: tag name or path to change (defaults to root node)
'''
xml = ElementTree.ElementTree()
xml.parse(file)
if element is None:
el = xml.getroot()
else:
el = xml.find(
element, dict(
(v, k) for k, v in ElementTree._namespace_map.items()))
el.text = utils.render_string(build.config, value).decode('utf8',
errors='replace')
xml.write(file)
def add_to_json_array(build, filename, key, value):
if isinstance(value, str):
value = utils.render_string(build.config, value)
build.log.debug("adding '{value}' to '{key}' in {files}".format(
key=key, value=value, files=filename
))
found_files = glob.glob(filename)
if len(found_files) == 0:
build.log.warning('No files were found to match pattern "%s"' % filename)
for found_file in found_files:
file_json = {}
with open(found_file, "r") as opened_file:
file_json = json.load(opened_file)
# TODO: . separated keys?
file_json[key].append(value)
with open(found_file, "w") as opened_file:
json.dump(file_json, opened_file, indent=2, sort_keys=True)
def write_config(build, filename, content, mapping_file=None):
# We hang various things we shouldn't off config, this is pretty horrible
clean_config = copy(build.config)
if 'json' in clean_config:
clean_config.pop('json')
if 'android_sdk_dir' in clean_config:
clean_config.pop('android_sdk_dir')
if mapping_file is None:
module_mapping = {}
else:
with open(mapping_file) as mapping_fileobj:
module_mapping = json.load(mapping_fileobj)
content = utils.render_string({
'config': json.dumps(clean_config, indent=4, sort_keys=True),
'module_mapping': json.dumps(module_mapping, indent=4, sort_keys=True)
}, content)
with open(filename, 'w') as fileobj:
fileobj.write(content)
def write_config(build, filename, content, mapping_file=None):
# We hang various things we shouldn't off config, this is pretty horrible
clean_config = copy(build.config)
if 'json' in clean_config:
clean_config.pop('json')
if 'android_sdk_dir' in clean_config:
clean_config.pop('android_sdk_dir')
if mapping_file is None:
module_mapping = {}
else:
with open(mapping_file) as mapping_fileobj:
module_mapping = json.load(mapping_fileobj)
content = utils.render_string(
{
'config': json.dumps(clean_config, indent=4, sort_keys=True),
'module_mapping': json.dumps(
module_mapping, indent=4, sort_keys=True)
}, content)
with open(filename, 'w') as fileobj:
fileobj.write(content)