def find_and_replace(build, *files, **kwargs): '''replace one string with another in a set of files :param kwargs: must contain ``find`` and ``replace`` keys, representing the string to look for, and string to replace with, respectively. :param kwargs: can also contain the ``template`` boolean argument, which determines if we will run the ``replace`` argument through genshi templating first (defaults to True). :param files: array of glob patterns to select files :param kwargs: must contain ``find`` and ``replace`` keys ''' if "in" in kwargs: files = kwargs['in'] if "find" not in kwargs: raise ConfigurationError("Find not passed in to find_and_replace") if "replace" not in kwargs: raise ConfigurationError("Replace not passed in to find_and_replace") template = kwargs.get('template', True) find = kwargs["find"] replace = kwargs['replace'] if template: replace = utils.render_string(build.config, replace) replace_summary = replace[:60]+'...' if len(replace) > 60 else replace build.log.debug("replacing %s with %s" % (find, repr(replace_summary))) for glob_str in files: found_files = glob.glob(utils.render_string(build.config, glob_str)) if len(found_files) == 0: build.log.warning('No files were found to match pattern "%s"' % glob_str) for _file in found_files: _replace_in_file(build, _file, find, replace)
def _rename_or_copy_files(build, frm, to, rename=True, ignore_patterns=None): if ignore_patterns is None: ignore_patterns = [] from_, to = utils.render_string(build.config, frm), utils.render_string( build.config, to) if path.isdir(from_): ignore_func = git_ignore(from_, ignore_patterns) else: ignore_func = None if rename: build.log.debug('renaming {from_} to {to}'.format(**locals())) shutil.move(from_, to) else: if '*' in to: # looks like a glob - last directory in path might not exist. tos = glob.glob(path.dirname(to)) tos = [path.join(t, path.basename(to)) for t in tos] else: # don't glob in case the to path doesn't exist yet tos = [to] for found_to in tos: build.log.debug('copying {from_} to {found_to}'.format(**locals())) if path.isdir(from_): shutil.copytree(from_, found_to, ignore=ignore_func) else: shutil.copy(from_, found_to)
def _rename_or_copy_files(build, frm, to, rename=True, ignore_patterns=None): if ignore_patterns is None: ignore_patterns = [] from_, to = utils.render_string(build.config, frm), utils.render_string(build.config, to) if path.isdir(from_): ignore_func = git_ignore(from_, ignore_patterns) else: ignore_func = None if rename: build.log.debug('renaming {from_} to {to}'.format(**locals())) shutil.move(from_, to) else: if '*' in to: # looks like a glob - last directory in path might not exist. tos = glob.glob(path.dirname(to)) tos = [path.join(t,path.basename(to)) for t in tos] else: # don't glob in case the to path doesn't exist yet tos = [to] for found_to in tos: build.log.debug('copying {from_} to {found_to}'.format(**locals())) if path.isdir(from_): shutil.copytree(from_, found_to, ignore=ignore_func) else: shutil.copy(from_, found_to)
def set_attribute_value_xml(build, file, value, attribute, element=None): '''set contents of an XML element's attribute :param build: the current build.Build :param file: filename or file object :param value: the new attribute value (will be templated) :param attribute: attribute name :param element: tag name or path to change (defaults to root node) ''' xml = ElementTree.ElementTree() xml.parse(file) if element is None: el = xml.getroot() else: el = xml.find(element, dict((v,k) for k,v in ElementTree._namespace_map.items())) # set is not aware of namespaces, so we have to replace "namespace" with "{schema}" namespaces = dict((v,k) for k,v in ElementTree._namespace_map.items()) if ":" in attribute: parts = attribute.split(":") attribute = "{%s}%s" % (namespaces[parts[0]], parts[1]) el.set(attribute, utils.render_string(build.config, value)) xml.write(file)
def find_and_replace_in_dir(build, root_dir, find, replace, file_suffixes=("html", ), template=False, **kw): 'For all files ending with one of the suffixes, under the root_dir, replace ``find`` with ``replace``' if template: replace = utils.render_string(build.config, replace) build.log.debug("replacing {find} with {replace} in {files}".format( find=find, replace=replace, files="{0}/**/*.{1}".format(root_dir, file_suffixes))) found_roots = glob.glob(root_dir) if len(found_roots) == 0: build.log.warning('No files were found to match pattern "%s"' % root_dir) for found_root in found_roots: for root, _, files, depth in walk_with_depth(found_root): for file_ in files: if file_.rpartition('.')[2] in file_suffixes: find_with_fixed_path = find.replace( "%{back_to_parent}%", "../" * (depth + 1)) replace_with_fixed_path = replace.replace( "%{back_to_parent}%", "../" * (depth + 1)) _replace_in_file(build, path.join(root, file_), find_with_fixed_path, replace_with_fixed_path)
def set_attribute_value_xml(build, file, value, attribute, element=None): '''set contents of an XML element's attribute :param build: the current build.Build :param file: filename or file object :param value: the new attribute value (will be templated) :param attribute: attribute name :param element: tag name or path to change (defaults to root node) ''' xml = ElementTree.ElementTree() xml.parse(file) if element is None: el = xml.getroot() else: el = xml.find( element, dict( (v, k) for k, v in ElementTree._namespace_map.items())) # set is not aware of namespaces, so we have to replace "namespace" with "{schema}" namespaces = dict((v, k) for k, v in ElementTree._namespace_map.items()) if ":" in attribute: parts = attribute.split(":") attribute = "{%s}%s" % (namespaces[parts[0]], parts[1]) el.set(attribute, utils.render_string(build.config, value)) xml.write(file)
def remove_files(build, *removes): build.log.info('deleting %d files' % len(removes)) for rem in removes: real_rem = utils.render_string(build.config, rem) build.log.debug('deleting %s' % real_rem) if path.isfile(real_rem): os.remove(real_rem) else: shutil.rmtree(real_rem, ignore_errors=True)
def write_config(build, filename, content): # We hang various things we shouldn't off config, this is pretty horrible clean_config = copy(build.config) if 'json' in clean_config: clean_config.pop('json') if 'android_sdk_dir' in clean_config: clean_config.pop('android_sdk_dir') content = utils.render_string({'config': json.dumps(clean_config, indent=4, sort_keys=True)}, content) with open(filename, 'w') as fileobj: fileobj.write(content)
def regex_replace_in_file(build, filename, find, replace, template=False): build.log.debug("regex replace in {filename}".format(**locals())) if template: replace = utils.render_string(build.config, replace) tmp_file = uuid.uuid4().hex in_file_contents = read_file_as_str(filename) in_file_contents = re.sub(find, replace, in_file_contents) with codecs.open(tmp_file, 'w', encoding='utf8') as out_file: out_file.write(in_file_contents) os.remove(filename) shutil.move(tmp_file, filename)
def write_config(build, filename, content): # We hang various things we shouldn't off config, this is pretty horrible clean_config = copy(build.config) if 'json' in clean_config: clean_config.pop('json') if 'android_sdk_dir' in clean_config: clean_config.pop('android_sdk_dir') content = utils.render_string( {'config': json.dumps(clean_config, indent=4, sort_keys=True)}, content) with open(filename, 'w') as fileobj: fileobj.write(content)
def find_and_replace(build, *files, **kwargs): '''replace one string with another in a set of files :param kwargs: must contain ``find`` and ``replace`` keys, representing the string to look for, and string to replace with, respectively. :param kwargs: can also contain the ``template`` boolean argument, which determines if we will run the ``replace`` argument through genshi templating first (defaults to True). :param files: array of glob patterns to select files :param kwargs: must contain ``find`` and ``replace`` keys ''' if "in" in kwargs: files = kwargs['in'] if "find" not in kwargs: raise ConfigurationError("Find not passed in to find_and_replace") if "replace" not in kwargs: raise ConfigurationError("Replace not passed in to find_and_replace") template = kwargs.get('template', True) find = kwargs["find"] replace = kwargs['replace'] if template: replace = utils.render_string(build.config, replace) replace_summary = replace[:60] + '...' if len(replace) > 60 else replace build.log.debug("replacing %s with %s" % (find, repr(replace_summary))) for glob_str in files: found_files = glob.glob(utils.render_string(build.config, glob_str)) if len(found_files) == 0: build.log.warning('No files were found to match pattern "%s"' % glob_str) for _file in found_files: _replace_in_file(build, _file, find, replace)
def set_in_config(build, key, value): if isinstance(value, str): value = utils.render_string(build.config, value) build.log.debug("Setting {key} to {value} in app_config.json".format(key=key, value=value)) key = key.split(".") last = key.pop() at = build.config for part in key: if not part in at or not isinstance(at[part], dict): at[part] = {} at = at[part] at[last] = value
def set_in_config(build, key, value): if isinstance(value, str): value = utils.render_string(build.config, value) build.log.debug("Setting {key} to {value} in app_config.json".format( key=key, value=value)) key = key.split(".") last = key.pop() at = build.config for part in key: if not part in at or not isinstance(at[part], dict): at[part] = {} at = at[part] at[last] = value
def set_element_value_xml(build, file, value, element=None): '''set text contents of an XML element :param build: the current build.Build :param file: filename or file object :param value: the new element contents (will be templated) :param element: tag name or path to change (defaults to root node) ''' xml = ElementTree.ElementTree() xml.parse(file) if element is None: el = xml.getroot() else: el = xml.find(element, dict((v,k) for k,v in ElementTree._namespace_map.items())) el.text = utils.render_string(build.config, value).decode('utf8', errors='replace') xml.write(file)
def set_in_biplist(build, filename, key, value): # biplist import must be done here, as in the server context, biplist doesn't exist import biplist if isinstance(value, str): value = utils.render_string(build.config, value) build.log.debug(u"setting {key} to {value} in {files}".format( key=key, value=value, files=filename)) found_files = glob.glob(filename) if len(found_files) == 0: build.log.warning('No files were found to match pattern "%s"' % filename) for found_file in found_files: plist = biplist.readPlist(found_file) plist = utils.transform(plist, key, lambda _: value, allow_set=True) biplist.writePlist(plist, found_file)
def set_in_biplist(build, filename, key, value): # biplist import must be done here, as in the server context, biplist doesn't exist import biplist if isinstance(value, str): value = utils.render_string(build.config, value) build.log.debug(u"setting {key} to {value} in {files}".format( key=key, value=value, files=filename )) found_files = glob.glob(filename) if len(found_files) == 0: build.log.warning('No files were found to match pattern "%s"' % filename) for found_file in found_files: plist = biplist.readPlist(found_file) plist = utils.transform(plist, key, lambda _: value, allow_set=True) biplist.writePlist(plist, found_file)
def add_to_json_array(build, filename, key, value): if isinstance(value, str): value = utils.render_string(build.config, value) build.log.debug("adding '{value}' to '{key}' in {files}".format( key=key, value=value, files=filename)) found_files = glob.glob(filename) if len(found_files) == 0: build.log.warning('No files were found to match pattern "%s"' % filename) for found_file in found_files: file_json = {} with open(found_file, "r") as opened_file: file_json = json.load(opened_file) # TODO: . separated keys? file_json[key].append(value) with open(found_file, "w") as opened_file: json.dump(file_json, opened_file, indent=2, sort_keys=True)
def find_and_replace_in_dir(build, root_dir, find, replace, file_suffixes=("html",), template=False, **kw): 'For all files ending with one of the suffixes, under the root_dir, replace ``find`` with ``replace``' if template: replace = utils.render_string(build.config, replace) build.log.debug("replacing {find} with {replace} in {files}".format( find=find, replace=replace, files="{0}/**/*.{1}".format(root_dir, file_suffixes) )) found_roots = glob.glob(root_dir) if len(found_roots) == 0: build.log.warning('No files were found to match pattern "%s"' % root_dir) for found_root in found_roots: for root, _, files, depth in walk_with_depth(found_root): for file_ in files: if file_.rpartition('.')[2] in file_suffixes: find_with_fixed_path = find.replace("%{back_to_parent}%", "../" * (depth+1)) replace_with_fixed_path = replace.replace("%{back_to_parent}%", "../" * (depth+1)) _replace_in_file(build, path.join(root, file_), find_with_fixed_path, replace_with_fixed_path)
def set_element_value_xml(build, file, value, element=None): '''set text contents of an XML element :param build: the current build.Build :param file: filename or file object :param value: the new element contents (will be templated) :param element: tag name or path to change (defaults to root node) ''' xml = ElementTree.ElementTree() xml.parse(file) if element is None: el = xml.getroot() else: el = xml.find( element, dict( (v, k) for k, v in ElementTree._namespace_map.items())) el.text = utils.render_string(build.config, value).decode('utf8', errors='replace') xml.write(file)
def add_to_json_array(build, filename, key, value): if isinstance(value, str): value = utils.render_string(build.config, value) build.log.debug("adding '{value}' to '{key}' in {files}".format( key=key, value=value, files=filename )) found_files = glob.glob(filename) if len(found_files) == 0: build.log.warning('No files were found to match pattern "%s"' % filename) for found_file in found_files: file_json = {} with open(found_file, "r") as opened_file: file_json = json.load(opened_file) # TODO: . separated keys? file_json[key].append(value) with open(found_file, "w") as opened_file: json.dump(file_json, opened_file, indent=2, sort_keys=True)
def write_config(build, filename, content, mapping_file=None): # We hang various things we shouldn't off config, this is pretty horrible clean_config = copy(build.config) if 'json' in clean_config: clean_config.pop('json') if 'android_sdk_dir' in clean_config: clean_config.pop('android_sdk_dir') if mapping_file is None: module_mapping = {} else: with open(mapping_file) as mapping_fileobj: module_mapping = json.load(mapping_fileobj) content = utils.render_string({ 'config': json.dumps(clean_config, indent=4, sort_keys=True), 'module_mapping': json.dumps(module_mapping, indent=4, sort_keys=True) }, content) with open(filename, 'w') as fileobj: fileobj.write(content)
def write_config(build, filename, content, mapping_file=None): # We hang various things we shouldn't off config, this is pretty horrible clean_config = copy(build.config) if 'json' in clean_config: clean_config.pop('json') if 'android_sdk_dir' in clean_config: clean_config.pop('android_sdk_dir') if mapping_file is None: module_mapping = {} else: with open(mapping_file) as mapping_fileobj: module_mapping = json.load(mapping_fileobj) content = utils.render_string( { 'config': json.dumps(clean_config, indent=4, sort_keys=True), 'module_mapping': json.dumps( module_mapping, indent=4, sort_keys=True) }, content) with open(filename, 'w') as fileobj: fileobj.write(content)