def upload_filtered(self, localpath, remotepath, include_list=[], exclude_list=[]):
# Create the base remote path first
try:
self.sftp.mkdir(remotepath)
except:
pass
# Switch to the directory we wnat to copy across
os.chdir(localpath)
for root, dirs, files in os.walk('.'):
#for root, dirs, files in os.walk(localpath):
if exclude_list:
#dirs[:] = [d for d in dirs if not os.path.join(root, d) in exclude_list]
dirs[:] = [d for d in dirs if not fileutil.check_include(root, d, exclude_list)]
files[:] = [f for f in files if not fileutil.check_include(root, f, exclude_list)]
if include_list:
dirs[:] = [d for d in dirs if fileutil.check_include(root, d, include_list)]
files[:] = [f for f in files if fileutil.check_include(root, f, include_list)]
try:
remote_create_path = os.path.join(remotepath, root)
logging.getLogger().debug("Creating remote dir {}".format(remote_create_path))
self.sftp.mkdir(remote_create_path)
except:
pass
for file in files:
source_file_path = os.path.join(root, file)
remote_file_path = os.path.join(remote_create_path, file)
logged_source_file_path = os.path.join(localpath, source_file_path)
logging.getLogger().debug("Uploading file FROM: {} TO: {}".format(source_file_path, remote_file_path))
self.upload(source_file_path, remote_file_path)
def test_contains_file_re_path(self):
rootdir = os.path.abspath('fileutil')
dir1dir = os.path.abspath('fileutil/dir1')
dir11dir = os.path.abspath('fileutil/dir1/dir1-1')
dir2dir = os.path.abspath('fileutil/dir2')
# standard tests of correct functionality
self.assertTrue(fileutil.check_include(rootdir, 'dir1', [rootdir + '/*']))
self.assertTrue(fileutil.check_include(rootdir, 'dir2/file2-1', [rootdir + '/dir2/f*le*1']))
self.assertTrue(fileutil.check_include(rootdir, 'dir2/file2-1', [rootdir + '*/dir2']))
self.assertTrue(fileutil.check_include(rootdir, 'dir1/dir1-1/abc', [rootdir + '*/dir2/../*/dir1-1']))
# check that dots don't get treated as a wildcard
self.assertFalse(fileutil.check_include(rootdir, 'dir1/agit', [rootdir + '/dir1/.git']))
def process_ansible_config_files(self, clouddata, output_path, include_list=[], exclude_list=[]):
"""
Generate a personalized ansible config environment based upon a clouddata config
:param clouddata: De-serialized YAML of clouddata
:param output_path: Output path for customized ansible config
:return:
"""
for rootdir, subdirList, fileList in os.walk(self.config_path):
env = Environment(block_start_string='[%',block_end_string='%]',
variable_start_string='[{',variable_end_string='}]',
loader=FileSystemLoader(rootdir))
if exclude_list:
subdirList[:] = [d for d in subdirList if not fileutil.check_include(rootdir, d, exclude_list)]
fileList[:] = [f for f in fileList if not fileutil.check_include(rootdir, f, exclude_list)]
if include_list:
subdirList[:] = [d for d in subdirList if fileutil.check_include(rootdir, d, include_list)]
fileList[:] = [f for f in fileList if fileutil.check_include(rootdir, f, include_list)]
for fname in fileList:
logging.getLogger().debug("Jinjafying ansible config file {}".format(fname))
template = env.get_template(fname)
try:
outputcontent = template.render(cloud=clouddata)
reldir = os.path.relpath(rootdir, self.config_path)
outputdir = os.path.join(output_path, reldir)
except Exception as error:
logging.getLogger().error("Applying clouddata template to file {} failed: {}".format(
fname, str(error)))
raise Exception("An error occurred processing ansible config area {}".format(
os.path.join(rootdir, fname)))
try:
os.makedirs(outputdir)
except OSError:
if not os.path.isdir(outputdir):
raise Exception("Unable to create output cached ansible directory {}".format(outputdir))
srcfile = os.path.join(rootdir, fname)
destfile = os.path.join(outputdir, fname)
with codecs.open(destfile, 'w', 'utf-8') as destination:
#with open(destfile, 'w') as destination:
destination.write(outputcontent)
os.chmod(destfile,os.stat(srcfile).st_mode)
logging.getLogger().info("Cached ansible config for {}".format(os.path.join(outputdir,fname)))
def process_overcloud_config_files(self,
clouddata,
output_path,
include_list=[],
exclude_list=[]):
"""
Generate a personalized overcloud config environment based upon a clouddata config
:param clouddata: De-serialized YAML of clouddata
:param output_path: Output path for customized overcloud config
:return:
"""
# For each directory under that path (recursively)
for rootdir, subdirList, fileList in os.walk(self.config_path):
ENV = Environment(loader=FileSystemLoader(rootdir))
# Process filters
if exclude_list:
subdirList[:] = [
d for d in subdirList
if not fileutil.check_include(rootdir, d, exclude_list)
]
fileList[:] = [
f for f in fileList
if not fileutil.check_include(rootdir, f, exclude_list)
]
if include_list:
subdirList[:] = [
d for d in subdirList
if fileutil.check_include(rootdir, d, include_list)
]
fileList[:] = [
f for f in fileList
if fileutil.check_include(rootdir, f, include_list)
]
# For each file found in that directory
for fname in fileList:
logging.getLogger().debug(
"Jinjafying overcloud config file {}".format(fname))
template = ENV.get_template(fname)
try:
outputcontent = template.render(cloud=clouddata)
reldir = os.path.relpath(rootdir, self.config_path)
outputdir = os.path.join(output_path, reldir)
except Exception as error:
logging.getLogger().error(
"Applying clouddata template to file {} failed: {}".
format(fname, str(error)))
raise Exception(
"An error occurred processing overcloud config area {}"
.format(self.config_path))
try:
os.makedirs(outputdir)
except OSError:
if not os.path.isdir(outputdir):
raise Exception(
"Unable to create output cached overcloud directory {}"
.format(outputdir))
srcfile = os.path.join(rootdir, fname)
destfile = os.path.join(outputdir, fname)
with codecs.open(destfile, 'w', 'utf-8') as destination:
#with open(destfile, 'w') as destination:
# first make the destination user-writable just in case it isn't
os.chmod(destfile, os.stat(srcfile).st_mode | stat.S_IWUSR)
destination.write(outputcontent)
os.chmod(destfile, os.stat(srcfile).st_mode)
logging.getLogger().info(
"Cached overcloud config for {} in {}".format(
fname, outputdir))
def test_contains_file_fullpath(self):
rootdir = os.path.abspath('fileutil')
dir1dir = os.path.abspath('fileutil/dir1')
dir11dir = os.path.abspath('fileutil/dir1/dir1-1')
dir2dir = os.path.abspath('fileutil/dir2')
# check that an empty include list results in no match
self.assertFalse(fileutil.check_include(rootdir, 'file0-1', []))
# checks if rootdir contains rootdir/file-1-1 (should succeed)
self.assertTrue(fileutil.check_include(rootdir, 'file-1-1', [rootdir]))
# checks if rootdir contains rootdir/file-1-1 (should succeed)
self.assertTrue(fileutil.check_include(rootdir, 'dir1/file-1-1', [dir1dir]))
# checks if rootdir/dir1 contains rootdir/file0-1 (should fail)
self.assertFalse(fileutil.check_include(rootdir, 'file0-1', [dir1dir]))
# checks if rootdir/dir2 contains rootdir/dir1/file1-1 (should fail)
self.assertFalse(fileutil.check_include(rootdir, 'dir1/file1-1', [dir2dir]))
# checks if rootdir contains rootdir/dir1
self.assertTrue(fileutil.check_include(rootdir, 'dir1', [rootdir]))
# checks if rootdir contains rootdir/dir2
self.assertTrue(fileutil.check_include(rootdir, 'dir2', [rootdir]))
# checks if rootdir/dir1 contains rootdir/dir1/dir1-1
self.assertTrue(fileutil.check_include(rootdir, 'dir1/dir1-1', [dir1dir]))
# checks if rootdir/dir2 contains rootdir/dir1/dir1-1
self.assertFalse(fileutil.check_include(rootdir, 'dir1/dir1-1', [dir2dir]))
# checks if rootdir contains rootdir/dir2/../dir1
self.assertTrue(fileutil.check_include(rootdir, 'dir2/../dir1', [rootdir]))
# checks if rootdir contains rootdir/../
self.assertFalse(fileutil.check_include(rootdir, '../', [rootdir]))
self.assertFalse(fileutil.check_include(rootdir, '..', [rootdir]))
# checks if rootdir/dir1/../dir2/ contains rootdir/dir2/file2-1
self.assertTrue(fileutil.check_include(rootdir, 'dir2/file2-1', [dir1dir + '/../dir2/']))
# check that directories are handled properly
self.assertTrue(fileutil.check_include(rootdir, 'dir1/.git', [rootdir + '/dir1/.git']))
# check that trailing slashes aren't a problem
self.assertTrue(fileutil.check_include(rootdir, 'dir1/.git', [rootdir + '/dir1/.git/']))
self.assertTrue(fileutil.check_include(rootdir, 'dir1/.git/', [rootdir + '/dir1/.git/']))