def setup_plugin(name, config): # pragma: no cover
pgm = PluginManager(config)
plugin = pgm.get_plugin(name)
if plugin is None:
sys.exit("Invalid plugin name...")
pgm.config_plugin(name, config)
def test_plugin_config(self):
class TPlugin(object):
name = 'test'
author = 'test1'
bucket = ''
def init_config(self, config):
self.bucket = config.get("bucket", "bad")
master_config = Config()
master_config['PLUGIN_CONFIGS'] = {
"test-test1": {
"bucket": "test_bucket"
}
}
pm = PluginManager(master_config, plugins=[TPlugin()])
p = pm.get_plugin('test', False)
assert p.bucket == ''
p = pm.get_plugin('test', True)
assert p.bucket == 'test_bucket'
def setup_plugin(name, config):
pgm = PluginManager(config)
plugin = pgm.get_plugin(name)
if plugin is None:
sys.exit('Invalid plugin name...')
pgm.config_plugin(name, config)
def setup_plugin(name, config):
pgm = PluginManager(config)
plugin = pgm.get_plugin(name)
if plugin is None:
sys.exit('Invalid plugin name...')
pgm.config_plugin(name, config)
def plugins(args):
plug_mgr = PluginManager({})
names = ['\n']
for n in plug_mgr.get_plugin_names():
out = '{} by {}\n'.format(n['name'], n['author'])
names.append(out)
output = ''.join(names)
print(output)
def print_plugin_settings(plugin_name, config):
pm = PluginManager(config)
config = pm.get_plugin_settings(plugin_name)
if len(config.keys()) == 0:
print('No config found for {}'.format(plugin_name))
else:
print(plugin_name)
print(config)
def plugins(args):
plug_mgr = PluginManager({})
names = ['\n']
for n in plug_mgr.get_plugin_names():
out = '{} by {}\n'.format(n['name'], n['author'])
names.append(out)
output = ''.join(names)
print(output)
def print_plugin_settings(plugin_name, config): # pragma: no cover
pm = PluginManager(config)
config = pm.get_plugin_settings(plugin_name)
if len(config.keys()) == 0:
print("No config found for {}".format(plugin_name))
else:
print(plugin_name)
print(config)
def print_plugin_settings(plugin_name, config):
pm = PluginManager(config)
config = pm.get_plugin_settings(plugin_name)
if len(config.keys()) == 0:
print('No config found for {}'.format(plugin_name))
else:
print(plugin_name)
print(config)
def _cmd_plugins(*args):
plug_mgr = PluginManager({})
# Doing some basic formatting. Design help here would be appreciated.
# By the way I just want to thank all the contributors and bug submitters.
names = ["\n"]
for n in plug_mgr.get_plugin_names():
out = "{} by {}\n".format(n["name"], n["author"])
names.append(out)
output = "".join(names)
log.info("Upload plugins:%s", output)
def _cmd_plugins(*args):
plug_mgr = PluginManager({})
# Doing some basic formatting. Design help here would be appreciated.
# By the way I just want to thank all the contributors and bug submitters.
names = ['\n']
for n in plug_mgr.get_plugin_names():
out = '{} by {}\n'.format(n['name'], n['author'])
names.append(out)
output = ''.join(names)
log.info('Upload plugins:%s', output)
def _cmd_plugins(*args):
plug_mgr = PluginManager({})
# Doing some basic formatting. Design help here would be appreciated.
# By the way I just want to thank all the contributors and bug submitters.
names = ['\n']
for n in plug_mgr.get_plugin_names():
out = '{} by {}\n'.format(n['name'], n['author'])
names.append(out)
output = ''.join(names)
print(output)
def init(self, obj):
"""Sets up client with config values from obj
Args:
obj (instance): config object
"""
data_dir = os.path.join(os.getcwd(), settings.USER_DATA_FOLDER)
self.deploy_dir = os.path.join(data_dir, 'deploy')
self.uploader = None
# Files to be uploaded
self.files = []
# Extension Manager
self.mgr = PluginManager(obj)
def __init__(self, config, plugins=None):
# Specifies whether to keep a file after uploading
self.keep = False
data_dir = os.path.join(os.getcwd(), settings.USER_DATA_FOLDER)
self.deploy_dir = os.path.join(data_dir, "deploy")
# The upload plugin that'll be used to upload our files
self.uploader = None
# Files to be uploaded
self.files = []
# Extension Manager
self.plg_mgr = PluginManager(config, plugins)
def init(self, obj):
"""Sets up client with config values from obj
Args:
obj (instance): config object
"""
data_dir = os.path.join(os.getcwd(), settings.USER_DATA_FOLDER)
self.deploy_dir = os.path.join(data_dir, 'deploy')
# The upload plugin that'll be used to upload our files
self.uploader = None
# Files to be uploaded
self.files = []
# Extension Manager
# ToDo: Make this a more descriptive variable name
self.mgr = PluginManager(obj)
def test_plugin_manager_load(self, n, a):
class Plugin(object):
name = n
author = a
pm = PluginManager(Config(), plugins=[Plugin()])
if isinstance(n, six.string_types) and isinstance(a, six.string_types):
assert len(pm.plugins) == 1
else:
assert len(pm.plugins) == 0
def test_plugin_unique_names(self):
class Plugin1(object):
name = "test"
author = "test1"
class Plugin2(object):
name = "test"
author = "test2"
pm = PluginManager(Config(), plugins=[Plugin1(), Plugin2()])
plugin_names = [n["name"] for n in pm.plugins]
assert "test" in plugin_names
assert "test2" in plugin_names
def test_plugin_config(self):
class TPlugin(object):
name = "test"
author = "test1"
bucket = ""
def init_config(self, config):
self.bucket = config.get("bucket", "bad")
master_config = Config()
master_config["PLUGIN_CONFIGS"] = {
"test-test1": {
"bucket": "test_bucket"
}
}
pm = PluginManager(master_config, plugins=[TPlugin()])
p = pm.get_plugin("test", False)
assert p.bucket == ""
p = pm.get_plugin("test", True)
assert p.bucket == "test_bucket"
def __init__(self, config, plugins=None):
# Specifies whether to keep a file after uploading
self.keep = False
data_dir = os.path.join(os.getcwd(), settings.USER_DATA_FOLDER)
self.deploy_dir = os.path.join(data_dir, 'deploy')
# The upload plugin that'll be used to upload our files
self.uploader = None
# Files to be uploaded
self.files = []
# Extension Manager
self.plg_mgr = PluginManager(config, plugins)
def test_plugin_unique_names(self):
class Plugin1(object):
name = 'test'
author = 'test1'
class Plugin2(object):
name = 'test'
author = 'test2'
pm = PluginManager(Config(), plugins=[Plugin1(), Plugin2()])
plugin_names = [n['name'] for n in pm.plugins]
assert 'test' in plugin_names
assert 'test2' in plugin_names
class Uploader(object):
"""Uploads updates to configured servers. SSH, SFTP, S3
Will automatically pick the correct uploader depending on
what is configured thorough the config object
Sets up client with config values from obj
Args:
obj (instance): config object
"""
def __init__(self, config=None):
# Specifies whether to keep a file after uploading
self.keep = False
if config:
self.init(config)
def init(self, obj):
"""Sets up client with config values from obj
Args:
obj (instance): config object
"""
data_dir = os.path.join(os.getcwd(), settings.USER_DATA_FOLDER)
self.deploy_dir = os.path.join(data_dir, 'deploy')
# The upload plugin that'll be used to upload our files
self.uploader = None
# Files to be uploaded
self.files = []
# Extension Manager
# ToDo: Make this a more descriptive variable name
self.mgr = PluginManager(obj)
def get_plugin_names(self):
return self.mgr.get_plugin_names()
def set_uploader(self, requested_uploader, keep=False):
"""Sets the named upload plugin.
Args:
requested_uploader (string): Either s3 or scp
"""
self.keep = keep
if isinstance(requested_uploader, six.string_types) is False:
raise UploaderError('Must pass str to set_uploader', expected=True)
self.uploader = self.mgr.get_plugin(requested_uploader, init=True)
if self.uploader is None:
log.debug('PLUGIN_NAMESPACE: %s', self.mgr.PLUGIN_NAMESPACE)
raise UploaderPluginError('Requested uploader is not installed',
expected=True)
msg = 'Requested uploader: {}'.format(requested_uploader)
log.debug(msg)
# ToDo: Move this into it's own function.
# Call this new function in the upload method
try:
_files = os.listdir(self.deploy_dir)
except OSError:
_files = []
files = []
for f in _files:
files.append(os.path.join(self.deploy_dir, f))
self.files = remove_dot_files(files)
# End ToDo
def upload(self):
"""Uploads all files in file_list"""
failed_uploads = []
self.files_completed = 1
self.file_count = len(self.files)
log.info('Plugin: %s', self.uploader.name)
log.info('Author: %s', self.uploader.author)
for f in self.files:
basename = os.path.basename(f)
msg = '\n\nUploading: {}'.format(basename)
msg2 = ' - File {} of {}\n'.format(self.files_completed,
self.file_count)
print(msg + msg2)
complete = self.uploader.upload_file(f)
if complete:
log.debug('%s uploaded successfully', basename)
if self.keep is False:
remove_any(f)
self.files_completed += 1
else:
log.debug('%s failed to upload. will retry', basename)
failed_uploads.append(f)
if len(failed_uploads) > 0:
failed_uploads = self._retry_upload(failed_uploads)
if len(failed_uploads) < 1:
print("\nUpload Complete")
return True
else:
print('The following files were not uploaded')
for i in failed_uploads:
log.error('%s failed to upload', os.path.basename(i))
print(i)
return False
def _retry_upload(self, failed_uploads):
# Takes list of failed downloads and tries to re upload them
retry = failed_uploads[:]
failed_uploads = []
failed_count = len(retry)
count = 1
for f in retry:
msg = '\n\nRetyring: {} - File {} of {}\n'.format(
f, count, failed_count)
print(msg)
complete = self.uploader.upload_file(f)
if complete:
log.debug('%s uploaded on retry', f)
if self.keep is False:
remove_any(f)
count += 1
else:
failed_uploads.append(f)
return failed_uploads
class Uploader(object):
"""Uploads updates to configured servers. SSH, SFTP, S3
Will automatically pick the correct uploader depending on
what is configured thorough the config object
Sets up client with config values from obj
Args:
config (instance): config object
"""
def __init__(self, config, plugins=None):
# Specifies whether to keep a file after uploading
self.keep = False
data_dir = os.path.join(os.getcwd(), settings.USER_DATA_FOLDER)
self.deploy_dir = os.path.join(data_dir, 'deploy')
# The upload plugin that'll be used to upload our files
self.uploader = None
# Files to be uploaded
self.files = []
# Extension Manager
self.plg_mgr = PluginManager(config, plugins)
def _get_files_to_upload(self, files=None):
if files:
self.files = files
else:
try:
_files = os.listdir(self.deploy_dir)
except OSError:
_files = []
files = []
for f in _files:
files.append(os.path.join(self.deploy_dir, f))
self.files = remove_dot_files(files)
def get_plugin_names(self):
return self.plg_mgr.get_plugin_names()
def set_uploader(self, requested_uploader, keep=False):
"""Sets the named upload plugin.
Args:
requested_uploader (string): Either s3 or scp
keep (bool): False to delete files after upload.
True to keep files. Default False.
"""
self.keep = keep
if isinstance(requested_uploader, six.string_types) is False:
raise UploaderError('Must pass str to set_uploader',
expected=True)
self.uploader = self.plg_mgr.get_plugin(requested_uploader, init=True)
if self.uploader is None:
log.debug('PLUGIN_NAMESPACE: %s', self.plg_mgr.PLUGIN_NAMESPACE)
raise UploaderPluginError('Requested uploader is not installed',
expected=True)
msg = 'Requested uploader: {}'.format(requested_uploader)
log.debug(msg)
def upload(self, files=None):
"""Uploads all files in file_list"""
self._get_files_to_upload(files)
failed_uploads = []
files_completed = 1
file_count = len(self.files)
log.info('Plugin: %s', self.uploader.name)
log.info('Author: %s', self.uploader.author)
for f in self.files:
basename = os.path.basename(f)
msg = '\n\nUploading: {}' .format(basename)
msg2 = ' - File {} of {}\n'.format(files_completed,
file_count)
print(msg + msg2)
complete = self.uploader.upload_file(f)
if complete:
log.debug('%s uploaded successfully', basename)
if self.keep is False:
remove_any(f)
files_completed += 1
else:
log.debug('%s failed to upload. will retry', basename)
failed_uploads.append(f)
if len(failed_uploads) > 0:
failed_uploads = self._retry_upload(failed_uploads)
if len(failed_uploads) < 1:
return True
else:
log.error('The following files were not uploaded')
for i in failed_uploads:
log.error('%s failed to upload', os.path.basename(i))
return False
def _retry_upload(self, failed_uploads):
# Takes list of failed downloads and tries to re upload them
retry = failed_uploads[:]
failed_uploads = []
failed_count = len(retry)
count = 1
for f in retry:
msg = 'Retyring: {} - File {} of {}'.format(f, count, failed_count)
log.info(msg)
complete = self.uploader.upload_file(f)
if complete:
log.debug('%s uploaded on retry', f)
if self.keep is False:
remove_any(f)
count += 1
else:
failed_uploads.append(f)
return failed_uploads
class Uploader(object):
"""Uploads updates to configured servers. SSH, SFTP, S3
Will automatically pick the correct uploader depending on
what is configured thorough the config object
Sets up client with config values from obj
Args:
config (instance): config object
"""
def __init__(self, config, plugins=None):
# Specifies whether to keep a file after uploading
self.keep = False
data_dir = os.path.join(os.getcwd(), settings.USER_DATA_FOLDER)
self.deploy_dir = os.path.join(data_dir, "deploy")
# The upload plugin that'll be used to upload our files
self.uploader = None
# Files to be uploaded
self.files = []
# Extension Manager
self.plg_mgr = PluginManager(config, plugins)
def _get_files_to_upload(self, files=None):
if files:
self.files = files
else:
try:
_files = os.listdir(self.deploy_dir)
except OSError:
_files = []
files = []
for f in _files:
files.append(os.path.join(self.deploy_dir, f))
self.files = remove_dot_files(files)
def get_plugin_names(self):
return self.plg_mgr.get_plugin_names()
def set_uploader(self, requested_uploader, keep=False):
"""Sets the named upload plugin.
Args:
requested_uploader (string): Either s3 or scp
keep (bool): False to delete files after upload.
True to keep files. Default False.
"""
self.keep = keep
if isinstance(requested_uploader, six.string_types) is False:
raise UploaderError("Must pass str to set_uploader", expected=True)
self.uploader = self.plg_mgr.get_plugin(requested_uploader, init=True)
if self.uploader is None:
log.debug("PLUGIN_NAMESPACE: %s", self.plg_mgr.PLUGIN_NAMESPACE)
raise UploaderPluginError(
"Requested uploader is not installed", expected=True
)
msg = "Requested uploader: {}".format(requested_uploader)
log.debug(msg)
def upload(self, files=None):
"""Uploads all files in file_list"""
self._get_files_to_upload(files)
failed_uploads = []
files_completed = 1
file_count = len(self.files)
log.info("Plugin: %s", self.uploader.name)
log.info("Author: %s", self.uploader.author)
for f in self.files:
basename = os.path.basename(f)
msg = "\n\nUploading: {}".format(basename)
msg2 = " - File {} of {}\n".format(files_completed, file_count)
print(msg + msg2)
complete = self.uploader.upload_file(f)
if complete:
log.debug("%s uploaded successfully", basename)
if self.keep is False:
remove_any(f)
files_completed += 1
else:
log.debug("%s failed to upload. will retry", basename)
failed_uploads.append(f)
if len(failed_uploads) > 0:
failed_uploads = self._retry_upload(failed_uploads)
if len(failed_uploads) < 1:
return True
else:
log.error("The following files were not uploaded")
for i in failed_uploads:
log.error("%s failed to upload", os.path.basename(i))
return False
def _retry_upload(self, failed_uploads):
# Takes list of failed downloads and tries to re upload them
retry = failed_uploads[:]
failed_uploads = []
failed_count = len(retry)
count = 1
for f in retry:
msg = "Retyring: {} - File {} of {}".format(f, count, failed_count)
log.info(msg)
complete = self.uploader.upload_file(f)
if complete:
log.debug("%s uploaded on retry", f)
if self.keep is False:
remove_any(f)
count += 1
else:
failed_uploads.append(f)
return failed_uploads
def test_default_plugin_detection_scp(self):
pm = PluginManager(Config())
plugin_names = [n['name'] for n in pm.plugins]
assert 'scp' in plugin_names
def test_default_plugin_detection_scp(self):
pm = PluginManager(Config())
plugin_names = [n["name"] for n in pm.plugins]
assert "scp" in plugin_names
