def test_plugin_config(self):
class TPlugin(object):
name = 'test'
author = 'test1'
bucket = ''
def init_config(self, config):
self.bucket = config.get("bucket", "bad")
master_config = Config()
master_config['PLUGIN_CONFIGS'] = {
"test-test1": {
"bucket": "test_bucket"
}
}
pm = PluginManager(master_config, plugins=[TPlugin()])
p = pm.get_plugin('test', False)
assert p.bucket == ''
p = pm.get_plugin('test', True)
assert p.bucket == 'test_bucket'
def setup_plugin(name, config):
pgm = PluginManager(config)
plugin = pgm.get_plugin(name)
if plugin is None:
sys.exit('Invalid plugin name...')
pgm.config_plugin(name, config)
def setup_plugin(name, config): # pragma: no cover
pgm = PluginManager(config)
plugin = pgm.get_plugin(name)
if plugin is None:
sys.exit("Invalid plugin name...")
pgm.config_plugin(name, config)
def setup_plugin(name, config):
pgm = PluginManager(config)
plugin = pgm.get_plugin(name)
if plugin is None:
sys.exit('Invalid plugin name...')
pgm.config_plugin(name, config)
def test_plugin_config(self):
class TPlugin(object):
name = "test"
author = "test1"
bucket = ""
def init_config(self, config):
self.bucket = config.get("bucket", "bad")
master_config = Config()
master_config["PLUGIN_CONFIGS"] = {
"test-test1": {
"bucket": "test_bucket"
}
}
pm = PluginManager(master_config, plugins=[TPlugin()])
p = pm.get_plugin("test", False)
assert p.bucket == ""
p = pm.get_plugin("test", True)
assert p.bucket == "test_bucket"
class Uploader(object):
"""Uploads updates to configured servers. SSH, SFTP, S3
Will automatically pick the correct uploader depending on
what is configured thorough the config object
Sets up client with config values from obj
Args:
obj (instance): config object
"""
def __init__(self, config=None):
# Specifies whether to keep a file after uploading
self.keep = False
if config:
self.init(config)
def init(self, obj):
"""Sets up client with config values from obj
Args:
obj (instance): config object
"""
data_dir = os.path.join(os.getcwd(), settings.USER_DATA_FOLDER)
self.deploy_dir = os.path.join(data_dir, 'deploy')
# The upload plugin that'll be used to upload our files
self.uploader = None
# Files to be uploaded
self.files = []
# Extension Manager
# ToDo: Make this a more descriptive variable name
self.mgr = PluginManager(obj)
def get_plugin_names(self):
return self.mgr.get_plugin_names()
def set_uploader(self, requested_uploader, keep=False):
"""Sets the named upload plugin.
Args:
requested_uploader (string): Either s3 or scp
"""
self.keep = keep
if isinstance(requested_uploader, six.string_types) is False:
raise UploaderError('Must pass str to set_uploader', expected=True)
self.uploader = self.mgr.get_plugin(requested_uploader, init=True)
if self.uploader is None:
log.debug('PLUGIN_NAMESPACE: %s', self.mgr.PLUGIN_NAMESPACE)
raise UploaderPluginError('Requested uploader is not installed',
expected=True)
msg = 'Requested uploader: {}'.format(requested_uploader)
log.debug(msg)
# ToDo: Move this into it's own function.
# Call this new function in the upload method
try:
_files = os.listdir(self.deploy_dir)
except OSError:
_files = []
files = []
for f in _files:
files.append(os.path.join(self.deploy_dir, f))
self.files = remove_dot_files(files)
# End ToDo
def upload(self):
"""Uploads all files in file_list"""
failed_uploads = []
self.files_completed = 1
self.file_count = len(self.files)
log.info('Plugin: %s', self.uploader.name)
log.info('Author: %s', self.uploader.author)
for f in self.files:
basename = os.path.basename(f)
msg = '\n\nUploading: {}'.format(basename)
msg2 = ' - File {} of {}\n'.format(self.files_completed,
self.file_count)
print(msg + msg2)
complete = self.uploader.upload_file(f)
if complete:
log.debug('%s uploaded successfully', basename)
if self.keep is False:
remove_any(f)
self.files_completed += 1
else:
log.debug('%s failed to upload. will retry', basename)
failed_uploads.append(f)
if len(failed_uploads) > 0:
failed_uploads = self._retry_upload(failed_uploads)
if len(failed_uploads) < 1:
print("\nUpload Complete")
return True
else:
print('The following files were not uploaded')
for i in failed_uploads:
log.error('%s failed to upload', os.path.basename(i))
print(i)
return False
def _retry_upload(self, failed_uploads):
# Takes list of failed downloads and tries to re upload them
retry = failed_uploads[:]
failed_uploads = []
failed_count = len(retry)
count = 1
for f in retry:
msg = '\n\nRetyring: {} - File {} of {}\n'.format(
f, count, failed_count)
print(msg)
complete = self.uploader.upload_file(f)
if complete:
log.debug('%s uploaded on retry', f)
if self.keep is False:
remove_any(f)
count += 1
else:
failed_uploads.append(f)
return failed_uploads
class Uploader(object):
"""Uploads updates to configured servers. SSH, SFTP, S3
Will automatically pick the correct uploader depending on
what is configured thorough the config object
Sets up client with config values from obj
Args:
config (instance): config object
"""
def __init__(self, config, plugins=None):
# Specifies whether to keep a file after uploading
self.keep = False
data_dir = os.path.join(os.getcwd(), settings.USER_DATA_FOLDER)
self.deploy_dir = os.path.join(data_dir, 'deploy')
# The upload plugin that'll be used to upload our files
self.uploader = None
# Files to be uploaded
self.files = []
# Extension Manager
self.plg_mgr = PluginManager(config, plugins)
def _get_files_to_upload(self, files=None):
if files:
self.files = files
else:
try:
_files = os.listdir(self.deploy_dir)
except OSError:
_files = []
files = []
for f in _files:
files.append(os.path.join(self.deploy_dir, f))
self.files = remove_dot_files(files)
def get_plugin_names(self):
return self.plg_mgr.get_plugin_names()
def set_uploader(self, requested_uploader, keep=False):
"""Sets the named upload plugin.
Args:
requested_uploader (string): Either s3 or scp
keep (bool): False to delete files after upload.
True to keep files. Default False.
"""
self.keep = keep
if isinstance(requested_uploader, six.string_types) is False:
raise UploaderError('Must pass str to set_uploader',
expected=True)
self.uploader = self.plg_mgr.get_plugin(requested_uploader, init=True)
if self.uploader is None:
log.debug('PLUGIN_NAMESPACE: %s', self.plg_mgr.PLUGIN_NAMESPACE)
raise UploaderPluginError('Requested uploader is not installed',
expected=True)
msg = 'Requested uploader: {}'.format(requested_uploader)
log.debug(msg)
def upload(self, files=None):
"""Uploads all files in file_list"""
self._get_files_to_upload(files)
failed_uploads = []
files_completed = 1
file_count = len(self.files)
log.info('Plugin: %s', self.uploader.name)
log.info('Author: %s', self.uploader.author)
for f in self.files:
basename = os.path.basename(f)
msg = '\n\nUploading: {}' .format(basename)
msg2 = ' - File {} of {}\n'.format(files_completed,
file_count)
print(msg + msg2)
complete = self.uploader.upload_file(f)
if complete:
log.debug('%s uploaded successfully', basename)
if self.keep is False:
remove_any(f)
files_completed += 1
else:
log.debug('%s failed to upload. will retry', basename)
failed_uploads.append(f)
if len(failed_uploads) > 0:
failed_uploads = self._retry_upload(failed_uploads)
if len(failed_uploads) < 1:
return True
else:
log.error('The following files were not uploaded')
for i in failed_uploads:
log.error('%s failed to upload', os.path.basename(i))
return False
def _retry_upload(self, failed_uploads):
# Takes list of failed downloads and tries to re upload them
retry = failed_uploads[:]
failed_uploads = []
failed_count = len(retry)
count = 1
for f in retry:
msg = 'Retyring: {} - File {} of {}'.format(f, count, failed_count)
log.info(msg)
complete = self.uploader.upload_file(f)
if complete:
log.debug('%s uploaded on retry', f)
if self.keep is False:
remove_any(f)
count += 1
else:
failed_uploads.append(f)
return failed_uploads
class Uploader(object):
"""Uploads updates to configured servers. SSH, SFTP, S3
Will automatically pick the correct uploader depending on
what is configured thorough the config object
Sets up client with config values from obj
Args:
config (instance): config object
"""
def __init__(self, config, plugins=None):
# Specifies whether to keep a file after uploading
self.keep = False
data_dir = os.path.join(os.getcwd(), settings.USER_DATA_FOLDER)
self.deploy_dir = os.path.join(data_dir, "deploy")
# The upload plugin that'll be used to upload our files
self.uploader = None
# Files to be uploaded
self.files = []
# Extension Manager
self.plg_mgr = PluginManager(config, plugins)
def _get_files_to_upload(self, files=None):
if files:
self.files = files
else:
try:
_files = os.listdir(self.deploy_dir)
except OSError:
_files = []
files = []
for f in _files:
files.append(os.path.join(self.deploy_dir, f))
self.files = remove_dot_files(files)
def get_plugin_names(self):
return self.plg_mgr.get_plugin_names()
def set_uploader(self, requested_uploader, keep=False):
"""Sets the named upload plugin.
Args:
requested_uploader (string): Either s3 or scp
keep (bool): False to delete files after upload.
True to keep files. Default False.
"""
self.keep = keep
if isinstance(requested_uploader, six.string_types) is False:
raise UploaderError("Must pass str to set_uploader", expected=True)
self.uploader = self.plg_mgr.get_plugin(requested_uploader, init=True)
if self.uploader is None:
log.debug("PLUGIN_NAMESPACE: %s", self.plg_mgr.PLUGIN_NAMESPACE)
raise UploaderPluginError(
"Requested uploader is not installed", expected=True
)
msg = "Requested uploader: {}".format(requested_uploader)
log.debug(msg)
def upload(self, files=None):
"""Uploads all files in file_list"""
self._get_files_to_upload(files)
failed_uploads = []
files_completed = 1
file_count = len(self.files)
log.info("Plugin: %s", self.uploader.name)
log.info("Author: %s", self.uploader.author)
for f in self.files:
basename = os.path.basename(f)
msg = "\n\nUploading: {}".format(basename)
msg2 = " - File {} of {}\n".format(files_completed, file_count)
print(msg + msg2)
complete = self.uploader.upload_file(f)
if complete:
log.debug("%s uploaded successfully", basename)
if self.keep is False:
remove_any(f)
files_completed += 1
else:
log.debug("%s failed to upload. will retry", basename)
failed_uploads.append(f)
if len(failed_uploads) > 0:
failed_uploads = self._retry_upload(failed_uploads)
if len(failed_uploads) < 1:
return True
else:
log.error("The following files were not uploaded")
for i in failed_uploads:
log.error("%s failed to upload", os.path.basename(i))
return False
def _retry_upload(self, failed_uploads):
# Takes list of failed downloads and tries to re upload them
retry = failed_uploads[:]
failed_uploads = []
failed_count = len(retry)
count = 1
for f in retry:
msg = "Retyring: {} - File {} of {}".format(f, count, failed_count)
log.info(msg)
complete = self.uploader.upload_file(f)
if complete:
log.debug("%s uploaded on retry", f)
if self.keep is False:
remove_any(f)
count += 1
else:
failed_uploads.append(f)
return failed_uploads