def start(self):
"""This is the upload method.
Uses file_upload is to simply upload all files and folders to a
specified container.
"""
# Index Local Files for Upload
with multi.spinner():
f_indexed = methods.get_local_files()
if ARGS.get('pattern_match'):
f_indexed = basic.match_filter(
idx_list=f_indexed, pattern=ARGS['pattern_match']
)
num_files = len(f_indexed)
# Get The rate of concurrency
concurrency = multi.set_concurrency(args=ARGS, file_count=num_files)
# Package up the Payload
payload = multi.manager_dict(
http.prep_payload(
auth=self.auth,
container=ARGS.get('container', basic.rand_string()),
source=basic.get_local_source(),
args=ARGS
)
)
LOG.info('MESSAGE\t: "%s" Files have been found.', num_files)
LOG.debug('PAYLOAD\t: "%s"', payload)
# Set the actions class up
self.go = actions.CloudActions(payload=payload)
kwargs = {'url': payload['url'],
'container': payload['c_name']}
# get that the container exists if not create it.
self.go.container_create(**kwargs)
kwargs['source'] = payload['source']
kwargs['cf_job'] = getattr(self.go, 'object_putter')
multi.job_processer(
num_jobs=num_files,
objects=f_indexed,
job_action=multi.doerator,
concur=concurrency,
kwargs=kwargs
)
if ARGS.get('delete_remote') is True:
self.remote_delete(payload=payload,
f_indexed=f_indexed)
def start(self):
"""This is the upload method.
Uses file_upload is to simply upload all files and folders to a
specified container.
"""
# Index Local Files for Upload
f_indexed = methods.get_local_files()
if ARGS.get('pattern_match'):
f_indexed = basic.match_filter(
idx_list=f_indexed, pattern=ARGS['pattern_match']
)
num_files = len(f_indexed)
# Get The rate of concurrency
concurrency = multi.set_concurrency(args=ARGS, file_count=num_files)
# Package up the Payload
payload = multi.manager_dict(
http.prep_payload(
auth=self.auth,
container=ARGS.get('container', basic.rand_string()),
source=basic.get_local_source(),
args=ARGS
)
)
LOG.info('MESSAGE\t: "%s" Files have been found.', num_files)
LOG.debug('PAYLOAD\t: "%s"', payload)
# Set the actions class up
self.go = actions.CloudActions(payload=payload)
kwargs = {'url': payload['url'],
'container': payload['c_name']}
# get that the container exists if not create it.
self.go.container_create(**kwargs)
kwargs['source'] = payload['source']
kwargs['cf_job'] = getattr(self.go, 'object_putter')
multi.job_processer(
num_jobs=num_files,
objects=f_indexed,
job_action=multi.doerator,
concur=concurrency,
kwargs=kwargs
)
if ARGS.get('delete_remote') is True:
self.remote_delete(payload=payload,
f_indexed=f_indexed)
def remote_delete(self, payload, f_indexed):
"""If Remote Delete was True run.
NOTE: Remote delete will delete ALL Objects in a remote container
which differ from the objects in the SOURCED LOCAL FILESYSTEM.
IE: If this option is used, on one directory and then another directory
and the files were different any difference would be deleted and based
on the index information found in LOCAL FILE SYSTEM on the LAST
command run.
:return:
"""
report.reporter(msg='Getting file list for REMOTE DELETE')
objects = self.go.object_lister(
url=payload['url'], container=payload['c_name']
)
source = payload['source']
obj_names = [basic.jpath(root=source, inode=obj.get('name'))
for obj in objects[0]]
# From the remote system see if we have differences in the local system
objects = multi.return_diff().difference(target=f_indexed,
source=obj_names)
if objects:
# Set Basic Data for file delete.
num_files = len(objects)
LOG.info('MESSAGE\t: "%s" Files have been found to be removed from'
' the REMOTE CONTAINER.', num_files)
concurrency = multi.set_concurrency(
args=ARGS, file_count=num_files
)
# Delete the difference in Files.
report.reporter(msg='Performing Remote Delete')
objects = [basic.get_sfile(
ufile=obj, source=payload['source']
) for obj in objects]
kwargs = {'url': payload['url'],
'container': payload['c_name'],
'cf_job': getattr(self.go, 'object_deleter')}
multi.job_processer(
num_jobs=num_files,
objects=objects,
job_action=multi.doerator,
concur=concurrency,
kwargs=kwargs
)
else:
report.reporter(
msg='No Difference between REMOTE and LOCAL Directories.'
)
def remote_delete(self, payload, f_indexed):
"""If Remote Delete was True run.
NOTE: Remote delete will delete ALL Objects in a remote container
which differ from the objects in the SOURCED LOCAL FILESYSTEM.
IE: If this option is used, on one directory and then another directory
and the files were different any difference would be deleted and based
on the index information found in LOCAL FILE SYSTEM on the LAST
command run.
:return:
"""
report.reporter(msg='Getting file list for REMOTE DELETE')
objects = self.go.object_lister(
url=payload['url'], container=payload['c_name']
)
source = payload['source']
obj_names = [basic.jpath(root=source, inode=obj.get('name'))
for obj in objects[0]]
# From the remote system see if we have differences in the local system
objects = multi.return_diff().difference(target=f_indexed,
source=obj_names)
if objects:
# Set Basic Data for file delete.
num_files = len(objects)
LOG.info('MESSAGE\t: "%s" Files have been found to be removed from'
' the REMOTE CONTAINER.', num_files)
concurrency = multi.set_concurrency(
args=ARGS, file_count=num_files
)
# Delete the difference in Files.
report.reporter(msg='Performing Remote Delete')
objects = [basic.get_sfile(
ufile=obj, source=payload['source']
) for obj in objects]
kwargs = {'url': payload['url'],
'container': payload['c_name'],
'cf_job': getattr(self.go, 'object_deleter')}
multi.job_processer(
num_jobs=num_files,
objects=objects,
job_action=multi.doerator,
concur=concurrency,
kwargs=kwargs
)
else:
report.reporter(
msg='No Difference between REMOTE and LOCAL Directories.'
)
def start(self):
"""Retrieve a long list of all files in a container."""
# Package up the Payload
payload = http.prep_payload(
auth=self.auth,
container=ARGS.get('container'),
source=ARGS.get('source'),
args=ARGS
)
self.go = actions.CloudActions(payload=payload)
self.action = getattr(self.go, 'object_lister')
LOG.info('Attempting Download of Remote path %s', payload['c_name'])
if ARGS.get('verbose'):
LOG.info(
'Accessing API for a list of Objects in %s', payload['c_name']
)
report.reporter(
msg='PAYLOAD\t: "%s"' % payload,
log=True,
lvl='debug',
prt=False
)
report.reporter(msg='getting file list')
with multi.spinner():
# Get all objects in a Container
objects, list_count, last_obj = self.action(
url=payload['url'],
container=payload['c_name'],
last_obj=ARGS.get('index_from')
)
if ARGS.get('pattern_match'):
objects = basic.match_filter(
idx_list=objects,
pattern=ARGS['pattern_match'],
dict_type=True
)
# Count the number of objects returned.
if objects is False:
report.reporter(msg='No Container found.')
return
elif objects is not None:
num_files = len(objects)
if num_files < 1:
report.reporter(msg='No Objects found.')
return
else:
report.reporter(msg='No Objects found.')
return
# Get The rate of concurrency
concurrency = multi.set_concurrency(args=ARGS,
file_count=num_files)
# Load the queue
obj_list = [obj['name'] for obj in objects if obj.get('name')]
report.reporter(msg='Building Directory Structure.')
with multi.spinner():
if ARGS.get('object'):
obj_names = ARGS.get('object')
obj_list = [obj for obj in obj_list if obj in obj_names]
num_files = len(obj_list)
elif ARGS.get('dir'):
objpath = ARGS.get('dir')
obj_list = [obj for obj in obj_list if obj.startswith(objpath)]
num_files = len(obj_list)
# from objects found set a unique list of directories
unique_dirs = basic.set_unique_dirs(object_list=obj_list,
root_dir=payload['source'])
for udir in unique_dirs:
basic.mkdir_p(path=udir)
kwargs = {'url': payload['url'],
'container': payload['c_name'],
'source': payload['source'],
'cf_job': getattr(self.go, 'object_downloader')}
report.reporter(msg='Performing Object Download.')
multi.job_processer(
num_jobs=num_files,
objects=obj_list,
job_action=multi.doerator,
concur=concurrency,
kwargs=kwargs
)
if ARGS.get('max_jobs') is not None:
report.reporter(
msg=('This is the last object downloaded. [ %s ]'
% last_obj),
log=True
)
def start(self):
"""Retrieve a long list of all files in a container."""
# Package up the Payload
payload = http.prep_payload(auth=self.auth,
container=ARGS.get('container'),
source=ARGS.get('source'),
args=ARGS)
self.go = actions.CloudActions(payload=payload)
self.action = getattr(self.go, 'object_lister')
LOG.info('Attempting Download of Remote path %s', payload['c_name'])
if ARGS.get('verbose'):
LOG.info('Accessing API for a list of Objects in %s',
payload['c_name'])
report.reporter(msg='PAYLOAD\t: "%s"' % payload,
log=True,
lvl='debug',
prt=False)
report.reporter(msg='getting file list')
with multi.spinner():
# Get all objects in a Container
objects, list_count, last_obj = self.action(
url=payload['url'], container=payload['c_name'])
if ARGS.get('pattern_match'):
objects = basic.match_filter(idx_list=objects,
pattern=ARGS['pattern_match'],
dict_type=True)
# Count the number of objects returned.
if objects is False:
report.reporter(msg='No Container found.')
return
elif objects is not None:
num_files = len(objects)
if num_files < 1:
report.reporter(msg='No Objects found.')
return
else:
report.reporter(msg='No Objects found.')
return
# Get The rate of concurrency
concurrency = multi.set_concurrency(args=ARGS,
file_count=num_files)
# Load the queue
obj_list = [obj['name'] for obj in objects]
report.reporter(msg='Building Directory Structure.')
with multi.spinner():
if ARGS.get('object'):
obj_names = ARGS.get('object')
obj_list = [obj for obj in obj_list if obj in obj_names]
num_files = len(obj_list)
elif ARGS.get('dir'):
objpath = ARGS.get('dir')
obj_list = [obj for obj in obj_list if obj.startswith(objpath)]
num_files = len(obj_list)
# from objects found set a unique list of directories
unique_dirs = basic.set_unique_dirs(object_list=obj_list,
root_dir=payload['source'])
for udir in unique_dirs:
basic.mkdir_p(path=udir)
kwargs = {
'url': payload['url'],
'container': payload['c_name'],
'source': payload['source'],
'cf_job': getattr(self.go, 'object_downloader')
}
report.reporter(msg='Performing Object Download.')
multi.job_processer(num_jobs=num_files,
objects=obj_list,
job_action=multi.doerator,
concur=concurrency,
kwargs=kwargs)
