Exemple #1
0
    def remote_delete(self, payload):
        """If Remote Delete was True run.

        NOTE: Remote delete will delete ALL Objects in a remote container
        which differ from the objects in the SOURCED LOCAL FILESYSTEM.

        IE: If this option is used, on one directory and then another directory
        and the files were different any difference would be deleted and based
        on the index information found in LOCAL FILE SYSTEM on the LAST
        command run.

        :param payload: ``dict``
        """

        report.reporter(msg='Getting file list for REMOTE DELETE')
        # From the remote system see if we have differences in the local system
        f_indexed = self._index_local_files()

        objects = self.go.object_lister(url=payload['url'],
                                        container=payload['c_name'])
        source = payload['source']
        obj_names = [
            basic.jpath(root=source, inode=obj.get('name'))
            for obj in objects[0]
        ]
        obj_names = set(obj_names)

        # Sort the difference between remote files and local files.
        objects = [obj for obj in obj_names if obj not in f_indexed]

        if objects:
            # Set Basic Data for file delete.
            num_files = len(objects)
            report.reporter(
                msg=('MESSAGE: "%d" Files have been found to be removed'
                     ' from the REMOTE CONTAINER.' % num_files))
            concurrency = multi.set_concurrency(args=ARGS,
                                                file_count=num_files)
            # Delete the difference in Files.
            report.reporter(msg='Performing REMOTE DELETE')

            del_objects = [
                basic.get_sfile(ufile=obj, source=payload['source'])
                for obj in objects if obj is not None
            ]

            kwargs = {
                'url': payload['url'],
                'container': payload['c_name'],
                'cf_job': getattr(self.go, 'object_deleter')
            }

            multi.job_processer(num_jobs=num_files,
                                objects=del_objects,
                                job_action=multi.doerator,
                                concur=concurrency,
                                kwargs=kwargs)
        else:
            report.reporter(
                msg='No Difference between REMOTE and LOCAL Directories.')
Exemple #2
0
        def _deleterator(payload):
            """Multipass Object Delete."""

            report.reporter(msg='Getting file list')
            with multi.spinner():
                # Get all objects in a Container
                objects, list_count, last_obj = self.action(
                    url=payload['url'],
                    container=payload['c_name']
                )

                if ARGS.get('pattern_match'):
                    objects = basic.match_filter(
                        idx_list=objects,
                        pattern=ARGS['pattern_match'],
                        dict_type=True
                    )

                # Count the number of objects returned.
                if objects is False:
                    report.reporter(msg='No Container found.')
                    return
                elif objects is not None:
                    # Load the queue
                    obj_list = [obj['name'] for obj in objects]
                    num_files = len(obj_list)
                    if num_files < 1:
                        report.reporter(msg='No Objects found.')
                        return
                else:
                    report.reporter(msg='Nothing found.')
                    return

                # Get The rate of concurrency
                concurrency = multi.set_concurrency(args=ARGS,
                                                    file_count=num_files)

                if ARGS.get('object'):
                    obj_names = ARGS.get('object')
                    obj_list = [obj for obj in obj_list if obj in obj_names]
                    if not obj_list:
                        return 'Nothing Found to Delete.'
                    num_files = len(obj_list)
                report.reporter(
                    msg=('Performing Object Delete for "%s" object(s)...'
                         % num_files)
                )
                kwargs = {'url': payload['url'],
                          'container': payload['c_name'],
                          'cf_job': getattr(self.go, 'object_deleter')}
            multi.job_processer(
                num_jobs=num_files,
                objects=obj_list,
                job_action=multi.doerator,
                concur=concurrency,
                kwargs=kwargs
            )
            _deleterator(payload=payload)
Exemple #3
0
    def start(self):
        """This is the upload method.

        Uses file_upload is to simply upload all files and folders to a
        specified container.
        """

        # Index Local Files for Upload
        with multi.spinner():
            f_indexed = methods.get_local_files()

        if ARGS.get('pattern_match'):
            f_indexed = basic.match_filter(
                idx_list=f_indexed, pattern=ARGS['pattern_match']
            )

        num_files = len(f_indexed)

        # Get The rate of concurrency
        concurrency = multi.set_concurrency(args=ARGS, file_count=num_files)

        # Package up the Payload
        payload = multi.manager_dict(
            http.prep_payload(
                auth=self.auth,
                container=ARGS.get('container', basic.rand_string()),
                source=basic.get_local_source(),
                args=ARGS
            )
        )

        LOG.info('MESSAGE\t: "%s" Files have been found.', num_files)
        LOG.debug('PAYLOAD\t: "%s"', payload)

        # Set the actions class up
        self.go = actions.CloudActions(payload=payload)

        kwargs = {'url': payload['url'],
                  'container': payload['c_name']}
        # get that the container exists if not create it.
        self.go.container_create(**kwargs)
        kwargs['source'] = payload['source']
        kwargs['cf_job'] = getattr(self.go, 'object_putter')

        multi.job_processer(
            num_jobs=num_files,
            objects=f_indexed,
            job_action=multi.doerator,
            concur=concurrency,
            kwargs=kwargs
        )

        if ARGS.get('delete_remote') is True:
            self.remote_delete(payload=payload,
                               f_indexed=f_indexed)
Exemple #4
0
    def start(self):
        """This is the upload method.

        Uses file_upload is to simply upload all files and folders to a
        specified container.
        """

        # Index Local Files for Upload
        f_indexed = methods.get_local_files()

        if ARGS.get('pattern_match'):
            f_indexed = basic.match_filter(
                idx_list=f_indexed, pattern=ARGS['pattern_match']
            )

        num_files = len(f_indexed)

        # Get The rate of concurrency
        concurrency = multi.set_concurrency(args=ARGS, file_count=num_files)

        # Package up the Payload
        payload = multi.manager_dict(
            http.prep_payload(
                auth=self.auth,
                container=ARGS.get('container', basic.rand_string()),
                source=basic.get_local_source(),
                args=ARGS
            )
        )

        LOG.info('MESSAGE\t: "%s" Files have been found.', num_files)
        LOG.debug('PAYLOAD\t: "%s"', payload)

        # Set the actions class up
        self.go = actions.CloudActions(payload=payload)

        kwargs = {'url': payload['url'],
                  'container': payload['c_name']}
        # get that the container exists if not create it.
        self.go.container_create(**kwargs)
        kwargs['source'] = payload['source']
        kwargs['cf_job'] = getattr(self.go, 'object_putter')

        multi.job_processer(
            num_jobs=num_files,
            objects=f_indexed,
            job_action=multi.doerator,
            concur=concurrency,
            kwargs=kwargs
        )

        if ARGS.get('delete_remote') is True:
            self.remote_delete(payload=payload,
                               f_indexed=f_indexed)
Exemple #5
0
    def remote_delete(self, payload, f_indexed):
        """If Remote Delete was True run.

        NOTE: Remote delete will delete ALL Objects in a remote container
        which differ from the objects in the SOURCED LOCAL FILESYSTEM.

        IE: If this option is used, on one directory and then another directory
        and the files were different any difference would be deleted and based
        on the index information found in LOCAL FILE SYSTEM on the LAST
        command run.

        :return:
        """

        report.reporter(msg='Getting file list for REMOTE DELETE')
        objects = self.go.object_lister(
            url=payload['url'], container=payload['c_name']
        )
        source = payload['source']
        obj_names = [basic.jpath(root=source, inode=obj.get('name'))
                     for obj in objects[0]]

        # From the remote system see if we have differences in the local system
        objects = multi.return_diff().difference(target=f_indexed,
                                                 source=obj_names)
        if objects:
            # Set Basic Data for file delete.
            num_files = len(objects)
            LOG.info('MESSAGE\t: "%s" Files have been found to be removed from'
                     ' the REMOTE CONTAINER.', num_files)
            concurrency = multi.set_concurrency(
                args=ARGS, file_count=num_files
            )
            # Delete the difference in Files.
            report.reporter(msg='Performing Remote Delete')

            objects = [basic.get_sfile(
                ufile=obj, source=payload['source']
            ) for obj in objects]
            kwargs = {'url': payload['url'],
                      'container': payload['c_name'],
                      'cf_job': getattr(self.go, 'object_deleter')}
            multi.job_processer(
                num_jobs=num_files,
                objects=objects,
                job_action=multi.doerator,
                concur=concurrency,
                kwargs=kwargs
            )
        else:
            report.reporter(
                msg='No Difference between REMOTE and LOCAL Directories.'
            )
Exemple #6
0
    def start(self):
        """This is the upload method.

        Uses file_upload is to simply upload all files and folders to a
        specified container.
        """

        f_indexed = self._index_local_files()
        num_files = len(f_indexed)

        # Get The rate of concurrency
        concurrency = multi.set_concurrency(args=ARGS, file_count=num_files)

        # Package up the Payload
        payload = multi.manager_dict(
            http.prep_payload(
                auth=self.auth,
                container=ARGS.get('container', basic.rand_string()),
                source=basic.get_local_source(),
                args=ARGS
            )
        )
        report.reporter(msg='MESSAGE : "%s" Files found.' % num_files)
        report.reporter(
            msg='PAYLOAD : [ %s ]' % payload,
            prt=False,
            lvl='debug'
        )

        # Set the actions class up
        self.go = actions.CloudActions(payload=payload)

        kwargs = {'url': payload['url'], 'container': payload['c_name']}
        # get that the container exists if not create it.
        self.go.container_create(**kwargs)
        kwargs['source'] = payload['source']
        kwargs['cf_job'] = getattr(self.go, 'object_putter')

        multi.job_processer(
            num_jobs=num_files,
            objects=f_indexed,
            job_action=multi.doerator,
            concur=concurrency,
            kwargs=kwargs
        )

        if ARGS.get('delete_remote') is True:
            self.remote_delete(payload=payload)
Exemple #7
0
    def start(self):
        """Return a list of objects from the API for a container."""

        def _check_list(list_object):
            if list_object:
                return list_object
            else:
                return None, None, None

        def _list(l_payload, go, l_last_obj):
            """Retrieve a long list of all files in a container.

            :return final_list, list_count, last_obj:
            """
            # object_lister(url, container, object_count=None, last_obj=None)
            return _check_list(
                list_object=go.object_lister(
                    url=l_payload['url'],
                    container=l_payload['c_name'],
                    last_obj=l_last_obj
                )
            )

        # Package up the Payload
        payload = http.prep_payload(
            auth=self.auth,
            container=ARGS.get('container'),
            source=None,
            args=ARGS
        )

        # Prep Actions.
        self.go = actions.CloudActions(payload=payload)

        report.reporter(
            msg='API Access for a list of Objects in %s' % payload['c_name'],
            log=True
        )
        report.reporter(
            msg='PAYLOAD\t: "%s"' % payload,
            log=True,
            lvl='debug',
            prt=False
        )

        last_obj = None
        with multi.spinner():
            objects, list_count, last_obj = _list(
                l_payload=payload, go=self.go, l_last_obj=last_obj
            )
            if 'pattern_match' in ARGS:
                objects = basic.match_filter(
                    idx_list=objects,
                    pattern=ARGS['pattern_match'],
                    dict_type=True
                )

            if ARGS.get('filter') is not None:
                objects = [obj for obj in objects
                           if ARGS.get('filter') in obj.get('name')]

        # Count the number of objects returned.
        if objects is False:
            report.reporter(msg='Nothing found.')
        elif len(objects) < 1:
            report.reporter(msg='Nothing found.')
        elif ARGS.get('object'):
            self.go.object_updater(
                url=payload['url'],
                container=payload['c_name'],
                u_file=last_obj
            )
        elif objects is not None:
            kwargs = {
                'url': payload['url'],
                'container': payload['c_name'],
                'cf_job': getattr(self.go, 'object_updater'),
            }

            object_names = [i['name'] for i in objects]
            num_files = len(object_names)
            concurrency = multi.set_concurrency(
                args=ARGS, file_count=num_files
            )
            multi.job_processer(
                num_jobs=num_files,
                objects=object_names,
                job_action=multi.doerator,
                concur=concurrency,
                kwargs=kwargs
            )
        else:
            report.reporter(msg='Nothing found.')
Exemple #8
0
    def start(self):
        """Clone onjects from one container to another.

        NOTE: This method was intended for use with inter-datacenter cloning of
        objects.
        """

        # Package up the Payload
        payload = http.prep_payload(
            auth=self.auth,
            container=ARGS.get('source_container'),
            source=None,
            args=ARGS
        )

        # Prep action class
        self.go = actions.CloudActions(payload=payload)

        # Ensure we have a target region.
        target_region = ARGS.get('target_region')
        if target_region is None:
            raise turbo.NoSource('No target Region was specified.')
        else:
            target_region = target_region.upper()

        # check for a target type URL
        if ARGS.get('target_snet') is True:
            target_type = 'internalURL'
        else:
            target_type = 'publicURL'

        # Format the target URL
        target_url = auth.get_surl(
            region=target_region, cf_list=payload['acfep'], lookup=target_type
        )
        if target_url is None:
            raise turbo.NoSource('No url was found from the target region')
        else:
            payload['turl'] = target_url

        # Ensure we have a target Container.
        target_container = ARGS.get('target_container')
        if target_container is None:
            raise turbo.NoSource('No target Container was specified.')
        else:
            payload['tc_name'] = target_container

        # Check if the source and target containers exist. If not Create them.
        # Source Container.
        self.go.container_create(url=payload['url'],
                                 container=payload['c_name'])
        # Target Container.
        self.go.container_create(url=target_url,
                                 container=target_container)

        report.reporter(msg='Getting Object list from the Source.')
        with multi.spinner():
            # Get a list of Objects from the Source/Target container.
            objects, list_count, last_obj = self.go.object_lister(
                url=payload['url'],
                container=payload['c_name']
            )

            if ARGS.get('pattern_match'):
                objects = basic.match_filter(
                    idx_list=objects,
                    pattern=ARGS['pattern_match'],
                    dict_type=True
                )

        if objects is None:
            raise turbo.NoSource('The source container is empty.')

        # Get the number of objects and set Concurrency
        num_files = len(objects)
        concurrency = multi.set_concurrency(args=ARGS,
                                            file_count=num_files)

        report.reporter(msg='Beginning Sync Operation.')
        kwargs = {'surl': payload['url'],
                  'turl': payload['turl'],
                  'scontainer': payload['c_name'],
                  'tcontainer': payload['tc_name'],
                  'cf_job': getattr(self.go, 'object_syncer')}

        multi.job_processer(
            num_jobs=num_files,
            objects=objects,
            job_action=multi.doerator,
            concur=concurrency,
            kwargs=kwargs
        )
Exemple #9
0
    def start(self):
        """Clone onjects from one container to another.

        NOTE: This method was intended for use with inter-datacenter cloning of
        objects.
        """

        # Package up the Payload
        payload = http.prep_payload(auth=self.auth,
                                    container=ARGS.get('source_container'),
                                    source=None,
                                    args=ARGS)

        # Prep action class
        self.go = actions.CloudActions(payload=payload)

        # Ensure we have a target region.
        target_region = ARGS.get('target_region')
        if target_region is None:
            raise turbo.NoSource('No target Region was specified.')
        else:
            target_region = target_region.upper()

        # check for a target type URL
        if ARGS.get('target_snet') is True:
            target_type = 'internalURL'
        else:
            target_type = 'publicURL'

        # Format the target URL
        target_url = auth.get_surl(region=target_region,
                                   cf_list=payload['acfep'],
                                   lookup=target_type)
        if target_url is None:
            raise turbo.NoSource('No url was found from the target region')
        else:
            payload['turl'] = target_url

        # Ensure we have a target Container.
        target_container = ARGS.get('target_container')
        if target_container is None:
            raise turbo.NoSource('No target Container was specified.')
        else:
            payload['tc_name'] = target_container

        # Check if the source and target containers exist. If not Create them.
        # Source Container.
        self.go.container_create(url=payload['url'],
                                 container=payload['c_name'])
        # Target Container.
        self.go.container_create(url=target_url, container=target_container)

        report.reporter(msg='Getting Object list from the Source.')
        with multi.spinner():
            # Get a list of Objects from the Source/Target container.
            objects, list_count, last_obj = self.go.object_lister(
                url=payload['url'], container=payload['c_name'])

            if ARGS.get('pattern_match'):
                objects = basic.match_filter(idx_list=objects,
                                             pattern=ARGS['pattern_match'],
                                             dict_type=True)

        if objects is None:
            raise turbo.NoSource('The source container is empty.')

        # Get the number of objects and set Concurrency
        num_files = len(objects)
        concurrency = multi.set_concurrency(args=ARGS, file_count=num_files)

        report.reporter(msg='Beginning Sync Operation.')
        kwargs = {
            'surl': payload['url'],
            'turl': payload['turl'],
            'scontainer': payload['c_name'],
            'tcontainer': payload['tc_name'],
            'cf_job': getattr(self.go, 'object_syncer')
        }

        multi.job_processer(num_jobs=num_files,
                            objects=objects,
                            job_action=multi.doerator,
                            concur=concurrency,
                            kwargs=kwargs)
Exemple #10
0
                            container=args.get('container'),
                            source=args.get('source'),
                            args=args)

# Load all of our available cloud actions
from turbolift.clouderator import actions

cf_actions = actions.CloudActions(payload=payload)

# Upload file(s)
# =============================================================================
import turbolift.utils.multi_utils as multi
from turbolift import methods

f_indexed = methods.get_local_files()  # Index all of the local files
num_files = len(f_indexed)  # counts the indexed files

kwargs = {
    'url': payload['url'],  # Defines the Upload URL
    'container': payload['c_name'],  # Sets the container
    'source': payload['source'],  # Defines the local source to upload
    'cf_job': cf_actions.object_putter  # sets the job
}

# Perform the upload job
multi.job_processer(num_jobs=num_files,
                    objects=f_indexed,
                    job_action=multi.doerator,
                    concur=25,
                    kwargs=kwargs)
Exemple #11
0
from turbolift.clouderator import actions
cf_actions = actions.CloudActions(payload=payload)


# Delete file(s)
# =============================================================================
import turbolift.utils.multi_utils as multi

kwargs = {
    'url': payload['url'],              # Defines the Upload URL
    'container': payload['c_name']      # Sets the container
}

# Return a list of all objects that we will delete
objects, list_count, last_obj = cf_actions.object_lister(**kwargs)

# Get a list of all of the object names
object_names = [obj['name'] for obj in objects]

# Set the delete job
kwargs['cf_job'] = cf_actions.object_deleter

# Perform the upload job
multi.job_processer(
    num_jobs=list_count,
    objects=object_names,
    job_action=multi.doerator,
    concur=50,
    kwargs=kwargs
)
Exemple #12
0
    def start(self):
        """Retrieve a long list of all files in a container."""

        # Package up the Payload
        payload = http.prep_payload(auth=self.auth,
                                    container=ARGS.get('container'),
                                    source=ARGS.get('source'),
                                    args=ARGS)
        self.go = actions.CloudActions(payload=payload)
        self.action = getattr(self.go, 'object_lister')

        LOG.info('Attempting Download of Remote path %s', payload['c_name'])

        if ARGS.get('verbose'):
            LOG.info('Accessing API for a list of Objects in %s',
                     payload['c_name'])

        report.reporter(
            msg='PAYLOAD : [ %s ]' % payload,
            prt=False,
            lvl='debug',
        )

        report.reporter(msg='getting file list')
        with multi.spinner():
            # Get all objects in a Container
            objects, list_count, last_obj = self.action(
                url=payload['url'],
                container=payload['c_name'],
                last_obj=ARGS.get('index_from'))

            if ARGS.get('pattern_match'):
                objects = basic.match_filter(idx_list=objects,
                                             pattern=ARGS['pattern_match'],
                                             dict_type=True)

            # Count the number of objects returned.
            if objects is False:
                report.reporter(msg='No Container found.')
                return
            elif objects is not None:
                num_files = len(objects)
                if num_files < 1:
                    report.reporter(msg='No Objects found.')
                    return
            else:
                report.reporter(msg='No Objects found.')
                return

            # Get The rate of concurrency
            concurrency = multi.set_concurrency(args=ARGS,
                                                file_count=num_files)
            # Load the queue
            obj_list = [obj['name'] for obj in objects if obj.get('name')]

        report.reporter(msg='Building Directory Structure.')
        with multi.spinner():
            if ARGS.get('object'):
                obj_names = ARGS.get('object')
                obj_list = [obj for obj in obj_list if obj in obj_names]
                num_files = len(obj_list)
            elif ARGS.get('dir'):
                objpath = ARGS.get('dir')
                obj_list = [obj for obj in obj_list if obj.startswith(objpath)]
                num_files = len(obj_list)

            # from objects found set a unique list of directories
            unique_dirs = basic.set_unique_dirs(object_list=obj_list,
                                                root_dir=payload['source'])
            for udir in unique_dirs:
                basic.mkdir_p(path=udir)

        kwargs = {
            'url': payload['url'],
            'container': payload['c_name'],
            'source': payload['source'],
            'cf_job': getattr(self.go, 'object_downloader')
        }

        report.reporter(msg='Performing Object Download.')
        multi.job_processer(num_jobs=num_files,
                            objects=obj_list,
                            job_action=multi.doerator,
                            concur=concurrency,
                            kwargs=kwargs)
        if ARGS.get('max_jobs') is not None:
            report.reporter(msg=('This is the last object downloaded. [ %s ]' %
                                 last_obj),
                            log=True)
Exemple #13
0
    def start(self):
        """Retrieve a long list of all files in a container."""

        # Package up the Payload
        payload = http.prep_payload(
            auth=self.auth,
            container=ARGS.get('container'),
            source=ARGS.get('source'),
            args=ARGS
        )
        self.go = actions.CloudActions(payload=payload)
        self.action = getattr(self.go, 'object_lister')

        LOG.info('Attempting Download of Remote path %s', payload['c_name'])

        if ARGS.get('verbose'):
            LOG.info(
                'Accessing API for a list of Objects in %s', payload['c_name']
            )

        report.reporter(
            msg='PAYLOAD\t: "%s"' % payload,
            log=True,
            lvl='debug',
            prt=False
        )

        report.reporter(msg='getting file list')
        with multi.spinner():
            # Get all objects in a Container
            objects, list_count, last_obj = self.action(
                url=payload['url'],
                container=payload['c_name'],
                last_obj=ARGS.get('index_from')
            )

            if ARGS.get('pattern_match'):
                objects = basic.match_filter(
                    idx_list=objects,
                    pattern=ARGS['pattern_match'],
                    dict_type=True
                )

            # Count the number of objects returned.
            if objects is False:
                report.reporter(msg='No Container found.')
                return
            elif objects is not None:
                num_files = len(objects)
                if num_files < 1:
                    report.reporter(msg='No Objects found.')
                    return
            else:
                report.reporter(msg='No Objects found.')
                return

            # Get The rate of concurrency
            concurrency = multi.set_concurrency(args=ARGS,
                                                file_count=num_files)
            # Load the queue
            obj_list = [obj['name'] for obj in objects if obj.get('name')]

        report.reporter(msg='Building Directory Structure.')
        with multi.spinner():
            if ARGS.get('object'):
                obj_names = ARGS.get('object')
                obj_list = [obj for obj in obj_list if obj in obj_names]
                num_files = len(obj_list)
            elif ARGS.get('dir'):
                objpath = ARGS.get('dir')
                obj_list = [obj for obj in obj_list if obj.startswith(objpath)]
                num_files = len(obj_list)

            # from objects found set a unique list of directories
            unique_dirs = basic.set_unique_dirs(object_list=obj_list,
                                                root_dir=payload['source'])
            for udir in unique_dirs:
                basic.mkdir_p(path=udir)

        kwargs = {'url': payload['url'],
                  'container': payload['c_name'],
                  'source': payload['source'],
                  'cf_job': getattr(self.go, 'object_downloader')}

        report.reporter(msg='Performing Object Download.')
        multi.job_processer(
            num_jobs=num_files,
            objects=obj_list,
            job_action=multi.doerator,
            concur=concurrency,
            kwargs=kwargs
        )
        if ARGS.get('max_jobs') is not None:
            report.reporter(
                msg=('This is the last object downloaded. [ %s ]'
                     % last_obj),
                log=True
            )
Exemple #14
0
# Package up the Payload
import turbolift.utils.http_utils as http

payload = http.prep_payload(auth=authentication, container=args.get("container"), source=args.get("source"), args=args)


# Load all of our available cloud actions
from turbolift.clouderator import actions

cf_actions = actions.CloudActions(payload=payload)


# Upload file(s)
# =============================================================================
import turbolift.utils.multi_utils as multi
from turbolift import methods

f_indexed = methods.get_local_files()  # Index all of the local files
num_files = len(f_indexed)  # counts the indexed files

kwargs = {
    "url": payload["url"],  # Defines the Upload URL
    "container": payload["c_name"],  # Sets the container
    "source": payload["source"],  # Defines the local source to upload
    "cf_job": cf_actions.object_putter,  # sets the job
}

# Perform the upload job
multi.job_processer(num_jobs=num_files, objects=f_indexed, job_action=multi.doerator, concur=25, kwargs=kwargs)
Exemple #15
0
                            args=args)

# Load all of our available cloud actions
from turbolift.clouderator import actions
cf_actions = actions.CloudActions(payload=payload)

# Delete file(s)
# =============================================================================
import turbolift.utils.multi_utils as multi

kwargs = {
    'url': payload['url'],  # Defines the Upload URL
    'container': payload['c_name']  # Sets the container
}

# Return a list of all objects that we will delete
objects, list_count, last_obj = cf_actions.object_lister(**kwargs)

# Get a list of all of the object names
object_names = [obj['name'] for obj in objects]

# Set the delete job
kwargs['cf_job'] = cf_actions.object_deleter

# Perform the upload job
multi.job_processer(num_jobs=list_count,
                    objects=object_names,
                    job_action=multi.doerator,
                    concur=50,
                    kwargs=kwargs)
Exemple #16
0
    def start(self):
        """Return a list of objects from the API for a container."""

        def _check_list(list_object):
            if list_object:
                return list_object
            else:
                return None, None, None

        def _list(l_payload, go, l_last_obj):
            """Retrieve a long list of all files in a container.

            :return final_list, list_count, last_obj:
            """
            # object_lister(url, container, object_count=None, last_obj=None)
            return _check_list(
                list_object=go.object_lister(
                    url=l_payload['url'],
                    container=l_payload['c_name'],
                    last_obj=l_last_obj
                )
            )

        # Package up the Payload
        payload = http.prep_payload(
            auth=self.auth,
            container=ARGS.get('container'),
            source=None,
            args=ARGS
        )

        # Prep Actions.
        self.go = actions.CloudActions(payload=payload)

        report.reporter(
            msg='API Access for a list of Objects in %s' % payload['c_name'],
            log=True
        )
        report.reporter(
            msg='PAYLOAD : [ %s ]' % payload,
            prt=False,
            lvl='debug',
        )

        last_obj = None
        with multi.spinner():
            objects, list_count, last_obj = _list(
                l_payload=payload, go=self.go, l_last_obj=last_obj
            )
            if 'pattern_match' in ARGS:
                objects = basic.match_filter(
                    idx_list=objects,
                    pattern=ARGS['pattern_match'],
                    dict_type=True
                )

            if ARGS.get('filter') is not None:
                objects = [obj for obj in objects
                           if ARGS.get('filter') in obj.get('name')]

        # Count the number of objects returned.
        if objects is False:
            report.reporter(msg='Nothing found.')
        elif len(objects) < 1:
            report.reporter(msg='Nothing found.')
        elif ARGS.get('object'):
            self.go.object_updater(
                url=payload['url'],
                container=payload['c_name'],
                u_file=last_obj
            )
        elif objects is not None:
            kwargs = {
                'url': payload['url'],
                'container': payload['c_name'],
                'cf_job': getattr(self.go, 'object_updater'),
            }

            object_names = [i['name'] for i in objects]
            num_files = len(object_names)
            concurrency = multi.set_concurrency(
                args=ARGS, file_count=num_files
            )
            multi.job_processer(
                num_jobs=num_files,
                objects=object_names,
                job_action=multi.doerator,
                concur=concurrency,
                kwargs=kwargs
            )
        else:
            report.reporter(msg='Nothing found.')