Example #1
0
def traverse_httpdir(self, path, task_name, only_files):
    """Traverses the file tree under the path given, within the CDMI service.
       Applies the named task to every path."""

    # reschedule this traverse if default queue is already large
    task_count = app.get_message_count('default')
    if(task_count > 50):
        exc = DelayedTraverseError("Delaying Traverse due to queue size: {0}"
                                   .format(task_count))
        raise self.retry(exc=exc)

    # Get directory
    res = requests.get(path)
    if not res.ok():
        logger.error("FTP-over-HTTP GET request failed: {0} at {1}"
                     .format(res.msg(), path))
        return

    dir_info = res.json()

    if only_files:
        for f in dir_info:
            if 'file' == f['type']:
                app.send_task('workers.tasks.'+task_name,
                              args=[str(path)+f], kwargs={})
    else:
        for o in dir_info:
            app.send_task('workers.tasks.'+task_name,
                          args=[str(path)+o], kwargs={})

    for x in dir_info:
        if 'directory' == x['type']:
            traverse_httpdir.apply_async((str(path)+x+'/', task_name, only_files))
Example #2
0
def traversal(self, path, task_name, only_files):
    """Traverses the file tree under the path given, within the CDMI service.
       Applies the named task to every path."""

    # reschedule this traverse if default queue is already large
    task_count = app.get_message_count('default')
    if(task_count > 50):
        exc = DelayedTraverseError("Delaying Traverse due to queue size: {0}"
                                   .format(task_count))
        raise self.retry(exc=exc)

    path = path[:-1] if path.endswith('?') else path

    client = get_client()
    res = client.ls(path)
    if not res.ok():
        logger.error("CDMI 'ls' request failed: {0} at {1}"
                     .format(res.msg(), path))
        return

    cdmi_info = res.json()
    if not cdmi_info[u'objectType'] == u'application/cdmi-container':
        logger.error("Cannot traverse a file path: {0}".format(path))
        return

    if only_files:
        for f in cdmi_info[u'children']:
            f = f[:-1] if f.endswith('?') else f
            if not f.endswith('/'):
                app.send_task('workers.tasks.'+task_name,
                              args=[str(path)+f], kwargs={})
    else:
        for o in cdmi_info[u'children']:
            o = o[:-1] if o.endswith('?') else o
            app.send_task('workers.tasks.'+task_name,
                          args=[str(path)+o], kwargs={})

    for x in cdmi_info[u'children']:
        x = x[:-1] if x.endswith('?') else x
        if x.endswith('/'):
            traversal.apply_async((str(path)+x, task_name, only_files))