def traverse_httpdir(self, path, task_name, only_files):
"""Traverses the file tree under the path given, within the CDMI service.
Applies the named task to every path."""
# reschedule this traverse if default queue is already large
task_count = app.get_message_count('default')
if(task_count > 50):
exc = DelayedTraverseError("Delaying Traverse due to queue size: {0}"
.format(task_count))
raise self.retry(exc=exc)
# Get directory
res = requests.get(path)
if not res.ok():
logger.error("FTP-over-HTTP GET request failed: {0} at {1}"
.format(res.msg(), path))
return
dir_info = res.json()
if only_files:
for f in dir_info:
if 'file' == f['type']:
app.send_task('workers.tasks.'+task_name,
args=[str(path)+f], kwargs={})
else:
for o in dir_info:
app.send_task('workers.tasks.'+task_name,
args=[str(path)+o], kwargs={})
for x in dir_info:
if 'directory' == x['type']:
traverse_httpdir.apply_async((str(path)+x+'/', task_name, only_files))
def traversal(self, path, task_name, only_files):
"""Traverses the file tree under the path given, within the CDMI service.
Applies the named task to every path."""
# reschedule this traverse if default queue is already large
task_count = app.get_message_count('default')
if(task_count > 50):
exc = DelayedTraverseError("Delaying Traverse due to queue size: {0}"
.format(task_count))
raise self.retry(exc=exc)
path = path[:-1] if path.endswith('?') else path
client = get_client()
res = client.ls(path)
if not res.ok():
logger.error("CDMI 'ls' request failed: {0} at {1}"
.format(res.msg(), path))
return
cdmi_info = res.json()
if not cdmi_info[u'objectType'] == u'application/cdmi-container':
logger.error("Cannot traverse a file path: {0}".format(path))
return
if only_files:
for f in cdmi_info[u'children']:
f = f[:-1] if f.endswith('?') else f
if not f.endswith('/'):
app.send_task('workers.tasks.'+task_name,
args=[str(path)+f], kwargs={})
else:
for o in cdmi_info[u'children']:
o = o[:-1] if o.endswith('?') else o
app.send_task('workers.tasks.'+task_name,
args=[str(path)+o], kwargs={})
for x in cdmi_info[u'children']:
x = x[:-1] if x.endswith('?') else x
if x.endswith('/'):
traversal.apply_async((str(path)+x, task_name, only_files))