def list(cls, group=DEFAULT_GROUP, limit=None, offset=None):
"""Returns a list of TaskManager objects.
Args:
group: The group ID of where to list task managers.
limit: An integer limit for the list query.
offset: An integer offset for the list query.
Returns:
A list of TaskManager objects in descending chronological order.
"""
titan_files = files.Files.list(dir_path=utils.safe_join(
_ROOT_DIR_PATH, group),
limit=limit,
offset=offset,
_internal=True)
titan_files.load()
# Sort in descending chronological order.
ordered_files = sorted(titan_files.itervalues(),
key=lambda f: f.created)
ordered_files.reverse()
task_manager_keys = [f.name_clean for f in ordered_files]
# Create TaskManager objects and inject the preloaded Titan file.
task_managers = []
for i, task_manager_key in enumerate(task_manager_keys):
task_manager = TaskManager(key=task_manager_key,
group=group,
_file=ordered_files[i])
task_managers.append(task_manager)
return task_managers
def list(cls, group=DEFAULT_GROUP, limit=None, offset=None):
"""Returns a list of TaskManager objects.
Args:
group: The group ID of where to list task managers.
limit: An integer limit for the list query.
offset: An integer offset for the list query.
Returns:
A list of TaskManager objects in descending chronological order.
"""
titan_files = files.Files.list(
dir_path=utils.safe_join(_ROOT_DIR_PATH, group),
limit=limit, offset=offset, _internal=True)
titan_files.load()
# Sort in descending chronological order.
ordered_files = sorted(titan_files.itervalues(), key=lambda f: f.created)
ordered_files.reverse()
task_manager_keys = [f.name_clean for f in ordered_files]
# Create TaskManager objects and inject the preloaded Titan file.
task_managers = []
for i, task_manager_key in enumerate(task_manager_keys):
task_manager = TaskManager(
key=task_manager_key, group=group, _file=ordered_files[i])
task_managers.append(task_manager)
return task_managers
def _file(self):
if self._internal_file is None:
# /_titan/tasks/<group>/<task_manager_key>/tasks/<task_key_hash>.json
filename = utils.safe_join(
_ROOT_DIR_PATH, self._task_manager.group, self._task_manager.key,
'tasks', self._internal_key + '.json')
self._internal_file = files.File(filename, _internal=True)
return self._internal_file
def _file(self):
if self._internal_file is None:
# /_titan/tasks/<group>/<task_manager_key>/tasks/<task_key_hash>.json
filename = utils.safe_join(_ROOT_DIR_PATH,
self._task_manager.group,
self._task_manager.key, 'tasks',
self._internal_key + '.json')
self._internal_file = files.File(filename, _internal=True)
return self._internal_file
def testSafeJoin(self):
self.assertEqual('foo/bar', utils.safe_join('foo', 'bar'))
self.assertEqual('foo/bar/baz/', utils.safe_join('foo', 'bar', 'baz/'))
self.assertEqual('foo.html', utils.safe_join('', 'foo.html'))
self.assertEqual('/foo/bar', utils.safe_join('/foo', 'bar'))
self.assertEqual('/foo/bar', utils.safe_join('/foo/', 'bar'))
self.assertEqual('/foo/bar/baz.html',
utils.safe_join('/foo', 'bar', 'baz.html'))
self.assertEqual('/foo/bar////baz.html',
utils.safe_join('/foo', 'bar////', 'baz.html'))
self.assertRaises(ValueError, utils.safe_join, '/foo', '/bar')
self.assertRaises(ValueError, utils.safe_join, 'foo', '/bar', 'baz')
self.assertRaises(ValueError, utils.safe_join, 'foo', 'bar', '/baz')
def testSafeJoin(self):
self.assertEqual('foo/bar', utils.safe_join('foo', 'bar'))
self.assertEqual('foo/bar/baz/', utils.safe_join('foo', 'bar', 'baz/'))
self.assertEqual('foo.html', utils.safe_join('', 'foo.html'))
self.assertEqual('/foo/bar', utils.safe_join('/foo', 'bar'))
self.assertEqual('/foo/bar', utils.safe_join('/foo/', 'bar'))
self.assertEqual('/foo/bar/baz.html',
utils.safe_join('/foo', 'bar', 'baz.html'))
self.assertEqual('/foo/bar////baz.html',
utils.safe_join('/foo', 'bar////', 'baz.html'))
self.assertRaises(ValueError, utils.safe_join, '/foo', '/bar')
self.assertRaises(ValueError, utils.safe_join, 'foo', '/bar', 'baz')
self.assertRaises(ValueError, utils.safe_join, 'foo', 'bar', '/baz')
def __init__(self, key, user=None, meta=None):
super(Activity, self).__init__()
self.timestamp = datetime.datetime.utcnow()
self.key = key
self.user = user
self.meta = meta
self._keys = utils.split_segments(self.key, sep='.')
# Make the activity_id relatively unique.
hashed = hashlib.md5()
hashed.update(self.key)
hashed.update(self.timestamp.isoformat(sep='T'))
hashed.update(os.environ.get('REQUEST_ID_HASH', ''))
if self.user:
hashed.update(self.user.email)
self.activity_id = utils.safe_join(
ACTIVITY_DIR, self.key, self.timestamp.strftime('%Y/%m/%d'),
'{}-{}.json'.format(self.timestamp.isoformat(sep='T'),
hashed.hexdigest()[:6]))
def __init__(self, key, user=None, meta=None):
super(Activity, self).__init__()
self.timestamp = datetime.datetime.utcnow()
self.key = key
self.user = user
self.meta = meta
self._keys = utils.split_segments(self.key, sep='.')
# Make the activity_id relatively unique.
hashed = hashlib.md5()
hashed.update(self.key)
hashed.update(self.timestamp.isoformat(sep='T'))
hashed.update(os.environ.get('REQUEST_ID_HASH', ''))
if self.user:
hashed.update(self.user.email)
self.activity_id = utils.safe_join(
ACTIVITY_DIR, self.key, self.timestamp.strftime('%Y/%m/%d'),
'{}-{}.json'.format(
self.timestamp.isoformat(sep='T'), hashed.hexdigest()[:6]))
def _make_log_path(date, counter):
# Make a path like:
# /_titan/activities/stats/counters/2015/05/15/page/view/data-60s.json
formatted_date = date.strftime(counter.date_format)
return utils.safe_join(
BASE_DIR, formatted_date, counter.name, counter.data_filename)
def _tasks_dir_path(self):
# /_titan/tasks/<group>/<task_manager_key>/tasks
if not self._internal_tasks_dir_path:
self._internal_tasks_dir_path = utils.safe_join(self._dir_path, 'tasks')
return self._internal_tasks_dir_path
def _dir_path(self):
# /_titan/tasks/<group>/<task_manager_key>
if not self._internal_dir_path:
self._internal_dir_path = utils.safe_join(
_ROOT_DIR_PATH, self.group, self.key)
return self._internal_dir_path
def _dir_path(self):
# /_titan/tasks/<group>/<task_manager_key>
if not self._internal_dir_path:
self._internal_dir_path = utils.safe_join(_ROOT_DIR_PATH,
self.group, self.key)
return self._internal_dir_path
def Run(self,
dir_path=None,
recursive=False,
depth=None,
file_paths=None,
target_dir=None):
"""Download runner.
Args:
dir_path: A list of remote directories to download.
recursive: Whether or not to download directory recursively.
depth: Depth of recursion if specified.
file_paths: A list of remote files to download.
target_dir: The target local directory to upload to.
Returns:
A list of mappings between remote_path -> local_path.
Raises:
DownloadFileError
"""
if not file_paths and not dir_path:
self.print_error('No files to download. Use --file_path or '
'--dir_path.')
return
if target_dir is None:
target_dir = '.'
path_map = []
if file_paths:
remote_files = self.remote_file_factory.make_remote_files(
paths=file_paths)
for remote_file in remote_files.itervalues():
if not remote_file.exists:
print 'File %s does not exist' % remote_file.path
target = os.path.abspath(
utils.safe_join(target_dir, remote_file.name))
path_map.append([remote_file, target])
elif dir_path:
dir_kwargs = {'recursive': recursive, 'depth': depth}
remote_files = self.remote_file_factory.make_remote_files(paths=[])
remote_files.list(dir_path, **dir_kwargs)
for remote_file in remote_files.itervalues():
target = os.path.abspath(
utils.safe_join(target_dir, remote_file.path[1:]))
path_map.append([remote_file, target])
conf_message = ['The following will be downloaded from %s' % self.host]
for remote_file, target in path_map:
conf_message.append(' %s --> %s' % (remote_file.path, target))
print '\n'.join(conf_message)
if not self.confirm(' Are you sure?'):
sys.exit('Download aborted.')
else:
print 'Downloading...'
utils.make_dirs(target_dir)
# Start the download.
start = time.time()
self.remote_file_factory.validate_client_auth()
future_results = []
with self.ThreadPoolExecutor() as executor:
for remote_file, target in path_map:
target_base_dir = os.path.dirname(target)
utils.make_dirs(target_base_dir)
future = executor.submit(self._DownloadFile, remote_file,
target)
future_results.append(future)
failed = False
for future in futures.as_completed(future_results):
try:
downloaded_file = future.result()
self.print_message(
'Downloaded %s to %s' %
(downloaded_file['path'], downloaded_file['target']))
except DownloadFileError as e:
self.print_error('Error downloading %s. Error was: %s %s' %
(e.target_path, e.__class__.__name__, str(e)))
failed = True
if failed:
self.print_error('Could not download one or more files.')
return
elapsed_time = time.time() - start
print 'Downloaded %d files in %s.' % (
len(path_map), utils.humanize_duration(elapsed_time))
def Run(self,
filenames,
root_dir=None,
target_path='/',
changeset=None,
commit=False,
confirm_manifest=False):
"""Upload runner.
Args:
filenames: A list of local filenames to upload.
root_dir: Local root dir containing all the files. Default: current dir.
target_path: Remote root target dir for uploaded documents.
changeset: A changeset number or "new".
commit: Whether or not to commit the changeset.
confirm_manifest: Whether or not the current list of files given to
upload can be trusted as a complete manifest of the files in the
changeset. Required when combining "commit" and "changeset=<num>".
Returns:
A dictionary containing "success", "manifest", "paths", and
"changeset_num" if uploading to a changeset.
"""
if root_dir is None:
root_dir = os.path.curdir
# Compose mapping of absolute local path to absolute remote path.
root_dir = os.path.abspath(root_dir)
filename_to_paths = {}
for filename in filenames:
absolute_filename = os.path.abspath(filename)
if not absolute_filename.startswith(root_dir):
self.print_error('Path "%s" not contained within "%s".' %
(absolute_filename, root_dir))
sys.exit()
remote_path = absolute_filename[len(root_dir) + 1:]
remote_path = utils.safe_join(target_path, remote_path)
filename_to_paths[absolute_filename] = remote_path
# Confirm action.
print '\nUploading files to %s (%s):' % (self.host, self.api_base_path)
for filename, path in filename_to_paths.iteritems():
if root_dir == os.path.abspath(os.path.curdir):
# Strip the root_dir from view if it's already the working directory.
filename = '.' + filename[len(root_dir):]
print ' %s --> %s' % (filename, path)
if not self.confirm('Upload files?'):
sys.exit('Upload aborted.')
# Versions Mixin:
file_kwargs = {}
changeset_num = None
if changeset == 'new':
self.vcs_factory.validate_client_auth()
vcs = self.vcs_factory.make_remote_vcs()
staging_changeset = vcs.new_staging_changeset()
changeset_num = staging_changeset.num
print 'New staging changeset created: %d' % changeset_num
elif changeset:
changeset_num = int(changeset)
if changeset and changeset_num:
file_kwargs['changeset'] = changeset_num
print 'Uploading %d files to changeset %d...' % (len(filenames),
changeset_num)
start = time.time()
self.remote_file_factory.validate_client_auth()
future_results = []
with self.ThreadPoolExecutor() as executor:
for filename, target_path in filename_to_paths.iteritems():
future = executor.submit(self._upload_file,
filename,
target_path,
file_kwargs=file_kwargs)
future_results.append(future)
failed = False
total_bytes = 0
for future in futures.as_completed(future_results):
try:
remote_file = future.result()
print 'Uploaded %s' % remote_file.real_path
total_bytes += remote_file.size
except UploadFileError as e:
self.print_error('Error uploading %s. Error was: %s %s' %
(e.target_path, e.__class__.__name__, str(e)))
failed = True
if failed:
self.print_error('Could not upload one or more files.')
return
manifest = filename_to_paths.values()
if commit:
if changeset != 'new' and not confirm_manifest:
self.print_error(
'Must use --changeset=new with --commit, or pass '
'--confirm_manifest.')
return
self._commit_changeset_or_exit(changeset_num, manifest=manifest)
elapsed_time = time.time() - start
print 'Uploaded %d files in %s.' % (
len(filenames), utils.humanize_duration(elapsed_time))
result = {}
if changeset_num:
result['changeset_num'] = changeset_num
result['success'] = not failed
result['manifest'] = manifest
result['paths'] = filename_to_paths.values()
return result
def _tasks_dir_path(self):
# /_titan/tasks/<group>/<task_manager_key>/tasks
if not self._internal_tasks_dir_path:
self._internal_tasks_dir_path = utils.safe_join(
self._dir_path, 'tasks')
return self._internal_tasks_dir_path
def Run(self, filenames, root_dir=None, target_path='/', changeset=None,
commit=False, confirm_manifest=False):
"""Upload runner.
Args:
filenames: A list of local filenames to upload.
root_dir: Local root dir containing all the files. Default: current dir.
target_path: Remote root target dir for uploaded documents.
changeset: A changeset number or "new".
commit: Whether or not to commit the changeset.
confirm_manifest: Whether or not the current list of files given to
upload can be trusted as a complete manifest of the files in the
changeset. Required when combining "commit" and "changeset=<num>".
Returns:
A dictionary containing "success", "manifest", "paths", and
"changeset_num" if uploading to a changeset.
"""
if root_dir is None:
root_dir = os.path.curdir
# Compose mapping of absolute local path to absolute remote path.
root_dir = os.path.abspath(root_dir)
filename_to_paths = {}
for filename in filenames:
absolute_filename = os.path.abspath(filename)
if not absolute_filename.startswith(root_dir):
self.print_error('Path "%s" not contained within "%s".'
% (absolute_filename, root_dir))
sys.exit()
remote_path = absolute_filename[len(root_dir) + 1:]
remote_path = utils.safe_join(target_path, remote_path)
filename_to_paths[absolute_filename] = remote_path
# Confirm action.
print '\nUploading files to %s (%s):' % (self.host, self.api_base_path)
for filename, path in filename_to_paths.iteritems():
if root_dir == os.path.abspath(os.path.curdir):
# Strip the root_dir from view if it's already the working directory.
filename = '.' + filename[len(root_dir):]
print ' %s --> %s' % (filename, path)
if not self.confirm('Upload files?'):
sys.exit('Upload aborted.')
# Versions Mixin:
file_kwargs = {}
changeset_num = None
if changeset == 'new':
self.vcs_factory.validate_client_auth()
vcs = self.vcs_factory.make_remote_vcs()
staging_changeset = vcs.new_staging_changeset()
changeset_num = staging_changeset.num
print 'New staging changeset created: %d' % changeset_num
elif changeset:
changeset_num = int(changeset)
if changeset and changeset_num:
file_kwargs['changeset'] = changeset_num
print 'Uploading %d files to changeset %d...' % (len(filenames),
changeset_num)
start = time.time()
self.remote_file_factory.validate_client_auth()
future_results = []
with self.ThreadPoolExecutor() as executor:
for filename, target_path in filename_to_paths.iteritems():
future = executor.submit(
self._upload_file, filename, target_path, file_kwargs=file_kwargs)
future_results.append(future)
failed = False
total_bytes = 0
for future in futures.as_completed(future_results):
try:
remote_file = future.result()
print 'Uploaded %s' % remote_file.real_path
total_bytes += remote_file.size
except UploadFileError as e:
self.print_error('Error uploading %s. Error was: %s %s'
% (e.target_path, e.__class__.__name__, str(e)))
failed = True
if failed:
self.print_error('Could not upload one or more files.')
return
manifest = filename_to_paths.values()
if commit:
if changeset != 'new' and not confirm_manifest:
self.print_error('Must use --changeset=new with --commit, or pass '
'--confirm_manifest.')
return
self._commit_changeset_or_exit(changeset_num, manifest=manifest)
elapsed_time = time.time() - start
print 'Uploaded %d files in %s.' % (
len(filenames),
utils.humanize_duration(elapsed_time))
result = {}
if changeset_num:
result['changeset_num'] = changeset_num
result['success'] = not failed
result['manifest'] = manifest
result['paths'] = filename_to_paths.values()
return result
def Run(self, dir_path=None, recursive=False, depth=None, file_paths=None,
target_dir=None):
"""Download runner.
Args:
dir_path: A list of remote directories to download.
recursive: Whether or not to download directory recursively.
depth: Depth of recursion if specified.
file_paths: A list of remote files to download.
target_dir: The target local directory to upload to.
Returns:
A list of mappings between remote_path -> local_path.
Raises:
DownloadFileError
"""
if not file_paths and not dir_path:
self.print_error('No files to download. Use --file_path or '
'--dir_path.')
return
if target_dir is None:
target_dir = '.'
path_map = []
if file_paths:
remote_files = self.remote_file_factory.make_remote_files(
paths=file_paths)
for remote_file in remote_files.itervalues():
if not remote_file.exists:
print 'File %s does not exist' % remote_file.path
target = os.path.abspath(utils.safe_join(target_dir, remote_file.name))
path_map.append([remote_file, target])
elif dir_path:
dir_kwargs = {'recursive': recursive, 'depth': depth}
remote_files = self.remote_file_factory.make_remote_files(paths=[])
remote_files.list(dir_path, **dir_kwargs)
for remote_file in remote_files.itervalues():
target = os.path.abspath(
utils.safe_join(target_dir, remote_file.path[1:]))
path_map.append([remote_file, target])
conf_message = ['The following will be downloaded from %s' % self.host]
for remote_file, target in path_map:
conf_message.append(' %s --> %s' % (remote_file.path, target))
print '\n'.join(conf_message)
if not self.confirm(' Are you sure?'):
sys.exit('Download aborted.')
else:
print 'Downloading...'
utils.make_dirs(target_dir)
# Start the download.
start = time.time()
self.remote_file_factory.validate_client_auth()
future_results = []
with self.ThreadPoolExecutor() as executor:
for remote_file, target in path_map:
target_base_dir = os.path.dirname(target)
utils.make_dirs(target_base_dir)
future = executor.submit(self._DownloadFile, remote_file, target)
future_results.append(future)
failed = False
for future in futures.as_completed(future_results):
try:
downloaded_file = future.result()
self.print_message(
'Downloaded %s to %s' %
(downloaded_file['path'], downloaded_file['target']))
except DownloadFileError as e:
self.print_error('Error downloading %s. Error was: %s %s' %
(e.target_path, e.__class__.__name__, str(e)))
failed = True
if failed:
self.print_error('Could not download one or more files.')
return
elapsed_time = time.time() - start
print 'Downloaded %d files in %s.' % (len(path_map),
utils.humanize_duration(elapsed_time))
def _make_log_path(date, counter):
# Make a path like:
# /_titan/activities/stats/counters/2015/05/15/page/view/data-60s.json
formatted_date = date.strftime(counter.date_format)
return utils.safe_join(BASE_DIR, formatted_date, counter.name,
counter.data_filename)
