def _commit_changeset_or_exit(self, changeset_num, force_commit=False, manifest=None, save_manifest=True): """If confirmed, commits the given changeset.""" assert not force_commit and manifest # Confirm action. msg = 'Commit changeset %d?' % changeset_num if not self.confirm(msg): sys.exit('Commit aborted.') start = time.time() remote_vcs = self.vcs_factory.make_remote_vcs() staging_changeset = self.vcs_factory.make_remote_changeset(changeset_num) if manifest: # We have full knowledge of the files uploaded to a changeset, associate # those files to staging_changeset to ensure a strong consistency commit. for path in manifest: assert not path.startswith('/_titan/ver/') remote_file = self.remote_file_factory.make_remote_file(path) staging_changeset.associate_file(remote_file) staging_changeset.finalize_associated_files() final_remote_changeset = remote_vcs.commit( staging_changeset, force=force_commit, save_manifest=save_manifest) elapsed_time = time.time() - start print 'Finished commit in %s.' % utils.humanize_duration(elapsed_time) return final_remote_changeset
def _commit_changeset_or_exit(self, changeset_num, force_commit=False, manifest=None, save_manifest=True): """If confirmed, commits the given changeset.""" assert not force_commit and manifest # Confirm action. msg = 'Commit changeset %d?' % changeset_num if not self.confirm(msg): sys.exit('Commit aborted.') start = time.time() remote_vcs = self.vcs_factory.make_remote_vcs() staging_changeset = self.vcs_factory.make_remote_changeset( changeset_num) if manifest: # We have full knowledge of the files uploaded to a changeset, associate # those files to staging_changeset to ensure a strong consistency commit. for path in manifest: assert not path.startswith('/_titan/ver/') remote_file = self.remote_file_factory.make_remote_file(path) staging_changeset.associate_file(remote_file) staging_changeset.finalize_associated_files() final_remote_changeset = remote_vcs.commit(staging_changeset, force=force_commit, save_manifest=save_manifest) elapsed_time = time.time() - start print 'Finished commit in %s.' % utils.humanize_duration(elapsed_time) return final_remote_changeset
def Run(self, dir_path=None, recursive=False, depth=None, file_paths=None, target_dir=None): """Download runner. Args: dir_path: A list of remote directories to download. recursive: Whether or not to download directory recursively. depth: Depth of recursion if specified. file_paths: A list of remote files to download. target_dir: The target local directory to upload to. Returns: A list of mappings between remote_path -> local_path. Raises: DownloadFileError """ if not file_paths and not dir_path: self.print_error('No files to download. Use --file_path or ' '--dir_path.') return if target_dir is None: target_dir = '.' path_map = [] if file_paths: remote_files = self.remote_file_factory.make_remote_files( paths=file_paths) for remote_file in remote_files.itervalues(): if not remote_file.exists: print 'File %s does not exist' % remote_file.path target = os.path.abspath(utils.safe_join(target_dir, remote_file.name)) path_map.append([remote_file, target]) elif dir_path: dir_kwargs = {'recursive': recursive, 'depth': depth} remote_files = self.remote_file_factory.make_remote_files(paths=[]) remote_files.list(dir_path, **dir_kwargs) for remote_file in remote_files.itervalues(): target = os.path.abspath( utils.safe_join(target_dir, remote_file.path[1:])) path_map.append([remote_file, target]) conf_message = ['The following will be downloaded from %s' % self.host] for remote_file, target in path_map: conf_message.append(' %s --> %s' % (remote_file.path, target)) print '\n'.join(conf_message) if not self.confirm(' Are you sure?'): sys.exit('Download aborted.') else: print 'Downloading...' utils.make_dirs(target_dir) # Start the download. start = time.time() self.remote_file_factory.validate_client_auth() future_results = [] with self.ThreadPoolExecutor() as executor: for remote_file, target in path_map: target_base_dir = os.path.dirname(target) utils.make_dirs(target_base_dir) future = executor.submit(self._DownloadFile, remote_file, target) future_results.append(future) failed = False for future in futures.as_completed(future_results): try: downloaded_file = future.result() self.print_message( 'Downloaded %s to %s' % (downloaded_file['path'], downloaded_file['target'])) except DownloadFileError as e: self.print_error('Error downloading %s. Error was: %s %s' % (e.target_path, e.__class__.__name__, str(e))) failed = True if failed: self.print_error('Could not download one or more files.') return elapsed_time = time.time() - start print 'Downloaded %d files in %s.' % (len(path_map), utils.humanize_duration(elapsed_time))
def Run(self, filenames, root_dir=None, target_path='/', changeset=None, commit=False, confirm_manifest=False): """Upload runner. Args: filenames: A list of local filenames to upload. root_dir: Local root dir containing all the files. Default: current dir. target_path: Remote root target dir for uploaded documents. changeset: A changeset number or "new". commit: Whether or not to commit the changeset. confirm_manifest: Whether or not the current list of files given to upload can be trusted as a complete manifest of the files in the changeset. Required when combining "commit" and "changeset=<num>". Returns: A dictionary containing "success", "manifest", "paths", and "changeset_num" if uploading to a changeset. """ if root_dir is None: root_dir = os.path.curdir # Compose mapping of absolute local path to absolute remote path. root_dir = os.path.abspath(root_dir) filename_to_paths = {} for filename in filenames: absolute_filename = os.path.abspath(filename) if not absolute_filename.startswith(root_dir): self.print_error('Path "%s" not contained within "%s".' % (absolute_filename, root_dir)) sys.exit() remote_path = absolute_filename[len(root_dir) + 1:] remote_path = utils.safe_join(target_path, remote_path) filename_to_paths[absolute_filename] = remote_path # Confirm action. print '\nUploading files to %s (%s):' % (self.host, self.api_base_path) for filename, path in filename_to_paths.iteritems(): if root_dir == os.path.abspath(os.path.curdir): # Strip the root_dir from view if it's already the working directory. filename = '.' + filename[len(root_dir):] print ' %s --> %s' % (filename, path) if not self.confirm('Upload files?'): sys.exit('Upload aborted.') # Versions Mixin: file_kwargs = {} changeset_num = None if changeset == 'new': self.vcs_factory.validate_client_auth() vcs = self.vcs_factory.make_remote_vcs() staging_changeset = vcs.new_staging_changeset() changeset_num = staging_changeset.num print 'New staging changeset created: %d' % changeset_num elif changeset: changeset_num = int(changeset) if changeset and changeset_num: file_kwargs['changeset'] = changeset_num print 'Uploading %d files to changeset %d...' % (len(filenames), changeset_num) start = time.time() self.remote_file_factory.validate_client_auth() future_results = [] with self.ThreadPoolExecutor() as executor: for filename, target_path in filename_to_paths.iteritems(): future = executor.submit( self._upload_file, filename, target_path, file_kwargs=file_kwargs) future_results.append(future) failed = False total_bytes = 0 for future in futures.as_completed(future_results): try: remote_file = future.result() print 'Uploaded %s' % remote_file.real_path total_bytes += remote_file.size except UploadFileError as e: self.print_error('Error uploading %s. Error was: %s %s' % (e.target_path, e.__class__.__name__, str(e))) failed = True if failed: self.print_error('Could not upload one or more files.') return manifest = filename_to_paths.values() if commit: if changeset != 'new' and not confirm_manifest: self.print_error('Must use --changeset=new with --commit, or pass ' '--confirm_manifest.') return self._commit_changeset_or_exit(changeset_num, manifest=manifest) elapsed_time = time.time() - start print 'Uploaded %d files in %s.' % ( len(filenames), utils.humanize_duration(elapsed_time)) result = {} if changeset_num: result['changeset_num'] = changeset_num result['success'] = not failed result['manifest'] = manifest result['paths'] = filename_to_paths.values() return result
def testHumanizeDuration(self): self.assertEqual('10s', utils.humanize_duration(10))
def Run(self, dir_path=None, recursive=False, depth=None, file_paths=None, target_dir=None): """Download runner. Args: dir_path: A list of remote directories to download. recursive: Whether or not to download directory recursively. depth: Depth of recursion if specified. file_paths: A list of remote files to download. target_dir: The target local directory to upload to. Returns: A list of mappings between remote_path -> local_path. Raises: DownloadFileError """ if not file_paths and not dir_path: self.print_error('No files to download. Use --file_path or ' '--dir_path.') return if target_dir is None: target_dir = '.' path_map = [] if file_paths: remote_files = self.remote_file_factory.make_remote_files( paths=file_paths) for remote_file in remote_files.itervalues(): if not remote_file.exists: print 'File %s does not exist' % remote_file.path target = os.path.abspath( utils.safe_join(target_dir, remote_file.name)) path_map.append([remote_file, target]) elif dir_path: dir_kwargs = {'recursive': recursive, 'depth': depth} remote_files = self.remote_file_factory.make_remote_files(paths=[]) remote_files.list(dir_path, **dir_kwargs) for remote_file in remote_files.itervalues(): target = os.path.abspath( utils.safe_join(target_dir, remote_file.path[1:])) path_map.append([remote_file, target]) conf_message = ['The following will be downloaded from %s' % self.host] for remote_file, target in path_map: conf_message.append(' %s --> %s' % (remote_file.path, target)) print '\n'.join(conf_message) if not self.confirm(' Are you sure?'): sys.exit('Download aborted.') else: print 'Downloading...' utils.make_dirs(target_dir) # Start the download. start = time.time() self.remote_file_factory.validate_client_auth() future_results = [] with self.ThreadPoolExecutor() as executor: for remote_file, target in path_map: target_base_dir = os.path.dirname(target) utils.make_dirs(target_base_dir) future = executor.submit(self._DownloadFile, remote_file, target) future_results.append(future) failed = False for future in futures.as_completed(future_results): try: downloaded_file = future.result() self.print_message( 'Downloaded %s to %s' % (downloaded_file['path'], downloaded_file['target'])) except DownloadFileError as e: self.print_error('Error downloading %s. Error was: %s %s' % (e.target_path, e.__class__.__name__, str(e))) failed = True if failed: self.print_error('Could not download one or more files.') return elapsed_time = time.time() - start print 'Downloaded %d files in %s.' % ( len(path_map), utils.humanize_duration(elapsed_time))
def Run(self, filenames, root_dir=None, target_path='/', changeset=None, commit=False, confirm_manifest=False): """Upload runner. Args: filenames: A list of local filenames to upload. root_dir: Local root dir containing all the files. Default: current dir. target_path: Remote root target dir for uploaded documents. changeset: A changeset number or "new". commit: Whether or not to commit the changeset. confirm_manifest: Whether or not the current list of files given to upload can be trusted as a complete manifest of the files in the changeset. Required when combining "commit" and "changeset=<num>". Returns: A dictionary containing "success", "manifest", "paths", and "changeset_num" if uploading to a changeset. """ if root_dir is None: root_dir = os.path.curdir # Compose mapping of absolute local path to absolute remote path. root_dir = os.path.abspath(root_dir) filename_to_paths = {} for filename in filenames: absolute_filename = os.path.abspath(filename) if not absolute_filename.startswith(root_dir): self.print_error('Path "%s" not contained within "%s".' % (absolute_filename, root_dir)) sys.exit() remote_path = absolute_filename[len(root_dir) + 1:] remote_path = utils.safe_join(target_path, remote_path) filename_to_paths[absolute_filename] = remote_path # Confirm action. print '\nUploading files to %s (%s):' % (self.host, self.api_base_path) for filename, path in filename_to_paths.iteritems(): if root_dir == os.path.abspath(os.path.curdir): # Strip the root_dir from view if it's already the working directory. filename = '.' + filename[len(root_dir):] print ' %s --> %s' % (filename, path) if not self.confirm('Upload files?'): sys.exit('Upload aborted.') # Versions Mixin: file_kwargs = {} changeset_num = None if changeset == 'new': self.vcs_factory.validate_client_auth() vcs = self.vcs_factory.make_remote_vcs() staging_changeset = vcs.new_staging_changeset() changeset_num = staging_changeset.num print 'New staging changeset created: %d' % changeset_num elif changeset: changeset_num = int(changeset) if changeset and changeset_num: file_kwargs['changeset'] = changeset_num print 'Uploading %d files to changeset %d...' % (len(filenames), changeset_num) start = time.time() self.remote_file_factory.validate_client_auth() future_results = [] with self.ThreadPoolExecutor() as executor: for filename, target_path in filename_to_paths.iteritems(): future = executor.submit(self._upload_file, filename, target_path, file_kwargs=file_kwargs) future_results.append(future) failed = False total_bytes = 0 for future in futures.as_completed(future_results): try: remote_file = future.result() print 'Uploaded %s' % remote_file.real_path total_bytes += remote_file.size except UploadFileError as e: self.print_error('Error uploading %s. Error was: %s %s' % (e.target_path, e.__class__.__name__, str(e))) failed = True if failed: self.print_error('Could not upload one or more files.') return manifest = filename_to_paths.values() if commit: if changeset != 'new' and not confirm_manifest: self.print_error( 'Must use --changeset=new with --commit, or pass ' '--confirm_manifest.') return self._commit_changeset_or_exit(changeset_num, manifest=manifest) elapsed_time = time.time() - start print 'Uploaded %d files in %s.' % ( len(filenames), utils.humanize_duration(elapsed_time)) result = {} if changeset_num: result['changeset_num'] = changeset_num result['success'] = not failed result['manifest'] = manifest result['paths'] = filename_to_paths.values() return result
def testHumanizeDuration(self): self.assertEqual('10s', utils.humanize_duration(10))