def hello(self, environment): config = Config() hostname = config.get(environment)['hostname'] _pulp = pulp.bindings.server_info.ServerInfoAPI(self.connection) response = _pulp.get_types() stargs = {'environment': environment, 'hostname': hostname} if response.response_code == Constants.PULP_GET_OK: self.output.info("{environment}: {hostname} OK".format(**stargs)) return True else: self.output.info("{environment}: {hostname} FAILED".format(**stargs)) return False
def hello(self, environment): config = Config() hostname = config.get(environment)['hostname'] _pulp = pulp.bindings.server_info.ServerInfoAPI(self.connection) response = _pulp.get_types() stargs = {'environment': environment, 'hostname': hostname} if response.response_code == Constants.PULP_GET_OK: self.output.info("{environment}: {hostname} OK".format(**stargs)) return True else: self.output.info( "{environment}: {hostname} FAILED".format(**stargs)) return False
def __init__(self, name, description=None, cart_type='rpm', autoload=False, autosync=False, autosave=False, force=False): self.output = logging.getLogger('juicer') self.name = name self.cart_file = os.path.join(juicer.common.Constants.CART_LOCATION, "{}.json".format(self.name)) self.repo_items_hash = {} self.remotes_storage = os.path.expanduser(os.path.join(juicer.common.Constants.CART_LOCATION, "{}-remotes".format(self.name))) self.config = Config() self.cart_type = cart_type if cart_type == 'rpm': self.type_object = juicer.types.RPM elif cart_type == 'docker': self.type_object = juicer.types.Docker elif cart_type == 'iso': self.type_object = juicer.types.Iso else: self.type_object = None if autoload: self.output.debug("[CART:{}] Auto-loading cart items".format(self.name)) self.load() if description is not None: for repo, items in description.iteritems(): self.output.debug("Processing {num} input items for repo {repo}".format(num=len(items), repo=repo)) self[repo] = items if autosave: self.save(remote_save=True, force=force)
def __init__(self, args): self.args = args self.config = Config() self.output = logging.getLogger('juicer') self.connections = {} for environment in self.config.keys(): cfg = self.config.get(environment) self.connections[environment] = PulpConnection( cfg['hostname'], int(cfg['port']), username=cfg['username'], password=cfg['password'], cert_filename=cfg['cert_filename'], verify_ssl=False, ca_path=cfg['ca_path'])
class JuicerCommand(object): def __init__(self, args): self.args = args self.config = Config() self.output = logging.getLogger('juicer') self.connections = {} for environment in self.config.keys(): cfg = self.config.get(environment) self.connections[environment] = PulpConnection( cfg['hostname'], int(cfg['port']), username=cfg['username'], password=cfg['password'], cert_filename=cfg['cert_filename'], verify_ssl=False, ca_path=cfg['ca_path'])
def __init__(self): self.parser = argparse.ArgumentParser( description='manage pulp and release carts') juicer.command.parser = self.parser # Create and test the config. self.config = Config() self.config.test() self._default_start_in = self.config.start_in() self._default_envs = self.config.environments() self.parser.add_argument('-q', '--quiet', action='store_true', dest='quiet', default=False, help='show no output') self.parser.add_argument('-v', '--verbose', action='store_true', dest='verbose', default=False, help='show verbose output') self.parser.add_argument('-V', '--version', action='version', version="juicer-{0}".format(juicer.__version__)) ################################################################## # Keep the different commands separate subparsers = self.parser.add_subparsers(title='commands', dest='command', description='\'%(prog)s COMMAND -h\' for individual help topics') ################################################################## # Create the 'cart' sub-parser parser_cart = subparsers.add_parser('cart', help='cart operations') subparser_cart = parser_cart.add_subparsers(dest='sub_command') ################################################################## # Create the 'rpm' sub-parser parser_rpm = subparsers.add_parser('rpm', help='rpm operations') subparser_rpm = parser_rpm.add_subparsers(dest='sub_command') ################################################################## # Create the 'repo' sub-parser parser_repo = subparsers.add_parser('repo', help='repo operations') subparser_repo = parser_repo.add_subparsers(dest='sub_command') ################################################################## # Create the 'role' sub-parser parser_role = subparsers.add_parser('role', help='role operations') subparser_role = parser_role.add_subparsers(dest='sub_command') ################################################################## # Create the 'user' sub-parser parser_user = subparsers.add_parser('user', help='user operations') subparser_user = parser_user.add_subparsers(dest='sub_command') ################################################################## # Create the 'cart create' sub-parser parser_cart_create = subparser_cart.add_parser('create', help='create cart with destination repositories and items', usage='%(prog)s CARTNAME [-r REPONAME ITEM ... [-r REPONAME ITEM ...]] [-t,-type CART-TYPE] [-f,--force]') parser_cart_create.add_argument('cartname', metavar='CARTNAME', help='cart name') cgroup = parser_cart_create.add_mutually_exclusive_group(required=True) cgroup.add_argument('-r', metavar=('REPONAME', 'ITEM'), action='append', nargs='+', help='destination repo name, items') parser_cart_create.add_argument('-t', '--type', metavar='CART-TYPE', default="rpm", dest='cart_type', help='type of cart items (one of: rpm, docker, iso)(default: rpm)') parser_cart_create.add_argument('-f', '--force', action='store_true', default=False, dest='force', help='force create') parser_cart_create.set_defaults(cmd=juicer.command.cart.CartCreateCommand) ################################################################## # Create the 'cart show' sub-parser parser_cart_show = subparser_cart.add_parser('show', usage='%(prog)s CARTNAME', help='show the contents of a cart') parser_cart_show.add_argument('cartname', metavar='CARTNAME', help='cart name') parser_cart_show.set_defaults(cmd=juicer.command.cart.CartShowCommand) ################################################################## # Create the 'cart list' sub-parser parser_cart_list = subparser_cart.add_parser('list', usage='%(prog)s [GLOB]', help='list local carts') parser_cart_list.add_argument('cart_glob', metavar='GLOB', nargs='*', default=['*'], help='pattern to match cart names against (default: *)') parser_cart_list.set_defaults(cmd=juicer.command.cart.CartListCommand) ################################################################## # Create the 'cart update' sub-parser parser_cart_update = subparser_cart.add_parser('update', help='update cart with new items', usage='%(prog)s CARTNAME [-r REPONAME ITEM ... [-r REPONAME ITEM ...]]') parser_cart_update.add_argument('cartname', metavar='CARTNAME', help='cart name') parser_cart_update.add_argument('-r', metavar=('REPONAME', 'ITEM'), action='append', nargs='+', help='destination repo name, items') parser_cart_update.set_defaults(cmd=juicer.command.cart.CartUpdateCommand) ################################################################## # Create the 'cart pull' sub-parser parser_cart_pull = subparser_cart.add_parser('pull', usage='%(prog)s CARTNAME', help='pull cart from remote') parser_cart_pull.add_argument('cartname', metavar='CARTNAME', help='cart name') parser_cart_pull.set_defaults(cmd=juicer.command.cart.CartPullCommand) ################################################################## # Create the 'cart push' sub-parser parser_cart_push = subparser_cart.add_parser('push', help='upload cart items to pulp', usage='%(prog)s CARTNAME [--in ENV [ENV ...]] [-f,--force]') parser_cart_push.add_argument('cartname', metavar='CARTNAME', help='cart name') parser_cart_push.add_argument('--in', nargs='*', metavar='ENV', default=[self._default_start_in], help='environments to push to', dest='environment') parser_cart_push.add_argument('-f', '--force', action='store_true', default=False, dest='force', help='force push') parser_cart_push.set_defaults(cmd=juicer.command.cart.CartPushCommand) ################################################################## # Create the 'cart delete' sub-parser parser_cart_delete = subparser_cart.add_parser('delete', help='delete cart locally and on remote', usage='%(prog)s CARTNAME [-l,--local] [-r,--remote]') parser_cart_delete.add_argument('cartname', metavar='CARTNAME', help='cart name') parser_cart_delete.add_argument('-l', '--local', action='store_true', default=False, dest='local', help='delete the cart which is cached locally (does not destroy remote cart definitions)') parser_cart_delete.add_argument('-r', '--remote', action='store_true', default=False, dest='remote', help='delete the cart on the remote side only (ignores any cached cart definitions)') parser_cart_delete.set_defaults(cmd=juicer.command.cart.CartDeleteCommand) ################################################################## # create the 'rpm upload' sub-parser parser_rpm_upload = subparser_rpm.add_parser('upload', help='upload rpms to repositories', usage='%(prog)s -r REPONAME ITEM ... [ -r REPONAME ITEM ...] [--in ENV [ENV ...]]') parser_rpm_upload.add_argument('-r', metavar=('REPONAME', 'ITEM'), action='append', nargs='+', required=True, help='destination repo name, items...') parser_rpm_upload.add_argument('--in', nargs='*', metavar='ENV', default=[self._default_start_in], help='environments to upload to', dest='environment') parser_rpm_upload.set_defaults(cmd=juicer.command.rpm.RPMUploadCommand) ################################################################## # create the 'hello' sub-parser parser_hello = subparsers.add_parser('hello', help='test your connection to the pulp server', usage='%(prog)s [--in ENV [ENV ...]]') parser_hello.add_argument('--in', nargs='*', metavar='ENV', help='environments to test agsint', default=self._default_envs, dest='environment') parser_hello.set_defaults(cmd=juicer.command.hello.HelloCommand) ################################################################## # create the 'rpm delete' sub-parser parser_rpm_delete = subparser_rpm.add_parser('delete', help='delete rpms from repositories', usage='%(prog)s -r REPONAME ITEM ... [-r REPONAME ITEM ...] [--in ENV [ENV ...]]') parser_rpm_delete.add_argument('-r', metavar=('REPONAME', 'ITEM'), required=True, action='append', nargs='+', help='target repo name, items (partial or full package names)') parser_rpm_delete.add_argument('--in', nargs='*', metavar='ENV', help='environments to delete from', default=self._default_envs, dest='environment') parser_rpm_delete.set_defaults(cmd=juicer.command.rpm.RPMDeleteCommand) ################################################################## # Create the 'repo delete' sub-parser parser_repo_delete = subparser_repo.add_parser('delete', usage='%(prog)s REPONAME [--in ENV [ENV ...]]', help='delete a repository') parser_repo_delete.add_argument('repo', metavar='REPONAME', type=str.lower, help='repo name') parser_repo_delete.add_argument('--in', metavar='ENV', nargs="+", dest='environment', default=self._default_envs, help='environments to delete from') parser_repo_delete.set_defaults(cmd=juicer.command.repo.RepoDeleteCommand) ################################################################## # create the 'repo publish' sub-parser parser_repo_publish = subparser_repo.add_parser('publish', help='publish a repository (this will regenerate metadata)', usage='%(prog)s REPONAME [--in ENV [ENV ...]]') parser_repo_publish.add_argument('repo', metavar='REPONAME', type=str.lower, help='repo name') parser_repo_publish.add_argument('--in', nargs='*', metavar='ENV', help='environments to publish in', default=self._default_envs, dest='environment') parser_repo_publish.set_defaults(cmd=juicer.command.repo.RepoPublishCommand) ################################################################## # Create the 'repo create' sub-parser parser_repo_create = subparser_repo.add_parser('create', help='create a repository', usage='%(prog)s REPONAME [-t,--type TYPE] [--checksum-type CHECKSUM-TYPE] [--in ENV [ENV ...]]') parser_repo_create.add_argument('repo', metavar='REPONAME', type=str.lower, help='repo name') parser_repo_create.add_argument('-t', '--type', metavar='TYPE', dest='repotype', default='rpm', choices=['rpm', 'docker', 'iso'], help='type used for repository creation (one of: rpm, docker, iso)(default: rpm)') parser_repo_create.add_argument('--checksum-type', metavar='CHECKSUM-TYPE', default='sha256', choices=['sha26', 'sha'], help='checksum-type used for metadata generation (one of: sha26, sha)') parser_repo_create.add_argument('--in', metavar='ENV', nargs="+", dest='environment', default=self._default_envs, help='environments to create in') parser_repo_create.set_defaults(cmd=juicer.command.repo.RepoCreateCommand) ################################################################## # Create the 'repo list' sub-parser parser_repo_list = subparser_repo.add_parser('list', usage='%(prog)s [--json] [--in ENV [ENV ...]]', help='list repositories') parser_repo_list.add_argument('--in', metavar='ENV', nargs="+", dest='environment', default=self._default_envs, help='environments to list from') parser_repo_list.add_argument('--json', action='store_true', default=False, help='output json') parser_repo_list.set_defaults(cmd=juicer.command.repo.RepoListCommand) ################################################################## # Create the 'repo show' sub-parser parser_repo_show = subparser_repo.add_parser('show', usage='%(prog)s REPONAME ... [--json] [--in ENV [ENV ...]]', help='show one or more repositories') parser_repo_show.add_argument('repo', metavar='REPONAME', type=str.lower, nargs="+", help='repo name') parser_repo_show.add_argument('--json', action='store_true', default=False, help='output json') parser_repo_show.add_argument('--in', metavar='ENV', nargs="+", dest='environment', default=self._default_envs, help='environments to show from') parser_repo_show.set_defaults(cmd=juicer.command.repo.RepoShowCommand) ################################################################## # Create the 'role add' sub-parser parser_role_add = subparser_role.add_parser('add', usage='%(prog)s --login LOGIN --role ROLE [--in ENV [ENV ...]]', help='add user to role') parser_role_add.add_argument('--login', metavar='LOGIN', type=str.lower, help='user\'s login', required=True) parser_role_add.add_argument('--role', metavar='ROLE', type=str.lower, help='role to add user to', required=True) parser_role_add.add_argument('--in', metavar='ENV', nargs="+", dest='environment', default=self._default_envs, help='environments to add roles in') parser_role_add.set_defaults(cmd=juicer.command.role.RoleAddCommand) ################################################################## # Create the 'role list' sub-parser parser_role_list = subparser_role.add_parser('list', usage='%(prog)s [--in ENV [ENV ...]]', help='list roles') parser_role_list.add_argument('--in', metavar='ENV', nargs="+", dest='environment', default=self._default_envs, help='environments to list from') parser_role_list.set_defaults(cmd=juicer.command.role.RoleListCommand) ################################################################## # Create the 'user create' sub-parser parser_user_create = subparser_user.add_parser('create', help='create a user', usage='%(prog)s LOGIN --name \"FULL NAME\" [--password [\"PASSWORD\"]] [--roles ROLE ...] [--in ENV [ENV ...]]') parser_user_create.add_argument('login', metavar='LOGIN', help='login id for user') parser_user_create.add_argument('--name', metavar='FULL NAME', dest='name', required=True, default=None, help='full name') parser_user_create.add_argument('--password', metavar='PASSWORD', dest='password', nargs='*', required=True, action=PromptAction, help='password (prompted if not argument not supplied)') parser_user_create.add_argument('--roles', metavar='ROLE', nargs="+", dest='roles', required=False, default=None, help='roles to apply to user (defaults to None)') parser_user_create.add_argument('--in', metavar='ENV', nargs="+", dest='environment', default=self._default_envs, help='environments to create in') parser_user_create.set_defaults(cmd=juicer.command.user.UserCreateCommand) ################################################################## # Create the 'user delete' sub-parser parser_user_delete = subparser_user.add_parser('delete', usage='%(prog)s LOGIN [--in ENV [ENV ...]]', help='delete a user') parser_user_delete.add_argument('login', metavar='LOGIN', type=str.lower, help='login id for user') parser_user_delete.add_argument('--in', metavar='ENV', nargs="+", dest='environment', default=self._default_envs, help='environments to delete from') parser_user_delete.set_defaults(cmd=juicer.command.user.UserDeleteCommand) ################################################################## # Create the 'user show' sub-parser parser_user_show = subparser_user.add_parser('show', usage='%(prog)s LOGIN [--in ENV [ENV ...]]', help='show a user') parser_user_show.add_argument('login', metavar='LOGIN', type=str.lower, help='login id for user') parser_user_show.add_argument('--in', metavar='ENV', nargs="+", dest='environment', default=self._default_envs, help='environments to show from') parser_user_show.set_defaults(cmd=juicer.command.user.UserShowCommand) ################################################################## # Create the 'user update' sub-parser parser_user_update = subparser_user.add_parser('update', help='change user information', usage='%(prog)s LOGIN [--name \"FULL NAME\"] [--password [\"PASSWORD\"]] [--roles ROLE ...] [--in ENV [ENV ...]]') parser_user_update.add_argument('login', metavar='LOGIN', type=str.lower, help='login id for user') parser_user_update.add_argument('--name', metavar='FULL NAME', dest='name', required=False, help='updated full name') parser_user_update.add_argument('--password', metavar='PASSWORD', dest='password', nargs='*', required=False, action=PromptAction, help='updated password (prompted if argument not supplied)') parser_user_update.add_argument('--roles', metavar='ROLE', nargs="+", dest='roles', required=False, default=None, help='updated roles') parser_user_update.add_argument('--in', metavar='ENV', nargs="+", dest='environment', default=self._default_envs, help='environments to update in') parser_user_update.set_defaults(cmd=juicer.command.user.UserUpdateCommand) ################################################################## # Create the 'user list' sub-parser parser_user_list = subparser_user.add_parser('list', usage='%(prog)s [--in ENV [ENV ...]]', help='list users') parser_user_list.add_argument('--in', metavar='ENV', nargs="+", dest='environment', default=self._default_envs, help='environments to list from') parser_user_list.set_defaults(cmd=juicer.command.user.UserListCommand)
class Cart(object): def __init__(self, name, description=None, cart_type='rpm', autoload=False, autosync=False, autosave=False, force=False): self.output = logging.getLogger('juicer') self.name = name self.cart_file = os.path.join(juicer.common.Constants.CART_LOCATION, "{}.json".format(self.name)) self.repo_items_hash = {} self.remotes_storage = os.path.expanduser(os.path.join(juicer.common.Constants.CART_LOCATION, "{}-remotes".format(self.name))) self.config = Config() self.cart_type = cart_type if cart_type == 'rpm': self.type_object = juicer.types.RPM elif cart_type == 'docker': self.type_object = juicer.types.Docker elif cart_type == 'iso': self.type_object = juicer.types.Iso else: self.type_object = None if autoload: self.output.debug("[CART:{}] Auto-loading cart items".format(self.name)) self.load() if description is not None: for repo, items in description.iteritems(): self.output.debug("Processing {num} input items for repo {repo}".format(num=len(items), repo=repo)) self[repo] = items if autosave: self.save(remote_save=True, force=force) def __getitem__(self, repo): """ Return the items in the given repo """ if repo in self.repo_items_hash: return self.repo_items_hash[repo] else: # TODO: Should this raise? return None def __setitem__(self, repo, items): """ Just provides a shorthand way to call add_repo: cart_object['repo'] = items """ self.add_repo(repo, items) def add_repo(self, repo_name, items): """ Build up repos `name` - Name of this repo. `items` - List of paths to rpm. """ self.output.debug("[CART:{name}] Adding {count} items to {repo}".format( name=self.name, count=len(items), repo=repo_name)) cart_items = [] for item in items: self.output.debug("Creating Cart for {}".format(item)) new_item = CartItem(item) cart_items.append(new_item) self.repo_items_hash[repo_name] = cart_items def keys(self): """ Get repo keys. """ return self.repo_items_hash.keys() def repos(self): """ Return all list of the repos it cart will upload items into. """ return self.repo_items_hash.keys() def items(self): item_list = [] for repo, items in self.repo_items_hash.iteritems(): item_list += items return item_list def iterrepos(self): """ A generator function that yields a (repo, [items]) tuple for each non-empty repo. """ for repo, items in self.repo_items_hash.iteritems(): if items: yield (repo, items) def is_empty(self): """ return True if the cart has no items, False otherwise """ for repo, items in self.iterrepos(): if items: return False return True def save(self, remote_save=True, warning=False, force=False): if self.is_empty(): self.output.error('Cart is empty, not saving anything') return False self.save_local(warning=warning) if remote_save: if 'cart_seeds' in self.config.get(self.config.keys()[0]).keys(): self.save_remote(force=force) else: self.output.warn('No cart_seeds found in config file. Cart not saved on remote.') self.output.info("Saved cart '{cart}'".format(cart=self.name)) return True def save_local(self, warning=False): json_body = json.dumps(self._cart_dict()) if warning and os.path.exists(self.cart_file): # Sometimes we don't want to show this warning. self.output.warn("Cart file '{cart}' already exists, overwriting with new data".format(cart=self.cart_file)) if not os.path.exists(juicer.common.Constants.CART_LOCATION): os.mkdir(juicer.common.Constants.CART_LOCATION) f = open(self.cart_file, 'w') f.write(json_body) f.flush() f.close() def save_remote(self, force=False): if not force: if not self.verify_remote(): raise SystemExit("A remote cart with the same name exists and has different content, use -f to force create.") cart_seeds = self.config.get(self.config.keys()[0])['cart_seeds'] connection_str = 'mongodb://' + cart_seeds mongo = pymongo.MongoClient(connection_str) db = mongo.carts try: db['carts'].save(self._cart_dict()) except pymongo.errors.AutoReconnect: self.output.error("Failed to save cart '{cart}' on remote".format(cart=self.name)) def pull(self): cart_seeds = self.config.get(self.config.keys()[0])['cart_seeds'] connection_str = 'mongodb://' + cart_seeds mongo = pymongo.MongoClient(connection_str) db = mongo.carts try: json_body = json.dumps(db['carts'].find_one({'_id': self.name})) if json_body == 'null': self.output.error("Cart {} does not exist on remote".format(self.name)) else: if os.path.exists(self.cart_file): self.output.warn("Cart file '{}' already exists, overwriting with new data.".format(self.cart_file)) f = open(self.cart_file, 'w') f.write(json_body) f.flush() f.close() self.output.info("Successfully pulled cart '{cart}' from remote".format(cart=self.name)) except pymongo.errors.AutoReconnect: self.output.error("Failed to find cart '{cart}' on remote".format(cart=self.name)) def load(self): """ Build a cart from a json file """ if not os.path.exists(self.cart_file): raise SystemExit("No cart file could be found: {cart_file}".format(cart_file=self.cart_file)) cart_file = open(self.cart_file) cart_body = json.loads(cart_file.read()) cart_file.close() self.cart_type = cart_body['type'] if self.cart_type == 'rpm': self.type_object = juicer.types.RPM elif self.cart_type == 'docker': self.type_object = juicer.types.Docker elif self.cart_type == 'iso': self.type_object = juicer.types.Iso else: self.type_object = None for repo, items in cart_body['repos_items'].iteritems(): self.add_repo(repo, items) def delete(self, local=False, remote=False): """ Remove all trace of this cart: delete the file(s) on the local filesystem and delete the entry from the database """ self.output.debug("Deleting cart {}".format(self.name)) if local and not remote: # User only specified local. self.delete_local() elif remote and not local: # User only specified remote self.delete_remote() else: # User didn't specify, do both. self.delete_local() self.delete_remote() def delete_local(self): # rm -r self.remotes_storage() if os.path.exists(self.remotes_storage): shutil.rmtree(self.remotes_storage) # rm cart_file() if os.path.exists(self.cart_file): self.output.debug("Removing {cart}'s cart file".format(cart=self.name)) os.remove(self.cart_file) self.output.info("Successfully deleted cart '{cart}' locally".format(cart=self.name)) def delete_remote(self): # rm in mongo if 'cart_seeds' in self.config.get(self.config.keys()[0]).keys(): cart_seeds = self.config.get(self.config.keys()[0])['cart_seeds'] connection_str = 'mongodb://' + cart_seeds mongo = pymongo.MongoClient(connection_str) db = mongo.carts try: db['carts'].remove({'_id': self.name}) self.output.info("Successfully deleted cart '{cart}' on remote".format(cart=self.name)) except pymongo.errors.AutoReconnect: self.output.error("Failed to delete cart '{cart}' on remote".format(self.name)) def update(self, description): for repo, items in description: if repo not in self.keys(): self[repo] = items else: for item in items: self[repo].append(CartItem(item)) self.save(remote_save=True, force=True) return True def upload_items(self, environment, connection, force): if not force: if not self.verify_remote(): raise SystemExit("Local cart differs from remote, use -f to force upload") ###################################################################### # Ensure repositories exist before we do any work ###################################################################### pulp_repo = juicer.pulp.Repo(connection) for repo, items in self.iterrepos(): # Make sure the repo exists before we upload items exists = pulp_repo.exists(repo, environment) if not exists: raise SystemExit("Repo '{repo}' does not exist in '{environment}'".format( repo=repo, environment=environment)) pulp_upload = juicer.pulp.Upload(connection) ###################################################################### # Sync remote items before we do anything else ###################################################################### for repo, items in self.iterrepos(): for item in items: if not item.synced: item.sync(self.remotes_storage) ###################################################################### # Execute pre plugins ###################################################################### plugins = juicer.plugins.Plugins() plugins.execute_pre_plugins(self.cart_type, environment, self.items()) ###################################################################### # Generate upload requests ###################################################################### widgets = ['Initializing ', progressbar.Percentage(), ' ', progressbar.Bar(marker=progressbar.RotatingMarker()), ' ', progressbar.ETA()] initialize_pbar = progressbar.ProgressBar( widgets=widgets, maxval=len(self.items())).start() item_count = 0 for repo, items in self.iterrepos(): for item in items: item.upload_id = pulp_upload.initialize_upload() item_count += 1 initialize_pbar.update(item_count) initialize_pbar.finish() ###################################################################### # Upload items ###################################################################### total_size = 0 for repo, items in self.iterrepos(): for item in items: total_size += os.path.getsize(item.path) widgets = ['Uploading ', progressbar.Percentage(), ' ', progressbar.Bar(marker=progressbar.RotatingMarker()), ' ', progressbar.ETA(), ' ', bitmath.integrations.BitmathFileTransferSpeed()] upload_pbar = progressbar.ProgressBar( widgets=widgets, maxval=int(total_size)).start() for repo, items in self.iterrepos(): for item in items: pulp_upload.upload(item.upload_id, item.path, repo, upload_pbar) upload_pbar.finish() ###################################################################### # Import uploads ###################################################################### widgets = ['Importing ', progressbar.Percentage(), ' ', progressbar.Bar(marker=progressbar.RotatingMarker()), ' ', progressbar.ETA()] import_pbar = progressbar.ProgressBar( widgets=widgets, maxval=len(self.items())).start() item_count = 0 tasks = [] for repo, items in self.iterrepos(): repo_id = "{0}-{1}".format(repo, environment) for item in items: if self.cart_type == 'docker': unit_type = 'docker_image' else: unit_type = self.cart_type unit_key, unit_metadata = self.type_object(item.path).generate_upload_data() # Keep tasks returned from import upload so we can make sure they've finished tasks.append(pulp_upload.import_upload(upload_id=item.upload_id, repo_id=repo_id, unit_type=unit_type, unit_key=unit_key, unit_metadata=unit_metadata)) item_count += 1 import_pbar.update(item_count) # Only update path to remote path if the item is iso or rpm if self.cart_type in ['rpm', 'iso']: item.path = "https://{host}/pulp/repos/{environment}/{repo}/{name}".format( host=connection.host, environment=environment, repo=repo, name=item.name) import_pbar.finish() ###################################################################### # Wait for the imports to finish before we delete remote upload ###################################################################### pulp_task = juicer.pulp.Task(connection) waiting_pbar = progressbar.ProgressBar(widgets=['Waiting for imports to finish ', progressbar.AnimatedMarker()]) for task in waiting_pbar(tasks): pulp_task.wait_for(task.spawned_tasks[0].task_id) ###################################################################### # Clean up upload requests ###################################################################### widgets = ['Cleaning ', progressbar.Percentage(), ' ', progressbar.Bar(marker=progressbar.RotatingMarker()), ' ', progressbar.ETA()] cleanup_pbar = progressbar.ProgressBar( widgets=widgets, maxval=len(self.items())).start() item_count = 0 for repo, items in self.iterrepos(): for item in items: pulp_upload.delete_upload(item.upload_id) item_count += 1 cleanup_pbar.update(item_count) cleanup_pbar.finish() ###################################################################### # Publish repositories ###################################################################### widgets = ['Publishing ', progressbar.Percentage(), ' ', progressbar.Bar(marker=progressbar.RotatingMarker()), ' ', progressbar.ETA()] publish_pbar = progressbar.ProgressBar( widgets=widgets, maxval=len(self.repos())).start() repo_count = 0 for repo, items in self.iterrepos(): pulp_repo.publish(repo, environment) repo_count += 1 publish_pbar.update(repo_count) publish_pbar.finish() ###################################################################### # Save the cart ###################################################################### self.save(warning=False) ###################################################################### # Execute post plugins ###################################################################### plugins = juicer.plugins.Plugins() plugins.execute_post_plugins(self.cart_type, environment, self.items()) def __str__(self): return json.dumps(self._cart_dict(), indent=4) def _cart_dict(self): output = {'_id': self.name, 'type': self.cart_type, 'repos_items': []} repos_items = {} for repo in self.repos(): repos_items[repo] = [str(i) for i in self[repo]] output['repos_items'] = repos_items return output def verify_remote(self): """ Checks if the remote cart exists. Checks if remote cart is different that the local cart. """ if 'cart_seeds' in self.config.get(self.config.keys()[0]).keys(): cart_seeds = self.config.get(self.config.keys()[0])['cart_seeds'] connection_str = 'mongodb://' + cart_seeds mongo = pymongo.MongoClient(connection_str) db = mongo.carts try: json_body = json.dumps(db['carts'].find_one({'_id': self.name})) if json_body == 'null': return True else: if json_body == json.dumps(self._cart_dict()): return True else: return False except pymongo.errors.AutoReconnect: self.output.error("Failed to find cart '{cart}' on remote".format(cart=self.name)) return False