def hello(self, environment):
config = Config()
hostname = config.get(environment)['hostname']
_pulp = pulp.bindings.server_info.ServerInfoAPI(self.connection)
response = _pulp.get_types()
stargs = {'environment': environment, 'hostname': hostname}
if response.response_code == Constants.PULP_GET_OK:
self.output.info("{environment}: {hostname} OK".format(**stargs))
return True
else:
self.output.info("{environment}: {hostname} FAILED".format(**stargs))
return False
def hello(self, environment):
config = Config()
hostname = config.get(environment)['hostname']
_pulp = pulp.bindings.server_info.ServerInfoAPI(self.connection)
response = _pulp.get_types()
stargs = {'environment': environment, 'hostname': hostname}
if response.response_code == Constants.PULP_GET_OK:
self.output.info("{environment}: {hostname} OK".format(**stargs))
return True
else:
self.output.info(
"{environment}: {hostname} FAILED".format(**stargs))
return False
class JuicerCommand(object):
def __init__(self, args):
self.args = args
self.config = Config()
self.output = logging.getLogger('juicer')
self.connections = {}
for environment in self.config.keys():
cfg = self.config.get(environment)
self.connections[environment] = PulpConnection(
cfg['hostname'],
int(cfg['port']),
username=cfg['username'],
password=cfg['password'],
cert_filename=cfg['cert_filename'],
verify_ssl=False,
ca_path=cfg['ca_path'])
class Cart(object):
def __init__(self, name, description=None, cart_type='rpm', autoload=False, autosync=False, autosave=False, force=False):
self.output = logging.getLogger('juicer')
self.name = name
self.cart_file = os.path.join(juicer.common.Constants.CART_LOCATION,
"{}.json".format(self.name))
self.repo_items_hash = {}
self.remotes_storage = os.path.expanduser(os.path.join(juicer.common.Constants.CART_LOCATION,
"{}-remotes".format(self.name)))
self.config = Config()
self.cart_type = cart_type
if cart_type == 'rpm':
self.type_object = juicer.types.RPM
elif cart_type == 'docker':
self.type_object = juicer.types.Docker
elif cart_type == 'iso':
self.type_object = juicer.types.Iso
else:
self.type_object = None
if autoload:
self.output.debug("[CART:{}] Auto-loading cart items".format(self.name))
self.load()
if description is not None:
for repo, items in description.iteritems():
self.output.debug("Processing {num} input items for repo {repo}".format(num=len(items), repo=repo))
self[repo] = items
if autosave:
self.save(remote_save=True, force=force)
def __getitem__(self, repo):
""" Return the items in the given repo """
if repo in self.repo_items_hash:
return self.repo_items_hash[repo]
else:
# TODO: Should this raise?
return None
def __setitem__(self, repo, items):
"""
Just provides a shorthand way to call add_repo:
cart_object['repo'] = items
"""
self.add_repo(repo, items)
def add_repo(self, repo_name, items):
"""
Build up repos
`name` - Name of this repo.
`items` - List of paths to rpm.
"""
self.output.debug("[CART:{name}] Adding {count} items to {repo}".format(
name=self.name,
count=len(items),
repo=repo_name))
cart_items = []
for item in items:
self.output.debug("Creating Cart for {}".format(item))
new_item = CartItem(item)
cart_items.append(new_item)
self.repo_items_hash[repo_name] = cart_items
def keys(self):
"""
Get repo keys.
"""
return self.repo_items_hash.keys()
def repos(self):
"""
Return all list of the repos it cart will upload items into.
"""
return self.repo_items_hash.keys()
def items(self):
item_list = []
for repo, items in self.repo_items_hash.iteritems():
item_list += items
return item_list
def iterrepos(self):
"""
A generator function that yields a (repo, [items]) tuple for
each non-empty repo.
"""
for repo, items in self.repo_items_hash.iteritems():
if items:
yield (repo, items)
def is_empty(self):
"""
return True if the cart has no items, False otherwise
"""
for repo, items in self.iterrepos():
if items:
return False
return True
def save(self, remote_save=True, warning=False, force=False):
if self.is_empty():
self.output.error('Cart is empty, not saving anything')
return False
self.save_local(warning=warning)
if remote_save:
if 'cart_seeds' in self.config.get(self.config.keys()[0]).keys():
self.save_remote(force=force)
else:
self.output.warn('No cart_seeds found in config file. Cart not saved on remote.')
self.output.info("Saved cart '{cart}'".format(cart=self.name))
return True
def save_local(self, warning=False):
json_body = json.dumps(self._cart_dict())
if warning and os.path.exists(self.cart_file): # Sometimes we don't want to show this warning.
self.output.warn("Cart file '{cart}' already exists, overwriting with new data".format(cart=self.cart_file))
if not os.path.exists(juicer.common.Constants.CART_LOCATION):
os.mkdir(juicer.common.Constants.CART_LOCATION)
f = open(self.cart_file, 'w')
f.write(json_body)
f.flush()
f.close()
def save_remote(self, force=False):
if not force:
if not self.verify_remote():
raise SystemExit("A remote cart with the same name exists and has different content, use -f to force create.")
cart_seeds = self.config.get(self.config.keys()[0])['cart_seeds']
connection_str = 'mongodb://' + cart_seeds
mongo = pymongo.MongoClient(connection_str)
db = mongo.carts
try:
db['carts'].save(self._cart_dict())
except pymongo.errors.AutoReconnect:
self.output.error("Failed to save cart '{cart}' on remote".format(cart=self.name))
def pull(self):
cart_seeds = self.config.get(self.config.keys()[0])['cart_seeds']
connection_str = 'mongodb://' + cart_seeds
mongo = pymongo.MongoClient(connection_str)
db = mongo.carts
try:
json_body = json.dumps(db['carts'].find_one({'_id': self.name}))
if json_body == 'null':
self.output.error("Cart {} does not exist on remote".format(self.name))
else:
if os.path.exists(self.cart_file):
self.output.warn("Cart file '{}' already exists, overwriting with new data.".format(self.cart_file))
f = open(self.cart_file, 'w')
f.write(json_body)
f.flush()
f.close()
self.output.info("Successfully pulled cart '{cart}' from remote".format(cart=self.name))
except pymongo.errors.AutoReconnect:
self.output.error("Failed to find cart '{cart}' on remote".format(cart=self.name))
def load(self):
"""
Build a cart from a json file
"""
if not os.path.exists(self.cart_file):
raise SystemExit("No cart file could be found: {cart_file}".format(cart_file=self.cart_file))
cart_file = open(self.cart_file)
cart_body = json.loads(cart_file.read())
cart_file.close()
self.cart_type = cart_body['type']
if self.cart_type == 'rpm':
self.type_object = juicer.types.RPM
elif self.cart_type == 'docker':
self.type_object = juicer.types.Docker
elif self.cart_type == 'iso':
self.type_object = juicer.types.Iso
else:
self.type_object = None
for repo, items in cart_body['repos_items'].iteritems():
self.add_repo(repo, items)
def delete(self, local=False, remote=False):
"""
Remove all trace of this cart: delete the file(s) on the local
filesystem and delete the entry from the database
"""
self.output.debug("Deleting cart {}".format(self.name))
if local and not remote: # User only specified local.
self.delete_local()
elif remote and not local: # User only specified remote
self.delete_remote()
else: # User didn't specify, do both.
self.delete_local()
self.delete_remote()
def delete_local(self):
# rm -r self.remotes_storage()
if os.path.exists(self.remotes_storage):
shutil.rmtree(self.remotes_storage)
# rm cart_file()
if os.path.exists(self.cart_file):
self.output.debug("Removing {cart}'s cart file".format(cart=self.name))
os.remove(self.cart_file)
self.output.info("Successfully deleted cart '{cart}' locally".format(cart=self.name))
def delete_remote(self):
# rm in mongo
if 'cart_seeds' in self.config.get(self.config.keys()[0]).keys():
cart_seeds = self.config.get(self.config.keys()[0])['cart_seeds']
connection_str = 'mongodb://' + cart_seeds
mongo = pymongo.MongoClient(connection_str)
db = mongo.carts
try:
db['carts'].remove({'_id': self.name})
self.output.info("Successfully deleted cart '{cart}' on remote".format(cart=self.name))
except pymongo.errors.AutoReconnect:
self.output.error("Failed to delete cart '{cart}' on remote".format(self.name))
def update(self, description):
for repo, items in description:
if repo not in self.keys():
self[repo] = items
else:
for item in items:
self[repo].append(CartItem(item))
self.save(remote_save=True, force=True)
return True
def upload_items(self, environment, connection, force):
if not force:
if not self.verify_remote():
raise SystemExit("Local cart differs from remote, use -f to force upload")
######################################################################
# Ensure repositories exist before we do any work
######################################################################
pulp_repo = juicer.pulp.Repo(connection)
for repo, items in self.iterrepos():
# Make sure the repo exists before we upload items
exists = pulp_repo.exists(repo, environment)
if not exists:
raise SystemExit("Repo '{repo}' does not exist in '{environment}'".format(
repo=repo, environment=environment))
pulp_upload = juicer.pulp.Upload(connection)
######################################################################
# Sync remote items before we do anything else
######################################################################
for repo, items in self.iterrepos():
for item in items:
if not item.synced:
item.sync(self.remotes_storage)
######################################################################
# Execute pre plugins
######################################################################
plugins = juicer.plugins.Plugins()
plugins.execute_pre_plugins(self.cart_type, environment, self.items())
######################################################################
# Generate upload requests
######################################################################
widgets = ['Initializing ',
progressbar.Percentage(), ' ',
progressbar.Bar(marker=progressbar.RotatingMarker()), ' ',
progressbar.ETA()]
initialize_pbar = progressbar.ProgressBar(
widgets=widgets,
maxval=len(self.items())).start()
item_count = 0
for repo, items in self.iterrepos():
for item in items:
item.upload_id = pulp_upload.initialize_upload()
item_count += 1
initialize_pbar.update(item_count)
initialize_pbar.finish()
######################################################################
# Upload items
######################################################################
total_size = 0
for repo, items in self.iterrepos():
for item in items:
total_size += os.path.getsize(item.path)
widgets = ['Uploading ',
progressbar.Percentage(), ' ',
progressbar.Bar(marker=progressbar.RotatingMarker()), ' ',
progressbar.ETA(), ' ',
bitmath.integrations.BitmathFileTransferSpeed()]
upload_pbar = progressbar.ProgressBar(
widgets=widgets,
maxval=int(total_size)).start()
for repo, items in self.iterrepos():
for item in items:
pulp_upload.upload(item.upload_id, item.path, repo, upload_pbar)
upload_pbar.finish()
######################################################################
# Import uploads
######################################################################
widgets = ['Importing ',
progressbar.Percentage(), ' ',
progressbar.Bar(marker=progressbar.RotatingMarker()), ' ',
progressbar.ETA()]
import_pbar = progressbar.ProgressBar(
widgets=widgets,
maxval=len(self.items())).start()
item_count = 0
tasks = []
for repo, items in self.iterrepos():
repo_id = "{0}-{1}".format(repo, environment)
for item in items:
if self.cart_type == 'docker':
unit_type = 'docker_image'
else:
unit_type = self.cart_type
unit_key, unit_metadata = self.type_object(item.path).generate_upload_data()
# Keep tasks returned from import upload so we can make sure they've finished
tasks.append(pulp_upload.import_upload(upload_id=item.upload_id,
repo_id=repo_id,
unit_type=unit_type,
unit_key=unit_key,
unit_metadata=unit_metadata))
item_count += 1
import_pbar.update(item_count)
# Only update path to remote path if the item is iso or rpm
if self.cart_type in ['rpm', 'iso']:
item.path = "https://{host}/pulp/repos/{environment}/{repo}/{name}".format(
host=connection.host,
environment=environment,
repo=repo,
name=item.name)
import_pbar.finish()
######################################################################
# Wait for the imports to finish before we delete remote upload
######################################################################
pulp_task = juicer.pulp.Task(connection)
waiting_pbar = progressbar.ProgressBar(widgets=['Waiting for imports to finish ', progressbar.AnimatedMarker()])
for task in waiting_pbar(tasks):
pulp_task.wait_for(task.spawned_tasks[0].task_id)
######################################################################
# Clean up upload requests
######################################################################
widgets = ['Cleaning ',
progressbar.Percentage(), ' ',
progressbar.Bar(marker=progressbar.RotatingMarker()), ' ',
progressbar.ETA()]
cleanup_pbar = progressbar.ProgressBar(
widgets=widgets,
maxval=len(self.items())).start()
item_count = 0
for repo, items in self.iterrepos():
for item in items:
pulp_upload.delete_upload(item.upload_id)
item_count += 1
cleanup_pbar.update(item_count)
cleanup_pbar.finish()
######################################################################
# Publish repositories
######################################################################
widgets = ['Publishing ',
progressbar.Percentage(), ' ',
progressbar.Bar(marker=progressbar.RotatingMarker()), ' ',
progressbar.ETA()]
publish_pbar = progressbar.ProgressBar(
widgets=widgets,
maxval=len(self.repos())).start()
repo_count = 0
for repo, items in self.iterrepos():
pulp_repo.publish(repo, environment)
repo_count += 1
publish_pbar.update(repo_count)
publish_pbar.finish()
######################################################################
# Save the cart
######################################################################
self.save(warning=False)
######################################################################
# Execute post plugins
######################################################################
plugins = juicer.plugins.Plugins()
plugins.execute_post_plugins(self.cart_type, environment, self.items())
def __str__(self):
return json.dumps(self._cart_dict(), indent=4)
def _cart_dict(self):
output = {'_id': self.name,
'type': self.cart_type,
'repos_items': []}
repos_items = {}
for repo in self.repos():
repos_items[repo] = [str(i) for i in self[repo]]
output['repos_items'] = repos_items
return output
def verify_remote(self):
"""
Checks if the remote cart exists.
Checks if remote cart is different that the local cart.
"""
if 'cart_seeds' in self.config.get(self.config.keys()[0]).keys():
cart_seeds = self.config.get(self.config.keys()[0])['cart_seeds']
connection_str = 'mongodb://' + cart_seeds
mongo = pymongo.MongoClient(connection_str)
db = mongo.carts
try:
json_body = json.dumps(db['carts'].find_one({'_id': self.name}))
if json_body == 'null':
return True
else:
if json_body == json.dumps(self._cart_dict()):
return True
else:
return False
except pymongo.errors.AutoReconnect:
self.output.error("Failed to find cart '{cart}' on remote".format(cart=self.name))
return False
