def __init__(self):
"""
Init contstructor
"""
self.api = AptlyApiRequests()
def __init__(self):
"""
Init contstructor
"""
self.api = AptlyApiRequests()
class Util(object):
""" Util
Instance for utils and tools.
"""
def __init__(self):
"""
Init contstructor
"""
self.api = AptlyApiRequests()
@staticmethod
def _atoi(text):
""" _atoi
Converts asci to int
"""
return int(text) if text.isdigit() else text
@staticmethod
def create_init_file():
""" create_init_file
Will create a config file at home folder, if it does not exist.
"""
home = expanduser("~")
name = home + '/aptly-cli.conf'
print "Look for already existing file..."
if not exists(name):
print 'Create_init_file'
try:
conf = open(name, 'a')
conf.write(
'[general]\nbasic_url=http://localhost\nport=:9003\nsave_last_snap=3\nsave_last_pkg=10\n \
prefixes_mirrors=\npackage_prefixes=\nrepos_to_clean=\n[general]\nrepos=\nstaging_snap_pre_post='
)
conf.close()
except:
print('Something went wrong! Can\'t tell what?')
else:
print "File already exists! Stop action"
print name
def _natural_keys(self, text):
""" _natural_keys
Split up string at int.
"""
return [self._atoi(c) for c in re.split('(\\d+)', text)]
def _sort_out_last_n_snap(self,
snaplist,
prefix,
nr_of_leftover,
postfix=None):
""" _sort_out_last_n_snap
Returns n sorted items from given input list by prefix.
"""
snapstaginglist = []
for x in snaplist:
if prefix in x[u'Name']:
if postfix is None:
# end of string contains
snapstaginglist.append(x[u'Name'])
else:
if postfix in x[u'Name']:
snapstaginglist.append(x[u'Name'])
slen = len(snapstaginglist)
snapstaginglist.sort(key=self._natural_keys)
ret = []
nr_o = int(nr_of_leftover)
if slen > (nr_o - 1):
for a in snapstaginglist[-nr_o:]:
ret.append(a)
else:
ret = snapstaginglist
return ret
def get_last_snapshots(self, prefix, nr_of_vers, postfix=None):
""" get_last_snapshots
Returns n versions of snapshots, sorted out by a prefix and optional postfix.
"""
snaplist = self.api.snapshot_list()
if postfix is None:
res = self._sort_out_last_n_snap(snaplist, prefix, nr_of_vers)
else:
res = self._sort_out_last_n_snap(snaplist, prefix, nr_of_vers,
postfix)
return res
def clean_last_snapshots(self, prefix, nr_of_vers, postfix=None):
""" clean_last_snapshots
Cleans n versions of snapshots, sorted out by a prefix and optional postfix.
"""
if postfix is None:
items_to_delete = self.get_last_snapshots(prefix, nr_of_vers)
else:
items_to_delete = self.get_last_snapshots(prefix, nr_of_vers,
postfix)
nr_to_left_over = int(
self.api.get_config_from_file()['save_last_snap'])
if len(items_to_delete) > nr_to_left_over:
for item in items_to_delete[:-nr_to_left_over]:
if item:
# force removal
self.api.snapshot_delete(item, '1')
else:
print prefix
print "Nothing to delete...."
def diff_both_last_snapshots_mirrors(self):
""" diff_both_last_snapshots_mirrors
Fetches out last two versions of snapshots from a given list of mirrors and diffs both.
Return, if all mirrors have new content to update or not (EMPTY).
"""
local_cfg = self.api.get_config_from_file()
if local_cfg['prefixes_mirrors']:
prefix_list = local_cfg['prefixes_mirrors'].split(', ')
else:
print "Error: Prefix list is empty: please add prefixes_mirrors to your configfile!"
snaplist = self.api.snapshot_list()
results = []
result = ""
for x in prefix_list:
res_list = self._sort_out_last_n_snap(snaplist, x, 2)
if len(res_list) >= 2:
res = self.api.snapshot_diff(res_list[0], res_list[1])
if not res:
results.append("EMPTY")
else:
results.append(res)
break
else:
results.append("EMPTY")
# print results
result = ""
for y in results:
if y == "EMPTY":
result = "EMPTY"
else:
result = y
break
print result
return result
def list_all_repos_and_packages(self):
""" list_all_repos_and_packages
"""
repos = self.api.repo_list()
for repo in repos:
print repo[u'Name']
packs = self.api.repo_show_packages(repo[u'Name'])
for pack in packs:
print pack
def get_last_packages(self,
repo_name,
pack_prefix,
nr_of_leftover,
postfix=None):
""" get_last_packages
"""
resp = None
packs = self.api.repo_show_packages(repo_name)
if postfix:
resp = self._sort_out_last_n_packages(packs, pack_prefix,
nr_of_leftover, postfix)
else:
resp = self._sort_out_last_n_packages(packs, pack_prefix,
nr_of_leftover)
return resp
def clean_last_packages(self,
repo_name,
pack_prefix,
nr_of_leftover,
postfix=None):
""" clean_last_packages
"""
items_to_delete = None
if postfix:
items_to_delete = self.get_last_packages(repo_name, pack_prefix,
nr_of_leftover, postfix)
else:
items_to_delete = self.get_last_packages(repo_name, pack_prefix,
nr_of_leftover)
nr_to_left_over = int(self.api.get_config_from_file()['save_last_pkg'])
print nr_to_left_over
if len(items_to_delete) > nr_to_left_over:
worklist = []
for item in items_to_delete[:-nr_to_left_over]:
if item:
print "Will remove..."
print item
worklist.append(item)
self.api.repo_delete_packages_by_key(repo_name, worklist)
else:
print "Nothing to delete..."
def _sort_out_last_n_packages(self,
packlist,
prefix,
nr_of_leftover,
postfix=None):
""" _sort_out_last_n_snap
Returns n sorted items from given input list by prefix.
"""
# print packlist
worklist = []
for pack_blob in packlist:
pack_tmp = pack_blob.split(' ')
if pack_tmp[1] in prefix:
worklist.append(pack_blob)
# print pack_tmp[1]
slen = len(worklist)
worklist.sort(key=self._natural_keys)
ret = []
nr_o = int(nr_of_leftover)
if slen > (nr_o - 1):
for a in worklist[-nr_o:]:
ret.append(a)
else:
ret = worklist
return ret
def clean_mirrored_snapshots(self):
""" clean_mirrored_snapshots
Clean out all snapshots that were taken from mirrors. The mirror entries are taken from config file.
"""
print "clean mirrored snapshots"
local_cfg = self.api.get_config_from_file()
if local_cfg['prefixes_mirrors']:
prefix_list = local_cfg['prefixes_mirrors'].split(', ')
else:
print "Error: Prefix list is empty: please add prefixes_mirrors to your configfile!"
for x in prefix_list:
self.clean_last_snapshots(x, 100)
def clean_repo_packages(self):
""" clean_repo_snapshots
Clean out all snapshots that were taken from repos. The repo entries are taken from config file.
"""
print "clean snapshots from repos"
local_cfg = self.api.get_config_from_file()
if local_cfg['repos_to_clean']:
repo_list = local_cfg['repos_to_clean'].split(', ')
else:
print "Error: Prefix list is empty: please add repos_to_clean to your configfile!"
if local_cfg['package_prefixes']:
pack_pref_list = local_cfg['package_prefixes'].split(', ')
else:
print "Error: Prefix list is empty: please add package_prefixes to your configfile!"
for repo_name in repo_list:
print repo_name
for pack_prefix in pack_pref_list:
print pack_prefix
self.clean_last_packages(repo_name, pack_prefix, 100)
def publish_switch_3rdparty_production(self):
""" publish_switch_s3_3rd_party_production
Publish the latest 3rd party snapshot from staging to production, only if there is new content available.
"""
print "publish_switch_s3_3rd_party_production"
# Get Config
local_cfg = self.api.get_config_from_file()
if local_cfg['repos']:
s3_list = local_cfg['repos'].split(', ')
else:
print "Error: Prefix list is empty: please add s3 buckets to your configfile!"
if local_cfg['staging_snap_pre_post']:
prefix_postfix = local_cfg['staging_snap_pre_post'].split(', ')
else:
print "Error: Prefix list is empty: please add staging_snap_pre_post to your configfile!"
# Diff snapshots from mirrors
# Temphack
res = 'something' # self.diff_both_last_snapshots_mirrors()
# Decide if it should be released to production
if res == "EMPTY":
print "New snapshot has no new packages. No need to release to production!"
else:
print "New packages were found...", res
# Get most actual snapshot from 3rdparty staging
last_snap = self.get_last_snapshots(prefix_postfix[0], 1,
prefix_postfix[1])
print "This is the new snapshot: ", last_snap
# publish snapshots to production on s3
print("Publish ", last_snap, s3_list[0])
self.api.publish_switch(s3_list[0], last_snap, "precise", "main",
0)
print("Publish ", last_snap, s3_list[1])
self.api.publish_switch(s3_list[1], last_snap, "precise", "main",
0)
# clean out
self.clean_mirrored_snapshots()
class Util(object):
""" Util
Instance for utils and tools.
"""
def __init__(self):
"""
Init contstructor
"""
self.api = AptlyApiRequests()
@staticmethod
def _atoi(text):
""" _atoi
Converts asci to int
"""
return int(text) if text.isdigit() else text
@staticmethod
def create_init_file():
""" create_init_file
Will create a config file at home folder, if it does not exist.
"""
home = expanduser("~")
name = home + '/aptly-cli.conf'
print "Look for already existing file..."
if not exists(name):
print 'Create_init_file'
try:
conf = open(name, 'a')
conf.write(
'[general]\nbasic_url=http://localhost\nport=:9003\nsave_last_snap=3\nsave_last_pkg=10\n \
prefixes_mirrors=\npackage_prefixes=\nrepos_to_clean=\n[general]\nrepos=\nstaging_snap_pre_post=')
conf.close()
except:
print('Something went wrong! Can\'t tell what?')
else:
print "File already exists! Stop action"
print name
def _natural_keys(self, text):
""" _natural_keys
Split up string at int.
"""
return [self._atoi(c) for c in re.split('(\\d+)', text)]
def _sort_out_last_n_snap(self, snaplist, prefix, nr_of_leftover, postfix=None):
""" _sort_out_last_n_snap
Returns n sorted items from given input list by prefix.
"""
snapstaginglist = []
for x in snaplist:
if prefix in x[u'Name']:
if postfix is None:
# end of string contains
snapstaginglist.append(x[u'Name'])
else:
if postfix in x[u'Name']:
snapstaginglist.append(x[u'Name'])
slen = len(snapstaginglist)
snapstaginglist.sort(key=self._natural_keys)
ret = []
nr_o = int(nr_of_leftover)
if slen > (nr_o - 1):
for a in snapstaginglist[-nr_o:]:
ret.append(a)
else:
ret = snapstaginglist
return ret
def get_last_snapshots(self, prefix, nr_of_vers, postfix=None):
""" get_last_snapshots
Returns n versions of snapshots, sorted out by a prefix and optional postfix.
"""
snaplist = self.api.snapshot_list()
if postfix is None:
res = self._sort_out_last_n_snap(snaplist, prefix, nr_of_vers)
else:
res = self._sort_out_last_n_snap(
snaplist, prefix, nr_of_vers, postfix)
return res
def clean_last_snapshots(self, prefix, nr_of_vers, postfix=None):
""" clean_last_snapshots
Cleans n versions of snapshots, sorted out by a prefix and optional postfix.
"""
if postfix is None:
items_to_delete = self.get_last_snapshots(prefix, nr_of_vers)
else:
items_to_delete = self.get_last_snapshots(
prefix, nr_of_vers, postfix)
nr_to_left_over = int(
self.api.get_config_from_file()['save_last_snap'])
if len(items_to_delete) > nr_to_left_over:
for item in items_to_delete[:-nr_to_left_over]:
if item:
# force removal
self.api.snapshot_delete(item, '1')
else:
print prefix
print "Nothing to delete...."
def diff_both_last_snapshots_mirrors(self):
""" diff_both_last_snapshots_mirrors
Fetches out last two versions of snapshots from a given list of mirrors and diffs both.
Return, if all mirrors have new content to update or not (EMPTY).
"""
local_cfg = self.api.get_config_from_file()
if local_cfg['prefixes_mirrors']:
prefix_list = local_cfg['prefixes_mirrors'].split(', ')
else:
print "Error: Prefix list is empty: please add prefixes_mirrors to your configfile!"
snaplist = self.api.snapshot_list()
results = []
result = ""
for x in prefix_list:
res_list = self._sort_out_last_n_snap(snaplist, x, 2)
if len(res_list) >= 2:
res = self.api.snapshot_diff(res_list[0], res_list[1])
if not res:
results.append("EMPTY")
else:
results.append(res)
break
else:
results.append("EMPTY")
# print results
result = ""
for y in results:
if y == "EMPTY":
result = "EMPTY"
else:
result = y
break
print result
return result
def list_all_repos_and_packages(self):
""" list_all_repos_and_packages
"""
repos = self.api.repo_list()
for repo in repos:
print repo[u'Name']
packs = self.api.repo_show_packages(repo[u'Name'])
for pack in packs:
print pack
def get_last_packages(self, repo_name, pack_prefix, nr_of_leftover, postfix=None):
""" get_last_packages
"""
resp = None
packs = self.api.repo_show_packages(repo_name)
if postfix:
resp = self._sort_out_last_n_packages(packs, pack_prefix, nr_of_leftover, postfix)
else:
resp = self._sort_out_last_n_packages(packs, pack_prefix, nr_of_leftover)
return resp
def clean_last_packages(self, repo_name, pack_prefix, nr_of_leftover, postfix=None):
""" clean_last_packages
"""
items_to_delete = None
if postfix:
items_to_delete = self.get_last_packages(repo_name, pack_prefix, nr_of_leftover, postfix)
else:
items_to_delete = self.get_last_packages(repo_name, pack_prefix, nr_of_leftover)
nr_to_left_over = int(
self.api.get_config_from_file()['save_last_pkg'])
print nr_to_left_over
if len(items_to_delete) > nr_to_left_over:
worklist = []
for item in items_to_delete[:-nr_to_left_over]:
if item:
print "Will remove..."
print item
worklist.append(item)
self.api.repo_delete_packages_by_key(repo_name, worklist)
else:
print "Nothing to delete..."
def _sort_out_last_n_packages(self, packlist, prefix, nr_of_leftover, postfix=None):
""" _sort_out_last_n_snap
Returns n sorted items from given input list by prefix.
"""
# print packlist
worklist = []
for pack_blob in packlist:
pack_tmp = pack_blob.split(' ')
if pack_tmp[1] in prefix:
worklist.append(pack_blob)
# print pack_tmp[1]
slen = len(worklist)
worklist.sort(key=self._natural_keys)
ret = []
nr_o = int(nr_of_leftover)
if slen > (nr_o - 1):
for a in worklist[-nr_o:]:
ret.append(a)
else:
ret = worklist
return ret
def clean_mirrored_snapshots(self):
""" clean_mirrored_snapshots
Clean out all snapshots that were taken from mirrors. The mirror entries are taken from config file.
"""
print "clean mirrored snapshots"
local_cfg = self.api.get_config_from_file()
if local_cfg['prefixes_mirrors']:
prefix_list = local_cfg['prefixes_mirrors'].split(', ')
else:
print "Error: Prefix list is empty: please add prefixes_mirrors to your configfile!"
for x in prefix_list:
self.clean_last_snapshots(x, 100)
def clean_repo_packages(self):
""" clean_repo_snapshots
Clean out all snapshots that were taken from repos. The repo entries are taken from config file.
"""
print "clean snapshots from repos"
local_cfg = self.api.get_config_from_file()
if local_cfg['repos_to_clean']:
repo_list = local_cfg['repos_to_clean'].split(', ')
else:
print "Error: Prefix list is empty: please add repos_to_clean to your configfile!"
if local_cfg['package_prefixes']:
pack_pref_list = local_cfg['package_prefixes'].split(', ')
else:
print "Error: Prefix list is empty: please add package_prefixes to your configfile!"
for repo_name in repo_list:
print repo_name
for pack_prefix in pack_pref_list:
print pack_prefix
self.clean_last_packages(repo_name, pack_prefix, 100)
def publish_switch_3rdparty_production(self):
""" publish_switch_s3_3rd_party_production
Publish the latest 3rd party snapshot from staging to production, only if there is new content available.
"""
print "publish_switch_s3_3rd_party_production"
# Get Config
local_cfg = self.api.get_config_from_file()
if local_cfg['repos']:
s3_list = local_cfg['repos'].split(', ')
else:
print "Error: Prefix list is empty: please add s3 buckets to your configfile!"
if local_cfg['staging_snap_pre_post']:
prefix_postfix = local_cfg['staging_snap_pre_post'].split(', ')
else:
print "Error: Prefix list is empty: please add staging_snap_pre_post to your configfile!"
# Diff snapshots from mirrors
# Temphack
res = 'something' # self.diff_both_last_snapshots_mirrors()
# Decide if it should be released to production
if res == "EMPTY":
print "New snapshot has no new packages. No need to release to production!"
else:
print "New packages were found...", res
# Get most actual snapshot from 3rdparty staging
last_snap = self.get_last_snapshots(prefix_postfix[0], 1, prefix_postfix[1])
print "This is the new snapshot: ", last_snap
# publish snapshots to production on s3
print ("Publish ", last_snap, s3_list[0])
self.api.publish_switch(s3_list[0], last_snap, "precise", "main", 0)
print ("Publish ", last_snap, s3_list[1])
self.api.publish_switch(s3_list[1], last_snap, "precise", "main", 0)
# clean out
self.clean_mirrored_snapshots()
