class RHNPackage(object):
def __init__(self, package_id, channel, **kw):
self.config = namespaces['mirror'].config
self.id = package_id
self.channel = channel
# get the proxy object
if kw.get('proxy', None):
self.proxy = kw['proxy']
else:
self.proxy = RHNSatelliteProxy()
# if we didn't pass the file_name, we need to look it up. but if we
# do pass it, and lookup_details=False we safe an api call.
if kw.get('file_name'):
self.file = kw['file_name']
else:
kw['lookup_details'] = True
if kw.get('lookup_details', None):
details = self.proxy.call('packages.getDetails', package_id)
for i in details:
setattr(self, i, details[i])
def fetch_file(self, local_path):
log.info("fetching %s/%s ...." % (self.channel, self.file))
url = self.proxy.call('packages.getPackageUrl', self.id)
f = open(local_path, 'w')
data = urllib2.urlopen(url).read()
f.write(data)
f.close()
class syncCommand(CementCommand):
config = namespaces['mirror'].config
def __init__(self, *args):
CementCommand.__init__(self, *args)
self.proxy = RHNSatelliteProxy()
if self.cli_opts.user:
self.proxy.get_session(use_cache=False)
def mirror_channel(self, channel, path):
local_dir = re.sub('\%\(mirror_dir\)', self.config['mirror_dir'], path)
chan = RHNSatelliteChannel(label=channel, local_dir=local_dir)
log.info("mirroring of %s started" % chan.label)
try:
chan.sync(verify=self.cli_opts.verify)
except RuntimeError, e:
log.error("RuntimeError => %s", e)
sys.exit(1)
except KeyboardInterrupt, e:
log.warn('Caught KeyboardInterrupt => Attempting to exit clean...')
# remove the last file attempted
if len(chan.attempted_files) > 0:
last_path = os.path.join(
chan.local_dir, chan.attempted_files[-1])
if os.path.exists(last_path):
log.debug('cleanup: removing last attempted file %s' \
% last_path)
os.remove(last_path)
chan._remove_lock()
sys.exit(1)
def run(self):
"""
Takes an API path (i.e. auth.login) and args (i.e username) and
attempts to make a call to the RHN Proxy. Useful for development.
"""
cmd = self.cli_args.pop(0)
path = self.cli_args.pop(0)
args = self.cli_args
proxy = RHNSatelliteProxy()
if self.cli_opts.user:
proxy.get_session(use_cache=False)
else:
proxy.get_session()
try:
res = proxy.call(path, *args)
except xmlrpclib.Fault, e:
res = proxy.noauth_call(path, *args)
def __init__(self, package_id, channel, **kw):
self.config = namespaces['mirror'].config
self.id = package_id
self.channel = channel
# get the proxy object
if kw.get('proxy', None):
self.proxy = kw['proxy']
else:
self.proxy = RHNSatelliteProxy()
# if we didn't pass the file_name, we need to look it up. but if we
# do pass it, and lookup_details=False we safe an api call.
if kw.get('file_name'):
self.file = kw['file_name']
else:
kw['lookup_details'] = True
if kw.get('lookup_details', None):
details = self.proxy.call('packages.getDetails', package_id)
for i in details:
setattr(self, i, details[i])
def __init__(self, label=None, local_dir=None, **kw):
self.config = namespaces['mirror'].config
self.label = label
self.local_dir = local_dir
self.proxy = RHNSatelliteProxy()
self.synced_files = []
self.attempted_files = []
self.modified = False
# lock file
self.lock_file = os.path.join(
self.config['lockfile_dir'], "%s.lock" % self.label)
if not os.path.exists(self.config['lockfile_dir']):
os.makedirs(self.config['lockfile_dir'])
# base mirror config
self.run_createrepo = self.config.get('run_createrepo', None)
self.run_yumarch = self.config.get('run_yumarch', None)
self.only_latest = self.config.get('only_latest', None)
# per channel config
if self.config.has_key('channels'):
if not kw.get('run_createrepo', None):
self.run_createrepo = self.config['channels'][self.label]\
.get('run_createrepo', self.run_createrepo)
if not kw.get('run_yumarch', None):
self.run_yumarch = self.config['channels'][self.label]\
.get('run_yumarch', self.run_yumarch)
if not kw.get('only_latest'):
self.only_latest = self.config['channels'][self.label]\
.get('only_latest', self.only_latest)
# create out local dir if missing
if not os.path.exists(self.local_dir):
os.makedirs(self.local_dir)
class RHNSatelliteChannel(object):
label = None
local_dir = None
def __init__(self, label=None, local_dir=None, **kw):
self.config = namespaces['mirror'].config
self.label = label
self.local_dir = local_dir
self.proxy = RHNSatelliteProxy()
self.synced_files = []
self.attempted_files = []
self.modified = False
# lock file
self.lock_file = os.path.join(
self.config['lockfile_dir'], "%s.lock" % self.label)
if not os.path.exists(self.config['lockfile_dir']):
os.makedirs(self.config['lockfile_dir'])
# base mirror config
self.run_createrepo = self.config.get('run_createrepo', None)
self.run_yumarch = self.config.get('run_yumarch', None)
self.only_latest = self.config.get('only_latest', None)
# per channel config
if self.config.has_key('channels'):
if not kw.get('run_createrepo', None):
self.run_createrepo = self.config['channels'][self.label]\
.get('run_createrepo', self.run_createrepo)
if not kw.get('run_yumarch', None):
self.run_yumarch = self.config['channels'][self.label]\
.get('run_yumarch', self.run_yumarch)
if not kw.get('only_latest'):
self.only_latest = self.config['channels'][self.label]\
.get('only_latest', self.only_latest)
# create out local dir if missing
if not os.path.exists(self.local_dir):
os.makedirs(self.local_dir)
def get_packages(self):
# only download the latest package, not all
if self.only_latest:
self.packages = self.proxy.call(
'channel.software.listLatestPackages', self.label)
else:
self.packages = self.proxy.call(\
'channel.software.listAllPackages', self.label)
return self.packages
def _create_lock(self):
log.debug('creating lock file %s' % self.lock_file)
if os.path.exists(self.lock_file):
raise RuntimeError, "lock file %s already exists" % self.lock_file
f = open(self.lock_file, 'w')
f.write(str(os.getpid()))
f.close()
def _remove_lock(self):
log.debug('removing lock file %s' % self.lock_file)
os.remove(self.lock_file)
def sync(self, verify=False):
self._create_lock()
for package in self.get_packages():
if verify:
self._slow_sync_package(package)
else:
self._fast_sync_package(package)
# finally, create the repo
if self.modified and self.run_createrepo:
log.info("running createrepo: %s" % self.label)
os.system("%s %s" % (self.config['createrepo_path'], self.local_dir))
if self.modified and self.run_yumarch:
log.info("running yum-arch: %s" % self.label)
os.system("%s %s" % (self.config['yumarch_path'], self.local_dir))
# clean up files that aren't in packages
for file in os.listdir(self.local_dir):
if file not in self.synced_files and file.endswith('.rpm'):
log.debug("cleanup: %s" % file)
os.remove(os.path.join(self.local_dir, file))
self._remove_lock()
def _fast_sync_package(self, package_dict):
# this is significantly faster
file_name = "%s-%s-%s.%s.rpm" % (
package_dict['name'],
package_dict['version'],
package_dict['release'],
package_dict['arch_label']
)
package = RHNPackage(package_dict['id'], self.label,
proxy=self.proxy, file_name=file_name)
self.attempted_files.append(package.file)
full_path = os.path.join(self.local_dir, file_name)
log.debug('processing %s/%s' % (self.label, file_name))
if not os.path.exists(full_path):
package.fetch_file(full_path)
self.modified = True
self.synced_files.append(package.file)
def _slow_sync_package(self, package_dict):
# this is significantly slower
package = RHNPackage(package_dict['id'], self.label,
lookup_details=True, proxy=self.proxy)
self.attempted_files.append(package.file)
full_path = os.path.join(self.local_dir, package.file)
log.debug('processing %s/%s' % (self.label, package.file))
if os.path.exists(full_path):
log.info('verifying %s/%s' % (self.label, package.file))
# fetch the file if md5 mismatch, and then re-verify
count = 0
while count < 3:
md5 = hashlib.md5(open(full_path).read()).hexdigest()
if md5 == package.md5sum:
break
else:
package.fetch_file(full_path)
self.modified = True
count += 1
if count >= 3:
log.error('failed to download %s/%s' % \
(self.label, package.file))
else:
# fetch the file cause it doesn't exist
package.fetch_file(full_path)
self.modified = True
count = 0
while count < 3:
md5 = hashlib.md5(open(full_path).read()).hexdigest()
if md5 == package.md5sum:
break
else:
package.fetch_file(full_path)
count += 1
if count >= 3:
log.error('failed to download %s/%s' % \
(self.label, package.file))
self.synced_files.append(package.file)
def __init__(self, *args):
CementCommand.__init__(self, *args)
self.proxy = RHNSatelliteProxy()
if self.cli_opts.user:
self.proxy.get_session(use_cache=False)
