def _rpc_call(self, function_name, params): get_server_obj = self.login() # Try a couple of times cfg = config.initUp2dateConfig() for i in range(cfg["networkRetries"]): try: ret = apply(getattr(get_server_obj, function_name), params) except rpclib.ProtocolError, e: # We have two codes to check: the HTTP error code, and the # combination (failtCode, faultString) encoded in the headers # of the request. http_error_code = e.errcode fault_code, fault_string = rpclib.reportError(e.headers) if http_error_code == 401 and fault_code == -34: # Login token expired get_server_obj = self.login(force=1) continue if http_error_code == 404 and fault_code == -17: # File not found self.extinctErrorYN = 1 return None log(-1, "ERROR: http error code :%s; fault code: %s; %s" % (http_error_code, fault_code, fault_string)) # XXX raise else: return ret
def import_channels(channels, orgid=None, master=None): collection = ChannelCollection() batch = [] import satCerts orgs = map(lambda a: a['id'], satCerts.get_all_orgs()) org_map = None my_backend = diskImportLib.get_backend() if master: org_map = my_backend.lookupOrgMap(master)['master-id-to-local-id'] for c in channels: try: timestamp = collection.get_channel_timestamp(c) except KeyError: raise Exception, "Could not find channel %s" % c, sys.exc_info()[2] c_obj = collection.get_channel(c, timestamp) if c_obj is None: raise Exception, "Channel not found in cache: %s" % c # Check to see if we're asked to sync to an orgid, # make sure the org from the export is not null org, # finally if the orgs differ so we might wanna use # requested org's channel-family. # TODO: Move these checks somewhere more appropriate if not orgid and c_obj['org_id'] is not None: #If the src org is not present default to org 1 orgid = DEFAULT_ORG if orgid is not None and c_obj['org_id'] is not None and \ c_obj['org_id'] != orgid: #If we know the master this is coming from and the master org #has been mapped to a local org, transform org_id to the local #org_id. Otherwise just put it in the default org. if (org_map and c_obj['org_id'] in org_map.keys() and org_map[c_obj['org_id']]): c_obj['org_id'] = org_map[c_obj['org_id']] else: c_obj['org_id'] = orgid if c_obj.has_key('trust_list'): del(c_obj['trust_list']) for family in c_obj['families']: family['label'] = 'private-channel-family-' + \ str(c_obj['org_id']) # If there's a trust list on the channel, transform the org ids to # the local ones if c_obj.has_key('trust_list') and c_obj['trust_list']: trusts = [] for trust in c_obj['trust_list']: if org_map.has_key(trust['org_trust_id']): trust['org_trust_id'] = org_map[trust['org_trust_id']] trusts.append(trust) c_obj['trust_list'] = trusts syncLib.log(6, "Syncing Channel %s to Org %s " % \ (c_obj['label'], c_obj['org_id'])) batch.append(c_obj) importer = channelImport.ChannelImport(batch, my_backend) # Don't commit just yet importer.will_commit = 0 importer.run() return importer
def _rpc_call(self, function_name, params): get_server_obj = self.login() # Try a couple of times fault_count = 0 expired_token = 0 cfg = config.initUp2dateConfig() while fault_count - expired_token < cfg['networkRetries']: try: ret = getattr(get_server_obj, function_name)(*params) except rpclib.xmlrpclib.ProtocolError: e = sys.exc_info()[1] # We have two codes to check: the HTTP error code, and the # combination (failtCode, faultString) encoded in the headers # of the request. http_error_code = e.errcode fault_code, fault_string = rpclib.reportError(e.headers) fault_count += 1 if http_error_code == 401 and fault_code == -34: # Login token expired get_server_obj = self.login(force=1) # allow exactly one respin for expired token expired_token = 1 continue if http_error_code == 404 and fault_code == -17: # File not found self.extinctErrorYN = 1 return None log(-1, 'ERROR: http error code :%s; fault code: %s; %s' % (http_error_code, fault_code, fault_string)) # XXX raise else: return ret raise Exception("Failed after multiple attempts!")
def endContainerCallback(self): arches = self.arches.keys() arches.sort() if arches: for arch in arches: syncLib.log(6, ' parsed arch: %s' % (arch)) diskImportLib.ChannelPackageArchCompatContainer.endContainerCallback(self)
def download(self): self.setServer(CFG.RHN_XMLRPC_HANDLER) #log(2, ' +++ Satellite synchronization tool downloading certificate.') try: cert = self._xmlrpc("certificate.download", (self.systemid, )) except rpclib.xmlrpclib.Fault, e: log(-1, ' --- Unable to download the satellite certificate') log(-1, ' ERROR: %s' % e, stream=sys.stderr) sys.exit(-1)
def _xmlrpc(function, params): try: retval = getattr(BaseWireSource.serverObj, function)(*params) except TypeError, e: log( -1, 'ERROR: during "getattr(BaseWireSource.serverObj, %s)(*(%s))"' % (function, params)) raise
def purge_extra_channel_families(): # Get rid of the extra channel families try: # Purge all unused private channel families with null org h = rhnSQL.prepare(_query_purge_private_channel_families) h.execute() except rhnSQL.SQLError, e: # Log it and move on - maybe we missed a FK; no reason to break the # sync completely for this. syncLib.log(-1, str(e))
def checkAuth(self): self.setServer(CFG.RHN_XMLRPC_HANDLER) authYN = None log(2, " +++ Satellite synchronization tool checking in.") try: authYN = self._xmlrpc("authentication.check", (self.systemid,)) except (rpclib.ProtocolError, rpclib.Fault), e: # bug 141197: the logging of all exceptions is handled higher up in # the call stack # log2(-1, 1, ' ERROR: %s' % e, stream=sys.stderr) raise
def _xmlrpc(function, params): try: retval = getattr(BaseWireSource.serverObj, function)(*params) except TypeError: e = sys.exc_info()[1] log(-1, 'ERROR: during "getattr(BaseWireSource.serverObj, %s)(*(%s))"' % (function, params)) raise except rpclib.xmlrpclib.ProtocolError: e = sys.exc_info()[1] log2(-1, 2, 'ERROR: ProtocolError: %s' % e, stream=sys.stderr) raise return retval
def import_channels(channels, orgid=None): collection = ChannelCollection() batch = [] import satCerts orgs = map(lambda a: a['id'], satCerts.get_all_orgs()) for c in channels: try: timestamp = collection.get_channel_timestamp(c) except KeyError: raise Exception, "Could not find channel %s" % c c_obj = collection.get_channel(c, timestamp) if c_obj is None: raise Exception, "Channel not found in cache: %s" % c # Check to see if we're asked to sync to an orgid, # make sure the org from the export is not null org, # finally if the orgs differ so we might wanna use # requested org's channel-family. # TODO: Move these checks somewhere more appropriate if not orgid and c_obj['org_id'] is not None: #If the src org is not present default to org 1 orgid = DEFAULT_ORG if orgid is not None and c_obj['org_id'] is not None and \ c_obj['org_id'] != orgid: #Only set the channel family if its a custom channel c_obj['org_id'] = orgid for family in c_obj['families']: family['label'] = 'private-channel-family-' + \ str(c_obj['org_id']) syncLib.log(6, "Syncing Channel %s to Org %s " % \ (c_obj['label'], c_obj['org_id'])) batch.append(c_obj) importer = channelImport.ChannelImport(batch, diskImportLib.get_backend()) # Don't commit just yet importer.will_commit = 0 importer.run() return importer
def _set_ssl_trusted_certs(self, serverObj): if not self.sslYN: return None # Check certificate if CFG.ISS_PARENT: caChain = CFG.ISS_CA_CHAIN else: caChain = CFG.CA_CHAIN if caChain: # require RHNS-CA-CERT file to be able to authenticate the SSL # connections. if not os.access(caChain, os.R_OK): message = "ERROR: can not find RHN CA file: %s" % caChain log(-1, message, stream=sys.stderr) raise Exception(message) # force the validation of the SSL cert serverObj.add_trusted_cert(caChain) return caChain message = '--- Warning: SSL connection made but no CA certificate used' log(1, message, stream=sys.stderr) return None
def import_groups(self, plug, url): groupsfile = plug.get_groups() if groupsfile: basename = os.path.basename(groupsfile) log(0, "Repo %s has comps file %s." % (url, basename)) relativedir = os.path.join(relative_comps_dir, self.channel_label) absdir = os.path.join(CFG.MOUNT_POINT, relativedir) if not os.path.exists(absdir): os.makedirs(absdir) relativepath = os.path.join(relativedir, basename) abspath = os.path.join(absdir, basename) for suffix in ['.gz', '.bz', '.xz']: if basename.endswith(suffix): abspath = abspath.rstrip(suffix) relativepath = relativepath.rstrip(suffix) src = fileutils.decompress_open(groupsfile) dst = open(abspath, "w") shutil.copyfileobj(src, dst) dst.close() src.close() # update or insert hu = rhnSQL.prepare("""update rhnChannelComps set relative_filename = :relpath, modified = current_timestamp where channel_id = :cid""") hu.execute(cid=self.channel['id'], relpath=relativepath) hi = rhnSQL.prepare("""insert into rhnChannelComps (id, channel_id, relative_filename) (select sequence_nextval('rhn_channelcomps_id_seq'), :cid, :relpath from dual where not exists (select 1 from rhnChannelComps where channel_id = :cid))""" ) hi.execute(cid=self.channel['id'], relpath=relativepath)
def import_groups(self, plug, url): groupsfile = plug.get_groups() if groupsfile: basename = os.path.basename(groupsfile) log(0, "Repo %s has comps file %s." % (url, basename)) relativedir = os.path.join(relative_comps_dir, self.channel_label) absdir = os.path.join(CFG.MOUNT_POINT, relativedir) if not os.path.exists(absdir): os.makedirs(absdir) relativepath = os.path.join(relativedir, basename) abspath = os.path.join(absdir, basename) for suffix in ['.gz', '.bz', '.xz']: if basename.endswith(suffix): abspath = abspath.rstrip(suffix) relativepath = relativepath.rstrip(suffix) src = fileutils.decompress_open(groupsfile) dst = open(abspath, "w") shutil.copyfileobj(src, dst) dst.close() src.close() # update or insert hu = rhnSQL.prepare("""update rhnChannelComps set relative_filename = :relpath, modified = current_timestamp where channel_id = :cid""") hu.execute(cid=self.channel['id'], relpath=relativepath) hi = rhnSQL.prepare("""insert into rhnChannelComps (id, channel_id, relative_filename) (select sequence_nextval('rhn_channelcomps_id_seq'), :cid, :relpath from dual where not exists (select 1 from rhnChannelComps where channel_id = :cid))""") hi.execute(cid=self.channel['id'], relpath=relativepath)
def checkAuth(self): self.setServer(CFG.RHN_XMLRPC_HANDLER) authYN = None log(2, ' +++ SUSE Manager Server synchronization tool checking in.') try: authYN = self._xmlrpc('authentication.check', (self.systemid,)) except (rpclib.xmlrpclib.ProtocolError, rpclib.xmlrpclib.Fault): raise if authYN: log(2, ' +++ Entitled SUSE Manager Server validated.', stream=sys.stderr) elif authYN is None: log(-1, ' --- An error occurred upon authentication of this SUSE Manager Server -- ' 'review the pertinent log file (%s) and/or submit a service request.' % CFG.LOG_FILE, stream=sys.stderr) sys.exit(-1) elif authYN == 0: log(-1, ' --- This server is not entitled.', stream=sys.stderr) sys.exit(-1) return authYN
def checkAuth(self): self.setServer(CFG.RHN_XMLRPC_HANDLER) authYN = None log(2, ' +++ Satellite synchronization tool checking in.') try: authYN = self._xmlrpc('authentication.check', (self.systemid,)) except (rpclib.xmlrpclib.ProtocolError, rpclib.xmlrpclib.Fault): raise if authYN: log(2, ' +++ Entitled satellite validated.', stream=sys.stderr) elif authYN is None: log(-1, ' --- An error occurred upon authentication of this satellite -- ' 'review the pertinent log file (%s) and/or alert RHN at [email protected].' % CFG.LOG_FILE, stream=sys.stderr) sys.exit(-1) elif authYN == 0: log(-1, ' --- This server is not an entitled satellite.', stream=sys.stderr) sys.exit(-1) return authYN
def checkAuth(self): self.setServer(CFG.RHN_XMLRPC_HANDLER) authYN = None log(2, ' +++ SUSE Manager Server synchronization tool checking in.') try: authYN = self._xmlrpc('authentication.check', (self.systemid, )) except (rpclib.xmlrpclib.ProtocolError, rpclib.xmlrpclib.Fault): raise if authYN: log(2, ' +++ Entitled SUSE Manager Server validated.', stream=sys.stderr) elif authYN is None: log(-1, ' --- An error occurred upon authentication of this SUSE Manager Server -- ' 'review the pertinent log file (%s) and/or submit a service request.' % CFG.LOG_FILE, stream=sys.stderr) sys.exit(-1) elif authYN == 0: log(-1, ' --- This server is not entitled.', stream=sys.stderr) sys.exit(-1) return authYN
def import_kickstart(self, plug, repo_label): ks_path = 'rhn/kickstart/' ks_tree_label = re.sub(r'[^-_0-9A-Za-z@.]', '', repo_label.replace(' ', '_')) if len(ks_tree_label) < 4: ks_tree_label += "_repo" # construct ks_path and check we already have this KS tree synced id_request = """ select id from rhnKickstartableTree where channel_id = :channel_id and label = :label """ if 'org_id' in self.channel and self.channel['org_id']: ks_path += str(self.channel['org_id']) + '/' + ks_tree_label # Trees synced from external repositories are expected to have full path it database db_path = os.path.join(CFG.MOUNT_POINT, ks_path) row = rhnSQL.fetchone_dict(id_request + " and org_id = :org_id", channel_id=self.channel['id'], label=ks_tree_label, org_id=self.channel['org_id']) else: ks_path += ks_tree_label db_path = ks_path row = rhnSQL.fetchone_dict(id_request + " and org_id is NULL", channel_id=self.channel['id'], label=ks_tree_label) treeinfo_path = ['treeinfo', '.treeinfo'] treeinfo_parser = None for path in treeinfo_path: log(1, "Trying " + path) treeinfo = plug.get_file(path, os.path.join(CFG.MOUNT_POINT, ks_path)) if treeinfo: try: treeinfo_parser = TreeInfoParser(treeinfo) break except TreeInfoError: pass if not treeinfo_parser: log(0, "Kickstartable tree not detected (no valid treeinfo file)") return if self.ks_install_type is None: family = treeinfo_parser.get_family() if family == 'Fedora': self.ks_install_type = 'fedora18' elif family == 'CentOS': self.ks_install_type = 'rhel_' + treeinfo_parser.get_major_version() else: self.ks_install_type = 'generic_rpm' # Make sure images are included to_download = [] for repo_path in treeinfo_parser.get_images(): local_path = os.path.join(CFG.MOUNT_POINT, ks_path, repo_path) # TODO: better check if not os.path.exists(local_path): to_download.append(repo_path) if row: log(0, "Kickstartable tree %s already synced. Updating content..." % ks_tree_label) ks_id = row['id'] else: row = rhnSQL.fetchone_dict(""" select sequence_nextval('rhn_kstree_id_seq') as id from dual """) ks_id = row['id'] rhnSQL.execute(""" insert into rhnKickstartableTree (id, org_id, label, base_path, channel_id, kstree_type, install_type, last_modified, created, modified) values (:id, :org_id, :label, :base_path, :channel_id, ( select id from rhnKSTreeType where label = :ks_tree_type), ( select id from rhnKSInstallType where label = :ks_install_type), current_timestamp, current_timestamp, current_timestamp)""", id=ks_id, org_id=self.channel['org_id'], label=ks_tree_label, base_path=db_path, channel_id=self.channel['id'], ks_tree_type=self.ks_tree_type, ks_install_type=self.ks_install_type) log(0, "Added new kickstartable tree %s. Downloading content..." % ks_tree_label) insert_h = rhnSQL.prepare(""" insert into rhnKSTreeFile (kstree_id, relative_filename, checksum_id, file_size, last_modified, created, modified) values (:id, :path, lookup_checksum('sha256', :checksum), :st_size, epoch_seconds_to_timestamp_tz(:st_time), current_timestamp, current_timestamp) """) delete_h = rhnSQL.prepare(""" delete from rhnKSTreeFile where kstree_id = :id and relative_filename = :path """) # Downloading/Updating content of KS Tree # start from root dir dirs_queue = [''] log(0, "Gathering all files in kickstart repository...") while len(dirs_queue) > 0: cur_dir_name = dirs_queue.pop(0) cur_dir_html = plug.get_file(cur_dir_name) if cur_dir_html is None: continue parser = KSDirParser(cur_dir_html) for ks_file in parser.get_content(): repo_path = cur_dir_name + ks_file['name'] # if this is a directory, just add a name into queue (like BFS algorithm) if ks_file['type'] == 'DIR': dirs_queue.append(repo_path) continue if repo_path not in to_download: to_download.append(repo_path) if to_download: log(0, "Downloading %d files." % len(to_download)) for item in to_download: for retry in range(3): try: log(1, "Retrieving %s" % item) plug.get_file(item, os.path.join(CFG.MOUNT_POINT, ks_path)) st = os.stat(os.path.join(CFG.MOUNT_POINT, ks_path, item)) break except OSError: # os.stat if the file wasn't downloaded if retry < 3: log(2, "Retry download %s: attempt #%d" % (item, retry + 1)) else: raise # update entity about current file in a database delete_h.execute(id=ks_id, path=item) insert_h.execute(id=ks_id, path=item, checksum=getFileChecksum('sha256', os.path.join(CFG.MOUNT_POINT, ks_path, item)), st_size=st.st_size, st_time=st.st_mtime) else: log(0, "Nothing to download.") rhnSQL.commit()
def endItemCallback(self): xmlSource.ChannelFamilyContainer.endItemCallback(self) if not self.batch: return syncLib.log(2, ' parsing family: %s' % (self.batch[-1]['name']))
def __init__(self, channel_label, repo_type, url=None, fail=False, filters=None, no_errata=False, sync_kickstart=False, latest=False, metadata_only=False, strict=0, excluded_urls=None, no_packages=False, log_dir="reposync", log_level=None): self.regen = False self.fail = fail self.filters = filters or [] self.no_packages = no_packages self.no_errata = no_errata self.sync_kickstart = sync_kickstart self.latest = latest self.metadata_only = metadata_only self.ks_tree_type = 'externally-managed' self.ks_install_type = 'generic_rpm' initCFG('server.satellite') rhnSQL.initDB() # setup logging log_filename = channel_label + '.log' log_path = default_log_location + log_dir + '/' + log_filename if log_level is None: log_level = 0 CFG.set('DEBUG', log_level) rhnLog.initLOG(log_path, log_level) # os.fchown isn't in 2.4 :/ if isSUSE(): os.system("chgrp www " + log_path) else: os.system("chgrp apache " + log_path) log2disk(0, "Command: %s" % str(sys.argv)) log2disk(0, "Sync of channel started.") self.channel_label = channel_label self.channel = self.load_channel() if not self.channel: log(0, "Channel %s does not exist." % channel_label) if not url: # TODO:need to look at user security across orgs h = rhnSQL.prepare("""select s.id, s.source_url, s.label, fm.channel_family_id from rhnContentSource s, rhnChannelContentSource cs, rhnChannelFamilyMembers fm where s.id = cs.source_id and cs.channel_id = fm.channel_id and cs.channel_id = :channel_id""") h.execute(channel_id=int(self.channel['id'])) source_data = h.fetchall_dict() self.urls = [] if excluded_urls is None: excluded_urls = [] if source_data: for row in source_data: if row['source_url'] not in excluded_urls: self.urls.append((row['id'], row['source_url'], row['label'], row['channel_family_id'])) else: self.urls = [(None, u, None, None) for u in url] if not self.urls: log2stderr(0, "Channel %s has no URL associated" % channel_label) self.repo_plugin = self.load_plugin(repo_type) self.strict = strict self.all_packages = []
def _xmlrpc(self, function, params): try: retval = apply(getattr(BaseWireSource.serverObj, function), params) except TypeError, e: log(-1, 'ERROR: during "apply(getattr(BaseWireSource.serverObj, %s), %s)"' % (function, params)) raise
def sync(self, update_repodata=False): """Trigger a reposync""" start_time = datetime.now() for (repo_id, url, repo_label) in self.urls: log(0, "Repo URL: %s" % url) plugin = None # If the repository uses a uln:// URL, switch to the ULN plugin, overriding the command-line if url.startswith("uln://"): self.repo_plugin = self.load_plugin("uln") # pylint: disable=W0703 try: # use modified relative_url as name of repo plugin, because # it used as name of cache directory as well relative_url = '_'.join(url.split('://')[1].split('/')[1:]) plugin_name = relative_url.replace("?", "_").replace("&", "_").replace("=", "_") plugin = self.repo_plugin(url, plugin_name) if update_repodata: plugin.clear_cache() if repo_id is not None: keys = rhnSQL.fetchone_dict(""" select k1.key as ca_cert, k2.key as client_cert, k3.key as client_key from rhncontentsource cs join rhncryptokey k1 on cs.ssl_ca_cert_id = k1.id left outer join rhncryptokey k2 on cs.ssl_client_cert_id = k2.id left outer join rhncryptokey k3 on cs.ssl_client_key_id = k3.id where cs.id = :repo_id """, repo_id=int(repo_id)) if keys and ('ca_cert' in keys): plugin.set_ssl_options(keys['ca_cert'], keys['client_cert'], keys['client_key']) if not self.no_packages: self.import_packages(plugin, repo_id, url) self.import_groups(plugin, url) if not self.no_errata: self.import_updates(plugin, url) # only for repos obtained from the DB if self.sync_kickstart and repo_label: try: self.import_kickstart(plugin, repo_label) except: rhnSQL.rollback() raise except Exception: e = sys.exc_info()[1] log2stderr(0, "ERROR: %s" % e) if plugin is not None: plugin.clear_ssl_cache() if self.regen: taskomatic.add_to_repodata_queue_for_channel_package_subscription( [self.channel_label], [], "server.app.yumreposync") taskomatic.add_to_erratacache_queue(self.channel_label) self.update_date() rhnSQL.commit() elapsed_time = datetime.now() - start_time log(0, "Sync of channel completed in %s." % str(elapsed_time).split('.')[0]) return elapsed_time
def upload_updates(self, notices): batch = [] typemap = { 'security': 'Security Advisory', 'recommended': 'Bug Fix Advisory', 'bugfix': 'Bug Fix Advisory', 'optional': 'Product Enhancement Advisory', 'feature': 'Product Enhancement Advisory', 'enhancement': 'Product Enhancement Advisory' } for notice in notices: notice = self.fix_notice(notice) advisory = notice['update_id'] + '-' + notice['version'] existing_errata = self.get_errata(notice['update_id']) e = Erratum() e['errata_from'] = notice['from'] e['advisory'] = advisory e['advisory_name'] = notice['update_id'] e['advisory_rel'] = notice['version'] e['advisory_type'] = typemap.get(notice['type'], 'Product Enhancement Advisory') e['product'] = notice['release'] or 'Unknown' e['description'] = notice['description'] e['synopsis'] = notice['title'] or notice['update_id'] if (notice['type'] == 'security' and notice['severity'] and not e['synopsis'].startswith(notice['severity'] + ': ')): e['synopsis'] = notice['severity'] + ': ' + e['synopsis'] if 'summary' in notice and not notice['summary'] is None: e['topic'] = notice['summary'] else: e['topic'] = ' ' if 'solution' in notice and not notice['solution'] is None: e['solution'] = notice['solution'] else: e['solution'] = ' ' e['issue_date'] = self._to_db_date(notice['issued']) if notice['updated']: e['update_date'] = self._to_db_date(notice['updated']) else: e['update_date'] = self._to_db_date(notice['issued']) e['org_id'] = self.channel['org_id'] e['notes'] = '' e['channels'] = [] e['packages'] = [] e['files'] = [] if existing_errata: e['channels'] = existing_errata['channels'] e['packages'] = existing_errata['packages'] e['channels'].append({'label': self.channel_label}) for pkg in notice['pkglist'][0]['packages']: param_dict = { 'name': pkg['name'], 'version': pkg['version'], 'release': pkg['release'], 'arch': pkg['arch'], 'channel_id': int(self.channel['id']), } if pkg['epoch'] == '0': epochStatement = "(pevr.epoch is NULL or pevr.epoch = '0')" elif pkg['epoch'] is None or pkg['epoch'] == '': epochStatement = "pevr.epoch is NULL" else: epochStatement = "pevr.epoch = :epoch" param_dict['epoch'] = pkg['epoch'] if self.channel['org_id']: param_dict['org_id'] = self.channel['org_id'] orgStatement = "= :org_id" else: orgStatement = "is NULL" h = rhnSQL.prepare(""" select p.id, pevr.epoch, c.checksum, c.checksum_type from rhnPackage p join rhnPackagename pn on p.name_id = pn.id join rhnpackageevr pevr on p.evr_id = pevr.id join rhnpackagearch pa on p.package_arch_id = pa.id join rhnArchType at on pa.arch_type_id = at.id join rhnChecksumView c on p.checksum_id = c.id join rhnChannelPackage cp on p.id = cp.package_id where pn.name = :name and p.org_id %s and pevr.version = :version and pevr.release = :release and pa.label = :arch and %s and at.label = 'rpm' and cp.channel_id = :channel_id """ % (orgStatement, epochStatement)) h.execute(**param_dict) cs = h.fetchone_dict() or None if not cs: if 'epoch' in param_dict: epoch = param_dict['epoch'] + ":" else: epoch = "" log(2, "No checksum found for %s-%s%s-%s.%s." " Skipping Package" % (param_dict['name'], epoch, param_dict['version'], param_dict['release'], param_dict['arch'])) continue newpkgs = [] for oldpkg in e['packages']: if oldpkg['package_id'] != cs['id']: newpkgs.append(oldpkg) package = IncompletePackage().populate(pkg) package['epoch'] = cs['epoch'] package['org_id'] = self.channel['org_id'] package['checksums'] = {cs['checksum_type']: cs['checksum']} package['checksum_type'] = cs['checksum_type'] package['checksum'] = cs['checksum'] package['package_id'] = cs['id'] newpkgs.append(package) e['packages'] = newpkgs if len(e['packages']) == 0: # FIXME: print only with higher debug option log(2, "Advisory %s has empty package list." % e['advisory_name']) e['keywords'] = [] if notice['reboot_suggested']: kw = Keyword() kw.populate({'keyword': 'reboot_suggested'}) e['keywords'].append(kw) if notice['restart_suggested']: kw = Keyword() kw.populate({'keyword': 'restart_suggested'}) e['keywords'].append(kw) e['bugs'] = [] e['cve'] = [] if notice['references']: bzs = [r for r in notice['references'] if r['type'] == 'bugzilla'] if len(bzs): tmp = {} for bz in bzs: try: bz_id = int(bz['id']) # This can happen in some incorrectly generated updateinfo, let's be smart except ValueError: log(2, "Bugzilla assigned to advisory %s has invalid id: %s, trying to get it from URL..." % (e['advisory_name'], bz['id'])) bz_id = int(re.search(r"\d+$", bz['href']).group(0)) if bz_id not in tmp: bug = Bug() bug.populate({'bug_id': bz_id, 'summary': bz['title'], 'href': bz['href']}) e['bugs'].append(bug) tmp[bz_id] = None cves = [r for r in notice['references'] if r['type'] == 'cve'] if len(cves): tmp = {} for cve in cves: if cve['id'] not in tmp: e['cve'].append(cve['id']) tmp[cve['id']] = None others = [r for r in notice['references'] if not r['type'] == 'bugzilla' and not r['type'] == 'cve'] if len(others): tmp = len(others) refers_to = "" for other in others: if refers_to: refers_to += "\n" refers_to += other['href'] e['refers_to'] = refers_to e['locally_modified'] = None batch.append(e) backend = SQLBackend() importer = ErrataImport(batch, backend) importer.run() self.regen = True
def import_packages(self, plug, source_id, url): failed_packages = 0 if (not self.filters) and source_id: h = rhnSQL.prepare(""" select flag, filter from rhnContentSourceFilter where source_id = :source_id order by sort_order """) h.execute(source_id=source_id) filter_data = h.fetchall_dict() or [] filters = [(row['flag'], re.split(r'[,\s]+', row['filter'])) for row in filter_data] else: filters = self.filters packages = plug.list_packages(filters, self.latest) self.all_packages.extend(packages) to_process = [] num_passed = len(packages) log(0, "Packages in repo: %5d" % plug.num_packages) if plug.num_excluded: log(0, "Packages passed filter rules: %5d" % num_passed) channel_id = int(self.channel['id']) for pack in packages: db_pack = rhnPackage.get_info_for_package( [pack.name, pack.version, pack.release, pack.epoch, pack.arch], channel_id, self.org_id) to_download = True to_link = True # Package exists in DB if db_pack: # Path in filesystem is defined if db_pack['path']: pack.path = os.path.join(CFG.MOUNT_POINT, db_pack['path']) else: pack.path = "" if self.metadata_only or self.match_package_checksum( db_pack['path'], pack.path, pack.checksum_type, pack.checksum): # package is already on disk or not required to_download = False if db_pack['channel_id'] == channel_id: # package is already in the channel to_link = False # just pass data from DB, they will be used in strict channel # linking if there is no new RPM downloaded pack.checksum = db_pack['checksum'] pack.checksum_type = db_pack['checksum_type'] pack.epoch = db_pack['epoch'] elif db_pack['channel_id'] == channel_id: # different package with SAME NVREA self.disassociate_package(db_pack) if to_download or to_link: to_process.append((pack, to_download, to_link)) num_to_process = len(to_process) if num_to_process == 0: log(0, "No new packages to sync.") # If we are just appending, we can exit if not self.strict: return failed_packages else: log( 0, "Packages already synced: %5d" % (num_passed - num_to_process)) log(0, "Packages to sync: %5d" % num_to_process) is_non_local_repo = (url.find("file:/") < 0) downloader = ThreadedDownloader() to_download_count = 0 for what in to_process: pack, to_download, to_link = what if to_download: target_file = os.path.join( plug.repo.pkgdir, os.path.basename(pack.unique_id.relativepath)) pack.path = target_file params = {} if self.metadata_only: bytes_range = (0, pack.unique_id.hdrend) checksum_type = None checksum = None else: bytes_range = None checksum_type = pack.checksum_type checksum = pack.checksum plug.set_download_parameters(params, pack.unique_id.relativepath, target_file, checksum_type=checksum_type, checksum_value=checksum, bytes_range=bytes_range) downloader.add(params) to_download_count += 1 if num_to_process != 0: log(0, "New packages to download: %5d" % to_download_count) logger = TextLogger(None, to_download_count) downloader.set_log_obj(logger) downloader.run() log2disk(0, "Importing packages started.") progress_bar = ProgressBarLogger("Importing packages: ", to_download_count) for (index, what) in enumerate(to_process): pack, to_download, to_link = what if not to_download: continue localpath = pack.path # pylint: disable=W0703 try: if os.path.exists(localpath): pack.load_checksum_from_header() rel_package_path = pack.upload_package( self.org_id, metadata_only=self.metadata_only) # Save uploaded package to cache with repository checksum type if rel_package_path: self.checksum_cache[rel_package_path] = { pack.checksum_type: pack.checksum } # we do not want to keep a whole 'a_pkg' object for every package in memory, # because we need only checksum. see BZ 1397417 pack.checksum = pack.a_pkg.checksum pack.checksum_type = pack.a_pkg.checksum_type pack.epoch = pack.a_pkg.header['epoch'] pack.a_pkg = None else: raise Exception progress_bar.log(True, None) except KeyboardInterrupt: raise except Exception: failed_packages += 1 e = str(sys.exc_info()[1]) if e: log2(0, 1, e, stream=sys.stderr) if self.fail: raise to_process[index] = (pack, False, False) self.all_packages.remove(pack) progress_bar.log(False, None) finally: if is_non_local_repo and localpath and os.path.exists( localpath): os.remove(localpath) log2disk(0, "Importing packages finished.") if self.strict: # Need to make sure all packages from all repositories are associated with channel import_batch = [ self.associate_package(pack) for pack in self.all_packages ] else: # Only packages from current repository are appended to channel import_batch = [ self.associate_package(pack) for (pack, to_download, to_link) in to_process if to_link ] # Do not re-link if nothing was marked to link if any([to_link for (pack, to_download, to_link) in to_process]): log(0, "Linking packages to channel.") backend = SQLBackend() caller = "server.app.yumreposync" importer = ChannelPackageSubscription(import_batch, backend, caller=caller, repogen=False, strict=self.strict) importer.run() backend.commit() self.regen = True return failed_packages
def upload_updates(self, notices): batch = [] typemap = { 'security': 'Security Advisory', 'recommended': 'Bug Fix Advisory', 'bugfix': 'Bug Fix Advisory', 'optional': 'Product Enhancement Advisory', 'feature': 'Product Enhancement Advisory', 'enhancement': 'Product Enhancement Advisory' } channel_advisory_names = self.list_errata() for notice in notices: notice = self.fix_notice(notice) if not self.force_all_errata and notice[ 'update_id'] in channel_advisory_names: continue advisory = notice['update_id'] + '-' + notice['version'] existing_errata = self.get_errata(notice['update_id']) e = Erratum() e['errata_from'] = notice['from'] e['advisory'] = advisory e['advisory_name'] = notice['update_id'] e['advisory_rel'] = notice['version'] e['advisory_type'] = typemap.get(notice['type'], 'Product Enhancement Advisory') e['product'] = notice['release'] or 'Unknown' e['description'] = notice['description'] e['synopsis'] = notice['title'] or notice['update_id'] if (notice['type'] == 'security' and notice['severity'] and not e['synopsis'].startswith(notice['severity'] + ': ')): e['synopsis'] = notice['severity'] + ': ' + e['synopsis'] if 'summary' in notice and not notice['summary'] is None: e['topic'] = notice['summary'] else: e['topic'] = ' ' if 'solution' in notice and not notice['solution'] is None: e['solution'] = notice['solution'] else: e['solution'] = ' ' e['issue_date'] = self._to_db_date(notice['issued']) if notice['updated']: e['update_date'] = self._to_db_date(notice['updated']) else: e['update_date'] = self._to_db_date(notice['issued']) e['org_id'] = self.org_id e['notes'] = '' e['channels'] = [] e['packages'] = [] e['files'] = [] if existing_errata: e['channels'] = existing_errata['channels'] e['packages'] = existing_errata['packages'] e['channels'].append({'label': self.channel_label}) for pkg in notice['pkglist'][0]['packages']: param_dict = { 'name': pkg['name'], 'version': pkg['version'], 'release': pkg['release'], 'arch': pkg['arch'], 'channel_id': int(self.channel['id']), } if pkg['epoch'] == '0': epochStatement = "(pevr.epoch is NULL or pevr.epoch = '0')" elif pkg['epoch'] is None or pkg['epoch'] == '': epochStatement = "pevr.epoch is NULL" else: epochStatement = "pevr.epoch = :epoch" param_dict['epoch'] = pkg['epoch'] if self.org_id: param_dict['org_id'] = self.org_id orgStatement = "= :org_id" else: orgStatement = "is NULL" h = rhnSQL.prepare(""" select p.id, pevr.epoch, c.checksum, c.checksum_type from rhnPackage p join rhnPackagename pn on p.name_id = pn.id join rhnpackageevr pevr on p.evr_id = pevr.id join rhnpackagearch pa on p.package_arch_id = pa.id join rhnArchType at on pa.arch_type_id = at.id join rhnChecksumView c on p.checksum_id = c.id join rhnChannelPackage cp on p.id = cp.package_id where pn.name = :name and p.org_id %s and pevr.version = :version and pevr.release = :release and pa.label = :arch and %s and at.label = 'rpm' and cp.channel_id = :channel_id """ % (orgStatement, epochStatement)) h.execute(**param_dict) cs = h.fetchone_dict() or None if not cs: if 'epoch' in param_dict: epoch = param_dict['epoch'] + ":" else: epoch = "" log( 2, "No checksum found for %s-%s%s-%s.%s." " Skipping Package" % (param_dict['name'], epoch, param_dict['version'], param_dict['release'], param_dict['arch'])) continue newpkgs = [] for oldpkg in e['packages']: if oldpkg['package_id'] != cs['id']: newpkgs.append(oldpkg) package = IncompletePackage().populate(pkg) package['epoch'] = cs['epoch'] package['org_id'] = self.org_id package['checksums'] = {cs['checksum_type']: cs['checksum']} package['checksum_type'] = cs['checksum_type'] package['checksum'] = cs['checksum'] package['package_id'] = cs['id'] newpkgs.append(package) e['packages'] = newpkgs if len(e['packages']) == 0: # FIXME: print only with higher debug option log(2, "Advisory %s has empty package list." % e['advisory_name']) e['keywords'] = [] if notice['reboot_suggested']: kw = Keyword() kw.populate({'keyword': 'reboot_suggested'}) e['keywords'].append(kw) if notice['restart_suggested']: kw = Keyword() kw.populate({'keyword': 'restart_suggested'}) e['keywords'].append(kw) e['bugs'] = [] e['cve'] = [] if notice['references']: bzs = [ r for r in notice['references'] if r['type'] == 'bugzilla' ] if len(bzs): tmp = {} for bz in bzs: try: bz_id = int(bz['id']) # This can happen in some incorrectly generated updateinfo, let's be smart except ValueError: log( 2, "Bugzilla assigned to advisory %s has invalid id: %s, trying to get it from URL..." % (e['advisory_name'], bz['id'])) bz_id = int( re.search(r"\d+$", bz['href']).group(0)) if bz_id not in tmp: bug = Bug() bug.populate({ 'bug_id': bz_id, 'summary': bz['title'], 'href': bz['href'] }) e['bugs'].append(bug) tmp[bz_id] = None cves = [r for r in notice['references'] if r['type'] == 'cve'] if len(cves): tmp = {} for cve in cves: if cve['id'] not in tmp: e['cve'].append(cve['id']) tmp[cve['id']] = None others = [ r for r in notice['references'] if not r['type'] == 'bugzilla' and not r['type'] == 'cve' ] if len(others): tmp = len(others) refers_to = "" for other in others: if refers_to: refers_to += "\n" refers_to += other['href'] e['refers_to'] = refers_to e['locally_modified'] = None batch.append(e) if batch: log(0, "Syncing %s new errata to channel." % len(batch)) backend = SQLBackend() importer = ErrataImport(batch, backend) importer.run() self.regen = True elif notices: log(0, "No new errata to sync.")
def import_packages(self, plug, source_id, url): if (not self.filters) and source_id: h = rhnSQL.prepare(""" select flag, filter from rhnContentSourceFilter where source_id = :source_id order by sort_order """) h.execute(source_id=source_id) filter_data = h.fetchall_dict() or [] filters = [(row['flag'], re.split(r'[,\s]+', row['filter'])) for row in filter_data] else: filters = self.filters packages = plug.list_packages(filters, self.latest) self.all_packages.extend(packages) to_process = [] num_passed = len(packages) log(0, "Packages in repo: %5d" % plug.num_packages) if plug.num_excluded: log(0, "Packages passed filter rules: %5d" % num_passed) channel_id = int(self.channel['id']) if self.channel['org_id']: self.channel['org_id'] = int(self.channel['org_id']) else: self.channel['org_id'] = None for pack in packages: db_pack = rhnPackage.get_info_for_package( [pack.name, pack.version, pack.release, pack.epoch, pack.arch], channel_id, self.channel['org_id']) to_download = True to_link = True # Package exists in DB if db_pack: # Path in filesystem is defined if db_pack['path']: pack.path = os.path.join(CFG.MOUNT_POINT, db_pack['path']) else: pack.path = "" if self.metadata_only or self.match_package_checksum(pack.path, pack.checksum_type, pack.checksum): # package is already on disk or not required to_download = False if db_pack['channel_id'] == channel_id: # package is already in the channel to_link = False elif db_pack['channel_id'] == channel_id: # different package with SAME NVREA self.disassociate_package(db_pack) # just pass data from DB, they will be used if there is no RPM available pack.checksum = db_pack['checksum'] pack.checksum_type = db_pack['checksum_type'] pack.epoch = db_pack['epoch'] if to_download or to_link: to_process.append((pack, to_download, to_link)) num_to_process = len(to_process) if num_to_process == 0: log(0, "No new packages to sync.") # If we are just appending, we can exit if not self.strict: return else: log(0, "Packages already synced: %5d" % (num_passed - num_to_process)) log(0, "Packages to sync: %5d" % num_to_process) self.regen = True is_non_local_repo = (url.find("file:/") < 0) for (index, what) in enumerate(to_process): pack, to_download, to_link = what localpath = None # pylint: disable=W0703 try: log(0, "%d/%d : %s" % (index + 1, num_to_process, pack.getNVREA())) if to_download: pack.path = localpath = plug.get_package(pack, metadata_only=self.metadata_only) pack.load_checksum_from_header() pack.upload_package(self.channel, metadata_only=self.metadata_only) except KeyboardInterrupt: raise except Exception: e = sys.exc_info()[1] log2stderr(0, e) if self.fail: raise to_process[index] = (pack, False, False) continue finally: if is_non_local_repo and localpath and os.path.exists(localpath): os.remove(localpath) log(0, "Linking packages to channel.") if self.strict: import_batch = [self.associate_package(pack) for pack in self.all_packages] else: import_batch = [self.associate_package(pack) for (pack, to_download, to_link) in to_process if to_link] backend = SQLBackend() caller = "server.app.yumreposync" importer = ChannelPackageSubscription(import_batch, backend, caller=caller, repogen=False, strict=self.strict) importer.run() backend.commit()
def import_updates(self, plug, url): notices = plug.get_updates() log(0, "Repo %s has %s errata." % (url, len(notices))) if notices: self.upload_updates(notices)
def sync(self, update_repodata=True): """Trigger a reposync""" failed_packages = 0 sync_error = 0 if not self.urls: sync_error = -1 start_time = datetime.now() for (repo_id, url, repo_label) in self.urls: log(0, "Repo URL: %s" % url) plugin = None # If the repository uses a uln:// URL, switch to the ULN plugin, overriding the command-line if url.startswith("uln://"): self.repo_plugin = self.load_plugin("uln") # pylint: disable=W0703 try: if repo_label: repo_name = repo_label else: # use modified relative_url as name of repo plugin, because # it used as name of cache directory as well relative_url = '_'.join(url.split('://')[1].split('/')[1:]) repo_name = relative_url.replace("?", "_").replace( "&", "_").replace("=", "_") plugin = self.repo_plugin(url, repo_name, org=str(self.org_id or ''), channel_label=self.channel_label) if update_repodata: plugin.clear_cache() if repo_id is not None: keys = rhnSQL.fetchall_dict(""" select k1.key as ca_cert, k2.key as client_cert, k3.key as client_key from rhncontentsource cs inner join rhncontentsourcessl csssl on cs.id = csssl.content_source_id inner join rhncryptokey k1 on csssl.ssl_ca_cert_id = k1.id left outer join rhncryptokey k2 on csssl.ssl_client_cert_id = k2.id left outer join rhncryptokey k3 on csssl.ssl_client_key_id = k3.id where cs.id = :repo_id """, repo_id=int(repo_id)) if keys: ssl_set = get_single_ssl_set( keys, check_dates=self.check_ssl_dates) if ssl_set: plugin.set_ssl_options(ssl_set['ca_cert'], ssl_set['client_cert'], ssl_set['client_key']) else: raise ValueError( "No valid SSL certificates were found for repository." ) if not self.no_packages: ret = self.import_packages(plugin, repo_id, url) failed_packages += ret self.import_groups(plugin, url) if not self.no_errata: self.import_updates(plugin, url) # only for repos obtained from the DB if self.sync_kickstart and repo_label: try: self.import_kickstart(plugin, repo_label) except: rhnSQL.rollback() raise except Exception: e = sys.exc_info()[1] log2(0, 0, "ERROR: %s" % e, stream=sys.stderr) log2disk(0, "ERROR: %s" % e) # pylint: disable=W0104 sync_error = -1 if plugin is not None: plugin.clear_ssl_cache() # Update cache with package checksums rhnCache.set(checksum_cache_filename, self.checksum_cache) if self.regen: taskomatic.add_to_repodata_queue_for_channel_package_subscription( [self.channel_label], [], "server.app.yumreposync") taskomatic.add_to_erratacache_queue(self.channel_label) self.update_date() rhnSQL.commit() # update permissions fileutils.createPath(os.path.join( CFG.MOUNT_POINT, 'rhn')) # if the directory exists update ownership only for root, dirs, files in os.walk(os.path.join(CFG.MOUNT_POINT, 'rhn')): for d in dirs: fileutils.setPermsPath(os.path.join(root, d), group='apache') for f in files: fileutils.setPermsPath(os.path.join(root, f), group='apache') elapsed_time = datetime.now() - start_time log( 0, "Sync of channel completed in %s." % str(elapsed_time).split('.')[0]) # if there is no global problems, but some packages weren't synced if sync_error == 0 and failed_packages > 0: sync_error = failed_packages return elapsed_time, sync_error
def __init__(self, channel_label, repo_type, url=None, fail=False, filters=None, no_errata=False, sync_kickstart=False, latest=False, metadata_only=False, strict=0, excluded_urls=None, no_packages=False, log_dir="reposync", log_level=None, force_kickstart=False, force_all_errata=False, check_ssl_dates=False, force_null_org_content=False): self.regen = False self.fail = fail self.filters = filters or [] self.no_packages = no_packages self.no_errata = no_errata self.sync_kickstart = sync_kickstart self.force_all_errata = force_all_errata self.force_kickstart = force_kickstart self.latest = latest self.metadata_only = metadata_only self.ks_tree_type = 'externally-managed' self.ks_install_type = None initCFG('server.satellite') rhnSQL.initDB() # setup logging log_filename = channel_label + '.log' log_path = default_log_location + log_dir + '/' + log_filename if log_level is None: log_level = 0 CFG.set('DEBUG', log_level) rhnLog.initLOG(log_path, log_level) # os.fchown isn't in 2.4 :/ if isSUSE(): os.system("chgrp www " + log_path) else: os.system("chgrp apache " + log_path) log2disk(0, "Command: %s" % str(sys.argv)) log2disk(0, "Sync of channel started.") self.channel_label = channel_label self.channel = self.load_channel() if not self.channel: log(0, "Channel %s does not exist." % channel_label) if not self.channel['org_id'] or force_null_org_content: self.org_id = None else: self.org_id = int(self.channel['org_id']) if not url: # TODO:need to look at user security across orgs h = rhnSQL.prepare("""select s.id, s.source_url, s.label from rhnContentSource s, rhnChannelContentSource cs where s.id = cs.source_id and cs.channel_id = :channel_id""") h.execute(channel_id=int(self.channel['id'])) source_data = h.fetchall_dict() self.urls = [] if excluded_urls is None: excluded_urls = [] if source_data: for row in source_data: if row['source_url'] not in excluded_urls: self.urls.append( (row['id'], row['source_url'], row['label'])) else: self.urls = [(None, u, None) for u in url] if not self.urls: log2(0, 0, "Channel %s has no URL associated" % channel_label, stream=sys.stderr) self.repo_plugin = self.load_plugin(repo_type) self.strict = strict self.all_packages = [] self.check_ssl_dates = check_ssl_dates # Init cache for computed checksums to not compute it on each reposync run again self.checksum_cache = rhnCache.get(checksum_cache_filename) if self.checksum_cache is None: self.checksum_cache = {}
def import_packages(self, plug, source_id, url): if (not self.filters) and source_id: h = rhnSQL.prepare(""" select flag, filter from rhnContentSourceFilter where source_id = :source_id order by sort_order """) h.execute(source_id=source_id) filter_data = h.fetchall_dict() or [] filters = [(row['flag'], re.split(r'[,\s]+', row['filter'])) for row in filter_data] else: filters = self.filters packages = plug.list_packages(filters, self.latest) self.all_packages.extend(packages) to_process = [] num_passed = len(packages) log(0, "Packages in repo: %5d" % plug.num_packages) if plug.num_excluded: log(0, "Packages passed filter rules: %5d" % num_passed) channel_id = int(self.channel['id']) for pack in packages: db_pack = rhnPackage.get_info_for_package( [pack.name, pack.version, pack.release, pack.epoch, pack.arch], channel_id, self.channel['org_id']) to_download = True to_link = True # Package exists in DB if db_pack: # Path in filesystem is defined if db_pack['path']: pack.path = os.path.join(CFG.MOUNT_POINT, db_pack['path']) else: pack.path = "" if self.metadata_only or self.match_package_checksum(pack.path, pack.checksum_type, pack.checksum): # package is already on disk or not required to_download = False if db_pack['channel_id'] == channel_id: # package is already in the channel to_link = False elif db_pack['channel_id'] == channel_id: # different package with SAME NVREA self.disassociate_package(db_pack) # just pass data from DB, they will be used if there is no RPM available pack.checksum = db_pack['checksum'] pack.checksum_type = db_pack['checksum_type'] pack.epoch = db_pack['epoch'] if to_download or to_link: to_process.append((pack, to_download, to_link)) num_to_process = len(to_process) if num_to_process == 0: log(0, "No new packages to sync.") # If we are just appending, we can exit if not self.strict: return else: log(0, "Packages already synced: %5d" % (num_passed - num_to_process)) log(0, "Packages to sync: %5d" % num_to_process) self.regen = True is_non_local_repo = (url.find("file:/") < 0) for (index, what) in enumerate(to_process): pack, to_download, to_link = what localpath = None # pylint: disable=W0703 try: log(0, "%d/%d : %s" % (index + 1, num_to_process, pack.getNVREA())) if to_download: pack.path = localpath = plug.get_package(pack, metadata_only=self.metadata_only) pack.load_checksum_from_header() pack.upload_package(self.channel, metadata_only=self.metadata_only) except KeyboardInterrupt: raise except Exception: e = sys.exc_info()[1] log2stderr(0, e) if self.fail: raise to_process[index] = (pack, False, False) continue finally: if is_non_local_repo and localpath and os.path.exists(localpath): os.remove(localpath) log(0, "Linking packages to channel.") if self.strict: import_batch = [self.associate_package(pack) for pack in self.all_packages] else: import_batch = [self.associate_package(pack) for (pack, to_download, to_link) in to_process if to_link] backend = SQLBackend() caller = "server.app.yumreposync" importer = ChannelPackageSubscription(import_batch, backend, caller=caller, repogen=False, strict=self.strict) importer.run() backend.commit()
def _xmlrpc(function, params): try: retval = getattr(BaseWireSource.serverObj, function)(*params) except TypeError, e: log(-1, 'ERROR: during "getattr(BaseWireSource.serverObj, %s)(*(%s))"' % (function, params)) raise
def sync(self, update_repodata=False): """Trigger a reposync""" start_time = datetime.now() for (repo_id, url, repo_label, channel_family_id) in self.urls: log(0, "Repo URL: %s" % url) plugin = None # If the repository uses a uln:// URL, switch to the ULN plugin, overriding the command-line if url.startswith("uln://"): self.repo_plugin = self.load_plugin("uln") # pylint: disable=W0703 try: # use modified relative_url as name of repo plugin, because # it used as name of cache directory as well relative_url = '_'.join(url.split('://')[1].split('/')[1:]) plugin_name = relative_url.replace("?", "_").replace("&", "_").replace("=", "_") plugin = self.repo_plugin(url, plugin_name) if update_repodata: plugin.clear_cache() if repo_id is not None: keys = rhnSQL.fetchone_dict(""" select k1.key as ca_cert, k2.key as client_cert, k3.key as client_key from rhncontentssl join rhncryptokey k1 on rhncontentssl.ssl_ca_cert_id = k1.id left outer join rhncryptokey k2 on rhncontentssl.ssl_client_cert_id = k2.id left outer join rhncryptokey k3 on rhncontentssl.ssl_client_key_id = k3.id where rhncontentssl.content_source_id = :repo_id or rhncontentssl.channel_family_id = :channel_family_id """, repo_id=int(repo_id), channel_family_id=int(channel_family_id)) if keys and ('ca_cert' in keys): plugin.set_ssl_options(keys['ca_cert'], keys['client_cert'], keys['client_key']) if not self.no_packages: self.import_packages(plugin, repo_id, url) self.import_groups(plugin, url) if not self.no_errata: self.import_updates(plugin, url) # only for repos obtained from the DB if self.sync_kickstart and repo_label: try: self.import_kickstart(plugin, repo_label) except: rhnSQL.rollback() raise except Exception: e = sys.exc_info()[1] log2stderr(0, "ERROR: %s" % e) if plugin is not None: plugin.clear_ssl_cache() if self.regen: taskomatic.add_to_repodata_queue_for_channel_package_subscription( [self.channel_label], [], "server.app.yumreposync") taskomatic.add_to_erratacache_queue(self.channel_label) self.update_date() rhnSQL.commit() elapsed_time = datetime.now() - start_time log(0, "Sync of channel completed in %s." % str(elapsed_time).split('.')[0]) return elapsed_time
def import_kickstart(self, plug, repo_label): ks_path = 'rhn/kickstart/' ks_tree_label = re.sub(r'[^-_0-9A-Za-z@.]', '', repo_label.replace(' ', '_')) if len(ks_tree_label) < 4: ks_tree_label += "_repo" # construct ks_path and check we already have this KS tree synced id_request = """ select id from rhnKickstartableTree where channel_id = :channel_id and label = :label """ if self.org_id: ks_path += str(self.org_id) + '/' + ks_tree_label # Trees synced from external repositories are expected to have full path it database db_path = os.path.join(CFG.MOUNT_POINT, ks_path) row = rhnSQL.fetchone_dict(id_request + " and org_id = :org_id", channel_id=self.channel['id'], label=ks_tree_label, org_id=self.org_id) else: ks_path += ks_tree_label db_path = ks_path row = rhnSQL.fetchone_dict(id_request + " and org_id is NULL", channel_id=self.channel['id'], label=ks_tree_label) treeinfo_path = ['treeinfo', '.treeinfo'] treeinfo_parser = None for path in treeinfo_path: log(1, "Trying " + path) treeinfo = plug.get_file( path, os.path.join(plug.repo.basecachedir, plug.name)) if treeinfo: try: treeinfo_parser = TreeInfoParser(treeinfo) break except TreeInfoError: pass if not treeinfo_parser: log(0, "Kickstartable tree not detected (no valid treeinfo file)") return if self.ks_install_type is None: family = treeinfo_parser.get_family() if family == 'Fedora': self.ks_install_type = 'fedora18' elif family == 'CentOS': self.ks_install_type = 'rhel_' + treeinfo_parser.get_major_version( ) else: self.ks_install_type = 'generic_rpm' fileutils.createPath(os.path.join(CFG.MOUNT_POINT, ks_path)) # Make sure images are included to_download = set() for repo_path in treeinfo_parser.get_images(): local_path = os.path.join(CFG.MOUNT_POINT, ks_path, repo_path) # TODO: better check if not os.path.exists(local_path) or self.force_kickstart: to_download.add(repo_path) if row: log( 0, "Kickstartable tree %s already synced. Updating content..." % ks_tree_label) ks_id = row['id'] else: row = rhnSQL.fetchone_dict(""" select sequence_nextval('rhn_kstree_id_seq') as id from dual """) ks_id = row['id'] rhnSQL.execute(""" insert into rhnKickstartableTree (id, org_id, label, base_path, channel_id, kstree_type, install_type, last_modified, created, modified) values (:id, :org_id, :label, :base_path, :channel_id, ( select id from rhnKSTreeType where label = :ks_tree_type), ( select id from rhnKSInstallType where label = :ks_install_type), current_timestamp, current_timestamp, current_timestamp)""", id=ks_id, org_id=self.org_id, label=ks_tree_label, base_path=db_path, channel_id=self.channel['id'], ks_tree_type=self.ks_tree_type, ks_install_type=self.ks_install_type) log( 0, "Added new kickstartable tree %s. Downloading content..." % ks_tree_label) insert_h = rhnSQL.prepare(""" insert into rhnKSTreeFile (kstree_id, relative_filename, checksum_id, file_size, last_modified, created, modified) values (:id, :path, lookup_checksum('sha256', :checksum), :st_size, epoch_seconds_to_timestamp_tz(:st_time), current_timestamp, current_timestamp) """) delete_h = rhnSQL.prepare(""" delete from rhnKSTreeFile where kstree_id = :id and relative_filename = :path """) # Downloading/Updating content of KS Tree # start from root dir is_root = True dirs_queue = [''] log(0, "Gathering all files in kickstart repository...") while len(dirs_queue) > 0: cur_dir_name = dirs_queue.pop(0) cur_dir_html = plug.get_file(cur_dir_name) if cur_dir_html is None: continue blacklist = None if is_root: blacklist = [treeinfo_parser.get_package_dir() + '/'] is_root = False parser = KSDirParser(cur_dir_html, blacklist) for ks_file in parser.get_content(): repo_path = cur_dir_name + ks_file['name'] # if this is a directory, just add a name into queue (like BFS algorithm) if ks_file['type'] == 'DIR': dirs_queue.append(repo_path) continue if not os.path.exists( os.path.join(CFG.MOUNT_POINT, ks_path, repo_path)) or self.force_kickstart: to_download.add(repo_path) if to_download: log(0, "Downloading %d kickstart files." % len(to_download)) progress_bar = ProgressBarLogger("Downloading kickstarts:", len(to_download)) downloader = ThreadedDownloader(force=self.force_kickstart) for item in to_download: params = {} plug.set_download_parameters( params, item, os.path.join(CFG.MOUNT_POINT, ks_path, item)) downloader.add(params) downloader.set_log_obj(progress_bar) downloader.run() log2disk(0, "Download finished.") for item in to_download: st = os.stat(os.path.join(CFG.MOUNT_POINT, ks_path, item)) # update entity about current file in a database delete_h.execute(id=ks_id, path=item) insert_h.execute(id=ks_id, path=item, checksum=getFileChecksum( 'sha256', os.path.join(CFG.MOUNT_POINT, ks_path, item)), st_size=st.st_size, st_time=st.st_mtime) else: log(0, "No new kickstart files to download.") # set permissions recursively rhnSQL.commit()
def import_kickstart(self, plug, repo_label): ks_path = 'rhn/kickstart/' ks_tree_label = re.sub(r'[^-_0-9A-Za-z@.]', '', repo_label.replace(' ', '_')) if len(ks_tree_label) < 4: ks_tree_label += "_repo" # construct ks_path and check we already have this KS tree synced id_request = """ select id from rhnKickstartableTree where channel_id = :channel_id and label = :label """ if 'org_id' in self.channel and self.channel['org_id']: ks_path += str(self.channel['org_id']) + '/' + ks_tree_label # Trees synced from external repositories are expected to have full path it database db_path = os.path.join(CFG.MOUNT_POINT, ks_path) row = rhnSQL.fetchone_dict(id_request + " and org_id = :org_id", channel_id=self.channel['id'], label=ks_tree_label, org_id=self.channel['org_id']) else: ks_path += ks_tree_label db_path = ks_path row = rhnSQL.fetchone_dict(id_request + " and org_id is NULL", channel_id=self.channel['id'], label=ks_tree_label) treeinfo_path = ['treeinfo', '.treeinfo'] treeinfo_parser = None for path in treeinfo_path: log(1, "Trying " + path) treeinfo = plug.get_file(path, os.path.join(CFG.MOUNT_POINT, ks_path)) if treeinfo: try: treeinfo_parser = TreeInfoParser(treeinfo) break except TreeInfoError: pass if not treeinfo_parser: log(0, "Kickstartable tree not detected (no valid treeinfo file)") return # Make sure images are included to_download = [] for repo_path in treeinfo_parser.get_images(): local_path = os.path.join(CFG.MOUNT_POINT, ks_path, repo_path) # TODO: better check if not os.path.exists(local_path): to_download.append(repo_path) if row: log(0, "Kickstartable tree %s already synced. Updating content..." % ks_tree_label) ks_id = row['id'] else: row = rhnSQL.fetchone_dict(""" select sequence_nextval('rhn_kstree_id_seq') as id from dual """) ks_id = row['id'] rhnSQL.execute(""" insert into rhnKickstartableTree (id, org_id, label, base_path, channel_id, kstree_type, install_type, last_modified, created, modified) values (:id, :org_id, :label, :base_path, :channel_id, ( select id from rhnKSTreeType where label = :ks_tree_type), ( select id from rhnKSInstallType where label = :ks_install_type), current_timestamp, current_timestamp, current_timestamp)""", id=ks_id, org_id=self.channel['org_id'], label=ks_tree_label, base_path=db_path, channel_id=self.channel['id'], ks_tree_type=self.ks_tree_type, ks_install_type=self.ks_install_type) log(0, "Added new kickstartable tree %s. Downloading content..." % ks_tree_label) insert_h = rhnSQL.prepare(""" insert into rhnKSTreeFile (kstree_id, relative_filename, checksum_id, file_size, last_modified, created, modified) values (:id, :path, lookup_checksum('sha256', :checksum), :st_size, epoch_seconds_to_timestamp_tz(:st_time), current_timestamp, current_timestamp) """) delete_h = rhnSQL.prepare(""" delete from rhnKSTreeFile where kstree_id = :id and relative_filename = :path """) # Downloading/Updating content of KS Tree # start from root dir dirs_queue = [''] log(0, "Gathering all files in kickstart repository...") while len(dirs_queue) > 0: cur_dir_name = dirs_queue.pop(0) cur_dir_html = plug.get_file(cur_dir_name) if cur_dir_html is None: continue parser = KSDirParser(cur_dir_html) for ks_file in parser.get_content(): repo_path = cur_dir_name + ks_file['name'] # if this is a directory, just add a name into queue (like BFS algorithm) if ks_file['type'] == 'DIR': dirs_queue.append(repo_path) continue if repo_path not in to_download: to_download.append(repo_path) if to_download: log(0, "Downloading %d files." % len(to_download)) for item in to_download: for retry in range(3): try: log(1, "Retrieving %s" % item) plug.get_file(item, os.path.join(CFG.MOUNT_POINT, ks_path)) st = os.stat(os.path.join(CFG.MOUNT_POINT, ks_path, item)) break except OSError: # os.stat if the file wasn't downloaded if retry < 3: log(2, "Retry download %s: attempt #%d" % (item, retry + 1)) else: raise # update entity about current file in a database delete_h.execute(id=ks_id, path=item) insert_h.execute(id=ks_id, path=item, checksum=getFileChecksum('sha256', os.path.join(CFG.MOUNT_POINT, ks_path, item)), st_size=st.st_size, st_time=st.st_mtime) else: log(0, "Nothing to download.") rhnSQL.commit()