def bulk_insert(cursor, rpm_nevra, filename, srpm_nevra=None): nvra = parse_nvra(rpm_nevra) if srpm_nevra: srpm_name = parse_nvra(srpm_nevra)["name"] else: srpm_name = nvra["name"] sql = add_returning("""INSERT INTO %s (name, epoch, version, release, arch, srpm_nevra, srpm_name, filename) VALUES (%%s, %%s, %%s, %%s, %%s, %%s, %%s, %%s)""" % RPM._meta.db_table) try: sid = transaction.savepoint() RPM.check_srpm_nevra(rpm_nevra, srpm_nevra) cursor.execute(sql, [nvra["name"], nvra["epoch"], nvra["version"], nvra["release"], nvra["arch"], srpm_nevra, srpm_name, filename]) if connection.features.can_return_id_from_insert: insert_id = connection.ops.fetch_returned_insert_id(cursor) else: insert_id = connection.ops.last_insert_id(cursor, RPM._meta.db_table, "id") except (IntegrityError, ValidationError): transaction.savepoint_rollback(sid) cursor.execute("""SELECT %s FROM %s WHERE name=%%s AND epoch=%%s AND version=%%s and release=%%s AND arch=%%s""" % ("id", RPM._meta.db_table), [nvra["name"], nvra["epoch"], nvra["version"], nvra["release"], nvra["arch"]]) insert_id = int(cursor.fetchone()[0]) transaction.savepoint_commit(sid) return insert_id
def test_valid_nvra(self): self.assertEqual(parse_nvra("net-snmp-5.3.2.2-5.el5.i386"), dict(name="net-snmp", version="5.3.2.2", release="5.el5", epoch="", arch="i386", src=False)) self.assertEqual(parse_nvra("net-snmp-5.3.2.2-5.el5.i386.rpm"), dict(name="net-snmp", version="5.3.2.2", release="5.el5", epoch="", arch="i386", src=False)) self.assertEqual(parse_nvra("net-snmp-5.3.2.2-5.el5.src.rpm"), dict(name="net-snmp", version="5.3.2.2", release="5.el5", epoch="", arch="src", src=True)) self.assertEqual(parse_nvra("/net-snmp-5.3.2.2-5.el5.src.rpm:1"), dict(name="net-snmp", version="5.3.2.2", release="5.el5", epoch="1", arch="src", src=True)) self.assertEqual(parse_nvra("/1:net-snmp-5.3.2.2-5.el5.src.rpm"), dict(name="net-snmp", version="5.3.2.2", release="5.el5", epoch="1", arch="src", src=True)) self.assertEqual(parse_nvra("foo/net-snmp-5.3.2.2-5.el5.src.rpm:1"), dict(name="net-snmp", version="5.3.2.2", release="5.el5", epoch="1", arch="src", src=True)) self.assertEqual(parse_nvra("foo/1:net-snmp-5.3.2.2-5.el5.src.rpm"), dict(name="net-snmp", version="5.3.2.2", release="5.el5", epoch="1", arch="src", src=True)) self.assertEqual(parse_nvra("/foo/bar/net-snmp-5.3.2.2-5.el5.src.rpm:1"), dict(name="net-snmp", version="5.3.2.2", release="5.el5", epoch="1", arch="src", src=True)) self.assertEqual(parse_nvra("/foo/bar/1:net-snmp-5.3.2.2-5.el5.src.rpm"), dict(name="net-snmp", version="5.3.2.2", release="5.el5", epoch="1", arch="src", src=True))
def fill_rpm_basic_info(hdr, rpm_name, rpm_path, rpm_name_to_path_dict, is_src_rpm): # It will return like # {'src': False, 'name': 'patternfly1', 'epoch': '', 'version': '1.0.5', 'release': '4.el7eng', 'arch': 'noarch', # 'filename': gtk-vnc2-0.5.2-7.el7.x86_64.rpm} result = parse_nvra(hdr[rpm.RPMTAG_NEVRA]) result['srpm_name'] = '' result['srpm_nevra'] = None result['filename'] = rpm_name if not result['epoch']: result['epoch'] = 0 if rpm_is_src(hdr, rpm_path) or is_src_rpm: # srpm_nevra should be empty if and only if arch is src. result['srpm_name'] = result["name"] result['srpm_nevra'] = None result['src'] = True result['arch'] = 'src' else: # Get srpm information source_rpm = hdr[rpm.RPMTAG_SOURCERPM] if source_rpm in rpm_name_to_path_dict: # mark as src rpm to prevent dead loop srpm_info = parse_rpm(source_rpm, rpm_name_to_path_dict[source_rpm], rpm_name_to_path_dict, True) result['srpm_name'] = srpm_info["name"] result['srpm_nevra'] = "%s-%s:%s-%s.%s" % (srpm_info['name'], srpm_info['epoch'], srpm_info['version'], srpm_info['release'], srpm_info['arch']) return result
def test_invalid_nvra(self): self.assertEqual( parse_nvra("net-snmp-5.3.2.2-5.el5.i386.rpm:1"), dict(name="net-snmp", version="5.3.2.2", release="5.el5", epoch="1", arch="i386", src=False)) self.assertEqual( parse_nvra("net-snmp-5.3.2.2-5.el5.i386:1.rpm"), dict(name="net-snmp", version="5.3.2.2", release="5.el5", epoch="1", arch="i386", src=False)) self.assertRaises(ValueError, parse_nvra, "net-snmp-5.3.2.2-5")
def parse_json_to_xml(self): result = self.downloade_manifest() assert result is True, "Failed to download package manifest." manifest_content = self.load_json() try: # XML - create a Dom object impl = minidom.getDOMImplementation() dom = impl.createDocument(None, 'rhel', None) root = dom.documentElement # Get Dict content if isinstance(manifest_content, list) == False: manifest_content = [manifest_content] for data in manifest_content: # Dict Key list['cdn','Compose' ,'rhn'] sat5_data = data["rhn"]["channels"] if isinstance(sat5_data.keys(), list) == False: assert False, "Failed to get content from package manifest." # Prepare to analyze manifest of rhn part for SAT5 testing for key in sat5_data.keys(): # XML - create Elements tag = repoid repoid_item = dom.createElement('repoid') repoid_item.setAttribute('value', str(key)) # XML - add child Element for repoid_item root.appendChild(repoid_item) # XML - create Elements tag = packagename if isinstance(sat5_data[key], list) == True and len(sat5_data[key]) > 0: packagename_item = dom.createElement('packagename') for rpm in sat5_data[key]: rpm_fmt = parse_nvra(rpm) wline = "%s %s %s %s" % ( rpm_fmt['name'], rpm_fmt['version'], rpm_fmt['release'], rpm_fmt['arch']) packagename_text = dom.createTextNode(wline) packagename_item.appendChild(packagename_text) # XML - add child Element for packagename_item repoid_item.appendChild(packagename_item) logger.info('Begin to write XML file {0} for SAT5.'.format( self.manifest_xml)) with open(self.manifest_xml, 'w') as f: dom.writexml(f, addindent=' ' * 4, newl='\n', encoding='utf-8') except Exception, e: logger.error(str(e)) logger.error("ERROR - Failed to parse json file when load it.") logger.error(traceback.format_exc()) assert False, str(e)
def parse_json_to_xml(self): result = self.downloade_manifest() assert result is True, "Failed to download package manifest." manifest_content = self.load_json() try: # XML - create a Dom object impl = minidom.getDOMImplementation() dom = impl.createDocument(None, 'rhel', None) root = dom.documentElement # Get Dict content if isinstance(manifest_content, list) == False: manifest_content = [manifest_content] for data in manifest_content: # Dict Key list['cdn','Compose' ,'rhn'] rhn_data = data["rhn"]["channels"] if isinstance(rhn_data.keys(), list) == False: assert False, "Failed to get content from package manifest." # Prepare to analyze manifest of rhn part for key in rhn_data.keys(): # XML - create Elements tag = repoid repoid_item = dom.createElement('repoid') repoid_item.setAttribute('value', str(key)) # XML - add child Element for repoid_item root.appendChild(repoid_item) # XML - create Elements tag = packagename if isinstance(rhn_data[key], list) == True and len (rhn_data[key]) > 0: packagename_item = dom.createElement('packagename') for rpm in rhn_data[key]: rpm_fmt = parse_nvra(rpm) wline = "%s %s %s %s" % (rpm_fmt['name'], rpm_fmt['version'], rpm_fmt['release'], rpm_fmt['arch']) packagename_text = dom.createTextNode(wline) packagename_item.appendChild(packagename_text) # XML - add child Element for packagename_item repoid_item.appendChild(packagename_item) logger.info('Begin to write RHN XML file {0}'.format(self.manifest_xml)) with open(self.manifest_xml, 'w') as f: dom.writexml(f, addindent=' '*4, newl='\n', encoding='utf-8') except Exception, e: logger.error(str(e)) logger.error("ERROR - Failed to parse json file when load it.") logger.error(traceback.format_exc()) assert False, str(e)
def test_invalid_nvra(self): self.assertEqual(parse_nvra("net-snmp-5.3.2.2-5.el5.i386.rpm:1"), dict(name="net-snmp", version="5.3.2.2", release="5.el5", epoch="1", arch="i386", src=False)) self.assertEqual(parse_nvra("net-snmp-5.3.2.2-5.el5.i386:1.rpm"), dict(name="net-snmp", version="5.3.2.2", release="5.el5", epoch="1", arch="i386", src=False)) self.assertRaises(ValueError, parse_nvra, "net-snmp-5.3.2.2-5")
def parse_json_to_xml(self): result = self.downloade_manifest() assert result is True, "Failed to download package manifest." manifest_content = self.load_json() try: # XML - create a Dom object impl = minidom.getDOMImplementation() dom = impl.createDocument(None, 'rhel', None) root = dom.documentElement # Get Dict content if isinstance(manifest_content,list) == False: manifest_content = [manifest_content] for data in manifest_content: # Dict Key list['cdn','Compose' ,'rhn'] cdn_data = data["cdn"]["products"] if isinstance(cdn_data.keys(), list) == False: assert False, "Failed to get content from package manifest." # Prepare to analyze manifest of cdn part # Get items from Product ID lists for key in cdn_data.keys(): oneDict = cdn_data[key] product_id_value = oneDict["Product ID"] product_name_value = oneDict["Name"] platform_value = oneDict["Platform"] if oneDict["Platform Version"] != None and "Platform Version" in oneDict.keys(): platform_version_value = oneDict["Platform Version"] product_version_value = oneDict["Product Version"] repo_paths_value = oneDict["Repo Paths"] # XML - create Elements tag = productid productid_item = dom.createElement('productid') # XML - set tag attributes for tag productid if product_id_value: productid_item.setAttribute('value', str(product_id_value)) else: productid_item.setAttribute('value', product_name_value) productid_item.setAttribute('name', product_name_value) productid_item.setAttribute('platform', platform_value) if oneDict["Platform Version"] != None and "Platform Version" in oneDict.keys(): productid_item.setAttribute('platform_version', platform_version_value) productid_item.setAttribute('product_version', product_version_value) # XML - add child item for root root.appendChild(productid_item) if isinstance(repo_paths_value.keys(),list) == True: # Json parse: add all repo_data in different arch lists basearch_list = [] basearch_dict = {} for key in repo_paths_value.keys(): basearch_value = repo_paths_value[key]["basearch"] if basearch_value not in basearch_list: basearch_dict[basearch_value] = [] basearch_list.append(basearch_value) repo_paths_value[key]["relativeurl"] = key basearch_dict[basearch_value].append(repo_paths_value[key]) for basearch_value in basearch_list: # XML - create Element tag = arch arch_item = dom.createElement('arch') arch_item.setAttribute('value', basearch_value) # XML - add child element arch_item for productid_item productid_item.appendChild(arch_item) for repo_dict in basearch_dict[basearch_value]: # XML - create Element tag = repoid repoid_item = dom.createElement('repoid') repoid_name = repo_dict["Label"] repoid_item.setAttribute('value', repoid_name) if "releasever" in repo_dict.keys(): releasever = repo_dict['releasever'] repoid_item.setAttribute('releasever', releasever) # XML - add child Element for arch_item arch_item.appendChild(repoid_item) # XML - create Element tag = relativeurl relativeurl_item = dom.createElement('relativeurl') relativeurl = repo_dict["relativeurl"] relativeurl_text = dom.createTextNode(relativeurl) relativeurl_item.appendChild(relativeurl_text) # XML - add child Element for relativeurl_item repoid_item.appendChild(relativeurl_item) # Json parse: parse each rpm with format %{name} %{version} %{release} %{arch} if isinstance(repo_dict["RPMs"], list) == True and len(repo_dict["RPMs"]) > 0: # XML - create Element tag = packagename packagename_item = dom.createElement('packagename') for rpm_pkg in repo_dict["RPMs"]: rpm_fmt = parse_nvra(rpm_pkg) wline = "%s %s %s %s" % (rpm_fmt['name'], rpm_fmt['version'], rpm_fmt['release'], rpm_fmt['arch']) packagename_text = dom.createTextNode(wline) packagename_item.appendChild(packagename_text) # XML - add child Element for packagename_item repoid_item.appendChild(packagename_item) logger.info('Begin to write CDN XML file {0}'.format(self.manifest_xml)) with open(self.manifest_xml, 'w') as f: dom.writexml(f, addindent=' '*4, newl='\n', encoding='utf-8') except Exception, e: logger.error(str(e)) logger.error(traceback.format_exc()) logger.error("ERROR - Failed to parse json file after load it.") assert False, str(e)