def bulk_insert(cursor, rpm_nevra, filename, srpm_nevra=None):
nvra = parse_nvra(rpm_nevra)
if srpm_nevra:
srpm_name = parse_nvra(srpm_nevra)["name"]
else:
srpm_name = nvra["name"]
sql = add_returning("""INSERT INTO %s (name, epoch, version, release, arch, srpm_nevra, srpm_name, filename)
VALUES (%%s, %%s, %%s, %%s, %%s, %%s, %%s, %%s)""" % RPM._meta.db_table)
try:
sid = transaction.savepoint()
RPM.check_srpm_nevra(rpm_nevra, srpm_nevra)
cursor.execute(sql, [nvra["name"], nvra["epoch"], nvra["version"], nvra["release"],
nvra["arch"], srpm_nevra, srpm_name, filename])
if connection.features.can_return_id_from_insert:
insert_id = connection.ops.fetch_returned_insert_id(cursor)
else:
insert_id = connection.ops.last_insert_id(cursor, RPM._meta.db_table, "id")
except (IntegrityError, ValidationError):
transaction.savepoint_rollback(sid)
cursor.execute("""SELECT %s FROM %s WHERE name=%%s AND epoch=%%s AND
version=%%s and release=%%s AND arch=%%s""" % ("id", RPM._meta.db_table),
[nvra["name"], nvra["epoch"], nvra["version"], nvra["release"], nvra["arch"]])
insert_id = int(cursor.fetchone()[0])
transaction.savepoint_commit(sid)
return insert_id
def bulk_insert(cursor, rpm_nevra, filename, srpm_nevra=None):
nvra = parse_nvra(rpm_nevra)
if srpm_nevra:
srpm_name = parse_nvra(srpm_nevra)["name"]
else:
srpm_name = nvra["name"]
sql = add_returning("""INSERT INTO %s (name, epoch, version, release, arch, srpm_nevra, srpm_name, filename)
VALUES (%%s, %%s, %%s, %%s, %%s, %%s, %%s, %%s)""" % RPM._meta.db_table)
try:
sid = transaction.savepoint()
RPM.check_srpm_nevra(rpm_nevra, srpm_nevra)
cursor.execute(sql, [nvra["name"], nvra["epoch"], nvra["version"], nvra["release"],
nvra["arch"], srpm_nevra, srpm_name, filename])
if connection.features.can_return_id_from_insert:
insert_id = connection.ops.fetch_returned_insert_id(cursor)
else:
insert_id = connection.ops.last_insert_id(cursor, RPM._meta.db_table, "id")
except (IntegrityError, ValidationError):
transaction.savepoint_rollback(sid)
cursor.execute("""SELECT %s FROM %s WHERE name=%%s AND epoch=%%s AND
version=%%s and release=%%s AND arch=%%s""" % ("id", RPM._meta.db_table),
[nvra["name"], nvra["epoch"], nvra["version"], nvra["release"], nvra["arch"]])
insert_id = int(cursor.fetchone()[0])
transaction.savepoint_commit(sid)
return insert_id
def test_valid_nvra(self):
self.assertEqual(parse_nvra("net-snmp-5.3.2.2-5.el5.i386"), dict(name="net-snmp", version="5.3.2.2", release="5.el5", epoch="", arch="i386", src=False))
self.assertEqual(parse_nvra("net-snmp-5.3.2.2-5.el5.i386.rpm"), dict(name="net-snmp", version="5.3.2.2", release="5.el5", epoch="", arch="i386", src=False))
self.assertEqual(parse_nvra("net-snmp-5.3.2.2-5.el5.src.rpm"), dict(name="net-snmp", version="5.3.2.2", release="5.el5", epoch="", arch="src", src=True))
self.assertEqual(parse_nvra("/net-snmp-5.3.2.2-5.el5.src.rpm:1"), dict(name="net-snmp", version="5.3.2.2", release="5.el5", epoch="1", arch="src", src=True))
self.assertEqual(parse_nvra("/1:net-snmp-5.3.2.2-5.el5.src.rpm"), dict(name="net-snmp", version="5.3.2.2", release="5.el5", epoch="1", arch="src", src=True))
self.assertEqual(parse_nvra("foo/net-snmp-5.3.2.2-5.el5.src.rpm:1"), dict(name="net-snmp", version="5.3.2.2", release="5.el5", epoch="1", arch="src", src=True))
self.assertEqual(parse_nvra("foo/1:net-snmp-5.3.2.2-5.el5.src.rpm"), dict(name="net-snmp", version="5.3.2.2", release="5.el5", epoch="1", arch="src", src=True))
self.assertEqual(parse_nvra("/foo/bar/net-snmp-5.3.2.2-5.el5.src.rpm:1"), dict(name="net-snmp", version="5.3.2.2", release="5.el5", epoch="1", arch="src", src=True))
self.assertEqual(parse_nvra("/foo/bar/1:net-snmp-5.3.2.2-5.el5.src.rpm"), dict(name="net-snmp", version="5.3.2.2", release="5.el5", epoch="1", arch="src", src=True))
def test_valid_nvra(self):
self.assertEqual(parse_nvra("net-snmp-5.3.2.2-5.el5.i386"), dict(name="net-snmp", version="5.3.2.2", release="5.el5", epoch="", arch="i386", src=False))
self.assertEqual(parse_nvra("net-snmp-5.3.2.2-5.el5.i386.rpm"), dict(name="net-snmp", version="5.3.2.2", release="5.el5", epoch="", arch="i386", src=False))
self.assertEqual(parse_nvra("net-snmp-5.3.2.2-5.el5.src.rpm"), dict(name="net-snmp", version="5.3.2.2", release="5.el5", epoch="", arch="src", src=True))
self.assertEqual(parse_nvra("/net-snmp-5.3.2.2-5.el5.src.rpm:1"), dict(name="net-snmp", version="5.3.2.2", release="5.el5", epoch="1", arch="src", src=True))
self.assertEqual(parse_nvra("/1:net-snmp-5.3.2.2-5.el5.src.rpm"), dict(name="net-snmp", version="5.3.2.2", release="5.el5", epoch="1", arch="src", src=True))
self.assertEqual(parse_nvra("foo/net-snmp-5.3.2.2-5.el5.src.rpm:1"), dict(name="net-snmp", version="5.3.2.2", release="5.el5", epoch="1", arch="src", src=True))
self.assertEqual(parse_nvra("foo/1:net-snmp-5.3.2.2-5.el5.src.rpm"), dict(name="net-snmp", version="5.3.2.2", release="5.el5", epoch="1", arch="src", src=True))
self.assertEqual(parse_nvra("/foo/bar/net-snmp-5.3.2.2-5.el5.src.rpm:1"), dict(name="net-snmp", version="5.3.2.2", release="5.el5", epoch="1", arch="src", src=True))
self.assertEqual(parse_nvra("/foo/bar/1:net-snmp-5.3.2.2-5.el5.src.rpm"), dict(name="net-snmp", version="5.3.2.2", release="5.el5", epoch="1", arch="src", src=True))
def fill_rpm_basic_info(hdr, rpm_name, rpm_path, rpm_name_to_path_dict, is_src_rpm):
# It will return like
# {'src': False, 'name': 'patternfly1', 'epoch': '', 'version': '1.0.5', 'release': '4.el7eng', 'arch': 'noarch',
# 'filename': gtk-vnc2-0.5.2-7.el7.x86_64.rpm}
result = parse_nvra(hdr[rpm.RPMTAG_NEVRA])
result['srpm_name'] = ''
result['srpm_nevra'] = None
result['filename'] = rpm_name
if not result['epoch']:
result['epoch'] = 0
if rpm_is_src(hdr, rpm_path) or is_src_rpm:
# srpm_nevra should be empty if and only if arch is src.
result['srpm_name'] = result["name"]
result['srpm_nevra'] = None
result['src'] = True
result['arch'] = 'src'
else:
# Get srpm information
source_rpm = hdr[rpm.RPMTAG_SOURCERPM]
if source_rpm in rpm_name_to_path_dict:
# mark as src rpm to prevent dead loop
srpm_info = parse_rpm(source_rpm, rpm_name_to_path_dict[source_rpm], rpm_name_to_path_dict, True)
result['srpm_name'] = srpm_info["name"]
result['srpm_nevra'] = "%s-%s:%s-%s.%s" % (srpm_info['name'], srpm_info['epoch'],
srpm_info['version'], srpm_info['release'], srpm_info['arch'])
return result
def test_invalid_nvra(self):
self.assertEqual(
parse_nvra("net-snmp-5.3.2.2-5.el5.i386.rpm:1"),
dict(name="net-snmp",
version="5.3.2.2",
release="5.el5",
epoch="1",
arch="i386",
src=False))
self.assertEqual(
parse_nvra("net-snmp-5.3.2.2-5.el5.i386:1.rpm"),
dict(name="net-snmp",
version="5.3.2.2",
release="5.el5",
epoch="1",
arch="i386",
src=False))
self.assertRaises(ValueError, parse_nvra, "net-snmp-5.3.2.2-5")
def parse_json_to_xml(self):
result = self.downloade_manifest()
assert result is True, "Failed to download package manifest."
manifest_content = self.load_json()
try:
# XML - create a Dom object
impl = minidom.getDOMImplementation()
dom = impl.createDocument(None, 'rhel', None)
root = dom.documentElement
# Get Dict content
if isinstance(manifest_content, list) == False:
manifest_content = [manifest_content]
for data in manifest_content:
# Dict Key list['cdn','Compose' ,'rhn']
sat5_data = data["rhn"]["channels"]
if isinstance(sat5_data.keys(), list) == False:
assert False, "Failed to get content from package manifest."
# Prepare to analyze manifest of rhn part for SAT5 testing
for key in sat5_data.keys():
# XML - create Elements tag = repoid
repoid_item = dom.createElement('repoid')
repoid_item.setAttribute('value', str(key))
# XML - add child Element for repoid_item
root.appendChild(repoid_item)
# XML - create Elements tag = packagename
if isinstance(sat5_data[key],
list) == True and len(sat5_data[key]) > 0:
packagename_item = dom.createElement('packagename')
for rpm in sat5_data[key]:
rpm_fmt = parse_nvra(rpm)
wline = "%s %s %s %s" % (
rpm_fmt['name'], rpm_fmt['version'],
rpm_fmt['release'], rpm_fmt['arch'])
packagename_text = dom.createTextNode(wline)
packagename_item.appendChild(packagename_text)
# XML - add child Element for packagename_item
repoid_item.appendChild(packagename_item)
logger.info('Begin to write XML file {0} for SAT5.'.format(
self.manifest_xml))
with open(self.manifest_xml, 'w') as f:
dom.writexml(f,
addindent=' ' * 4,
newl='\n',
encoding='utf-8')
except Exception, e:
logger.error(str(e))
logger.error("ERROR - Failed to parse json file when load it.")
logger.error(traceback.format_exc())
assert False, str(e)
def parse_json_to_xml(self):
result = self.downloade_manifest()
assert result is True, "Failed to download package manifest."
manifest_content = self.load_json()
try:
# XML - create a Dom object
impl = minidom.getDOMImplementation()
dom = impl.createDocument(None, 'rhel', None)
root = dom.documentElement
# Get Dict content
if isinstance(manifest_content, list) == False:
manifest_content = [manifest_content]
for data in manifest_content:
# Dict Key list['cdn','Compose' ,'rhn']
rhn_data = data["rhn"]["channels"]
if isinstance(rhn_data.keys(), list) == False:
assert False, "Failed to get content from package manifest."
# Prepare to analyze manifest of rhn part
for key in rhn_data.keys():
# XML - create Elements tag = repoid
repoid_item = dom.createElement('repoid')
repoid_item.setAttribute('value', str(key))
# XML - add child Element for repoid_item
root.appendChild(repoid_item)
# XML - create Elements tag = packagename
if isinstance(rhn_data[key], list) == True and len (rhn_data[key]) > 0:
packagename_item = dom.createElement('packagename')
for rpm in rhn_data[key]:
rpm_fmt = parse_nvra(rpm)
wline = "%s %s %s %s" % (rpm_fmt['name'], rpm_fmt['version'], rpm_fmt['release'], rpm_fmt['arch'])
packagename_text = dom.createTextNode(wline)
packagename_item.appendChild(packagename_text)
# XML - add child Element for packagename_item
repoid_item.appendChild(packagename_item)
logger.info('Begin to write RHN XML file {0}'.format(self.manifest_xml))
with open(self.manifest_xml, 'w') as f:
dom.writexml(f, addindent=' '*4, newl='\n', encoding='utf-8')
except Exception, e:
logger.error(str(e))
logger.error("ERROR - Failed to parse json file when load it.")
logger.error(traceback.format_exc())
assert False, str(e)
def test_invalid_nvra(self):
self.assertEqual(parse_nvra("net-snmp-5.3.2.2-5.el5.i386.rpm:1"), dict(name="net-snmp", version="5.3.2.2", release="5.el5", epoch="1", arch="i386", src=False))
self.assertEqual(parse_nvra("net-snmp-5.3.2.2-5.el5.i386:1.rpm"), dict(name="net-snmp", version="5.3.2.2", release="5.el5", epoch="1", arch="i386", src=False))
self.assertRaises(ValueError, parse_nvra, "net-snmp-5.3.2.2-5")
def parse_json_to_xml(self):
result = self.downloade_manifest()
assert result is True, "Failed to download package manifest."
manifest_content = self.load_json()
try:
# XML - create a Dom object
impl = minidom.getDOMImplementation()
dom = impl.createDocument(None, 'rhel', None)
root = dom.documentElement
# Get Dict content
if isinstance(manifest_content,list) == False:
manifest_content = [manifest_content]
for data in manifest_content:
# Dict Key list['cdn','Compose' ,'rhn']
cdn_data = data["cdn"]["products"]
if isinstance(cdn_data.keys(), list) == False:
assert False, "Failed to get content from package manifest."
# Prepare to analyze manifest of cdn part
# Get items from Product ID lists
for key in cdn_data.keys():
oneDict = cdn_data[key]
product_id_value = oneDict["Product ID"]
product_name_value = oneDict["Name"]
platform_value = oneDict["Platform"]
if oneDict["Platform Version"] != None and "Platform Version" in oneDict.keys():
platform_version_value = oneDict["Platform Version"]
product_version_value = oneDict["Product Version"]
repo_paths_value = oneDict["Repo Paths"]
# XML - create Elements tag = productid
productid_item = dom.createElement('productid')
# XML - set tag attributes for tag productid
if product_id_value:
productid_item.setAttribute('value', str(product_id_value))
else:
productid_item.setAttribute('value', product_name_value)
productid_item.setAttribute('name', product_name_value)
productid_item.setAttribute('platform', platform_value)
if oneDict["Platform Version"] != None and "Platform Version" in oneDict.keys():
productid_item.setAttribute('platform_version', platform_version_value)
productid_item.setAttribute('product_version', product_version_value)
# XML - add child item for root
root.appendChild(productid_item)
if isinstance(repo_paths_value.keys(),list) == True:
# Json parse: add all repo_data in different arch lists
basearch_list = []
basearch_dict = {}
for key in repo_paths_value.keys():
basearch_value = repo_paths_value[key]["basearch"]
if basearch_value not in basearch_list:
basearch_dict[basearch_value] = []
basearch_list.append(basearch_value)
repo_paths_value[key]["relativeurl"] = key
basearch_dict[basearch_value].append(repo_paths_value[key])
for basearch_value in basearch_list:
# XML - create Element tag = arch
arch_item = dom.createElement('arch')
arch_item.setAttribute('value', basearch_value)
# XML - add child element arch_item for productid_item
productid_item.appendChild(arch_item)
for repo_dict in basearch_dict[basearch_value]:
# XML - create Element tag = repoid
repoid_item = dom.createElement('repoid')
repoid_name = repo_dict["Label"]
repoid_item.setAttribute('value', repoid_name)
if "releasever" in repo_dict.keys():
releasever = repo_dict['releasever']
repoid_item.setAttribute('releasever', releasever)
# XML - add child Element for arch_item
arch_item.appendChild(repoid_item)
# XML - create Element tag = relativeurl
relativeurl_item = dom.createElement('relativeurl')
relativeurl = repo_dict["relativeurl"]
relativeurl_text = dom.createTextNode(relativeurl)
relativeurl_item.appendChild(relativeurl_text)
# XML - add child Element for relativeurl_item
repoid_item.appendChild(relativeurl_item)
# Json parse: parse each rpm with format %{name} %{version} %{release} %{arch}
if isinstance(repo_dict["RPMs"], list) == True and len(repo_dict["RPMs"]) > 0:
# XML - create Element tag = packagename
packagename_item = dom.createElement('packagename')
for rpm_pkg in repo_dict["RPMs"]:
rpm_fmt = parse_nvra(rpm_pkg)
wline = "%s %s %s %s" % (rpm_fmt['name'], rpm_fmt['version'], rpm_fmt['release'], rpm_fmt['arch'])
packagename_text = dom.createTextNode(wline)
packagename_item.appendChild(packagename_text)
# XML - add child Element for packagename_item
repoid_item.appendChild(packagename_item)
logger.info('Begin to write CDN XML file {0}'.format(self.manifest_xml))
with open(self.manifest_xml, 'w') as f:
dom.writexml(f, addindent=' '*4, newl='\n', encoding='utf-8')
except Exception, e:
logger.error(str(e))
logger.error(traceback.format_exc())
logger.error("ERROR - Failed to parse json file after load it.")
assert False, str(e)