def updateDist(self, kwargs, username, password):
log_debug(3)
self._auth(username, password)
if not kwargs.get('release'):
raise rhnFault(23, "Insufficient data, release missing to update dist", explain=0)
if not kwargs.get('os'):
kwargs['os'] = 'Red Hat Linux'
if kwargs.get('channel_id') is None:
# Missing stuff
raise rhnFault(23, "Insufficient data, channel_id missing to update dist", explain=0)
if kwargs.get('channel_arch_id') is None:
# Missing stuff
raise rhnFault(23, "Insufficient data, channel arch id missing to update dist", explain=0)
try:
rhnSQL.execute("""
insert into rhnDistChannelMap
(channel_id, channel_arch_id, os, release)
values
(:channel_id, :channel_arch_id, :os, :release)
""", kwargs)
except rhnSQL.SQLError, e:
rhnSQL.rollback()
raise rhnFault(23, str(e.args[1]), explain=0 )
def management_disable_file(self, dict):
log_debug(1)
self._get_and_validate_session(dict)
config_channel = dict.get('config_channel')
# XXX Validate the namespace
path = dict.get('path')
t = rhnSQL.Table('rhnConfigFileState', 'label')
state_id_dead = t['dead']['id']
row = rhnSQL.fetchone_dict(self._query_lookup_config_file_by_channel,
config_channel=config_channel,
path=path)
if not row or row['state_id'] == state_id_dead:
raise rhnFault(4011,
"File %s does not exist in channel %s" %
(path, config_channel),
explain=0)
rhnSQL.execute(self._query_update_file_state,
config_file_id=row['id'],
state_id=state_id_dead)
rhnSQL.commit()
return {}
def moveChannelDownloads(self, channel_id, old_channel_family_id,
new_channel_family_id, username, password):
log_debug(3)
self._auth(username, password)
if old_channel_family_id is None or \
old_channel_family_id == new_channel_family_id:
# Nothing to be done here, same channel family
return 0
log_debug(3, " Migrating downloads")
try:
rhnSQL.execute("""
update rhnDownloads
set channel_family_id = :new_channel_family_id
where channel_family_id = :old_channel_family_id
and id in (
select downloads_id
from rhnChannelDownloads
where channel_id = :channel_id)
""",
channel_id=channel_id,
old_channel_family_id=old_channel_family_id,
new_channel_family_id=new_channel_family_id,
)
except rhnSQL.SQLError, e:
rhnSQL.rollback()
raise rhnFault(23, str(e.args[1]), explain=0 )
def _channelPermission(self, label, role, commit, username, password, query):
log_debug(3)
self._auth(username, password)
try:
rhnSQL.execute(query, username = username, label = label, role_label = role)
except rhnSQL.SQLError, e:
rhnSQL.rollback()
raise rhnFault(23, str(e.args[1]), explain=0 )
def setUp(self):
self.table_name = "misatest_%d" % os.getpid()
rhnSQL.initDB(DB)
self._cleanup()
rhnSQL.execute("create table %s (id int, val varchar2(10))" %
self.table_name)
rhnSQL.commit()
def test_ddl_1(self):
"""Tests table creation/table removal"""
table_name = self.table_name + "_1"
rhnSQL.execute("create table %s (id int)" % table_name)
tables = self._list_tables()
self.failUnless(string.upper(table_name) in tables,
"Table %s not created" % table_name)
rhnSQL.execute("drop table %s" % table_name)
def test_ddl_1(self):
"""Tests table creation/table removal"""
table_name = self.table_name + "_1"
rhnSQL.execute("create table %s (id int)" % table_name)
tables = self._list_tables()
self.failUnless(string.upper(table_name) in tables,
"Table %s not created" % table_name)
rhnSQL.execute("drop table %s" % table_name)
def setUp(self):
rhnSQL.initDB(backend="oracle",
username=DB_SETTINGS["user"],
password=DB_SETTINGS["password"],
database=DB_SETTINGS["database"])
self._cleanup()
rhnSQL.execute("create table misatestlob (id int, val blob)")
rhnSQL.execute("create sequence misatestlob_id_seq")
def deleteDist(self, channel_id, username, password):
log_debug(3)
self._auth(username, password)
try:
rhnSQL.execute("""
delete from rhnDistChannelMap where channel_id = :channel_id
""", channel_id=channel_id)
except rhnSQL.SQLError, e:
rhnSQL.rollback()
raise rhnFault(23, str(e.args[1]), explain=0 )
def setUp(self):
self.table_name = "misatest_%d" % os.getpid()
rhnSQL.initDB(backend="oracle",
username=DB_SETTINGS["user"],
password=DB_SETTINGS["password"],
database=DB_SETTINGS["database"])
self._cleanup()
rhnSQL.execute("create table %s (id int, val varchar2(10))" %
self.table_name)
rhnSQL.commit()
def setUp(self):
rhnSQL.initDB(
backend="oracle",
username=DB_SETTINGS["user"],
password=DB_SETTINGS["password"],
database=DB_SETTINGS["database"]
)
self._cleanup()
rhnSQL.execute("create table misatestlob (id int, val blob)")
rhnSQL.execute("create sequence misatestlob_id_seq")
def _cleanup(self):
try:
rhnSQL.execute("drop table misatestlob")
except rhnSQL.SQLStatementPrepareError:
pass
try:
rhnSQL.execute("drop sequence misatestlob_id_seq")
except rhnSQL.SQLStatementPrepareError:
pass
except rhnSQL.SQLError, e:
if e.args[0] != 2289:
raise
def setUp(self):
self.table_name = "misatest_%d" % os.getpid()
rhnSQL.initDB(
backend="oracle",
username=DB_SETTINGS["user"],
password=DB_SETTINGS["password"],
database=DB_SETTINGS["database"]
)
self._cleanup()
rhnSQL.execute("create table %s (id int, val varchar2(10))" %
self.table_name)
rhnSQL.commit()
def test_execute_rowcount(self):
"""Tests row counts"""
# XXX
table_name = "misatest"
rhnSQL.execute("delete from misatest")
ret = rhnSQL.execute("insert into misatest values (1, 1)")
self.assertEqual(ret, 1)
ret = rhnSQL.execute("insert into misatest values (2, 2)")
self.assertEqual(ret, 1)
ret = rhnSQL.execute("delete from misatest")
self.assertEqual(ret, 2)
rhnSQL.commit()
def _cleanup(self):
try:
rhnSQL.execute("drop table misatestlob")
except rhnSQL.SQLStatementPrepareError:
pass
try:
rhnSQL.execute("drop sequence misatestlob_id_seq")
except rhnSQL.SQLStatementPrepareError:
pass
except rhnSQL.SQLError, e:
if e.args[0] != 2289:
raise
def updateChannelMembership(self, channel_id, channel_family_id,
kargs, commit, username, password):
log_debug(3)
authobj = self._auth(username, password)
authobj.isChannelAdmin()
rhnSQL.execute("""
delete from rhnChannelFamilyMembers where channel_id = :channel_id""",
channel_id=channel_id)
try:
rhnSQL.execute("""
insert into rhnChannelFamilyMembers (channel_id, channel_family_id )
values (:channel_id, :channel_family_id)
""", channel_id=channel_id, channel_family_id=channel_family_id)
except rhnSQL.SQLError, e:
rhnSQL.rollback()
raise rhnFault(23, str(e.args[1]), explain=0 )
def management_disable_file(self, dict):
log_debug(1)
self._get_and_validate_session(dict)
config_channel = dict.get("config_channel")
# XXX Validate the namespace
path = dict.get("path")
t = rhnSQL.Table("rhnConfigFileState", "label")
state_id_dead = t["dead"]["id"]
row = rhnSQL.fetchone_dict(self._query_lookup_config_file_by_channel, config_channel=config_channel, path=path)
if not row or row["state_id"] == state_id_dead:
raise rhnFault(4011, "File %s does not exist in channel %s" % (path, config_channel), explain=0)
rhnSQL.execute(self._query_update_file_state, config_file_id=row["id"], state_id=state_id_dead)
rhnSQL.commit()
return {}
def createChannel(self, params, commit, username, password):
log_debug(3)
self._auth(username, password)
params['id'] = rhnSQL.Sequence("rhn_channel_id_seq").next()
fields = []
for f in rhnChannel_fields:
if params.has_key(f):
fields.append(f)
field_names = string.join(fields, ", ")
bind_vars = string.join(map(lambda x: ':' + x, fields), ', ')
try:
rhnSQL.execute("insert into rhnChannel (%s) values (%s)" %
(field_names, bind_vars),
**params)
except rhnSQL.SQLError, e:
rhnSQL.rollback()
raise rhnFault(23, str(e.args[1]), explain=0 )
def test_execute_rowcount(self):
"""Tests row counts"""
table_name = "misatest"
try:
tables = self._list_tables()
if not table_name in tables:
rhnSQL.execute("create table %s (id int, value int)" % table_name)
else:
rhnSQL.execute("delete from %s" % table_name)
insert_statement = rhnSQL.Statement(
"insert into %s values (:item_id, :value)" % table_name
)
h = rhnSQL.prepare(insert_statement)
ret = h.execute(item_id=1, value=2)
self.assertEqual(ret, 1)
ret = h.execute(item_id=2, value=2)
self.assertEqual(ret, 1)
delete_statement = rhnSQL.Statement("delete from %s" % table_name)
h = rhnSQL.prepare(delete_statement)
ret = h.execute()
self.assertEqual(ret, 2)
rhnSQL.commit()
finally:
rhnSQL.execute("drop table %s" % table_name)
def test_execute_rowcount(self):
"""Tests row counts"""
table_name = "misatest"
try:
tables = self._list_tables()
if not table_name in tables:
rhnSQL.execute("create table %s (id int, value int)" %
table_name)
else:
rhnSQL.execute("delete from %s" % table_name)
insert_statement = rhnSQL.Statement(
"insert into %s values (:item_id, :value)" % table_name)
h = rhnSQL.prepare(insert_statement)
ret = h.execute(item_id=1, value=2)
self.assertEqual(ret, 1)
ret = h.execute(item_id=2, value=2)
self.assertEqual(ret, 1)
delete_statement = rhnSQL.Statement("delete from %s" % table_name)
h = rhnSQL.prepare(delete_statement)
ret = h.execute()
self.assertEqual(ret, 2)
rhnSQL.commit()
finally:
rhnSQL.execute("drop table %s" % table_name)
def updateChannel(self, params, channel_id, old_channel_family_id,
new_channel_family_id, commit, username, password):
log_debug(3)
global rhnChannel_fields
authobj = self._auth(username, password)
authobj.isChannelAdmin()
fields = []
for f in rhnChannel_fields:
if params.has_key(f):
fields.append(f)
set_clause = string.join(
map(lambda x: "%s = :%s" % (x, x), fields), ', ')
try:
rhnSQL.execute("update rhnChannel set %s where id = :id" % set_clause,
**params)
except rhnSQL.SQLError, e:
rhnSQL.rollback()
raise rhnFault(23, str(e.args[1]), explain=0 )
def _cleanup(self):
try:
rhnSQL.execute("drop table %s" % self.table_name)
except rhnSQL.SQLStatementPrepareError:
pass
def import_kickstart(self, plug, url, repo_label):
ks_tree_label = re.sub(r'[^-_0-9A-Za-z@.]', '', repo_label.replace(' ', '_'))
if len(ks_tree_label) < 4:
ks_tree_label += "_repo"
pxeboot_path = 'images/pxeboot/'
pxeboot = plug.get_file(pxeboot_path)
if pxeboot is None:
if not re.search(r'/$', url):
url = url + '/'
self.print_msg("Kickstartable tree not detected (no %s%s)" % (url, pxeboot_path))
return
if rhnSQL.fetchone_dict("""
select id
from rhnKickstartableTree
where org_id = :org_id and channel_id = :channel_id and label = :label
""", org_id=self.channel['org_id'], channel_id=self.channel['id'], label=ks_tree_label):
print("Kickstartable tree %s already synced." % ks_tree_label)
return
row = rhnSQL.fetchone_dict("""
select sequence_nextval('rhn_kstree_id_seq') as id from dual
""")
ks_id = row['id']
ks_path = 'rhn/kickstart/%s/%s' % (self.channel['org_id'], ks_tree_label)
row = rhnSQL.execute("""
insert into rhnKickstartableTree (id, org_id, label, base_path, channel_id,
kstree_type, install_type, last_modified, created, modified)
values (:id, :org_id, :label, :base_path, :channel_id,
( select id from rhnKSTreeType where label = 'externally-managed'),
( select id from rhnKSInstallType where label = 'generic_rpm'),
current_timestamp, current_timestamp, current_timestamp)
""", id=ks_id, org_id=self.channel['org_id'], label=ks_tree_label,
base_path=os.path.join(CFG.MOUNT_POINT, ks_path), channel_id=self.channel['id'])
insert_h = rhnSQL.prepare("""
insert into rhnKSTreeFile (kstree_id, relative_filename, checksum_id, file_size, last_modified, created, modified)
values (:id, :path, lookup_checksum('sha256', :checksum), :st_size, epoch_seconds_to_timestamp_tz(:st_time), current_timestamp, current_timestamp)
""")
dirs = ['']
while len(dirs) > 0:
d = dirs.pop(0)
v = None
if d == pxeboot_path:
v = pxeboot
else:
v = plug.get_file(d)
if v is None:
continue
for s in (m.group(1) for m in re.finditer(r'(?i)<a href="(.+?)"', v)):
if (re.match(r'/', s) or re.search(r'\?', s) or re.search(r'\.\.', s)
or re.match(r'[a-zA-Z]+:', s) or re.search(r'\.rpm$', s)):
continue
if re.search(r'/$', s):
dirs.append(d + s)
continue
local_path = os.path.join(CFG.MOUNT_POINT, ks_path, d, s)
if os.path.exists(local_path):
print("File %s%s already present locally" % (d, s))
else:
print("Retrieving %s" % d + s)
plug.get_file(d + s, os.path.join(CFG.MOUNT_POINT, ks_path))
st = os.stat(local_path)
insert_h.execute(id=ks_id, path=d + s, checksum=getFileChecksum('sha256', local_path),
st_size=st.st_size, st_time=st.st_mtime)
rhnSQL.commit()
def import_kickstart(self, plug, repo_label):
ks_path = 'rhn/kickstart/'
ks_tree_label = re.sub(r'[^-_0-9A-Za-z@.]', '', repo_label.replace(' ', '_'))
if len(ks_tree_label) < 4:
ks_tree_label += "_repo"
# construct ks_path and check we already have this KS tree synced
id_request = """
select id
from rhnKickstartableTree
where channel_id = :channel_id and label = :label
"""
if 'org_id' in self.channel and self.channel['org_id']:
ks_path += str(self.channel['org_id']) + '/' + ks_tree_label
# Trees synced from external repositories are expected to have full path it database
db_path = os.path.join(CFG.MOUNT_POINT, ks_path)
row = rhnSQL.fetchone_dict(id_request + " and org_id = :org_id", channel_id=self.channel['id'],
label=ks_tree_label, org_id=self.channel['org_id'])
else:
ks_path += ks_tree_label
db_path = ks_path
row = rhnSQL.fetchone_dict(id_request + " and org_id is NULL", channel_id=self.channel['id'],
label=ks_tree_label)
treeinfo_path = ['treeinfo', '.treeinfo']
treeinfo_parser = None
for path in treeinfo_path:
log(1, "Trying " + path)
treeinfo = plug.get_file(path, os.path.join(CFG.MOUNT_POINT, ks_path))
if treeinfo:
try:
treeinfo_parser = TreeInfoParser(treeinfo)
break
except TreeInfoError:
pass
if not treeinfo_parser:
log(0, "Kickstartable tree not detected (no valid treeinfo file)")
return
if self.ks_install_type is None:
family = treeinfo_parser.get_family()
if family == 'Fedora':
self.ks_install_type = 'fedora18'
elif family == 'CentOS':
self.ks_install_type = 'rhel_' + treeinfo_parser.get_major_version()
else:
self.ks_install_type = 'generic_rpm'
# Make sure images are included
to_download = []
for repo_path in treeinfo_parser.get_images():
local_path = os.path.join(CFG.MOUNT_POINT, ks_path, repo_path)
# TODO: better check
if not os.path.exists(local_path):
to_download.append(repo_path)
if row:
log(0, "Kickstartable tree %s already synced. Updating content..." % ks_tree_label)
ks_id = row['id']
else:
row = rhnSQL.fetchone_dict("""
select sequence_nextval('rhn_kstree_id_seq') as id from dual
""")
ks_id = row['id']
rhnSQL.execute("""
insert into rhnKickstartableTree (id, org_id, label, base_path, channel_id, kstree_type,
install_type, last_modified, created, modified)
values (:id, :org_id, :label, :base_path, :channel_id,
( select id from rhnKSTreeType where label = :ks_tree_type),
( select id from rhnKSInstallType where label = :ks_install_type),
current_timestamp, current_timestamp, current_timestamp)""", id=ks_id,
org_id=self.channel['org_id'], label=ks_tree_label, base_path=db_path,
channel_id=self.channel['id'], ks_tree_type=self.ks_tree_type,
ks_install_type=self.ks_install_type)
log(0, "Added new kickstartable tree %s. Downloading content..." % ks_tree_label)
insert_h = rhnSQL.prepare("""
insert into rhnKSTreeFile (kstree_id, relative_filename, checksum_id, file_size, last_modified, created,
modified) values (:id, :path, lookup_checksum('sha256', :checksum), :st_size,
epoch_seconds_to_timestamp_tz(:st_time), current_timestamp, current_timestamp)
""")
delete_h = rhnSQL.prepare("""
delete from rhnKSTreeFile where kstree_id = :id and relative_filename = :path
""")
# Downloading/Updating content of KS Tree
# start from root dir
dirs_queue = ['']
log(0, "Gathering all files in kickstart repository...")
while len(dirs_queue) > 0:
cur_dir_name = dirs_queue.pop(0)
cur_dir_html = plug.get_file(cur_dir_name)
if cur_dir_html is None:
continue
parser = KSDirParser(cur_dir_html)
for ks_file in parser.get_content():
repo_path = cur_dir_name + ks_file['name']
# if this is a directory, just add a name into queue (like BFS algorithm)
if ks_file['type'] == 'DIR':
dirs_queue.append(repo_path)
continue
if repo_path not in to_download:
to_download.append(repo_path)
if to_download:
log(0, "Downloading %d files." % len(to_download))
for item in to_download:
for retry in range(3):
try:
log(1, "Retrieving %s" % item)
plug.get_file(item, os.path.join(CFG.MOUNT_POINT, ks_path))
st = os.stat(os.path.join(CFG.MOUNT_POINT, ks_path, item))
break
except OSError: # os.stat if the file wasn't downloaded
if retry < 3:
log(2, "Retry download %s: attempt #%d" % (item, retry + 1))
else:
raise
# update entity about current file in a database
delete_h.execute(id=ks_id, path=item)
insert_h.execute(id=ks_id, path=item,
checksum=getFileChecksum('sha256', os.path.join(CFG.MOUNT_POINT, ks_path, item)),
st_size=st.st_size, st_time=st.st_mtime)
else:
log(0, "Nothing to download.")
rhnSQL.commit()
def import_kickstart(self, plug, repo_label):
ks_path = 'rhn/kickstart/'
ks_tree_label = re.sub(r'[^-_0-9A-Za-z@.]', '', repo_label.replace(' ', '_'))
if len(ks_tree_label) < 4:
ks_tree_label += "_repo"
# construct ks_path and check we already have this KS tree synced
id_request = """
select id
from rhnKickstartableTree
where channel_id = :channel_id and label = :label
"""
if 'org_id' in self.channel and self.channel['org_id']:
ks_path += str(self.channel['org_id']) + '/' + ks_tree_label
# Trees synced from external repositories are expected to have full path it database
db_path = os.path.join(CFG.MOUNT_POINT, ks_path)
row = rhnSQL.fetchone_dict(id_request + " and org_id = :org_id", channel_id=self.channel['id'],
label=ks_tree_label, org_id=self.channel['org_id'])
else:
ks_path += ks_tree_label
db_path = ks_path
row = rhnSQL.fetchone_dict(id_request + " and org_id is NULL", channel_id=self.channel['id'],
label=ks_tree_label)
treeinfo_path = ['treeinfo', '.treeinfo']
treeinfo_parser = None
for path in treeinfo_path:
log(1, "Trying " + path)
treeinfo = plug.get_file(path, os.path.join(CFG.MOUNT_POINT, ks_path))
if treeinfo:
try:
treeinfo_parser = TreeInfoParser(treeinfo)
break
except TreeInfoError:
pass
if not treeinfo_parser:
log(0, "Kickstartable tree not detected (no valid treeinfo file)")
return
# Make sure images are included
to_download = []
for repo_path in treeinfo_parser.get_images():
local_path = os.path.join(CFG.MOUNT_POINT, ks_path, repo_path)
# TODO: better check
if not os.path.exists(local_path):
to_download.append(repo_path)
if row:
log(0, "Kickstartable tree %s already synced. Updating content..." % ks_tree_label)
ks_id = row['id']
else:
row = rhnSQL.fetchone_dict("""
select sequence_nextval('rhn_kstree_id_seq') as id from dual
""")
ks_id = row['id']
rhnSQL.execute("""
insert into rhnKickstartableTree (id, org_id, label, base_path, channel_id, kstree_type,
install_type, last_modified, created, modified)
values (:id, :org_id, :label, :base_path, :channel_id,
( select id from rhnKSTreeType where label = :ks_tree_type),
( select id from rhnKSInstallType where label = :ks_install_type),
current_timestamp, current_timestamp, current_timestamp)""", id=ks_id,
org_id=self.channel['org_id'], label=ks_tree_label, base_path=db_path,
channel_id=self.channel['id'], ks_tree_type=self.ks_tree_type,
ks_install_type=self.ks_install_type)
log(0, "Added new kickstartable tree %s. Downloading content..." % ks_tree_label)
insert_h = rhnSQL.prepare("""
insert into rhnKSTreeFile (kstree_id, relative_filename, checksum_id, file_size, last_modified, created,
modified) values (:id, :path, lookup_checksum('sha256', :checksum), :st_size,
epoch_seconds_to_timestamp_tz(:st_time), current_timestamp, current_timestamp)
""")
delete_h = rhnSQL.prepare("""
delete from rhnKSTreeFile where kstree_id = :id and relative_filename = :path
""")
# Downloading/Updating content of KS Tree
# start from root dir
dirs_queue = ['']
log(0, "Gathering all files in kickstart repository...")
while len(dirs_queue) > 0:
cur_dir_name = dirs_queue.pop(0)
cur_dir_html = plug.get_file(cur_dir_name)
if cur_dir_html is None:
continue
parser = KSDirParser(cur_dir_html)
for ks_file in parser.get_content():
repo_path = cur_dir_name + ks_file['name']
# if this is a directory, just add a name into queue (like BFS algorithm)
if ks_file['type'] == 'DIR':
dirs_queue.append(repo_path)
continue
if repo_path not in to_download:
to_download.append(repo_path)
if to_download:
log(0, "Downloading %d files." % len(to_download))
for item in to_download:
for retry in range(3):
try:
log(1, "Retrieving %s" % item)
plug.get_file(item, os.path.join(CFG.MOUNT_POINT, ks_path))
st = os.stat(os.path.join(CFG.MOUNT_POINT, ks_path, item))
break
except OSError: # os.stat if the file wasn't downloaded
if retry < 3:
log(2, "Retry download %s: attempt #%d" % (item, retry + 1))
else:
raise
# update entity about current file in a database
delete_h.execute(id=ks_id, path=item)
insert_h.execute(id=ks_id, path=item,
checksum=getFileChecksum('sha256', os.path.join(CFG.MOUNT_POINT, ks_path, item)),
st_size=st.st_size, st_time=st.st_mtime)
else:
log(0, "Nothing to download.")
rhnSQL.commit()
def _cleanup(self):
try:
rhnSQL.execute("drop table %s" % self.table_name)
except rhnSQL.SQLStatementPrepareError:
pass
def setUp(self):
rhnSQL.initDB(DB)
self._cleanup()
rhnSQL.execute("create table misatestlob (id int, val blob)")
rhnSQL.execute("create sequence misatestlob_id_seq")
def import_kickstart(self, plug, url, repo_label):
pxeboot_path = 'images/pxeboot/'
pxeboot = plug.get_file(pxeboot_path)
if pxeboot is None:
if not re.search(r'/$', url):
url += '/'
self.print_msg("Kickstartable tree not detected (no %s%s)" % (url, pxeboot_path))
return
ks_path = 'rhn/kickstart/'
ks_tree_label = re.sub(r'[^-_0-9A-Za-z@.]', '', repo_label.replace(' ', '_'))
if len(ks_tree_label) < 4:
ks_tree_label += "_repo"
id_request = """
select id
from rhnKickstartableTree
where channel_id = :channel_id and label = :label
"""
if 'org_id' in self.channel and self.channel['org_id']:
ks_path += str(self.channel['org_id']) + '/' + CFG.MOUNT_POINT + ks_tree_label
row = rhnSQL.fetchone_dict(id_request + " and org_id = :org_id", channel_id=self.channel['id'],
label=ks_tree_label, org_id=self.channel['org_id'])
else:
ks_path += ks_tree_label
row = rhnSQL.fetchone_dict(id_request + " and org_id is NULL", channel_id=self.channel['id'],
label=ks_tree_label)
if row:
print("Kickstartable tree %s already synced with id = %d. Updating content..." % (ks_tree_label, row['id']))
ks_id = row['id']
else:
row = rhnSQL.fetchone_dict("""
select sequence_nextval('rhn_kstree_id_seq') as id from dual
""")
ks_id = row['id']
rhnSQL.execute("""
insert into rhnKickstartableTree (id, org_id, label, base_path, channel_id, kstree_type,
install_type, last_modified, created, modified)
values (:id, :org_id, :label, :base_path, :channel_id,
( select id from rhnKSTreeType where label = :ks_tree_type),
( select id from rhnKSInstallType where label = :ks_install_type),
current_timestamp, current_timestamp, current_timestamp)""", id=ks_id,
org_id=self.channel['org_id'], label=ks_tree_label, base_path=ks_path,
channel_id=self.channel['id'], ks_tree_type=self.ks_tree_type,
ks_install_type=self.ks_install_type)
print("Added new kickstartable tree %s with id = %d. Downloading content..." % (ks_tree_label, row['id']))
insert_h = rhnSQL.prepare("""
insert into rhnKSTreeFile (kstree_id, relative_filename, checksum_id, file_size, last_modified, created,
modified) values (:id, :path, lookup_checksum('sha256', :checksum), :st_size,
epoch_seconds_to_timestamp_tz(:st_time), current_timestamp, current_timestamp)
""")
delete_h = rhnSQL.prepare("""
delete from rhnKSTreeFile where kstree_id = :id and relative_filename = :path
""")
# Downloading/Updating content of KS Tree
# start from root dir
dirs_queue = ['']
while len(dirs_queue) > 0:
cur_dir_name = dirs_queue.pop(0)
cur_dir_html = None
if cur_dir_name == pxeboot_path:
cur_dir_html = pxeboot
else:
cur_dir_html = plug.get_file(cur_dir_name)
if cur_dir_html is None:
continue
parser = KSDirParser()
parser.feed(cur_dir_html.split('<HR>')[1])
for ks_file in parser.get_content():
# do not download rpms, they are already downloaded by self.import_packages()
if re.search(r'\.rpm$', ks_file['name']) or re.search(r'\.\.', ks_file['name']):
continue
# if this is a directory, just add a name into queue (like BFS algorithm)
if ks_file['type'] == 'DIR':
dirs_queue.append(cur_dir_name + ks_file['name'])
continue
else:
local_path = os.path.join(CFG.MOUNT_POINT, ks_path, cur_dir_name, ks_file['name'])
need_download = True
if os.path.exists(local_path):
t = os.path.getmtime(local_path)
if ks_file['datetime'] == datetime.utcfromtimestamp(t).strftime('%d-%b-%Y %H:%M'):
print("File %s%s already present locally" % (cur_dir_name, ks_file['name']))
need_download = False
st = os.stat(local_path)
else:
os.unlink(os.path.join(CFG.MOUNT_POINT, ks_path, cur_dir_name + ks_file['name']))
if need_download:
for retry in range(3):
try:
print("Retrieving %s" % cur_dir_name + ks_file['name'])
plug.get_file(cur_dir_name + ks_file['name'], os.path.join(CFG.MOUNT_POINT, ks_path))
st = os.stat(local_path)
break
except OSError: # os.stat if the file wasn't downloaded
if retry < 3:
print("Retry download %s: attempt #%d" % (cur_dir_name + ks_file['name'], retry+1))
else:
raise
# update entity about current file in a database
delete_h.execute(id=ks_id, path=(cur_dir_name + ks_file['name']))
insert_h.execute(id=ks_id, path=(cur_dir_name + ks_file['name']),
checksum=getFileChecksum('sha256', local_path),
st_size=st.st_size, st_time=st.st_mtime)
rhnSQL.commit()
def import_kickstart(self, plug, url, repo_label):
ks_tree_label = re.sub(r'[^-_0-9A-Za-z@.]', '',
repo_label.replace(' ', '_'))
if len(ks_tree_label) < 4:
ks_tree_label += "_repo"
pxeboot_path = 'images/pxeboot/'
pxeboot = plug.get_file(pxeboot_path)
if pxeboot is None:
if not re.search(r'/$', url):
url = url + '/'
self.error_msg("ERROR: kickstartable tree not detected (no %s%s)" %
(url, pxeboot_path))
return
if rhnSQL.fetchone_dict("""
select id
from rhnKickstartableTree
where org_id = :org_id and channel_id = :channel_id and label = :label
""",
org_id=self.channel['org_id'],
channel_id=self.channel['id'],
label=ks_tree_label):
print "Kickstartable tree %s already synced." % ks_tree_label
return
row = rhnSQL.fetchone_dict("""
select sequence_nextval('rhn_kstree_id_seq') as id from dual
""")
ks_id = row['id']
ks_path = 'rhn/kickstart/%s/%s' % (self.channel['org_id'],
ks_tree_label)
row = rhnSQL.execute("""
insert into rhnKickstartableTree (id, org_id, label, base_path, channel_id,
kstree_type, install_type, last_modified, created, modified)
values (:id, :org_id, :label, :base_path, :channel_id,
( select id from rhnKSTreeType where label = 'externally-managed'),
( select id from rhnKSInstallType where label = 'generic_rpm'),
current_timestamp, current_timestamp, current_timestamp)
""",
id=ks_id,
org_id=self.channel['org_id'],
label=ks_tree_label,
base_path=os.path.join(CFG.MOUNT_POINT, ks_path),
channel_id=self.channel['id'])
insert_h = rhnSQL.prepare("""
insert into rhnKSTreeFile (kstree_id, relative_filename, checksum_id, file_size, last_modified, created, modified)
values (:id, :path, lookup_checksum('sha256', :checksum), :st_size, epoch_seconds_to_timestamp_tz(:st_time), current_timestamp, current_timestamp)
""")
dirs = ['']
while len(dirs) > 0:
d = dirs.pop(0)
v = None
if d == pxeboot_path:
v = pxeboot
else:
v = plug.get_file(d)
if v is None:
continue
for s in (m.group(1)
for m in re.finditer(r'(?i)<a href="(.+?)"', v)):
if (re.match(r'/', s) or re.search(r'\?', s)
or re.search(r'\.\.', s) or re.match(r'[a-zA-Z]+:', s)
or re.search(r'\.rpm$', s)):
continue
if re.search(r'/$', s):
dirs.append(d + s)
continue
local_path = os.path.join(CFG.MOUNT_POINT, ks_path, d, s)
if os.path.exists(local_path):
print "File %s%s already present locally" % (d, s)
else:
print "Retrieving %s" % d + s
plug.get_file(d + s, os.path.join(CFG.MOUNT_POINT,
ks_path))
st = os.stat(local_path)
insert_h.execute(id=ks_id,
path=d + s,
checksum=getFileChecksum(
'sha256', local_path),
st_size=st.st_size,
st_time=st.st_mtime)
rhnSQL.commit()
def import_kickstart(self, plug, repo_label):
ks_path = 'rhn/kickstart/'
ks_tree_label = re.sub(r'[^-_0-9A-Za-z@.]', '',
repo_label.replace(' ', '_'))
if len(ks_tree_label) < 4:
ks_tree_label += "_repo"
# construct ks_path and check we already have this KS tree synced
id_request = """
select id
from rhnKickstartableTree
where channel_id = :channel_id and label = :label
"""
if self.org_id:
ks_path += str(self.org_id) + '/' + ks_tree_label
# Trees synced from external repositories are expected to have full path it database
db_path = os.path.join(CFG.MOUNT_POINT, ks_path)
row = rhnSQL.fetchone_dict(id_request + " and org_id = :org_id",
channel_id=self.channel['id'],
label=ks_tree_label,
org_id=self.org_id)
else:
ks_path += ks_tree_label
db_path = ks_path
row = rhnSQL.fetchone_dict(id_request + " and org_id is NULL",
channel_id=self.channel['id'],
label=ks_tree_label)
treeinfo_path = ['treeinfo', '.treeinfo']
treeinfo_parser = None
for path in treeinfo_path:
log(1, "Trying " + path)
treeinfo = plug.get_file(
path, os.path.join(plug.repo.basecachedir, plug.name))
if treeinfo:
try:
treeinfo_parser = TreeInfoParser(treeinfo)
break
except TreeInfoError:
pass
if not treeinfo_parser:
log(0, "Kickstartable tree not detected (no valid treeinfo file)")
return
if self.ks_install_type is None:
family = treeinfo_parser.get_family()
if family == 'Fedora':
self.ks_install_type = 'fedora18'
elif family == 'CentOS':
self.ks_install_type = 'rhel_' + treeinfo_parser.get_major_version(
)
else:
self.ks_install_type = 'generic_rpm'
fileutils.createPath(os.path.join(CFG.MOUNT_POINT, ks_path))
# Make sure images are included
to_download = set()
for repo_path in treeinfo_parser.get_images():
local_path = os.path.join(CFG.MOUNT_POINT, ks_path, repo_path)
# TODO: better check
if not os.path.exists(local_path) or self.force_kickstart:
to_download.add(repo_path)
if row:
log(
0,
"Kickstartable tree %s already synced. Updating content..." %
ks_tree_label)
ks_id = row['id']
else:
row = rhnSQL.fetchone_dict("""
select sequence_nextval('rhn_kstree_id_seq') as id from dual
""")
ks_id = row['id']
rhnSQL.execute("""
insert into rhnKickstartableTree (id, org_id, label, base_path, channel_id, kstree_type,
install_type, last_modified, created, modified)
values (:id, :org_id, :label, :base_path, :channel_id,
( select id from rhnKSTreeType where label = :ks_tree_type),
( select id from rhnKSInstallType where label = :ks_install_type),
current_timestamp, current_timestamp, current_timestamp)""",
id=ks_id,
org_id=self.org_id,
label=ks_tree_label,
base_path=db_path,
channel_id=self.channel['id'],
ks_tree_type=self.ks_tree_type,
ks_install_type=self.ks_install_type)
log(
0, "Added new kickstartable tree %s. Downloading content..." %
ks_tree_label)
insert_h = rhnSQL.prepare("""
insert into rhnKSTreeFile (kstree_id, relative_filename, checksum_id, file_size, last_modified, created,
modified) values (:id, :path, lookup_checksum('sha256', :checksum), :st_size,
epoch_seconds_to_timestamp_tz(:st_time), current_timestamp, current_timestamp)
""")
delete_h = rhnSQL.prepare("""
delete from rhnKSTreeFile where kstree_id = :id and relative_filename = :path
""")
# Downloading/Updating content of KS Tree
# start from root dir
is_root = True
dirs_queue = ['']
log(0, "Gathering all files in kickstart repository...")
while len(dirs_queue) > 0:
cur_dir_name = dirs_queue.pop(0)
cur_dir_html = plug.get_file(cur_dir_name)
if cur_dir_html is None:
continue
blacklist = None
if is_root:
blacklist = [treeinfo_parser.get_package_dir() + '/']
is_root = False
parser = KSDirParser(cur_dir_html, blacklist)
for ks_file in parser.get_content():
repo_path = cur_dir_name + ks_file['name']
# if this is a directory, just add a name into queue (like BFS algorithm)
if ks_file['type'] == 'DIR':
dirs_queue.append(repo_path)
continue
if not os.path.exists(
os.path.join(CFG.MOUNT_POINT, ks_path,
repo_path)) or self.force_kickstart:
to_download.add(repo_path)
if to_download:
log(0, "Downloading %d kickstart files." % len(to_download))
progress_bar = ProgressBarLogger("Downloading kickstarts:",
len(to_download))
downloader = ThreadedDownloader(force=self.force_kickstart)
for item in to_download:
params = {}
plug.set_download_parameters(
params, item, os.path.join(CFG.MOUNT_POINT, ks_path, item))
downloader.add(params)
downloader.set_log_obj(progress_bar)
downloader.run()
log2disk(0, "Download finished.")
for item in to_download:
st = os.stat(os.path.join(CFG.MOUNT_POINT, ks_path, item))
# update entity about current file in a database
delete_h.execute(id=ks_id, path=item)
insert_h.execute(id=ks_id,
path=item,
checksum=getFileChecksum(
'sha256',
os.path.join(CFG.MOUNT_POINT, ks_path,
item)),
st_size=st.st_size,
st_time=st.st_mtime)
else:
log(0, "No new kickstart files to download.")
# set permissions recursively
rhnSQL.commit()
