def start_schema(conn): cursor = StatementCursor(conn, 'start_schema') map(cursor.create_sequence, primary_sequences()) tables, mapping = primary_tables() map(cursor.create_table, tables) priorities_table = mapping['priorities'] insert_list(cursor, priorities_table.name, 'priority', PRIORITIES) insert_list(cursor, 'scriptnames', 'script', SCRIPTS) cursor.execute(grant_public([x.name for x in tables])) cursor.execute(grant_public(['current_environment'], 'ALL')) cursor.execute(grant_public(['partition_workspace'], 'ALL')) cursor.execute(plpgsql_delete_profile) cursor.execute(plpgsql_delete_trait) cursor.execute(plpgsql_delete_family)
def start_schema(conn): cursor = StatementCursor(conn, 'start_schema') tables, mapping = primary_tables() map(cursor.create_table, tables) priorities_table = mapping['priorities'] insert_list(cursor, priorities_table.name, 'priority', PRIORITIES) insert_list(cursor, 'scriptnames', 'script', SCRIPTS) cursor.execute(grant_public([x.name for x in tables])) cursor.execute(grant_public(['current_environment'], 'ALL')) cursor.execute(grant_public(['partition_workspace'], 'ALL')) cursor.execute(plpgsql_delete_profile) cursor.execute(plpgsql_delete_trait)
def remove_client(self, client): profiles, families, traits = self._client_schema(client) disks, mtypes, machines = self._client_mdata(client) cursor = StatementCursor(self.conn) if machines: cursor.delete(table='machines', clause=In('machine', machines)) for mtype in mtypes: cursor.execute("select * from delete_mtype('%s')" % mtype) for disk in disks: cursor.execute("select * from delete_disk('%s')" % disk) for profile in profiles: cursor.execute("select * from delete_profile('%s')" % profile) for family in families: cursor.execute("select * from delete_family('%s')" % family)
def create_database(cfg, default_traits): dsn = cfg.get_dsn() dsn['dbname'] = 'mishmash' conn = QuickConn(dsn) cmd = StatementCursor(conn, 'create_database') for table in cmd.tables(): cmd.execute('drop table %s' %table) start_schema(conn, default_traits) make_suites(conn) cmd.execute(grant_public(cmd.tables())) cmd.execute(grant_public(['current_environment'], 'ALL'))
def create_database(cfg, default_traits): dsn = cfg.get_dsn() dsn['dbname'] = 'mishmash' conn = QuickConn(dsn) cmd = StatementCursor(conn, 'create_database') for table in cmd.tables(): cmd.execute('drop table %s' % table) start_schema(conn, default_traits) make_suites(conn) cmd.execute(grant_public(cmd.tables())) cmd.execute(grant_public(['current_environment'], 'ALL'))
class PaellaProcessor(object): def __init__(self, conn): object.__init__(self) self.conn = conn self.__set_cursors__() self.main_path = None def parse_xml(self, filename): self.dbdata = PaellaParser(filename) self.main_path = dirname(filename) def start_schema(self): start_schema(self.conn) self._sync_suites() def _sync_suites(self): self.main.set_table('suites') current_suites = [row.suite for row in self.main.select()] for suite in self.dbdata.suites: if suite.name not in current_suites: self.main.insert(data=suite) make_suite(self.main, suite.name) insert_packages(self.main, suite.name) else: self.main.update(data=suite) def insert_profiles(self): path = join(self.main_path, 'profiles') print 'path is in insert_profiles', path xmlfiles = [join(path, x) for x in os.listdir(path) if x[-4:] == '.xml'] profiles = PaellaProfiles(self.conn) for xmlfile in xmlfiles: xml = parse_file(xmlfile) elements = xml.getElementsByTagName('profile') if len(elements) != 1: raise Error, 'bad profile number %s' % len(elements) element = elements[0] parsed = ProfileParser(element) profiles.insert_profile(parsed) def insert_profile(self, profile): idata = {'profile' : profile.name, 'suite' : profile.suite} self.main.insert(table='profiles', data=idata) idata = {'profile' : profile.name, 'trait' : None, 'ord' : 0} for trait, ord in profile.traits: print trait, ord idata['trait'] = trait idata['ord'] = ord #str(ord) self.main.insert(table='profile_trait', data=idata) idata = {'profile' : profile.name, 'trait' : None, 'name' : None, 'value': None} for trait, name, value in profile.vars: idata['trait'] = trait idata['name'] = name idata['value'] = value self.main.insert(table='profile_variables', data=idata) def insert_traits(self, suite): self.__set_suite_cursors__(suite) traits = [trait for trait in self.dbdata.get_traits(suite)] self._insert_traits_(traits, suite) def _clear_traits(self, suite): self.__set_suite_cursors__(suite) self.traitparent.cmd.delete() self.traitpackage.cmd.delete() self.main.delete(table=ujoin(suite, 'variables')) self.traits.delete() def _insert_traits_(self, traits, suite): while len(traits): trait = traits[0] print 'inserting %s' %trait, len(traits) try: self._insert_trait_(trait, suite) except UnbornError: traits.append(trait) del traits[0] def _insert_trait_(self, trait, suite): traitdb = Trait(self.conn, suite) path = join(self.main_path, suite, trait + '.tar') traitdb.insert_trait(path, suite) def __set_cursors__(self): self.main = StatementCursor(self.conn, 'main_paella_cursor') self.all_traits = StatementCursor(self.conn, 'all_traits') self.all_traits.set_table('traits') def __set_suite_cursors__(self, suite): self.traits = StatementCursor(self.conn, 'traits') self.traits.set_table(ujoin(suite, 'traits')) self.traitparent = TraitParent(self.conn, suite) self.traitpackage = TraitPackage(self.conn, suite) self.traittemplate = TraitTemplate(self.conn, suite) self.traitdebconf = TraitDebconf(self.conn, suite) def create(self, filename): for table in self.main.tables(): self.main.execute('drop table %s' %table) self.parse_xml(filename) self.start_schema() for suite in self.dbdata.suites: self.insert_traits(suite.name) self.insert_profiles()
class Family(object): def __init__(self, conn): object.__init__(self) self.conn = conn self.suites = Suites(conn).list() self.cursor = StatementCursor(self.conn) self.current = None self.parent = SimpleRelation(self.conn, 'family_parent', 'family') self.env = FamilyEnvironment(self.conn) def set_family(self, family): self.current = family self.parent.set_current(family) self.env.set_family(family) def add_family(self, family, type='general'): pass def get_related_families(self, families=[]): rows = self.cursor.select(table='family_parent') graph = kjGraph([(r.family, r.parent) for r in rows]) dfamilies = Set() for fam in families: dfamilies |= Set([fam]) | Set(graph.reachable(fam).items()) return dfamilies def parent_rows(self, family=None): if family is None: family = self.current self.parent.set_clause(family) rows = self.parent.cmd.select(fields=['parent'], order='parent') self.parent.reset_clause() return rows def parents(self, family=None): rows = self.parent_rows(family) return [x.parent for x in rows] def environment_rows(self, family=None): if family is None: family = self.current clause = Eq('family', family) args = dict(fields=['trait', 'name', 'value'], clause=clause, order=['trait', 'name']) return self.env.cursor.select(**args) def family_rows(self): return self.cursor.select(fields=['family'], table='families', order='family') def all_families(self): return [r.family for r in self.family_rows()] def get_all_defaults(self): stmt = select_multisuite_union(self.suites, 'variables') print stmt self.cursor.execute(stmt) return self.cursor.fetchall() def create_family(self, family): if family not in self.all_families(): self.cursor.insert(table='families', data=dict(family=family)) else: raise ExistsError, '%s already exists' % family def insert_parents(self, parents): self.parent.insert('parent', parents) def FamilyData(self, families=[]): if families is None: families = [self.current] all = self.make_familylist(families) superdict = {} for f in all: superdict.update(self.env.make_tagdict(f)) return superdict def make_familylist(self, families): deps = families all = list(self.get_related_families(families)) setfun = self.set_family parfun = self.parents return make_deplist(deps, all, setfun, parfun) def export_family(self, family=None): if family is None: family = self.current element = FamilyElement(family) element.append_parents(self.parents(family)) element.append_variables(self.environment_rows(family)) return element def write_family(self, family, path): fxml = file(join(path, '%s.xml' % family), 'w') data = self.export_family(family) data.writexml(fxml, indent='\t', newl='\n', addindent='\t') fxml.close() def export_families(self, path): families = self.all_families() for f in families: self.write_family(f, path) def import_family(self, element): parsed = FamilyParser(element) print 'inserting family', parsed.name all = self.all_families() for p in parsed.parents: if p not in all: print 'insertion failed for', parsed.name raise UnbornError self.create_family(parsed.name) self.set_family(parsed.name) self.insert_parents(parsed.parents) row = dict(family=parsed.name) for var in parsed.environ: row.update(var) self.cursor.insert(table='family_environment', data=row) def import_families(self, path): xmlfiles = [ join(path, x) for x in os.listdir(path) if x[-4:] == '.xml' ] families = [] for f in xmlfiles: xml = parse_file(f) elements = xml.getElementsByTagName('family') if len(elements) != 1: raise Error, 'bad number of family tags %s' % len(elements) element = elements[0] families.append(element) while len(families): f = families[0] try: self.import_family(f) except UnbornError: families.append(f) del families[0] print len(families), 'families inserted'
class PaellaProcessor(object): def __init__(self, conn): object.__init__(self) self.conn = conn self.__set_cursors__() self.main_path = None def parse_xml(self, filename): self.dbdata = PaellaParser(filename) self.main_path = dirname(filename) def start_schema(self): start_schema(self.conn) self._sync_suites() def _sync_suites(self): self.main.set_table('suites') current_suites = [row.suite for row in self.main.select()] for suite in self.dbdata.suites: if suite.name not in current_suites: self.main.insert(data=suite) make_suite(self.main, suite.name) insert_packages(self.main, suite.name) else: self.main.update(data=suite) def insert_profiles(self): path = join(self.main_path, 'profiles') print 'path is in insert_profiles', path xmlfiles = [ join(path, x) for x in os.listdir(path) if x[-4:] == '.xml' ] profiles = PaellaProfiles(self.conn) for xmlfile in xmlfiles: xml = parse_file(xmlfile) elements = xml.getElementsByTagName('profile') if len(elements) != 1: raise Error, 'bad profile number %s' % len(elements) element = elements[0] parsed = ProfileParser(element) profiles.insert_profile(parsed) def insert_profile(self, profile): idata = {'profile': profile.name, 'suite': profile.suite} self.main.insert(table='profiles', data=idata) idata = {'profile': profile.name, 'trait': None, 'ord': 0} for trait, ord in profile.traits: print trait, ord idata['trait'] = trait idata['ord'] = ord #str(ord) self.main.insert(table='profile_trait', data=idata) idata = { 'profile': profile.name, 'trait': None, 'name': None, 'value': None } for trait, name, value in profile.vars: idata['trait'] = trait idata['name'] = name idata['value'] = value self.main.insert(table='profile_variables', data=idata) def insert_traits(self, suite): self.__set_suite_cursors__(suite) traits = [trait for trait in self.dbdata.get_traits(suite)] self._insert_traits_(traits, suite) def _clear_traits(self, suite): self.__set_suite_cursors__(suite) self.traitparent.cmd.delete() self.traitpackage.cmd.delete() self.main.delete(table=ujoin(suite, 'variables')) self.traits.delete() def _insert_traits_(self, traits, suite): while len(traits): trait = traits[0] print 'inserting %s' % trait, len(traits) try: self._insert_trait_(trait, suite) except UnbornError: traits.append(trait) del traits[0] def _insert_trait_(self, trait, suite): traitdb = Trait(self.conn, suite) path = join(self.main_path, suite, trait + '.tar') traitdb.insert_trait(path, suite) def __set_cursors__(self): self.main = StatementCursor(self.conn, 'main_paella_cursor') self.all_traits = StatementCursor(self.conn, 'all_traits') self.all_traits.set_table('traits') def __set_suite_cursors__(self, suite): self.traits = StatementCursor(self.conn, 'traits') self.traits.set_table(ujoin(suite, 'traits')) self.traitparent = TraitParent(self.conn, suite) self.traitpackage = TraitPackage(self.conn, suite) self.traittemplate = TraitTemplate(self.conn, suite) self.traitdebconf = TraitDebconf(self.conn, suite) def create(self, filename): for table in self.main.tables(): self.main.execute('drop table %s' % table) self.parse_xml(filename) self.start_schema() for suite in self.dbdata.suites: self.insert_traits(suite.name) self.insert_profiles()
class BaseDatabase(QSqlDatabase): def __init__(self, dsn, name, parent=None, objname=None): QSqlDatabase.__init__(self, "QPSQL7", name, parent, objname) self.conn = BasicConnection(**dsn) self.setDatabaseName(dsn["dbname"]) self.setHostName(dsn["host"]) self.dbuser = dsn["user"] self.setUserName(self.dbuser) self.stmt = Statement() self.mcursor = StatementCursor(self.conn) def set_table(self, table): self.stmt.table = table def set_clause(self, items, cmp="=", join="and"): self.stmt.set_clause(items, cmp=cmp, join=join) def set_data(self, data): self.stmt.set(data) def set_fields(self, fields): self.stmt.fields = fields def qdelete(self, table=None, clause=None): query = self.stmt.delete(table=table, clause=clause) return self.execStatement(query) def qinsert(self, table=None, data=None): query = self.stmt.insert(table=table, data=data) return self.execStatement(query) def qupdate(self, table=None, data=None, clause=None): query = self.stmt.update(table=table, data=data, clause=clause) return self.execStatement(query) def qselect(self, fields=None, table=None, clause=None, group=None, having=None, order=None): query = self.stmt.select(fields=fields, table=table, clause=clause, group=group, having=having, order=order) return self.execStatement(query) def delete(self, table=None, clause=None): query = self.stmt.delete(table=table, clause=clause) return self.mcursor.execute(query) def insert(self, table=None, data=None): query = self.stmt.insert(table=table, data=data) return self.mcursor.execute(query) def update(self, table=None, data=None, clause=None): query = self.stmt.update(table=table, data=data, clause=clause) return self.mcursor.execute(query) def select(self, fields=None, table=None, clause=None, group=None, having=None, order=None): query = self.stmt.select(fields=fields, table=table, clause=clause, group=group, having=having, order=order) self.mcursor.execute(query) return self.mcursor.fetchall() def select_row(self, fields=None, table=None, clause=None, group=None, having=None, order=None): query = self.stmt.select(fields=fields, table=table, clause=clause, group=group, having=having, order=order) self.mcursor.execute(query) rows = len(self.mcursor) if rows == 1: return self.mcursor.next() elif rows == 0: raise NoExistError else: raise Error, "bad row count %s" % rows def clear(self, **args): self.stmt.clear(**args) def stdclause(self, data): return reduce(and_, [Eq(k, v) for k, v in data.items()]) def insertData(self, idcol, table, data, commit=True): clause = self.stdclause(data) try: self.mcursor.select_row(fields=[idcol], table=table, clause=clause) except NoExistError: self.mcursor.insert(table=table, data=data) if commit: self.conn.commit() def identifyData(self, idcol, table, data, commit=True): clause = self.stdclause(data) self.insertData(idcol, table, data, commit=commit) return self.mcursor.select_row(fields=["*"], table=table, clause=clause)
class DiskManager(object): def __init__(self, conn): self.conn = conn self.parser = PartitionParser() self.cursor = StatementCursor(self.conn) def _quick_partition(self, device, data): i, o = os.popen2('sfdisk %s' % device) i.write(data) i.close() def get_partition_info(self, device, parser=None): if parser is None: parser = self.parser command = 'bash -c "sfdisk -d %s | grep %s"' % (device, device) part_info = commands.getoutput(command) return self._parse_diskconfig(device, part_info) def _parse_diskconfig(self, device, astring): parsed = self.parser.parseString(astring) partitions = [] for p in parsed: pnum = p[0].split(device)[1] pdict = dict(partition=pnum, start=p[1], size=p[2], Id=p[3]) partitions.append(pdict) return partitions def _submit_diskconfig(self, diskname, device, astring): workspace = 'partition_workspace' self.cursor.delete(table=workspace, clause=(Eq('diskname', diskname))) row = dict(diskname=diskname) for partition in self._parse_diskconfig(device, astring): print 'submitting', partition row.update(partition) self.cursor.insert(table=workspace, data=row) def submit_partitions(self, diskname, device): self.cursor.set_table('partition_workspace') self.cursor.delete(clause=(Eq('diskname', diskname))) row = dict(diskname=diskname) print 'submitting', device for partition in self.get_partition_info(device): print 'submitting', partition row.update(partition) self.cursor.insert(data=row) def approve_disk(self, diskname): clause = Eq('diskname', diskname) workspace = 'partition_workspace' sql = Statement('select') sql.table = workspace sql.clause = clause new_rows = sql.select(order='partition') if diskname not in [ r.diskname for r in self.cursor.select(table='disks') ]: self.cursor.insert(table='disks', data=dict(diskname=diskname)) else: self.cursor.delete(table='partitions', clause=clause) self.cursor.execute('insert into partitions %s' % new_rows) def get_partitions_by_name(self, diskname): return self.cursor.select(table='partitions', clause=Eq('diskname', diskname), order='partition') def make_partition_dump(self, device, partitions): output = '# partition table of %s\n' output += 'unit: sectors\n' for p in partitions: line = '%s%s : start= %8d, size= %8d, Id=%2d' % \ (device, p.partition, p.start, p.size, p.id) output += line + '\n' return output def partition_disk(self, diskname, device): partitions = self.get_partitions_by_name(diskname) data = self.make_partition_dump(device, partitions) self._quick_partition(device, data) def clear_partition_table(self, device): command = 'dd if=/dev/zero of=%s count=1 bs=512' % device os.system(command)
class DiskManager(object): def __init__(self, conn): self.conn = conn self.parser = PartitionParser() self.cursor = StatementCursor(self.conn) def _quick_partition(self, device, data): i, o = os.popen2('sfdisk %s' % device) i.write(data) i.close() def get_partition_info(self, device, parser=None): if parser is None: parser = self.parser command = 'bash -c "sfdisk -d %s | grep %s"' % (device, device) part_info = commands.getoutput(command) return self._parse_diskconfig(device, part_info) def _parse_diskconfig(self, device, astring): parsed = self.parser.parseString(astring) partitions = [] for p in parsed: pnum = p[0].split(device)[1] pdict = dict(partition=pnum, start=p[1], size=p[2], Id=p[3]) partitions.append(pdict) return partitions def _submit_diskconfig(self, diskname, device, astring): workspace = 'partition_workspace' self.cursor.delete(table=workspace, clause=(Eq('diskname', diskname))) row = dict(diskname=diskname) for partition in self._parse_diskconfig(device, astring): print 'submitting', partition row.update(partition) self.cursor.insert(table=workspace, data=row) def submit_partitions(self, diskname, device): self.cursor.set_table('partition_workspace') self.cursor.delete(clause=(Eq('diskname', diskname))) row = dict(diskname=diskname) print 'submitting', device for partition in self.get_partition_info(device): print 'submitting', partition row.update(partition) self.cursor.insert(data=row) def approve_disk(self, diskname): clause = Eq('diskname', diskname) workspace = 'partition_workspace' sql = Statement('select') sql.table = workspace sql.clause = clause new_rows = sql.select(order='partition') if diskname not in [r.diskname for r in self.cursor.select(table='disks')]: self.cursor.insert(table='disks', data=dict(diskname=diskname)) else: self.cursor.delete(table='partitions', clause=clause) self.cursor.execute('insert into partitions %s' % new_rows) def get_partitions_by_name(self, diskname): return self.cursor.select(table='partitions', clause=Eq('diskname', diskname), order='partition') def make_partition_dump(self, device, partitions): output = '# partition table of %s\n' output += 'unit: sectors\n' for p in partitions: line = '%s%s : start= %8d, size= %8d, Id=%2d' % \ (device, p.partition, p.start, p.size, p.id) output += line + '\n' return output def partition_disk(self, diskname, device): partitions = self.get_partitions_by_name(diskname) data = self.make_partition_dump(device, partitions) self._quick_partition(device, data) def clear_partition_table(self, device): command = 'dd if=/dev/zero of=%s count=1 bs=512' % device os.system(command)
class Family(object): def __init__(self, conn): object.__init__(self) self.conn = conn self.suites = Suites(conn).list() self.cursor = StatementCursor(self.conn) self.current = None self.parent = SimpleRelation(self.conn, 'family_parent', 'family') self.env = FamilyEnvironment(self.conn) def set_family(self, family): self.current = family self.parent.set_current(family) self.env.set_family(family) def add_family(self, family, type='general'): pass def get_related_families(self, families=[]): rows = self.cursor.select(table='family_parent') graph = kjGraph([(r.family, r.parent) for r in rows]) dfamilies = Set() for fam in families: dfamilies |= Set([fam]) | Set(graph.reachable(fam).items()) return dfamilies def parent_rows(self, family=None): if family is None: family = self.current self.parent.set_clause(family) rows = self.parent.cmd.select(fields=['parent'], order='parent') self.parent.reset_clause() return rows def parents(self, family=None): rows = self.parent_rows(family) return [x.parent for x in rows] def environment_rows(self, family=None): if family is None: family = self.current clause = Eq('family', family) args = dict(fields=['trait', 'name', 'value'], clause=clause, order=['trait', 'name']) return self.env.cursor.select(**args) def family_rows(self): return self.cursor.select(fields=['family'], table='families', order='family') def all_families(self): return [r.family for r in self.family_rows()] def get_all_defaults(self): stmt = select_multisuite_union(self.suites, 'variables') print stmt self.cursor.execute(stmt) return self.cursor.fetchall() def create_family(self, family): if family not in self.all_families(): self.cursor.insert(table='families', data=dict(family=family)) else: raise ExistsError, '%s already exists' % family def insert_parents(self, parents): self.parent.insert('parent', parents) def FamilyData(self, families=[]): if families is None: families = [self.current] all = self.make_familylist(families) superdict = {} for f in all: superdict.update(self.env.make_tagdict(f)) return superdict def make_familylist(self, families): deps = families all = list(self.get_related_families(families)) setfun = self.set_family parfun = self.parents return make_deplist(deps, all, setfun, parfun) def export_family(self, family=None): if family is None: family = self.current element = FamilyElement(family) element.append_parents(self.parents(family)) element.append_variables(self.environment_rows(family)) return element def export_families(self, path): families = self.all_families() for f in families: fxml = file(join(path, '%s.xml' % f), 'w') data = self.export_family(f) data.writexml(fxml, indent='\t', newl='\n', addindent='\t') fxml.close() def import_family(self, element): parsed = FamilyParser(element) print 'inserting family', parsed.name all = self.all_families() for p in parsed.parents: if p not in all: print 'insertion failed for', parsed.name raise UnbornError self.create_family(parsed.name) self.set_family(parsed.name) self.insert_parents(parsed.parents) row = dict(family=parsed.name) for var in parsed.environ: row.update(var) self.cursor.insert(table='family_environment', data=row) def import_families(self, path): xmlfiles = [join(path, x) for x in os.listdir(path) if x[-4:] == '.xml'] families = [] for f in xmlfiles: xml = parse_file(f) elements = xml.getElementsByTagName('family') if len(elements) != 1: raise Error, 'bad number of family tags %s' % len(elements) element = elements[0] families.append(element) while len(families): f = families[0] try: self.import_family(f) except UnbornError: families.append(f) del families[0] print len(families), 'families inserted'