def get_udoc_oids(self): """ @brief Get the list of changed/removed UDOC OIDs for comparison """ ret = [] changed_opcs = set() for opc, li in self._udoc.items(): for e in li: changed_opcs.add((opc, e['index'])) gte_gpdb5 = (self._portid == 'greenplum' and is_rev_gte(get_rev_num(self._dbver), get_rev_num('5.0'))) if (self._portid == 'postgres' or gte_gpdb5): method_col = 'opcmethod' else: method_col = 'opcamid' rows = self._run_sql(""" SELECT oc.oid, opcname, amname AS index FROM pg_opclass AS oc, pg_am as am WHERE oc.opcnamespace = {madlib_schema_oid} AND oc.{method_col} = am.oid; """.format(method_col=method_col, madlib_schema_oid=self._schema_oid)) for row in rows: if (row['opcname'], row['index']) in changed_opcs: ret.append(row['oid']) return ret
def get_udoc_oids(self): """ @brief Get the list of changed/removed UDOC OIDs for comparison """ ret = [] changed_opcs = set() for opc, li in self._udoc.items(): for e in li: changed_opcs.add((opc, e['index'])) gte_gpdb5 = (self._portid == 'greenplum' and is_rev_gte( get_rev_num(self._dbver), get_rev_num('5.0'))) if (self._portid == 'postgres' or gte_gpdb5): method_col = 'opcmethod' else: method_col = 'opcamid' rows = self._run_sql(""" SELECT oc.oid, opcname, amname AS index FROM pg_opclass AS oc, pg_am as am WHERE oc.opcnamespace = {madlib_schema_oid} AND oc.{method_col} = am.oid; """.format(method_col=method_col, madlib_schema_oid=self._schema_oid)) for row in rows: if (row['opcname'], row['index']) in changed_opcs: ret.append(row['oid']) return ret
def _load(self): """ @brief Load the configuration file """ rev = get_rev_num(self._mad_dbrev) # _mad_dbrev = 1.9.1 if is_rev_gte([1,9,1],rev): filename = os.path.join(self._maddir, 'madpack', 'changelist_1.9.1_1.12.yaml') # _mad_dbrev = 1.10.0 elif is_rev_gte([1,10],rev): filename = os.path.join(self._maddir, 'madpack', 'changelist_1.10.0_1.12.yaml') # _mad_dbrev = 1.11 else: filename = os.path.join(self._maddir, 'madpack', 'changelist.yaml') config = yaml.load(open(filename)) self._newmodule = config['new module'] if config['new module'] else {} self._udt = config['udt'] if config['udt'] else {} self._udc = config['udc'] if config['udc'] else {} self._udf = self._load_config_param(config['udf']) self._uda = self._load_config_param(config['uda']) # FIXME remove the following special handling for HAWQ after svec is # removed from catalog if self._portid != 'hawq' and not self._is_hawq2: self._udo = self._load_config_param(config['udo']) self._udoc = self._load_config_param(config['udoc'])
def _get_relevant_filenames(self, upgrade_from): """ Get all changelist files that together describe the upgrade process Args: @param upgrade_from: List. Version to upgrade from - the format is expected to be per the output of get_rev_num Details: Changelist files are named in the format changelist_<src>_<dest>.yaml When upgrading from 'upgrade_from_rev' to 'self._curr_rev', all intermediate changelist files need to be followed to get all upgrade steps. This function globs for such files and filters in changelists that lie between the desired versions. Additional verification: The function also ensures that a valid upgrade path exists. Each version in the changelist files needs to be seen twice (except upgrade_from and upgrade_to) for a valid path. This is verified by performing an xor-like operation by adding/deleting from a list. """ output_filenames = [] upgrade_to = self._curr_rev verify_list = [upgrade_from, upgrade_to] # assuming that changelists are in the same directory as this file glob_filter = os.path.abspath( os.path.join(self._maddir, 'madpack', 'changelist*.yaml')) all_changelists = glob.glob(glob_filter) for each_ch in all_changelists: # split file names to get dest versions # Assumption: changelist format is # changelist_<src>_<dest>.yaml ch_basename = os.path.splitext(os.path.basename(each_ch))[0] # remove extension ch_splits = ch_basename.split('_') # underscore delineates sections if len(ch_splits) >= 3: src_version, dest_version = [get_rev_num(i) for i in ch_splits[1:3]] # file is part of upgrade if # upgrade_to >= dest >= src >= upgrade_from is_part_of_upgrade = ( is_rev_gte(src_version, upgrade_from) and is_rev_gte(upgrade_to, dest_version)) if is_part_of_upgrade: for ver in (src_version, dest_version): if ver in verify_list: verify_list.remove(ver) else: verify_list.append(ver) abs_path = os.path.join(self._maddir, 'src', 'madpack', each_ch) output_filenames.append(abs_path) if verify_list: # any version remaining in verify_list implies upgrade path is broken raise RuntimeError("Upgrade from {0} to {1} broken due to missing " "changelist files ({2}). ". format(upgrade_from, upgrade_to, verify_list)) return output_filenames
def test_invalid_path(self): with self.assertRaises(RuntimeError): ChangeHandler(self._dummy_schema, self._dummy_portid, self._dummy_con_args, self.maddir, '1.9', self._dummy_hawq2, upgrade_to=get_rev_num('1.12'))
def _load(self): """ @brief Load the configuration file """ rev = get_rev_num(self._mad_dbrev) upgrade_filenames = self._get_relevant_filenames(rev) for f in upgrade_filenames: with open(f) as handle: config = yaml.load(handle) self._update_objects(config)
def _load(self): """ @brief Load the configuration file """ rev = get_rev_num(self._mad_dbrev) upgrade_filenames = self._get_relevant_filenames(rev) for f in upgrade_filenames: with open(f) as handle: config = yaml.load(handle) self._update_objects(config)
def _get_current_version(self): """ Get current version of MADlib This currently assumes that version is available in '$MADLIB_HOME/src/config/Version.yml' """ version_filepath = os.path.abspath( os.path.join(self._maddir, 'config', 'Version.yml')) with open(version_filepath) as ver_file: version_str = str(yaml.load(ver_file)['version']) return get_rev_num(version_str)
def _get_current_version(self): """ Get current version of MADlib This currently assumes that version is available in '$MADLIB_HOME/src/config/Version.yml' """ version_filepath = os.path.abspath( os.path.join(self._maddir, 'config', 'Version.yml')) with open(version_filepath) as ver_file: version_str = str(yaml.load(ver_file)['version']) return get_rev_num(version_str)
def _get_existing_udoc(self): """ @brief Get the existing UDOCs in the current version """ gte_gpdb5 = (self._portid == 'greenplum' and is_rev_gte( get_rev_num(self._dbver), get_rev_num('5.0'))) if (self._portid == 'postgres' or gte_gpdb5): method_col = 'opcmethod' else: method_col = 'opcamid' rows = self._run_sql(""" SELECT opcname, amname AS index FROM pg_opclass AS oc, pg_namespace AS ns, pg_am as am WHERE oc.opcnamespace = ns.oid AND oc.{method_col} = am.oid AND ns.nspname = '{schema}'; """.format(schema=self._schema.lower(), **locals())) self._existing_udoc = defaultdict(list) for row in rows: self._existing_udoc[row['opcname']].append({'index': row['index']})
def _get_existing_udoc(self): """ @brief Get the existing UDOCs in the current version """ gte_gpdb5 = (self._portid == 'greenplum' and is_rev_gte(get_rev_num(self._dbver), get_rev_num('5.0'))) if (self._portid == 'postgres' or gte_gpdb5): method_col = 'opcmethod' else: method_col = 'opcamid' rows = self._run_sql(""" SELECT opcname, amname AS index FROM pg_opclass AS oc, pg_namespace AS ns, pg_am as am WHERE oc.opcnamespace = ns.oid AND oc.{method_col} = am.oid AND ns.nspname = '{schema}'; """.format(schema=self._schema.lower(), **locals())) self._existing_udoc = defaultdict(list) for row in rows: self._existing_udoc[row['opcname']].append({'index': row['index']})
def test_valid_path(self): ch = ChangeHandler(self._dummy_schema, self._dummy_portid, self._dummy_con_args, self.maddir, '1.9.1', self._dummy_hawq2, upgrade_to=get_rev_num('1.12')) self.assertEqual(ch.newmodule.keys(), [ 'knn', 'sssp', 'apsp', 'measures', 'stratified_sample', 'encode_categorical', 'bfs', 'mlp', 'pagerank', 'train_test_split', 'wcc' ]) self.assertEqual(ch.udt, {'kmeans_result': None, 'kmeans_state': None}) self.assertEqual(ch.udf['forest_train'], [{ 'argument': 'text, text, text, text, text, text, text, ' 'integer, integer, boolean, integer, integer, ' 'integer, integer, integer, text, boolean, ' 'double precision', 'rettype': 'void' }, { 'argument': 'text, text, text, text, text, text, text, ' 'integer, integer, boolean, integer, integer, ' 'integer, integer, integer, text, boolean', 'rettype': 'void' }, { 'argument': 'text, text, text, text, text, text, text, ' 'integer, integer, boolean, integer, integer, ' 'integer, integer, integer, text', 'rettype': 'void' }])
def test_valid_path(self): ch = ChangeHandler(self._dummy_schema, self._dummy_portid, self._dummy_con_args, self.maddir, '1.9.1', self._dummy_hawq2, upgrade_to=get_rev_num('1.12')) self.assertEqual(ch.newmodule.keys(), ['knn', 'sssp', 'apsp', 'measures', 'stratified_sample', 'encode_categorical', 'bfs', 'mlp', 'pagerank', 'train_test_split', 'wcc']) self.assertEqual(ch.udt, {'kmeans_result': None, 'kmeans_state': None}) self.assertEqual(ch.udf['forest_train'], [{'argument': 'text, text, text, text, text, text, text, ' 'integer, integer, boolean, integer, integer, ' 'integer, integer, integer, text, boolean, ' 'double precision', 'rettype': 'void'}, {'argument': 'text, text, text, text, text, text, text, ' 'integer, integer, boolean, integer, integer, ' 'integer, integer, integer, text, boolean', 'rettype': 'void'}, {'argument': 'text, text, text, text, text, text, text, ' 'integer, integer, boolean, integer, integer, ' 'integer, integer, integer, text', 'rettype': 'void'}])
def test_invalid_path(self): with self.assertRaises(RuntimeError): ChangeHandler(self._dummy_schema, self._dummy_portid, self._dummy_con_args, self.maddir, '1.9', self._dummy_hawq2, upgrade_to=get_rev_num('1.12'))