class Taggable(ABC): def __init__(self): self._tags = OrderedSet() @abstractmethod def get_required_tags(self): raise NotImplementedError def get_tags(self): """Gets the comma-separated list of tags. Returns: A comma separated list of tags, or an empty string if there are no tags. """ return ','.join(self.get_required_tags() | self._tags) def set_tags(self, tags): """Sets from a comma-separated list of tags. """ if tags is None: # Weird, but for compatibility with the Java version return self._tags.clear() self._tags.update(tags.split(',')) def add_tags(self, *tags): self._tags.update(tags) def remove_tag(self, tag): if tag: self._tags.remove(tag) def has_tag(self, tag): return tag in self._tags
class TurnGame(object): def __init__(self, advance_func=None): self._player_queue = OrderedSet() self.advance_func = advance_func or self._default_advance_turn def reset(self): self._player_queue.clear() def advance_turn(self): self.advance_func(self._player_queue) @staticmethod def _default_advance_turn(player_queue): # Move current player to end of queue current_player = player_queue.pop(last=False) player_queue.add(current_player) def get_current_player(self): return self._player_queue[0] def get_next_player(self): return self.advance_func(self.get_players())[0] def fast_forward(self, player): # Ensure player is in the queue self.add_player(player) # Rotate player to the front for _ in range(len(self._player_queue)): if self._player_queue[0] == player: break TurnGame._default_advance_turn(self._player_queue) def get_players(self): return OrderedSet(self._player_queue) def get_player_count(self): return len(self._player_queue) def add_player(self, player) -> bool: if player not in self._player_queue: self._player_queue.add(player) return True else: return False def remove_player(self, player) -> bool: if player in self._player_queue: self._player_queue.remove(player) return True else: return False
class __TransactionSet: def __init__(self): """Initialize transaction store.""" self.store = OrderedSet() def add(self, transaction): self.store.add(transaction) def add_multiple(self, transaction_list): """Add multiple transactions to the set. Since the transactions are re-added to the set (they were in it once), we add them at the front of the existing set. """ transaction_list = OrderedSet(transaction_list) self.store = transaction_list.union(self.store) def contains(self, transaction): return self.store.__contains__(transaction) def pop(self): """Remove and return a transaction from the set.""" try: return self.store.pop(0) # Catch KeyError if set is empty except KeyError: return None def discard(self, transaction): """Remove the transaction if it was present in the set.""" self.store.discard(transaction) def discard_multiple(self, transaction_list): """Remove multiple transactions from the set.""" [self.discard(tx) for tx in transaction_list] def clear(self): """Remove all transactions from the set.""" self.store.clear() def __len__(self): return len(self.store) def __iter__(self): yield from self.store def __repr__(self): return self.store.__repr__()
class ContextExtractor(ExpressionEvaluator): """ Context extractor is used in conjunction with extract_context_names to extract argument context as a token value """ def __init__(self): # Init evaluator with fake empty context super(ContextExtractor, self).__init__({}) self.infix_rules = super(ContextExtractor, self).infix_rules for k, v in super(ContextExtractor, self).prefix_rules.items(): self.prefix_rules.setdefault(k, v) # Init empty context set self.contexts = OrderedSet() @prefix("identifier") def identifier(self, token, pc): """ Get corresponding value for token In this function, we return the same token value without using context Args: token: token object pc: parse context Returns: same token value """ self.contexts.add(token.value) return token.value def parse(self, expression): """ Parse current expression, return contexts inside this expression Args: expression (str): expression Returns: set of contexts """ self.contexts.clear() super(ContextExtractor, self).parse(expression) return self.contexts
def test_clear(self): s = OrderedSet((1,2,3,4)) s.clear()
def test_clear(self): s = OrderedSet((1, 2, 3, 4)) s.clear()
class QueryModel(object): """ The QueryModel class represents the intermediate object between the DAG graph and the ultimate SPARQL query. """ def __init__(self): """ Initializing the QueryModel QueryModel is a representation of a sparql query. It has a place holder for every possible componne of a sparql query """ self.prefixes = {} # a dictionary of prefix_name: prefix_URI self.variables = set() # a set of all variables in the query. self.from_clause = set() # a list of graph URIs self.filter_clause = { } # a dictionary of column name as key and associated conditions as a value self.groupBy_columns = OrderedSet( ) # a set of columns for the groupby modifier, it's a subset of self.variables self.aggregate_clause = { } # a dictionary of new_aggregation_col_name: (aggregate function, src_column_name) self.having_clause = { } # a dictionary of new_aggregation_col_name : condition self.order_clause = OrderedDict( ) # a dictionary of columns and the specifier (ASC, DSC) self.limit = 0 # represents the number of rows to be returned by the query. self.offset = 0 # represents the offset in terms of the number of rows self.triples = [ ] # list of basic graph patterns in the form (subject, predicate, object) tuples self.optionals = [] # list of optional query models. self.subqueries = [ ] # list of subqueries. each subquery is a query model self.optional_subqueries = [ ] # list of optional subqueries. each subquery is a query model self.unions = [ ] # list of subqueries to union with the current query model self.graph_triples = { } # dict of graph: list of triples. When there is more than one triple in the graph self.select_columns = OrderedSet( ) # list of columns to be selected , set() self.auto_generated_select_columns = OrderedSet() self.select_all = False self.querybuilder = None # a SPARQLbuilder that converts the query model to a string self.parent_query_model = None # a pointer to the parent query if this is a subquery self.is_optional = False def add_prefixes(self, prefixes): """ Add a dictionary of prefixs to the sparql queries :param prefixes: a dictionary of prefixes where the key is the prefix name and the value is the prefix URI """ if not self.is_subquery(): self.prefixes.update(prefixes) def add_graphs(self, graphs): """ Add a list of graphs to the from clause :param graphs: a list of graphs' URIs """ if not self.is_subquery(): self.from_clause = self.from_clause.union(graphs) #extend def add_optional_triples(self, triples, graph=None): """ add a triple to the list of the optional triples in the query model. :param subject: subject of the triple :param object: object of the triple :param predicate: predicate of the triple """ if len(triples) > 0: optional_query_model = OptionalQueryModel() for (subject, predicate, object) in triples: optional_query_model.add_triple(subject, predicate, object) self.optionals.append(optional_query_model) return optional_query_model def add_optional_block(self, optional_query_model): """ add a triple to the list of the optional triples in the query model. :param subject: subject of the triple :param object: object of the triple :param predicate: predicate of the triple """ self.optionals.append(optional_query_model) def add_triple(self, subject, predicate, object): """ add a triple to the list of the triples in the query model. :param subject: subject of the triple :param object: object of the triple :param predicate: predicate of the triple """ if (subject, predicate, object) not in self.triples: self.triples.append((subject, predicate, object)) self.add_variable(subject) self.add_variable(object) self.add_variable(predicate) def add_graph_triple(self, graph, triples): self.graph_triples[graph] = triples def add_unions(self, unionquery): # subquery type is query_builder """ adds a subquery to the query model :param subquery: :return: """ if len(unionquery.select_columns) <= 0 and len( unionquery.auto_generated_select_columns) <= 0: #if len(unionquery.select_columns) <= 0: unionquery.select_all = True self.unions.append(unionquery) unionquery.parent_query_model = weakref.ref(self) #unionquery.from_clause.clear() def add_subquery(self, subquery): # subquery type is query_builder """ adds a subquery to the query model :param subquery: :return: """ self.subqueries.append(subquery) subquery.parent_query_model = weakref.ref(self) subquery.from_clause.clear() def add_optional_subquery(self, subquery): # subquery type is query_builder """ adds a subquery to the query model :param subquery: :return: """ self.optional_subqueries.append(subquery) subquery.parent_query_model = weakref.ref(self) subquery.from_clause.clear() def add_variable(self, col_name): """ add a variable (column name) to the list of the variables of a single SPARQL query (mainly to represent Select variables) . :param col_name: represents the column name after being parced from the corresponding DAG node. """ if not is_uri(col_name): if col_name.find(":") < 0: self.variables.add(col_name) elif col_name[:col_name.find(":")] not in self.prefixes: self.variables.add(col_name) def add_group_columns(self, col_names): """ add a columns to the list of the group by columns. :param col_names: represents the column name that will group the records based on it. """ self.groupBy_columns = self.groupBy_columns.union(col_names) def add_aggregate_pair(self, src_col_name, func_name, new_col_name, agg_param=None): """ add a pair of column, function name to the list that forms the aggregation clause :param src_col_name: the source column name to be aggregated :param new_col_name: the new column name :param func_name: represents aggregation function on the corresponding column :param agg_param: aggregation parameter like distinct with count """ if new_col_name not in self.aggregate_clause: self.aggregate_clause[new_col_name] = [] self.aggregate_clause[new_col_name].append( (func_name, agg_param, src_col_name)) self.variables.add(new_col_name) def add_filter_condition(self, col_name, condition): """ add a pair of (column, condition) to the list of conditions of the filter clause :param col_name: represents the column name at which the condition will be applied. :param condition: represents the filtering criterion ( Operator, Value) """ if col_name not in self.variables: self.add_variable(col_name) if col_name in self.filter_clause: self.filter_clause[col_name].append(condition) else: self.filter_clause[col_name] = [condition] def add_having_condition(self, agg_col_name, condition): """ add a pair of (column, condition) to the list of conditions of the filter clause :param agg_col_name: represents the column name where the filtering will occur. :param condition: represents the having criterion ( Operator Value) """ if agg_col_name not in self.having_clause: self.having_clause[agg_col_name] = [] func_name, agg_param, src_col_name = self.aggregate_clause[ agg_col_name][0] self.having_clause[agg_col_name].append( [func_name, agg_param, src_col_name, condition]) def add_order_columns(self, sorting_cols): """ add a pair of (column, specifier) to the list of sorting options. :param sorting_cols: list of pairs of (column name, sort order) that will be used for sorting """ for col, order in sorting_cols.items(): self.order_clause[col] = order def set_limit(self, limit): """ :param limit: the value that represents the number of results to be returned :return: none """ self.limit = limit def set_offset(self, offset): """ :param offset: the value that represents the number of NEXT results to be returned :return: none """ self.offset = offset def add_select_column(self, col_name): """ :param col_name: :return: """ self.select_columns.add(col_name) def auto_add_select_column(self, col_name): #print("Auto adding {}".format(col_name)) self.auto_generated_select_columns.add(col_name) def rem_select_column(self, col_name): self.select_columns.remove(col_name) def rem_all_triples(self): self.triples = [] def rem_graph_triples(self): self.graph_triples = {} def rem_from_clause(self): self.from_clause = set() def rem_prefixes(self): self.prefixes = {} def rem_optional_triples(self): self.optionals = [] def rem_filters(self): self.filter_clause = {} def rem_subqueries(self): self.subqueries = [] def rem_optional_subqueries(self): self.optional_subqueries = [] def rem_unions(self): self.unions = [] def transfer_grouping_to_subquery(self, subquery): grouping_cols = self.groupBy_columns for g_col in grouping_cols: involved_triples = [ triple for triple in self.triples if g_col == triple[0] or g_col == triple[2] ] for t in involved_triples: subquery.add_triple(*t) subquery.groupBy_columns = OrderedSet(grouping_cols) subquery.select_columns = set(grouping_cols) subquery.having_clause = dict(self.having_clause) subquery.aggregate_clause = dict(self.aggregate_clause) self.groupBy_columns.clear() self.having_clause.clear() self.aggregate_clause.clear() self.add_subquery(subquery) @staticmethod def clean_inner_qm(qm): # clean the inner query (self) qm.rem_prefixes() qm.rem_from_clause() qm.limit = 0 qm.offset = 0 qm.order_clause = OrderedDict() def wrap_in_a_parent_query(self): """ wraps the current query in a subquery and returns a new query model that contains one graph pattern which is the current query as a subquery :return: a new QueryModel that wraps the current query model """ # initialize the parent query with the graph uri, the prefixes and the variables in the inner query parent_query = QueryModel() parent_query.add_prefixes(self.prefixes) parent_query.add_graphs(self.from_clause) to_add_to_select = [] to_rem_from_select = [] for var in self.select_columns: # if select column in groupby or aggregation: add it to selected columns by the user in inner query # and the outer query. # else: find the relevant graph patterns and move them to the outer query and # remove the select column from select clause in inner query if (var in self.groupBy_columns) or (var in self.aggregate_clause): to_add_to_select.append(var) else: # add basic graph patterns involved_triples = [ triple for triple in self.triples if var == triple[0] or var == triple[2] ] for t in involved_triples: parent_query.add_triple(*t) # add filter patterns if var in self.filter_clause: for condition in self.filter_clause[var]: parent_query.add_filter_condition(var, condition) # add subqueries # Is it query.select or query.variables for subquery in self.subqueries: if var in subquery.select_columns: parent_query.add_subquery(subquery) to_rem_from_select.append(var) parent_query.auto_add_select_column(var) parent_query.add_variable(var) for var in to_add_to_select: self.auto_add_select_column(var) for var in to_rem_from_select: self.rem_select_column(var) # set the limit and offset of the outer query. don't allow limit and offset in the inner query parent_query.set_limit(self.limit) parent_query.set_offset(self.offset) parent_query.add_order_columns(self.order_clause) # add self to the subqueries in the parent subquery parent_query.add_subquery(self) self.parent_query_model = parent_query # clean the inner query (self) QueryModel.clean_inner_qm(self) #self.prefixes = {} #self.rem_from_clause() #self.limit = 0 #self.offset = 0 #self.order_clause = OrderedDict() #self.rem_from_clause() return parent_query def transfer_select_triples_to_parent_query(self, parent_ds_cols): # transfer the order by, the filter clause, for col in parent_ds_cols: triples_list = [subquery.triples for subquery in self.subqueries] involved_triples = [ triple for triples in triples_list for triple in triples if col == triple[0] or col == triple[2] ] for t in involved_triples: if t not in self.triples: self.add_triple(*t) def is_defined_variable(self, var): return var in self.variables or any([ subquery.is_defined_variable(var) for subquery in self.subqueries ]) def is_grouped(self): return len(self.groupBy_columns) > 0 def is_sorted(self): return len(self.order_clause) > 0 def to_sparql(self): #self.validate() self.querybuilder = SPARQLBuilder() return self.querybuilder.to_sparql(self) def is_aggregate_col(self, src_col_name): if src_col_name in self.aggregate_clause: return True return False def is_subquery(self): return self.parent_query_model is not None def all_variables(self): if len(self.subqueries) == 0: return self.variables all_vars = set().union(self.variables) for subq in self.subqueries: all_vars = all_vars.union(subq.all_variables()) return all_vars def rename_variable(self, old_name, new_name): self.triples = [[ new_name if element == old_name else element for element in triple ] for triple in self.triples] for query in self.optionals: query.rename_variable(old_name, new_name) self.select_columns = OrderedSet([ new_name if var == old_name else var for var in self.select_columns ]) self.auto_generated_select_columns = OrderedSet([ new_name if var == old_name else var for var in self.auto_generated_select_columns ]) self.groupBy_columns = OrderedSet([ new_name if var == old_name else var for var in self.groupBy_columns ]) self.variables = { new_name if var == old_name else var for var in self.variables } if old_name in self.order_clause: self.order_clause[new_name] = self.order_clause[old_name] del self.order_clause[old_name] if old_name in self.filter_clause: self.filter_clause[new_name] = self.filter_clause[old_name] del self.filter_clause[old_name] if old_name in self.having_clause: self.having_clause[new_name] = self.having_clause[old_name] del self.having_clause[old_name] for var in self.aggregate_clause: self.aggregate_clause[var] = [[ new_name if element == old_name else element for element in triple ] for triple in self.aggregate_clause[var]] if var == old_name: self.aggregate_clause[new_name] = self.aggregate_clause[ old_name] del self.aggregate_clause[old_name] for query in self.subqueries: query.rename_variable(old_name, new_name) for query in self.unions: query.rename_variable(old_name, new_name) for query in self.optional_subqueries: query.rename_variable(old_name, new_name) def is_valid_prefix(self, prefix): if prefix in self.prefixes.keys(): return True else: return False """ def get_triples(self): triple_string = "" for triple in self.triples: triple1 = triple[1] triple2 = triple[2] if not is_uri(triple[1]) and triple[1].find(":") < 0: triple1 = "?" + triple[1] if not is_uri(triple[2]) and triple[2].find(":") < 0: triple2 = "?" + triple[2] triple = (triple[0], triple1, triple2) triple_string += '\t?%s %s %s' % (triple[0], triple[1], triple[2]) + " .\n" optional_string = self.get_optional_triples() triple_string += '\t'.join(('\n' + optional_string.lstrip()).splitlines(True)) return triple_string def get_optional_triples(self): optional_string = "" if len(self.optionals) > 0: optional_string = "OPTIONAL { \n" for triple in self.optionals: triple1 = triple[1] triple2 = triple[2] if not is_uri(triple[1]) and triple[1].find(":") < 0: triple1 = "?" + triple[1] if not is_uri(triple[2]) and triple[2].find(":") < 0: triple2 = "?" + triple[2] triple = (triple[0], triple1, triple2) optional_string += '\t?%s %s %s' % (triple[0], triple[1], triple[2]) + " .\n" optional_string += "}" return optional_string """ def union(self, qm2): """ union this query model with query model (qm2) :param qm2: :return: a query model that unions the current query model and qm2 """ final_qm = QueryModel() if self.from_clause == qm2.from_clause: # same graph # add the graphs to the outer qm and remove them from the inner qms final_qm.add_graphs(self.from_clause) final_qm.add_graphs(qm2.from_clause) # union the prefixes and remove them from the inner qms # TODO: check that all namespaces that have the same prefix have the same uri final_qm.add_prefixes(self.prefixes) final_qm.add_prefixes(qm2.prefixes) final_qm.variables = final_qm.variables.union(self.variables) final_qm.variables = final_qm.variables.union(qm2.variables) final_qm.set_offset(min(self.offset, qm2.offset)) final_qm.set_limit(max(self.limit, qm2.limit)) final_qm.add_order_columns(self.order_clause) final_qm.add_order_columns(qm2.order_clause) QueryModel.clean_inner_qm(self) QueryModel.clean_inner_qm(qm2) final_qm.add_unions(self) final_qm.add_unions(qm2) return final_qm def validate(self): """ validate the columns and parameters data in the query model and reports and inconsistencies. 1) validate the namespance in the preidcate to match the given in the graphs' prefixes 2) :return: True if valid and False if not """ ## add the aggregation to expandable group ## group by in expandable dataset, raise exception if the select cols not in the group by ### validate the prefix in the triple if self.parent_query_model is None: for triple in self.triples: if not is_uri(triple[1]): if triple[1].find(":") >= 0: prefix = triple[1].split(":") if (len(prefix) >= 1): if not self.is_valid_prefix(prefix[0]): raise Exception( "Not a valid Prefix in triple {}".format( triple)) else: # predicate is a variable pass if self.parent_query_model is None: for col_name in self.filter_clause: if col_name.find(':') != -1: prefix = col_name.split(":") if (len(prefix) >= 1): if not self.is_valid_prefix(prefix[0]): raise Exception( "Not a valid Prefix in filter {}".format( col_name)) for subquery in self.subqueries: subquery_variables_set = set(subquery.variables) my_variables_set = set(self.variables) intersection_variables = my_variables_set.intersection( subquery_variables_set) if len(intersection_variables) < 0: raise Exception( "No common variables between the main query and the subquery" ) all_vars = self.all_variables() missing_vars = set() for sel_col in self.select_columns: if sel_col not in all_vars: missing_vars.add(sel_col) if len(missing_vars) > 0: raise Exception( 'Variables {} are not defined in the query\'s body'.format( ', '.join(missing_vars))) # filter_clause validation for col_name in self.filter_clause: if col_name not in all_vars: raise Warning( 'Cannot add filter on {}, is not part of the query variables' .format(col_name)) for col in self.order_clause: if col not in self.variables: raise Warning( '{} cannot be a sorting column, it should be part of variables' .format(col)) for col_name in self.having_clause: if not self.is_aggregate_col(col_name): raise Warning( '{} is not an aggregate column, cannot be added to having clause' .format(col_name)) def copy(self): return copy.deepcopy(self) def __repr__(self): return self.to_sparql() def __str__(self): return self.to_sparql()