def __init__(self, name='DynFilterSrv', condition_srv_name='ConditionSrv'): ''' @param name: The unique name of the service ''' self._dfilters = Case().tag(name).tag(DynFilter.__name__) self._conditions = Showcase.instance().get_case(condition_srv_name) INamed.__init__(self, name)
def __init__(self, name='TableSrv'): ''' @param name: The unique name of the service ''' self._tables = Case().tag(name) \ .tag(Table.__name__) \ .tag(TableView.__name__) INamed.__init__(self, name)
def __init__(self, name='ConditionSrv'): ''' @param name: The unique name of the service ''' self._conditions = Case().tag(name) \ .tag(Condition.__name__) \ .tag(CategoricalCondition.__name__) \ .tag(AttributeCondition.__name__) \ .tag(RangeCondition.__name__) \ .tag(QueryCondition.__name__) INamed.__init__(self, name)
class ConditionService(INamed): ''' This class provide a facade for managing Condition objects ''' def __init__(self, name='ConditionSrv'): ''' @param name: The unique name of the service ''' self._conditions = Case().tag(name) \ .tag(Condition.__name__) \ .tag(CategoricalCondition.__name__) \ .tag(AttributeCondition.__name__) \ .tag(RangeCondition.__name__) \ .tag(QueryCondition.__name__) INamed.__init__(self, name) def register_in(self, dispatcher): dispatcher.add_method(self.new_condition) dispatcher.add_method(self.expose_condition) dispatcher.add_method(self.del_condition) # Condition Properties dispatcher.add_method(partial(self._proxy_property, 'name'), 'name') dispatcher.add_method(partial(self._proxy_property, 'data'), 'data') dispatcher.add_method(partial(self._proxy_property, 'grammar'), 'grammar') dispatcher.add_method(partial(self._proxy_property, 'enabled'), 'enabled') # Condition Methods dispatcher.add_method(partial(self._proxy, 'include_all'), 'include_all') dispatcher.add_method(partial(self._proxy, 'exclude_all'), 'exclude_all') dispatcher.add_method(partial(self._proxy, 'enable'), 'enable') dispatcher.add_method(partial(self._proxy, 'disable'), 'disable') # CategoricalCondition Properties dispatcher.add_method(partial(self._proxy_property, 'attr'), 'attr') dispatcher.add_method(partial(self._proxy, 'get_condition'), 'get_condition') # CategoricalCondition Methods dispatcher.add_method(partial(self._proxy, 'included_categories'), 'included_categories') dispatcher.add_method(partial(self._proxy, 'excluded_categories'), 'excluded_categories') dispatcher.add_method(partial(self._proxy, 'included_items'), 'included_items') dispatcher.add_method(partial(self._proxy, 'excluded_items'), 'excluded_items') dispatcher.add_method(partial(self._proxy, 'add_category'), 'add_category') dispatcher.add_method(partial(self._proxy, 'remove_category'), 'remove_category') dispatcher.add_method(partial(self._proxy, 'toggle_category'), 'toggle_category') # AttributeCondition Methods dispatcher.add_method(partial(self._proxy, 'included_attributes'), 'included_attributes') dispatcher.add_method(partial(self._proxy, 'excluded_attributes'), 'excluded_attributes') dispatcher.add_method(partial(self._proxy, 'add_attribute'), 'add_attribute') dispatcher.add_method(partial(self._proxy, 'remove_attribute'), 'remove_attribute') dispatcher.add_method(partial(self._proxy, 'toggle_attribute'), 'toggle_attribute') # RangeCondition Properties dispatcher.add_method(partial(self._proxy_property, 'range'), 'range') dispatcher.add_method(partial(self._proxy_property, 'domain'), 'domain') # RangeCondition Methods dispatcher.add_method(partial(self._proxy, 'set_range'), 'set_range') # QueryCondition Properties dispatcher.add_method(partial(self._proxy_property, 'query'), 'query') # QueryCondition Methods dispatcher.add_method(partial(self._proxy, 'set_query'), 'set_query') def _proxy(self, method, condition_oid, *args, **kwargs): condition = self._conditions[condition_oid] result = condition.__getattribute__(method)(*args, **kwargs) if isinstance(result, Condition): self._conditions[result.oid] = result return result def _proxy_property(self, method, condition_oid): condition = self._conditions[condition_oid] result = condition.__getattribute__(method) if isinstance(result, Condition): self._conditions[result.oid] = result return result def new_condition(self, kind, data, *args, **kwargs): dataset = Showcase.instance().get(data) if kind == 'categorical': new_condition = CategoricalCondition(dataset, *args, **kwargs) if kind == 'attribute': new_condition = AttributeCondition(dataset, *args, **kwargs) if kind == 'range': new_condition = RangeCondition(dataset, *args, **kwargs) if kind == 'query': new_condition = QueryCondition(dataset, *args, **kwargs) self._conditions[new_condition.oid] = new_condition return new_condition def expose_condition(self, condition): self._conditions[condition.oid] = condition return condition def del_condition(self, oid): self._conditions.pop(oid)
class TableService(INamed): ''' This class provide a facade for managing table objects ''' def __init__(self, name='TableSrv'): ''' @param name: The unique name of the service ''' self._tables = Case().tag(name) \ .tag(Table.__name__) \ .tag(TableView.__name__) INamed.__init__(self, name) def register_in(self, dispatcher): dispatcher.add_method(self.new_table) dispatcher.add_method(self.expose_table) dispatcher.add_method(self.del_table) dispatcher.add_method(self.load_data) dispatcher.add_method(self.concat_data) dispatcher.add_method(self.load_schema) # TableView properties dispatcher.add_method(partial(self._proxy_property, 'name'), 'name') dispatcher.add_method(partial(self._proxy_property, 'index'), 'index') dispatcher.add_method(partial(self._proxy_property, 'schema'), 'schema') dispatcher.add_method(partial(self._proxy_property, 'view_args'), 'view_args') # TableView methods dispatcher.add_method(partial(self._proxy, 'get_data'), 'get_data') dispatcher.add_method(partial(self._proxy, 'find'), 'find') dispatcher.add_method(partial(self._proxy, 'find_one'), 'find_one') dispatcher.add_method(partial(self._proxy, 'distinct'), 'distinct') dispatcher.add_method(partial(self._proxy, 'aggregate'), 'aggregate') dispatcher.add_method(partial(self._proxy, 'row_count'), 'row_count') dispatcher.add_method(partial(self._proxy, 'column_count'), 'column_count') dispatcher.add_method(partial(self._proxy, 'column_names'), 'column_names') # Table methods dispatcher.add_method(partial(self._proxy, 'data'), 'data') dispatcher.add_method(partial(self._proxy, 'insert'), 'insert') dispatcher.add_method(partial(self._proxy, 'update'), 'update') dispatcher.add_method(partial(self._proxy, 'remove'), 'remove') dispatcher.add_method(partial(self._proxy, 'add_column'), 'add_column') dispatcher.add_method(partial(self._proxy, 'add_derived_column'), 'add_derived_column') dispatcher.add_method(partial(self._proxy, 'rename_columns'), 'rename_columns') def _proxy(self, method, table_oid, *args, **kwargs): table = self._tables[table_oid] result = table.__getattribute__(method)(*args, **kwargs) if isinstance(result, ITableView): self._tables[result.oid] = result return result def _proxy_property(self, method, table_oid): table = self._tables[table_oid] result = table.__getattribute__(method) if isinstance(result, ITableView): self._tables[result.oid] = result return result def new_table(self, name, data, schema=None, prefix=''): ''' :param str name: If a name is not provided, an uuid is generated :param dict data: A list of dicts, each dict is a row. :param schema: The schema associated to the data. :param str prefix: Prepended to the name creates the oid ''' new_table = Table(name, schema, prefix=prefix).data(data) self._tables[new_table.oid] = new_table return new_table def expose_table(self, table): self._tables[table.oid] = table return table def del_table(self, oid): self._tables.pop(oid) def __getattr__(self, method): if method in ['name', 'index', 'view_args', 'schema']: return partial(self._proxy_property, method) else: return partial(self._proxy, method) def load_data(self, str_data, table_name, infer, sch): # Load new data to dataset (removing old data) index_col = 'id_index' # Clean CSV (extra columns with comma ',') lines = [line.rstrip(',') for line in str_data.encode('utf-8').split('\n')] str_data = '\n'.join(lines) # Receive the data as string data=StringIO(str_data) df = pd.read_csv(data, sep=",") # Create id_index column if index_col in df.columns: df.drop(labels=[index_col], axis=1, inplace = True) df.insert(0, index_col, df.index) df.fillna("NaN", inplace=True) print ":::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::" print ":::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::" from pprint import pprint pprint(df) print ":::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::" print ":::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::" table = self._tables[table_name] if (infer == True): table._schema = None table.data(df) _backend = MongoTable(table_name, table._schema, prefix='') _backend.data(df) return table._schema def concat_data(self, str_data, table_name, new_cols=None): # Concat new data to current dataset index_col = 'id_index' # Clean CSV (extra columns with comma ',') lines = [line.rstrip(',') for line in str_data.encode('utf-8').split('\n')] str_data = '\n'.join(lines) # Receive the data as string data=StringIO(str_data) df = pd.read_csv(data, sep=",") df.fillna("NaN", inplace=True) table = self._tables[table_name] try: max_id_index = table.find_one(sort=[(index_col, -1)])[index_col] except: pass # Create id_index column if index_col in df.columns: df.drop(labels=[index_col], axis=1, inplace = True) df.insert(0, index_col, range(max_id_index+1, max_id_index+df.shape[0]+1)) # Add columns of the new data if new_cols is not None: if index_col in new_cols: new_cols.remove(index_col) concat_schema = {} for attr in new_cols: attr_utf = attr.encode('utf-8') concat_schema[attr_utf] = dict(AttributeSchema.infer_from_data(df[attr_utf])._schema) table._schema._schema['attributes'].update(collections.OrderedDict(concat_schema)) dict_data = df.to_dict() list_all_insert = [] data_insert = {} continue_bool = True i = 0 while continue_bool: for data in dict_data.keys(): try: data_insert[data] = dict_data[data][i] except: continue_bool = False break i += 1 if continue_bool: list_all_insert.append(data_insert) data_insert = {} table.insert(list_all_insert) return table._schema def load_schema(self, table_name, schema): # Load new schema # Receive the schema as string table = self._tables[table_name] # String to OrderedDict schema = json.loads(schema, object_pairs_hook=collections.OrderedDict) table._schema = TableSchema(schema['attributes'], schema['index']) return table._schema
def __init__(self, name='SharedObjectSrv'): ''' @param name: The unique name of the service ''' self._shared_objects = Case().tag(name).tag(SharedObject.__name__) INamed.__init__(self, name)
class SharedObjectService(INamed): ''' This class provide a facade for managing SharedObjects ''' def __init__(self, name='SharedObjectSrv'): ''' @param name: The unique name of the service ''' self._shared_objects = Case().tag(name).tag(SharedObject.__name__) INamed.__init__(self, name) def register_in(self, dispatcher): dispatcher.add_method(self.new_shared_object) dispatcher.add_method(self.expose_shared_object) dispatcher.add_method(self.del_shared_object) dispatcher.add_method(self.clear) dispatcher.add_method(self.push) # SharedObject properties dispatcher.add_method(partial(self._proxy_property, 'name'), 'name') dispatcher.add_method(partial(self._proxy_property, 'grammar'), 'grammar') # SharedObject methods dispatcher.add_method(partial(self._proxy, 'pull'), 'pull') def _proxy(self, method, shared_object_oid, *args, **kwargs): shared_object = self._shared_objects[shared_object_oid] result = shared_object.__getattribute__(method)(*args, **kwargs) return result def _proxy_property(self, method, shared_object_oid): shared_object = self._shared_objects[shared_object_oid] result = shared_object.__getattribute__(method) return result def push(self, oid, data, version): ''' :param oid: The SharedObject oid :param data: list or dict. The updated data. :param version: This should be the returned by pull or push methods. :returns: tuple(conflict_ocurred, new_version) conflict_ocurred: True if provided version is older than current False otherwise new_version: The version generated after the modification. Becomes the current one. ''' shared_object = self._shared_objects[oid] conflict = False try: new_version = shared_object.push(data, version) except VersionError: conflict = True new_version = None return (conflict, new_version) def new_shared_object(self, name, data, prefix=''): ''' :param str name: If a name is not provided, an uuid is generated :param data: A dict or list :param str prefix: Prepended to the name creates the oid ''' new_shared_object = SharedObject(name, data, prefix=prefix) self._shared_objects[new_shared_object.oid] = new_shared_object return new_shared_object def expose_shared_object(self, shared_object): self._shared_objects[shared_object.oid] = shared_object return shared_object def del_shared_object(self, oid): self._shared_objects.pop(oid) def clear(self): self._shared_objects.clear() def __getattr__(self, method): if method in ['name', 'grammar']: return partial(self._proxy_property, method) else: return partial(self._proxy, method)
class DynFilterService(INamed): ''' This class provide a facade for managing DynFilter objects ''' def __init__(self, name='DynFilterSrv', condition_srv_name='ConditionSrv'): ''' @param name: The unique name of the service ''' self._dfilters = Case().tag(name).tag(DynFilter.__name__) self._conditions = Showcase.instance().get_case(condition_srv_name) INamed.__init__(self, name) def register_in(self, dispatcher): dispatcher.add_method(self.new_dfilter) dispatcher.add_method(self.expose_dfilter) dispatcher.add_method(self.del_dfilter) dispatcher.add_method(self.clear) # DynFilter Methods dispatcher.add_method(partial(self._proxy, 'new_categorical_condition'), 'new_categorical_condition') dispatcher.add_method(partial(self._proxy, 'new_attribute_condition'), 'new_attribute_condition') dispatcher.add_method(partial(self._proxy, 'new_range_condition'), 'new_range_condition') dispatcher.add_method(partial(self._proxy, 'new_query_condition'), 'new_query_condition') # ConditionSet Methods dispatcher.add_method(partial(self._proxy, 'add_condition'), 'add_condition') dispatcher.add_method(partial(self._proxy, 'set_condition'), 'set_condition') dispatcher.add_method(partial(self._proxy, 'update'), 'update') dispatcher.add_method(partial(self._proxy, 'remove_condition'), 'remove_condition') dispatcher.add_method(partial(self._proxy, 'has_condition'), 'has_condition') dispatcher.add_method(partial(self._proxy, 'get_condition'), 'get_condition') dispatcher.add_method(partial(self._proxy, 'get_conditions'), 'get_conditions') # ConditionSet Properties dispatcher.add_method(partial(self._proxy_property, 'name'), 'name') dispatcher.add_method(partial(self._proxy_property, 'grammar'), 'grammar') dispatcher.add_method(partial(self._proxy_property, 'grammar_of_conditions'), 'grammar_of_conditions') dispatcher.add_method(partial(self._proxy_property, 'reference'), 'reference') dispatcher.add_method(partial(self._proxy_property, 'projection'), 'projection') dispatcher.add_method(partial(self._proxy_property, 'query'), 'query') dispatcher.add_method(partial(self._proxy_property, 'view_args'), 'view_args') def _proxy(self, method, dfilter_oid, *args, **kwargs): dfilter = self._dfilters[dfilter_oid] result = dfilter.__getattribute__(method)(*args, **kwargs) if isinstance(result, Condition): self._conditions[result.oid] = result return result def _proxy_property(self, method, dfilter_oid): dfilter = self._dfilters[dfilter_oid] result = dfilter.__getattribute__(method) if isinstance(result, Condition): self._conditions[result.oid] = result return result def new_dfilter(self, name, data, setop='AND', prefix=''): dataset = Showcase.instance().get(data) new_dfilter = DynFilter(name, dataset, setop, prefix=prefix) self._dfilters[new_dfilter.oid] = new_dfilter return new_dfilter def expose_dfilter(self, dfilter): self._dfilters[dfilter.oid] = dfilter return dfilter def del_dfilter(self, oid): self._dfilters.pop(oid) def clear(self): self._dfilters.clear() self._conditions.clear()
def __init__(self, name='GrammarSrv'): ''' :param name: The unique name of the service ''' self._roots = Case().tag(name).tag(Root.__name__) INamed.__init__(self, name)
class GrammarService(INamed): ''' This class let clients create ''' def __init__(self, name='GrammarSrv'): ''' :param name: The unique name of the service ''' self._roots = Case().tag(name).tag(Root.__name__) INamed.__init__(self, name) def register_in(self, dispatcher): dispatcher.add_method(self.new_root) dispatcher.add_method(self.expose_root) dispatcher.add_method(self.del_root) dispatcher.add_method(self.build) dispatcher.add_method(partial(self._proxy_instanciator, 'add_dataset'), 'add_dataset') dispatcher.add_method(partial(self._proxy_instanciator, 'add_dynamic'), 'add_dynamic') dispatcher.add_method(partial(self._proxy_instanciator, 'add_condition'), 'add_condition') dispatcher.add_method(partial(self._proxy_property, 'grammar'), 'grammar') def build(self, grammar, objects=None): showcase = Showcase.instance() if objects: objects = {o:showcase.get(o) for o in objects} instances = Root.build(grammar, objects) print '////', instances for name, instance in instances.items(): case = showcase.get_case(instance.__class__.__name__) case[name] = instance return instances.values() def _proxy(self, method, root_oid, *args, **kwargs): root = self._roots[root_oid] result = root.__getattribute__(method)(*args, **kwargs) return result def _proxy_property(self, method, root_oid): root = self._roots[root_oid] result = root.__getattribute__(method) return result def _proxy_instanciator(self, method, root_oid, object_or_objects): showcase = Showcase.instance() names = object_or_objects if isinstance(object_or_objects, list) else [object_or_objects] objects = [showcase.get(n) for n in names] return self._proxy(method, root_oid, objects) def new_root(self, name, prefix=''): ''' :param str name: The name is ussed as identifier :param str prefix: Prepended to the name creates the oid ''' new_root = Root(name, prefix=prefix) self._roots[new_root.oid] = new_root return new_root def expose_root(self, root): self._roots[root.oid] = root return root def del_root(self, oid): self._roots.pop(oid)
class ConditionService(INamed): ''' This class provide a facade for managing Condition objects ''' def __init__(self, name='ConditionSrv'): ''' @param name: The unique name of the service ''' self._conditions = Case().tag(name) \ .tag(Condition.__name__) \ .tag(CategoricalCondition.__name__) \ .tag(AttributeCondition.__name__) \ .tag(RangeCondition.__name__) \ .tag(QueryCondition.__name__) INamed.__init__(self, name) def register_in(self, dispatcher): dispatcher.add_method(self.new_condition) dispatcher.add_method(self.expose_condition) dispatcher.add_method(self.del_condition) dispatcher.add_method(self.update_conditions) dispatcher.add_method(self.update_conditions2) # Condition Properties dispatcher.add_method(partial(self._proxy_property, 'name'), 'name') dispatcher.add_method(partial(self._proxy_property, 'data'), 'data') dispatcher.add_method(partial(self._proxy_property, 'grammar'), 'grammar') dispatcher.add_method(partial(self._proxy_property, 'enabled'), 'enabled') # Condition Methods dispatcher.add_method(partial(self._proxy, 'include_all'), 'include_all') dispatcher.add_method(partial(self._proxy, 'exclude_all'), 'exclude_all') dispatcher.add_method(partial(self._proxy, 'enable'), 'enable') dispatcher.add_method(partial(self._proxy, 'disable'), 'disable') # CategoricalCondition Properties dispatcher.add_method(partial(self._proxy_property, 'attr'), 'attr') dispatcher.add_method(partial(self._proxy, 'get_condition'), 'get_condition') # CategoricalCondition Methods dispatcher.add_method(partial(self._proxy, 'included_categories'), 'included_categories') dispatcher.add_method(partial(self._proxy, 'excluded_categories'), 'excluded_categories') dispatcher.add_method(partial(self._proxy, 'included_items'), 'included_items') dispatcher.add_method(partial(self._proxy, 'excluded_items'), 'excluded_items') dispatcher.add_method(partial(self._proxy, 'add_category'), 'add_category') dispatcher.add_method(partial(self._proxy, 'remove_category'), 'remove_category') dispatcher.add_method(partial(self._proxy, 'toggle_category'), 'toggle_category') # AttributeCondition Methods dispatcher.add_method(partial(self._proxy, 'included_attributes'), 'included_attributes') dispatcher.add_method(partial(self._proxy, 'excluded_attributes'), 'excluded_attributes') dispatcher.add_method(partial(self._proxy, 'add_attribute'), 'add_attribute') dispatcher.add_method(partial(self._proxy, 'remove_attribute'), 'remove_attribute') dispatcher.add_method(partial(self._proxy, 'toggle_attribute'), 'toggle_attribute') # RangeCondition Properties dispatcher.add_method(partial(self._proxy_property, 'range'), 'range') dispatcher.add_method(partial(self._proxy_property, 'domain'), 'domain') # RangeCondition Methods dispatcher.add_method(partial(self._proxy, 'set_range'), 'set_range') # QueryCondition Properties dispatcher.add_method(partial(self._proxy_property, 'query'), 'query') # QueryCondition Methods dispatcher.add_method(partial(self._proxy, 'set_query'), 'set_query') def _proxy(self, method, condition_oid, *args, **kwargs): condition = self._conditions[condition_oid] result = condition.__getattribute__(method)(*args, **kwargs) if isinstance(result, Condition): self._conditions[result.oid] = result return result def _proxy_property(self, method, condition_oid): condition = self._conditions[condition_oid] result = condition.__getattribute__(method) if isinstance(result, Condition): self._conditions[result.oid] = result return result def new_condition(self, kind, data, *args, **kwargs): dataset = Showcase.instance().get(data) if kind == 'categorical': new_condition = CategoricalCondition(dataset, *args, **kwargs) if kind == 'attribute': new_condition = AttributeCondition(dataset, *args, **kwargs) if kind == 'range': new_condition = RangeCondition(dataset, *args, **kwargs) if kind == 'query': new_condition = QueryCondition(dataset, *args, **kwargs) self._conditions[new_condition.oid] = new_condition return new_condition def expose_condition(self, condition): self._conditions[condition.oid] = condition return condition def del_condition(self, oid): self._conditions.pop(oid) def update_conditions(self): # Update all conditions (variable _domain) from pprint import pprint for ind in self._conditions.keys(): cond = self._conditions[ind] cond._sieve._domain = set(cond._sieve._data.distinct(cond._sieve._data_index)) # UPDATE #domm = cond._sieve._data.aggregate([{'$match': {cond._attr: {'$type': 1}}}, # Only numbers # {'$group': # {'_id': {}, # 'min': {'$min': "$"+cond._attr}, # 'max': {'$max': "$"+cond._attr}}}]).get_data() #ran = cond._sieve._data.distinct(cond._attr) #min_ran = min(ran) #max_ran = max(ran) #cond._range = {'min': min_ran, 'max': max_ran} #cond._sieve._domain = {'min': min_ran, 'max': max_ran} #pprint(cond._sieve._domain) return self._conditions def update_conditions2(self, conditions): # Update all conditions (variable _domain) from pprint import pprint for ind in conditions.keys(): cond = conditions[ind]['grammar'] condition = self._conditions[cond['name']] if cond['type'] == "categorical": condition._sieve._domain = set(condition._sieve._data.distinct(condition._sieve._data_index)) # UPDATE cond['included_categories'] = list(set(condition._sieve._data.distinct(condition._sieve._data_index))) cond['excluded_categories'] = [] elif cond['type'] == "range": ran = condition._sieve._data.distinct(condition._attr) min_ran = min(ran) max_ran = max(ran) if max_ran == min_ran: min_ran -= 1 max_ran += 1 relative_max = condition._to_relative(condition._range['max']) relative_min = condition._to_relative(condition._range['min']) condition._domain = {'min': min_ran, 'max': max_ran} condition._range = {'min': min_ran, 'max': max_ran, 'relative_min': relative_min, 'relative_max': relative_max} cond['domain'] = condition._domain cond['range'] = condition._range return conditions