class Portlet(Persistent): __name__ = None __parent__ = None type_name = u"Portlet" type_title = _(u"Portlet") type_description = _(u"A mini view rendered ") portlet_type = u"" add_permission = "Add %s" % type_name def __init__(self, portlet_type, **kw): self.uid = unicode(uuid4()) self.portlet_type = portlet_type self.__settings__ = OOBTree() settings = kw.pop('settings', {}) self.settings = settings self.__dict__.update(**kw) super(Portlet, self).__init__() @property def title(self): return self.settings.get('title', getattr(self.portlet_adapter, 'title', u'')) @property def description(self): return self.settings.get( 'description', getattr(self.portlet_adapter, 'description', u'')) @property def settings(self): return self.__settings__ @settings.setter def settings(self, value): self.__settings__.clear() self.__settings__.update(value) @property def schema_factory(self): return self.portlet_adapter.schema_factory @property def portlet_adapter(self): reg = get_current_registry() return reg.getAdapter(self, IPortletType, name=self.portlet_type) @property def slot(self): try: return self.__parent__.slot except AttributeError: pass def render(self, context, request, view, **kw): try: return self.portlet_adapter.render(context, request, view, **kw) except ComponentLookupError: _logger.error("portlet %r not found for context %r" % (self.portlet_type, context)) return ""
def index(table_name, attr, data_structure): if data_structure == "Btree": tree = OOBTree() T = tables[table_name] indexes = np.arange(len(T.table)) header = T.header header_idx = {k: v for v, k in enumerate(header)} attr_idx = header_idx[attr] T_table = T.table for i in range(len(T_table)): value = T_table[i][attr_idx] if tree.has_key(value): current_idxs = tree[value] current_idxs.append(i) tree.update({value: current_idxs}) else: tree.update({value: [i]}) tables[table_name].index = tree tables[table_name].index_att = attr else: dict = {} T = tables[table_name] indexes = np.arange(len(T.table)) header = T.header header_idx = {k: v for v, k in enumerate(header)} attr_idx = header_idx[attr] T_table = T.table for i in range(len(T_table)): value = T_table[i][attr_idx] dict.setdefault(value, []).append(i) tables[table_name].index = dict tables[table_name].index_att = attr
def pushCondition(database, table_schema, tables, conditions): newDatabase = {} for table in tables: temp = database[table] inCondition = False for c in conditions: if c[0] == table: condition = c inCondition = True break if inCondition and len(condition) < 6: #not two var newTree = OOBTree() for key in temp: if condition[1] == 0: #key column value = key else: value = temp[key][condition[1] - 1] if evaluateCondition(value, condition[3], condition[2]): newTree.update({key: temp[key]}) newDatabase[table] = newTree else: newDatabase[table] = temp return newDatabase
def loadDatabase(database, table): #load the data into a btree (assuming that the first column is the primary key) file = open("database_files/" + table + ".csv", "r") t = OOBTree() for line in file: data = line[:-1].split(',') fixed_data = [] i = 0 while i < len(data): if data[i][0] != '"' and data[i][0] != "'": fixed_data.append(float(data[i])) elif (data[i][0] == "'" or data[i][0] == '"') and (data[i][-1] == "'" or data[i][-1] == '"'): fixed_data.append(data[i][1:-1]) else: j = i + 1 temp = data[i][1:] while j < len(data): temp += "," + data[j] if data[j][-1] == "'" or data[j][-1] == '"': fixed_data.append(temp[:-1]) break j += 1 i = j i += 1 t.update({fixed_data[0]: fixed_data[1:]}) database[table] = t file.close()
class Portlet(Persistent): __name__ = None __parent__ = None type_name = u"Portlet" type_title = _(u"Portlet") type_description = _(u"A mini view rendered ") portlet_type = u"" add_permission = "Add %s" % type_name def __init__(self, portlet_type, **kw): self.uid = unicode(uuid4()) self.portlet_type = portlet_type self.__settings__ = OOBTree() settings = kw.pop('settings', {}) self.settings = settings self.__dict__.update(**kw) super(Portlet, self).__init__() @property def title(self): return self.settings.get('title', getattr(self.portlet_adapter, 'title', u'')) @property def description(self): return self.settings.get('description', getattr(self.portlet_adapter, 'description', u'')) @property def settings(self): return self.__settings__ @settings.setter def settings(self, value): self.__settings__.clear() self.__settings__.update(value) @property def schema_factory(self): return self.portlet_adapter.schema_factory @property def portlet_adapter(self): reg = get_current_registry() return reg.getAdapter(self, IPortletType, name = self.portlet_type) @property def slot(self): try: return self.__parent__.slot except AttributeError: pass def render(self, context, request, view, **kw): try: return self.portlet_adapter.render(context, request, view, **kw) except ComponentLookupError: _logger.error("portlet %r not found for context %r" % (self.portlet_type, context)) return ""
def setupNewSubsystemsFromMovedFileInfo(loggingMixin, mocker): """ Setup for testing create new runs and subsystems based on moved file information. """ # Ensure that processing clasess doesn't actually create any files or folders... mocker.patch("overwatch.processing.processingClasses.os.makedirs") # Set the relevant subsystems # We want the TPC to use HLT subsystem files. subsystems = ["EMC", "HLT", "TPC"] processRuns.processingParameters["subsystemList"] = subsystems runDir = "Run123" hltMode = "C" runDict = { runDir: { "hltMode": hltMode, "HLT": [ "HLThists.2015_11_24_18_05_10.root", "HLThists.2015_11_24_18_09_12.root" ], "EMC": [ "EMChists.2015_11_24_18_05_12.root", "EMChists.2015_11_24_18_09_14.root" ], } } # Same as the runDict above, but with an additional set of files. additionalRunDict = { runDir: { "hltMode": hltMode, "HLT": [ "HLThists.2015_11_24_19_05_10.root", "HLThists.2015_11_24_19_09_12.root" ], "EMC": [ "EMChists.2015_11_24_19_05_11.root", "EMChists.2015_11_24_19_09_13.root" ], } } # Ensure we don't create random directories mocker.patch("overwatch.processing.processingClasses.os.path.exists") # Create the runs container runs = OOBTree() runs.update({ runDir: processingClasses.runContainer(runDir=runDir, fileMode=True, hltMode=hltMode) }) # Add some subsystems runs[runDir].subsystems return runs, runDir, runDict, additionalRunDict, subsystems
def compressDict(originalDict): newDict = {} for key in originalDict: (_, a, b, _) = originalDict[key] newDict[key] = (a, b) t = OOBTree() t.update(newDict) del newDict return t
def unionResultSets(sets): """ perform union of ResultSets """ docids = None words = OOBTree() sets = list(sets) sets.sort(lambda s1,s2: cmp(len(s1.docIds()),len(s2.docIds()))) for set in sets: docids = IntUnion(docids, set.docIds()) words.update(set.words()) return ResultSet(docids, words)
def create_index_btree(): """ Build b-tree for efficient index file search """ index_path = 'Index/' index_file_names = sorted(listdir(index_path)) t = OOBTree() # Create a dictionary where there are file names on key and index on values name_id_dict = {k: v for v, k in enumerate(index_file_names)} t.update(name_id_dict) # Build tree return t
def add_index(self, attr, idex_name): if attr[0] not in self.uniqueattr.keys(): raise Exception( 'ERROR: The attr is not unique and cannot create index') # If unique: if attr[0] not in self.index: self.index[attr[0]] = idex_name # Get pairs {v1:p1, v2:p2,...} nodes = self.uniqueattr[attr[0]] # Create a b tree t = OOBTree() t.update(nodes) self.BTree[idex_name] = t return t
def test_create_tree(self): array = list(range(self.complexity)) random.shuffle(array) st = time.clock() d = {i: -i for i in array} elapse1 = time.clock() - st t = OOBTree() st = time.clock() t.update(d) elapse2 = time.clock() - st print("creating a dict using %.6f sec, a tree %.6f sec" % (elapse1, elapse2)) self.assertLess(elapse1, elapse2)
def testFailedJoin(self): """ Tests if the join works correctly. """ b_tree = OOBTree() b_tree.update({1: "Monkey D. Luffy", 2: "Roronoa Zoro", 3: "Nami"}) failed_counter = 0 key = 10 data = {"from": "East Blue"} (mod_data, mod_tree, failed_counter) = self.processing.join(b_tree, key, data, failed_counter) self.assertEqual(mod_data, {"from": "East Blue"}) self.assertEqual(len(mod_tree), 3) self.assertEqual(failed_counter, 1)
def BTreeJoin(orderedPK=False, orderedFK=False): #Arquivos que usaremos no JOIN PKFile = "" FKFile = "" if (orderedPK): PKFile = tg.PKOrderedFileName else: PKFile = tg.PKUnorderedFileName if (orderedFK): FKFile = tg.FKOrderedFileName else: FKFile = tg.FKUnorderedFileName #Carregando o indice da BTree btree = OOBTree() lineCounter = 0 with open(FKFile) as f: for line in f: l = line.split(tg.fieldSeparator) btree.update({l[1]: lineCounter}) lineCounter += 1 # checking the size print("Btree size: " + str(len(btree))) lineSize = tg.registrySize + 2 start = dt.datetime.now() matches = 0 with open(FKFile) as right: with open(PKFile) as left: for line in left: leftRegistry = line.split(tg.fieldSeparator) position = btree[leftRegistry[0]] * lineSize right.seek(position) rightRegistry = right.read(tg.registrySize).split( tg.fieldSeparator) if leftRegistry[0] == rightRegistry[1]: matches += 1 joined = leftRegistry + rightRegistry else: print("This should never happen!") end = dt.datetime.now() print("Total joined registers: " + str(matches)) print("Time taken: " + str((end - start).total_seconds()) + "s")
def file_to_tree(root_guarda, file): ''' Função responsável por, a partir de um arquivo, criar uma árvore B obedecendo os padrões de arquivos e diretórios :param root_guarda : caminho da pasta do programa guarda onde está o arquivo :param file : arquivo a ser lido ''' tree = OOBTree() file = open(root_guarda+file,"r") for line in file: file_hash = line.split("\t") if file_hash[1] == "dir\n": tree.update({file_hash[0]:file_to_tree(root_guarda, file_hash[0]+".txt")}) continue else: tree.update({file_hash[0]:str(file_hash[1])[:-1]}) file.close() return tree
def test_getitem(self): array = list(range(self.complexity)) random.shuffle(array) d = {i: -i for i in array} t = OOBTree() t.update(d) st = time.clock() for k in array: d[k] elapse1 = time.clock() - st st = time.clock() for k in array: t[k] elapse2 = time.clock() - st print("get item from a dict using %.6f sec, a tree %.6f sec" % (elapse1, elapse2)) self.assertLess(elapse1, elapse2)
class IX_IndexHandle: def __init__(self, fileName, indexNo): self.filename = fileName + '.' + str(indexNo) self.indexNo = indexNo self.fileName = fileName with open(self.filename, "r", encoding="utf-8") as f: tree_dict = json.load(f) keys = list(map(int, tree_dict.keys())) values = list(tree_dict.values()) new_tree_dict = dict(zip(keys, values)) self.tree = Tree() self.tree.update(new_tree_dict) pass def __del__(self): pass def InsertEntry(self, record, rid): key = record[self.indexNo] l = self.tree.get(key) pageNum = rid.getPageNum() slotNum = rid.getSlotNum() if l: l.append((pageNum, slotNum)) else: self.tree[key] = [(pageNum, slotNum)] def DeleteEntry(self, record, rid): key = record[self.indexNo] l = self.tree.get(key) pageNum = rid.getPageNum() slotNum = rid.getSlotNum() if l: l.remove((pageNum, slotNum)) else: return False def ForcePages(self): with open(self.filename, "w", encoding="utf-8") as f: tree_dict = dict(self.tree.items()) json.dump(tree_dict, f)
def test_range_seach(self): """ """ array = list(range(self.complexity)) random.shuffle(array) d = {i: -i for i in array} t = OOBTree() t.update(d) # === dict === st = time.clock() result1 = list() for k, v in d.items(): if (self.lower <= k <= self.upper): result1.append(v) elapse1 = time.clock() - st # === bisearch === sorted_keys = list(d.keys()) # sort the list of keys sorted_keys.sort() st = time.clock() lower_ind = bisect.bisect_left(sorted_keys, self.lower) # find the min index upper_ind = bisect.bisect_right(sorted_keys, self.upper) - 1 # find the max index result2 = list() # fetch item for ind in range(lower_ind, upper_ind + 1): result2.append(d[sorted_keys[ind]]) elapse2 = time.clock() - st # === tree === st = time.clock() result3 = t.values(min=self.lower, max=self.upper, excludemin=False, excludemax=False) elapse3 = time.clock() - st print("results are:", result1, result2, list(result3)) print("dict method = %.6f, bisearch = %.6f, tree = %.6f" % (elapse1, elapse2, elapse3)) self.assertGreater(elapse1, elapse2) self.assertGreater(elapse2, elapse3)
def buildTreeForAttr(dictAttr, fileName, attrIndx, file_path, return_tree): """ This funciton accepts a dctionary dictAttr, build an OOBtree, store the value in OOBTree and then dump the Btree object into a file with name defined as fileName + _Attr_ + attrIndx + _.tree Args: dictAttr: a dictionary that store the (attribute, offset set) pair fileName: csvfile name without '.csv' attrIndx: index of attribute file_path: path to store the btree file Returns: None """ sys.setrecursionlimit(100000) t = OOBTree() t.update(dictAttr) os.makedirs(file_path, exist_ok=True) with open(file_path + '/' + fileName + '_Attr_' + str(attrIndx) + '_.tree', "wb") as f: cPickle.dump(t, f) if return_tree: return t
def createBTree(collectionName, column): """Creates a BTree for the given collection based on specific column Parameters ---------- collectionName : str The name of the collection we want to create BTree for column : str Column we use to create BTree """ # Creating a defaultdict to prevent KeyError and if a key is not present, it is intialized to empty array hashmap = defaultdict(lambda: []) collection = allCollections[collectionName] for row in range(len(collection)): hashmap[collection[row][column]].append(row) # Intializing the empty BTree tree = OOBTree() # To update the tree we need hash of all values which is calculated above tree.update(hashmap) bTreedKeys[collectionName + '_' + column] = tree print('Created a BTree on ' + column + ' for collection ' + collectionName)
class Root(Content, LocalRolesMixin, DCMetadataMixin, ContextACLMixin): type_name = "Root" type_title = _("Site root") add_permission = "Add %s" % type_name search_visible = False is_permanent = True def __init__(self, data=None, **kwargs): self.catalog = Catalog() self.document_map = DocumentMap() populate_catalog(self.catalog) self.__site_settings__ = OOBTree() super(Root, self).__init__(data=data, **kwargs) @property def site_settings(self): return getattr(self, "__site_settings__", {}) @site_settings.setter def site_settings(self, value): self.__site_settings__.clear() self.__site_settings__.update(value)
class Root(Content, LocalRolesMixin, DCMetadataMixin, ContextACLMixin): type_name = u"Root" type_title = _(u"Site root") add_permission = "Add %s" % type_name search_visible = False is_permanent = True def __init__(self, data=None, **kwargs): self.catalog = Catalog() self.document_map = DocumentMap() populate_catalog(self.catalog) self.__site_settings__ = OOBTree() super(Root, self).__init__(data=data, **kwargs) @property def site_settings(self): return getattr(self, '__site_settings__', {}) @site_settings.setter def site_settings(self, value): self.__site_settings__.clear() self.__site_settings__.update(value)
def Btree(items, dict, btree_dict, col_dict): var = re.split(r'[\s\,\(\)]+', items) while '' in var: var.remove('') ori_table = var[1] index_col = var[2] table_col = ori_table + '_' + index_col col_dict[table_col] = 'Btree' col = find_col(dict[ori_table], index_col) dict = {} for index in range(0, len(col)): if col[index] not in dict.keys(): res = [] res.append(index) dict[col[index]] = res else: res = dict[col[index]] res.append(index) dict[col[index]] = res t = OOBTree() t.update(dict) btree_dict[table_col] = t return 1
def create_hash_dir(name, directory, method, key, root_guarda): dir_tree = OOBTree() path = directory+name+"/" for i in os.listdir(path): if i == ".guarda": continue if method == '--hash': hasher = hashlib.sha1() elif method == '--hmac': hasher = hmac.new(key.encode(), digestmod=hashlib.sha1) if (os.path.isfile(path+i)): with open(path+i, 'rb') as node: buf = None buf = node.read() hasher.update(buf) dir_tree.update({i:hasher.hexdigest()}) continue if (os.path.isdir(path+i)): dir_tree.update({i:create_hash_dir(i, path, method, key, root_guarda)}) return dir_tree
def index(self): for table_def in table_data.table_data.values(): print ("Looking for Indexes on " + table_def["name"]) + "..." table_data.indexes[table_def["name"]] = {} for column_def in table_def["columns"].values(): if column_def["index"]: print "Indexing table " + table_def["name"] + " column " + column_def["name"] index_tree = OOBTree() table_data.indexes[table_def["name"]][column_def["name"]] = index_tree i = 0 for data_file in listdir(table_def["data_store"]): with open(table_def["data_store"] + data_file) as indexing_data: page_index = 0 table_reader = csv.reader(indexing_data, delimiter=",", quotechar="|") for row in table_reader: index_data = row[column_def["position"]] if not index_data in index_tree: index_tree.update({index_data : [(data_file, page_index)]}) else: index_tree[index_data].append((data_file, page_index)) page_index += 1 i += 1 print "Indexed " + str(i) + " files..."
def create_index(self, field_to_index: str, index_type="btree") -> None: index_of_field = self.get_index_of_field(field_to_index) with open(f"{DB_ROOT}/{self.name}.csv", "r") as csv_file: csv_reader = csv.reader(csv_file) index_values = {} next(csv_reader) for record in csv_reader: if record: index_values[record[index_of_field]] = { "path": "{DB_ROOT}/{self.name}.csv", "line": csv_reader.line_num } if index_type == "btree": btree_index = OOBTree() btree_index.update(index_values) self.btree_index_info[field_to_index] = btree_index if index_type == "hash": self.hash_index_info[field_to_index] = index_values else: raise ValueError("No such index type")
def Btree(self, condition, return_=False): ''' (i) Create a btree with an index on an attribute from a table that user specified. (ii) Inputs: An attribute from a table (iii) Outputs: output a btree that is indexed with the attribute specified. (iv) There is a global effect that a btree is created ''' condition = condition.replace(' ', '') condition = re.split('[,\(\)\n\s]', condition) unwanted = ['', 'Btree'] condition = [e for e in condition if e not in unwanted] table = eval(condition[0]) btree = OOBTree() for i, value in enumerate(table[condition[1]]): if value in btree: btree.update({value: btree[value] + [i]}) else: btree.update({value: [i]}) self.flag = 'btree' self.index['btree'] = condition[1] self.btree = btree if return_: return btree
def test_range_seach(self): """ """ array = list(range(self.complexity)) random.shuffle(array) d = {i: -i for i in array} t = OOBTree() t.update(d) # === dict === st = time.clock() result1 = list() for k, v in d.items(): if (self.lower <= k <= self.upper): result1.append(v) elapse1 = time.clock() - st # === bisearch === sorted_keys = list(d.keys()) # sort the list of keys sorted_keys.sort() st = time.clock() lower_ind = bisect.bisect_left(sorted_keys, self.lower) # find the min index upper_ind = bisect.bisect_right(sorted_keys, self.upper) - 1 # find the max index result2 = list() # fetch item for ind in range(lower_ind, upper_ind+1): result2.append(d[sorted_keys[ind]]) elapse2 = time.clock() - st # === tree === st = time.clock() result3 = t.values(min=self.lower, max=self.upper, excludemin=False, excludemax=False) elapse3 = time.clock() - st print("results are:", result1, result2, list(result3)) print("dict method = %.6f, bisearch = %.6f, tree = %.6f" % (elapse1, elapse2, elapse3)) self.assertGreater(elapse1, elapse2) self.assertGreater(elapse2, elapse3)
def init(method, directory,out, key): path = None try: path_guarda = str(directory) +".guarda/" #se já existir programa guarda : apaga if os.path.isdir(path_guarda): shutil.rmtree(path_guarda) os.makedirs(path_guarda) guarda = open(path_guarda +"guarda.txt", "w+") guarda.close() guarda_tree = OOBTree() for i in os.listdir(directory): if i == ".guarda" or i == ".tmp": continue if method == '--hash': hasher = hashlib.sha1() elif method == '--hmac': hasher = hmac.new(key.encode(), digestmod=hashlib.sha1) if (os.path.isfile(directory+i)): with open(directory+i, 'rb') as node: buf = None buf = node.read() hasher.update(buf) guarda_tree.update({i:hasher.hexdigest()}) continue if (os.path.isdir(directory+i)): guarda_tree.update({i:create_hash_dir(i, directory, method, key, directory+".guarda/")}) tree_to_file("guarda", directory, directory+".guarda/", guarda_tree) except OSError as error: print(error)
class Subscriptions(SimpleItem): security = ClassSecurityInfo() title = "Meeting registrations" def __init__(self, id): """ """ super(SimpleItem, self).__init__(id) self.id = id self._signups = OOBTree() self._account_subscriptions = OOBTree() security.declarePublic('getMeeting') def getMeeting(self): return self.aq_parent.aq_parent def _validate_signup(self, form): """ """ formdata = {} formerrors = {} keys = ('first_name', 'last_name', 'email', 'organization', 'phone') formdata = dict( (key, form.get(key, '')) for key in keys ) for key in formdata: if formdata[key] == '': formerrors[key] = 'This field is mandatory' if formerrors == {}: if formdata['email'].count('@') != 1: formerrors['email'] = 'An email address must contain a single @' if formerrors == {}: formerrors = None return formdata, formerrors def _add_signup(self, formdata): """ """ meeting = self.getMeeting() key = random_key() name = formdata['first_name'] + ' ' + formdata['last_name'] email = formdata['email'] organization = formdata['organization'] phone = formdata['phone'] signup = SignUp(key, name, email, organization, phone) self._signups.insert(key, signup) if meeting.auto_register: self._accept_signup(key) email_sender = self.getMeeting().getEmailSender() email_sender.send_signup_email(signup) security.declareProtected(view, 'signup') def signup(self, REQUEST): """ """ meeting = self.getMeeting() if not meeting.allow_register: return REQUEST.RESPONSE.redirect(self.absolute_url() + '/subscription_not_allowed') if REQUEST.REQUEST_METHOD == 'GET': return self.getFormsTool().getContent({'here': self}, 'naaya.content.meeting.subscription_signup') if REQUEST.REQUEST_METHOD == 'POST': formdata, formerrors = self._validate_signup(REQUEST.form) #check Captcha/reCaptcha if not self.checkPermissionSkipCaptcha(): contact_word = REQUEST.form.get('contact_word', '') captcha_validator = self.validateCaptcha(contact_word, REQUEST) if captcha_validator: if formerrors is None: formerrors = {} formerrors['captcha'] = captcha_validator if formerrors is not None: return self.getFormsTool().getContent({'here': self, 'formdata': formdata, 'formerrors': formerrors}, 'naaya.content.meeting.subscription_signup') else: self._add_signup(formdata) REQUEST.RESPONSE.redirect(self.absolute_url() + '/signup_successful') security.declareProtected(view, 'signup_successful') def signup_successful(self, REQUEST): """ """ return self.getFormsTool().getContent({'here': self}, 'naaya.content.meeting.subscription_signup_successful') security.declareProtected(view, 'subscribe') def subscribe(self, REQUEST): """ """ meeting = self.getMeeting() if not meeting.allow_register: return REQUEST.RESPONSE.redirect(self.absolute_url() + '/subscription_not_allowed') return self.getFormsTool().getContent({'here': self}, 'naaya.content.meeting.subscription_subscribe') security.declareProtected(PERMISSION_ADMIN_MEETING, 'getSignups') def getSignups(self): """ """ return self._signups.itervalues() security.declareProtected(PERMISSION_ADMIN_MEETING, 'getSignup') def getSignup(self, key): """ """ return self._signups.get(key, None) security.declareProtected(PERMISSION_ADMIN_MEETING, 'index_html') def index_html(self, REQUEST): """ """ return self.getFormsTool().getContent({'here': self}, 'naaya.content.meeting.subscription_index') def _accept_signup(self, key): """ """ meeting = self.getMeeting() meeting.getParticipants()._set_attendee(key, PARTICIPANT_ROLE) signup = self._signups[key] signup.accepted = 'accepted' email_sender = meeting.getEmailSender() result = email_sender.send_signup_accepted_email(signup) def _reject_signup(self, key): """ """ meeting = self.getMeeting() signup = self._signups[key] signup.accepted = 'rejected' participants = meeting.getParticipants() if key in participants._get_attendees(): participants._del_attendee(key) email_sender = meeting.getEmailSender() result = email_sender.send_signup_rejected_email(signup) def _delete_signup(self, key): """ """ meeting = self.getMeeting() signup = self._signups.pop(key, None) if signup is None: return participants = meeting.getParticipants() if key in participants._get_attendees(): participants._del_attendee(key) email_sender = meeting.getEmailSender() result = email_sender.send_signup_rejected_email(signup) def _is_signup(self, key): """ """ return self._signups.has_key(key) and self._signups[key].accepted == 'accepted' security.declareProtected(PERMISSION_ADMIN_MEETING, 'manageSignups') def manageSignups(self, REQUEST): """ """ keys = REQUEST.form.get('keys', []) assert isinstance(keys, list) if 'accept' in REQUEST.form: for key in keys: self._accept_signup(key) elif 'reject' in REQUEST.form: for key in keys: self._reject_signup(key) elif 'delete' in REQUEST.form: for key in keys: self._delete_signup(key) return REQUEST.RESPONSE.redirect(self.absolute_url()) security.declarePublic('welcome') def welcome(self, REQUEST): """ """ if 'logout' in REQUEST.form: REQUEST.SESSION['nymt-current-key'] = None return REQUEST.RESPONSE.redirect(self.getMeeting().absolute_url()) key = REQUEST.get('key', None) signup = self.getSignup(key) if self._is_signup(key): REQUEST.SESSION['nymt-current-key'] = key return self.getFormsTool().getContent({'here': self, 'signup': signup}, 'naaya.content.meeting.subscription_welcome') def _add_account_subscription(self, uid): """ """ site = self.getSite() meeting = self.getMeeting() name = getUserFullName(site, uid) email = getUserEmail(site, uid) organization = getUserOrganization(site, uid) if not organization: organization = self.get_survey_answer(uid, 'w_organization') if not organization: organization = self.get_survey_answer(uid, 'w_organisation') phone = getUserPhoneNumber(site, uid) if not phone: phone = self.get_survey_answer(uid, 'w_telephone') if not phone: phone = self.get_survey_answer(uid, 'w_phone') account_subscription = AccountSubscription(uid, name, email, organization, phone) self._account_subscriptions.insert(uid, account_subscription) if meeting.auto_register: self._accept_account_subscription(uid) email_sender = self.getMeeting().getEmailSender() email_sender.send_account_subscription_email(account_subscription) security.declareProtected(PERMISSION_ADMIN_MEETING, 'update_account_subscription') def update_account_subscription(self, uid): """ """ site = self.getSite() name = getUserFullName(site, uid) email = getUserEmail(site, uid) organization = getUserOrganization(site, uid) phone = getUserPhoneNumber(site, uid) account_subscription = AccountSubscription(uid, name, email, organization, phone) self._account_subscriptions.update({uid: account_subscription}) security.declareProtected(view, 'subscribe_account') def subscribe_account(self, REQUEST): """ """ meeting = self.getMeeting() if not meeting.allow_register: return REQUEST.RESPONSE.redirect(self.absolute_url() + '/subscription_not_allowed') self._add_account_subscription(REQUEST.AUTHENTICATED_USER.getId()) if self.survey_required: site = self.getSite() path = str(self.survey_pointer) survey_ob = site.unrestrictedTraverse(path, None) if survey_ob is not None and survey_ob.meta_type == 'Naaya Mega Survey': answers = survey_ob.getAnswers() respondents = [a.respondent for a in answers] current_user = REQUEST.AUTHENTICATED_USER.getUserName() if current_user not in respondents: self.setSessionInfoTrans( 'Registration successfully sent for approval. ' 'Please also respond to the following questionaire.') return REQUEST.RESPONSE.redirect('%s/%s' % (self.getSite().absolute_url(), self.survey_pointer)) return REQUEST.RESPONSE.redirect(self.absolute_url() + '/subscribe_account_successful') security.declareProtected(view, 'subscribe_account_successful') def subscribe_account_successful(self, REQUEST): """ """ return self.getFormsTool().getContent({'here': self}, 'naaya.content.meeting.subscription_subscribe_successful') security.declareProtected(PERMISSION_ADMIN_MEETING, 'getAccountSubscriptions') def getAccountSubscriptions(self): """ """ return self._account_subscriptions.itervalues() security.declareProtected(PERMISSION_ADMIN_MEETING, 'getAccountSubscription') def getAccountSubscription(self, uid): """ """ return self._account_subscriptions.get(uid, None) def _is_account_subscription(self, uid): """ """ return self._account_subscriptions.has_key(uid) and self._account_subscriptions[uid].accepted == 'accepted' security.declareProtected(PERMISSION_ADMIN_MEETING, 'manageSignups') def manageAccountSubscriptions(self, REQUEST): """ """ uids = REQUEST.form.get('uids', []) assert isinstance(uids, list) if 'accept' in REQUEST.form: for uid in uids: self._accept_account_subscription(uid) elif 'reject' in REQUEST.form: for uid in uids: self._reject_account_subscription(uid) elif 'delete' in REQUEST.form: for uid in uids: self._delete_account_subscription(uid) return REQUEST.RESPONSE.redirect(self.absolute_url()) def _accept_account_subscription(self, uid): """ """ meeting = self.getMeeting() meeting.getParticipants()._set_attendee(uid, PARTICIPANT_ROLE) account_subscription = self._account_subscriptions[uid] account_subscription.accepted = 'accepted' email_sender = meeting.getEmailSender() result = email_sender.send_account_subscription_accepted_email(account_subscription) def _reject_account_subscription(self, uid): """ """ meeting = self.getMeeting() account_subscription = self._account_subscriptions[uid] account_subscription.accepted = 'rejected' participants = meeting.getParticipants() if uid in participants._get_attendees(): participants._del_attendee(uid) email_sender = meeting.getEmailSender() result = email_sender.send_account_subscription_rejected_email(account_subscription) def _delete_account_subscription(self, uid): """ """ meeting = self.getMeeting() account_subscription = self._account_subscriptions.pop(uid, None) if account_subscription is None: return participants = meeting.getParticipants() if uid in participants._get_attendees(): participants._del_attendee(uid) email_sender = meeting.getEmailSender() result = email_sender.send_account_subscription_rejected_email(account_subscription) security.declareProtected(view, 'subscription_not_allowed') def subscription_not_allowed(self, REQUEST): """ """ return self.getFormsTool().getContent({'here': self}, 'naaya.content.meeting.subscription_not_allowed')
class Index: """ Indexes the specified column of the specified table to speed up select queries This data structure is usually a B-Tree """ def __init__(self): # the column this index operates on self.column_number = 0 # the actual tree that does the work associated with indexing self.index = OOBTree() # the table this index operates on self.table = None def locate(self, value): """ returns the location of all records with the given value :param value: int # the value of the key for which we need the RID """ return self.index.get(value, False) def create_index(self, table, column_number): """ Create index on specific column :param table: table object # the table the index will operate on :param column_number: int # the column number in the table the index will operate on :return: index object # the created index object """ self.table = table self.column_number = column_number page = 0 offset = 0 # for each page range in the table for page_range in self.table.ranges: # for each record in the page range for i in range(page_range.num_of_records): # get the key for this record relative to the chosen column key = page_range.columns[self.column_number].pages[page].read( offset) # get the RID for this record RID = page_range.columns[RID_COLUMN].pages[page].read(offset) # increment the offset offset += 8 # if reached the end of the page increment page and reset offset if offset == PAGE_SIZE: page += 1 offset = 0 self.add_index_item(key, RID) return self def add_index_item(self, key, RID): """ Add to the index the key value and RID pair :param key: int # the key to store this RID at :param RID: int # the RID of the key """ # if there are RID/s at this key get them, else false rids = self.index.get(key, False) # if there are rids at this key already append the new RID if rids: self.index.update({key: rids.append(RID)}) return # otherwise add key and RID self.index.update({key: [RID]}) def drop_index(self, table, column_number): """ Drop index of specific column """ self.table.indexes[self.column_number] = None self.index = None self.table = None self.column_number = 0
class ZODBContinuousIncreasingIdGenerator(IdGenerator): """ Create some Ids with the zodb storage """ zope.interface.implements(interfaces.IIdGenerator) # CMF Type Definition meta_type = 'ERP5 ZODB Continous Increasing Id Generator' portal_type = 'ZODB Continous Increasing Id Generator' add_permission = Permissions.AddPortalContent # Declarative security security = ClassSecurityInfo() security.declareObjectProtected(Permissions.AccessContentsInformation) def _generateNewId(self, id_group, id_count=1, default=None, poison=False): """ Return the new_id from the last_id of the zodb Use int to store the last_id, use also a persistant mapping for to be persistent. """ if id_group in (None, 'None'): raise ValueError, '%s is not a valid group Id.' % ( repr(id_group), ) if default is None: default = 0 last_id_dict = getattr(self, 'last_id_dict', None) if last_id_dict is None: # If the dictionary not exist initialize generator self.initializeGenerator() last_id_dict = self.last_id_dict # Retrieve the last id and increment new_id = last_id_dict.get(id_group, default - 1) + id_count # Store the new_id in the dictionary last_id_dict[id_group] = None if poison else new_id return new_id security.declareProtected(Permissions.AccessContentsInformation, 'generateNewId') def generateNewId(self, id_group=None, default=None, poison=False): """ Generate the next id in the sequence of ids of a particular group """ return self._generateNewId(id_group=id_group, default=default, poison=poison) security.declareProtected(Permissions.AccessContentsInformation, 'generateNewIdList') def generateNewIdList(self, id_group=None, id_count=1, default=None, poison=False): """ Generate a list of next ids in the sequence of ids of a particular group """ new_id = 1 + self._generateNewId(id_group=id_group, id_count=id_count, default=default, poison=poison) return range(new_id - id_count, new_id) security.declareProtected(Permissions.ModifyPortalContent, 'initializeGenerator') def initializeGenerator(self): """ Initialize generator. This is mostly used when a new ERP5 site is created. Some generators will need to do some initialization like prepare some data in ZODB """ LOG('initialize ZODB Generator', INFO, 'Id Generator: %s' % (self, )) if getattr(self, 'last_id_dict', None) is None: self.last_id_dict = OOBTree() # XXX compatiblity code below, dump the old dictionnaries portal_ids = getattr(self, 'portal_ids', None) # Dump the dict_ids dictionary if getattr(portal_ids, 'dict_ids', None) is not None: for id_group, last_id in portal_ids.dict_ids.items(): if not isinstance(id_group, str): id_group = repr(id_group) if self.last_id_dict.has_key(id_group) and \ self.last_id_dict[id_group] > last_id: continue self.last_id_dict[id_group] = last_id security.declareProtected(Permissions.ModifyPortalContent, 'clearGenerator') def clearGenerator(self): """ Clear generators data. This can be usefull when working on a development instance or in some other rare cases. This will loose data and must be use with caution This can be incompatible with some particular generator implementation, in this case a particular error will be raised (to be determined and added here) """ # Remove dictionary self.last_id_dict = OOBTree() security.declareProtected(Permissions.ModifyPortalContent, 'exportGeneratorIdDict') def exportGeneratorIdDict(self): """ Export last id values in a dictionnary in the form { group_id : last_id } """ return dict(self.last_id_dict) security.declareProtected(Permissions.ModifyPortalContent, 'importGeneratorIdDict') def importGeneratorIdDict(self, id_dict, clear=False): """ Import data, this is usefull if we want to replace a generator by another one. """ if clear: self.clearGenerator() if not isinstance(id_dict, dict): raise TypeError, 'the argument given is not a dictionary' for value in id_dict.values(): if not isinstance(value, (int, long)): raise TypeError, 'the value given in dictionary is not a integer' self.last_id_dict.update(id_dict) security.declareProtected(Permissions.ModifyPortalContent, 'rebuildGeneratorIdDict') def rebuildGeneratorIdDict(self): """ Rebuild generator id dict. In fact, export it, clear it and import it into new dict. This is mostly intendted to use when we are migrating the id dict structure. """ id_dict = self.exportGeneratorIdDict() self.importGeneratorIdDict(id_dict=id_dict, clear=True)
print "Joined " + str(joinnum) + " lines." #1d. BTree Join (Index-Based Join) #We'll be using BTrees.OOBTree module. The library file is included in the project folder #to install it, do "pip install BTrees" #this method is very similar to the method studied before, but it uses the BTree data structure #instead of hash tables print "Calculating BTree Join time" start = datetime.datetime.now() btree = OOBTree() position = 1 right = open(right_un) for line in right: r_data = line.split(delimiter) btree.update({r_data[1]: position}) position += 1 joinnum = 0 left = open(left_un) for line in left: l_data = line.split(delimiter) position = btree[l_data[0]] r_data = linecache.getline(right_un, position).split(delimiter) if l_data[0] == r_data[1]: joinnum += 1 end = datetime.datetime.now() print "BTree Join Time: " + str((end - start).total_seconds()) + "sec." print "Joined " + str(joinnum) + " lines." print "All done!"
class SiteData(Persistent): """ Utility used to store and access site use data/logs """ implements(ISiteData) def __init__(self): self.user_data = OOBTree() self.log_data = OOBTree() def store_data(self, data): """ Extracted and store the data in OOBTtrees """ # extract csv files from received zip file zipped_file = StringIO() zipped_file.write(data[2:]) # [2:] removes leading '\r\n' zipped_file.seek(0) zf = zipfile.ZipFile(zipped_file, mode='r') users = StringIO() logs = StringIO() esheets = StringIO() for filename in zf.namelist(): # store user data in persistent storage if filename == 'users.csv': users.write(zf.read(filename)) users.seek(0) user_data = users.getvalue().splitlines() # as a reference from export-user-profiles view: # fieldnames=['username', 'fullname', 'email', # 'teacher_mobile_number', 'school', # 'province', 'EMIS', # 'school_contact_number', # 'school_email', 'qualification', # 'years_teaching','last_login_time', 'uuid' # sort data by province, school and teacher user_data.sort(key= lambda line: ( line.split(",")[5], line.split(",")[4], line.split(",")[1])) # place data in an organised dictionary for user in user_data: username, fullname, email, mobile, school_name, \ province_name, EMIS, school_contact_number, school_email, \ qualification, years_teaching, last_login_time, \ teacher_uuid = user.split(',') if self.user_data.get(province_name) is None: # province entry did not yet exist, initialise to {} self.user_data.update({province_name:{}}) school_dict = self.user_data.get(province_name) try: x = school_dict[school_name] # if this school exists, we do not set it to blank except KeyError: # school entry did not yet exist, initialise to {} school_dict.update({school_name: {}}) teacher_dict = school_dict[school_name] try: x = teacher_dict[teacher_uuid] except KeyError: teacher_dict.update({teacher_uuid: {}}) teacher_data_dict = teacher_dict[teacher_uuid] try: x = teacher_data_dict['evaluationsheets'] except KeyError: teacher_data_dict.update({'evaluationsheets': {}}) e_list = teacher_data_dict['evaluationsheets'] teacher_data = { 'username' : username, 'fullname' : fullname, 'email' : email, 'mobile' : mobile, 'qualification' : qualification, 'years_teaching' : years_teaching, 'last_login_time' : last_login_time, 'evaluationsheets' : e_list, } teacher_dict.update({teacher_uuid: teacher_data}) school_dict.update({school_name: teacher_dict}) self.user_data.update({province_name:school_dict}) # store log data in persistent storage if filename == 'logs.csv': logs.write(zf.read(filename)) logs.seek(0) log_data = logs.getvalue().splitlines() # as reference from exportloggedrequests # fieldnames=['time', 'path', 'username','province','school'], # delete all log values that do not contain a province or school # entry log_data_clean = [] for entry in range(len(log_data)): time, path, username, province_name, school_name = \ log_data[entry].split(',') if province_name != '': log_data_clean.append(log_data[entry]) # sort data by province, school and time log_data_clean.sort(key= lambda line: ( line.split(",")[3], line.split(",")[4], line.split(",")[0])) # place data in an organised dictionary for entry in range(len(log_data_clean)): time, path, username, province_name, school_name = \ log_data_clean[entry].split(',') if self.log_data.get(province_name) is None: # province entry did not yet exist, initialise to {} self.log_data.update({province_name:{}}) school_dict = self.log_data.get(province_name) try: x = school_dict[school_name] # if this school exists, we do not set it to blank except KeyError: # school entry did not yet exist, initialise to {} school_dict.update({school_name: {}}) date_dict = school_dict[school_name] date_uuid = time[:10] try: x = date_dict[date_uuid] # if this date exists, we do not set it to blank except KeyError: # date entry did not yet exist, initialise to {} date_dict.update({date_uuid: []}) date_data = date_dict[date_uuid] # get existing list of # paths # append another path to existing list of paths date_data.append(path) date_dict.update({date_uuid: date_data}) school_dict.update({school_name: date_dict}) self.log_data.update({province_name:school_dict}) # 2nd pass to extract data out of evaluationsheets and store in # persistent structure (requires users.csv to have been parsed already) for filename in zf.namelist(): if filename == 'evaluation_sheets.csv': esheets.write(zf.read(filename)) esheets.seek(0) esheet_data = esheets.getvalue().splitlines() # as a reference from export-evaluationsheets view: # fieldnames=['assessment', 'assessment_date', # 'classlist','learner','learner_uid', # 'activity_number','rating','school', # 'province','uuid','esheet_uid'] # sort data by uuid, assessment, learner, activity_number esheet_data.sort(key= lambda line: ( line.split(",")[9], line.split(",")[0], line.split(",")[3], line.split(",")[5])) # place data in the same organised dictionary as user data for esheet in esheet_data: assessment, assessment_date, classlist, learner, \ learner_uid, activity_number, rating, school_name, \ province_name, teacher_uuid, esheet_uid = esheet.split(',') school_dict = self.user_data.get(province_name) teacher_dict = school_dict[school_name] teacher_data_dict = teacher_dict[teacher_uuid] try: x = teacher_data_dict['evaluationsheets'] except KeyError: teacher_data_dict.update({'evaluationsheets': {}}) e_list = teacher_data_dict['evaluationsheets'] evalsheet_key = assessment + '_' + classlist + '_' +\ esheet_uid try: x = e_list[evalsheet_key] except KeyError: e_list.update({evalsheet_key: {}}) assessment_obj = e_list[evalsheet_key] try: x = assessment_obj['learners'] except KeyError: assessment_obj.update({'learners': {}}) l_list = assessment_obj['learners'] try: x = l_list[learner_uid] except KeyError: l_list.update({learner_uid: {}}) learner_obj = l_list[learner_uid] try: x = learner_obj['activity_data'] except KeyError: learner_obj.update({'activity_data': []}) activities = learner_obj['activity_data'] activity = { 'activity_number' : activity_number, 'rating' : rating } activities.append(activity) learner_obj.update({'activity_data': activities}) learner_obj.update({'learner_name': learner}) l_list.update({learner_uid: learner_obj}) assessment_obj.update({'learners': l_list}) assessment_obj.update({'assessment_date': assessment_date}) assessment_obj.update({'assessment': assessment}) assessment_obj.update({'classlist': classlist}) e_list.update({evalsheet_key: assessment_obj}) teacher_data_dict.update({'evaluationsheets': e_list}) teacher_dict.update({teacher_uuid: teacher_data_dict}) school_dict.update({school_name: teacher_dict}) self.user_data.update({province_name:school_dict}) zf.close() return
def insert(self,row,b_id): self.row_num.append(row) self.business_id.append(b_id) datas = pandas.read_csv(path+"review-1m.csv") for i in range (datas.shape[0]): print(i) row = i b_id = datas.iloc[row]['business_id'] key = datas.iloc[row]['useful'] if t.has_key(key): c = t.__getitem__(key) c.insert(row,b_id) t.update({key:c}) else: c = content() c.insert(row,b_id) t.update({key:c}) funny_file = open(r'../test/useful.pkl', 'wb') pickle.dump(t, funny_file) funny_file.close() #c = content(1) #c.insert(1,100,200) #c.insert(1,200,300) #print(c.key,c.row_num,c.business_id) # #t.update({1:c}) #c = t.__getitem__(1)
def migrate_PersistentDict(dict): result = OOBTree() result.update(dict) return result
def new_register(self, userids): reg = OOBTree() reg.update({'userids': frozenset(userids), 'time': utcnow()}) self.registers[self.get_next_key()] = reg
class Table: def __init__(self, name, num_columns, key): """ The actual table holding the records :param name: string # Table name :param num_columns: int # Number of user Columns: all columns are integer :param key: int # Index of primary key column """ # name of the table self.name = name # column number of the primary key column self.prim_key_col_num = key + NUMBER_OF_META_COLUMNS # the number user columns self.num_columns = num_columns # the total number of columns self.number_of_columns = self.num_columns + NUMBER_OF_META_COLUMNS # accepts a record's RID and returns page_range_index, page_number and offset self.page_directory = OOBTree() # a list containing the page ranges for the table self.ranges = [ PageRange(self.number_of_columns, self.prim_key_col_num, 0, os.path.expanduser("~/ECS165/" + self.name)) ] # the number of records in the tale self.num_records = 0 # highest used rid number self.rid = 0 # lowest used lid number self.lid = LID_MAX # a list of indexes for the table self.indexes = make_indexes(self.number_of_columns, self.prim_key_col_num, table=self) # name of directory that table is in self.directory_name = "~/ECS165/" # stack of deleted, available RIDs self.rid_stack = [] def get_rid_value(self): """ Manage the RIDs of the table :return: int # an integer value to represent the next RID """ self.rid += 1 return self.rid def get_lid_value(self): """ Manage the LIDs of the table :return: int # an integer value to represent the next LID """ self.lid -= 1 return self.lid def add_record(self, columns): """ Add a record to the table :param columns: [] # the column values of the record """ # TODO: Check to see if primary key already exists in table # TA said this error check isn't necessary # check if can grab old, deleted rid from rid_stack if len(self.rid_stack) > 0: RID, page_range_index, page_num, offset = self.rid_stack.pop(0) columns = [0, RID, int(time() * 1000000), 0, 0] + columns self.ranges[page_range_index].update_base_record( page_num, offset, columns) else: # get new rid and add meta-data columns RID = self.get_rid_value() columns = [0, RID, int(time() * 1000000), 0, 0] + columns # get a hold of the last page range page_range = self.ranges[-1] #page_range_index = len(self.ranges) - 1 # if it is full if not page_range.has_capacity(): # create a new one and append it page_range = PageRange( num_of_columns=self.number_of_columns, primary_key_column=self.prim_key_col_num, page_range_number=page_range.my_index + 1, directory_name=os.path.expanduser("~/ECS165/" + self.name)) self.ranges.append(page_range) # write record to page range and return page number and offset of record page_num, offset = page_range.add_base_record(columns) page_range_index = len(self.ranges) - 1 # increment the number of records self.num_records += 1 # update page directory self.page_directory.update({RID: [page_range_index, page_num, offset]}) # update primary key index self.indexes[self.prim_key_col_num].add_index_item( columns[self.prim_key_col_num], RID) def read_record(self, key, column_number, query_columns): """ Read the record from the table :param key: int # the value to select records based on :param column_number: int # the column number to match keys on :param query_columns: [] # a list of Nones and 1s defining which column values to return :return: [] # a list of records matching the arguments """ # a list to hold the matching records records = [] # if no index exists for this column, create it index = self.indexes[column_number] if not index: self.indexes[column_number] = index = Index().create_index( table=self, column_number=column_number) # get the matching rids rids = index.locate(value=key) # if there are no matching rids return an empty list if not rids: return records # for each matching record for RID in rids: # get the location in the table page_range_num, page_num, offset = self.page_directory.get(RID) # TODO - double check that logic is correct # check to see if the record has been updated LID = self.ranges[page_range_num].read_column( page_range_num, page_num, offset, INDIRECTION_COLUMN) tps = self.ranges[page_range_num].tps # if a merge hasn't occurred, check for an update if self.ranges[page_range_num].num_merges == 0 and LID != 0: # get the updated records location page_range_num, page_num, offset = self.page_directory.get(LID) # if a merge has occurred, check TPS and LID values elif self.ranges[page_range_num].num_merges > 0 and 0 < LID < tps: # get the updated records location page_range_num, page_num, offset = self.page_directory.get(LID) # get the record record = self.ranges[page_range_num].read_record( [[page_range_num, page_num, offset]], query_columns) # append the record to records records = records + record # return the records return records def update_record(self, key, columns): """ Add a tail record for a specific record :param key: int # the primary key value for finding the record :param columns: [] # a list of the new values to be added """ # create an LID for the tail record LID = self.get_lid_value() # get the RID of the base record RID = self.indexes[self.prim_key_col_num].locate(value=key) # get the location of the base record page_range_num, page_num, offset = self.page_directory.get(RID[0]) # get current schema encoding schema_encoding = self.ranges[page_range_num].read_column( page_range_num, page_num, offset, SCHEMA_ENCODING_COLUMN) # get the new schema encoding by ORing the new one with the existing one new_schema_encoding = schema_encoding | get_schema_encoding(columns) # if there is already a tail record get it's LID indirection_value = self.ranges[page_range_num].read_column( page_range_num, page_num, offset, INDIRECTION_COLUMN) # update the base record with the new indirection value and schema encoding self.ranges[page_range_num].update_schema_indirection( new_schema_encoding, LID, page_num, offset) # if there was originally an indirection value in the base record if indirection_value: # find it _, page_num, offset = self.page_directory.get( indirection_value, [0, page_num, offset]) # get the base or tail record # TODO: improve efficiency by only getting record values we need record = \ self.ranges[page_range_num].read_record([[page_range_num, page_num, offset]], [1] * self.number_of_columns)[0] columns = [ indirection_value, LID, int(time() * 1000000), new_schema_encoding, RID[0] ] + list(columns) # for every column, if we have a new value save it, otherwise use old value for i in range(NUMBER_OF_META_COLUMNS, len(columns)): if columns[i] is None: columns[i] = record.columns[i] # add tail record page_num, offset = self.ranges[page_range_num].add_tail_record(columns) # update page directory self.page_directory.update({LID: [page_range_num, page_num, offset]}) # if we've reached update threshold, start merge if self.ranges[page_range_num].check_threshold(): merge_thread = Thread(target=self.__merge, args=(self.ranges[page_range_num], )) merge_thread.start() def sum_records(self, start_range, end_range, column_number): """ Sum all the records of the given column from the starting rid to the ending rid :param start_range: int # the key of the first record to accumulate :param end_range: int # the rid of the last record to accumulate(inclusive) :param column_number: int # the column to accumulate on :return: int # the outcome of summing all the records """ sum = 0 # TODO: TA said this check isn't necessary # if start_range >= end_range: # return 0 # for each possible key within the given range for key_in_range in range(start_range, end_range + 1): # get the list of RIDs that match rids = self.indexes[self.prim_key_col_num].locate(key_in_range) # if no matching RIDs break from this iteration of the loop if not rids: continue # otherwise for every matching RID for RID in rids: # get the location of the record and check to see if there has been an update page_range_num, page_num, offset = self.page_directory.get(RID) LID = self.ranges[page_range_num].read_column( page_range_num, page_num, offset, INDIRECTION_COLUMN) # if an update get the location of the latest update if LID: page_range_num, page_num, offset = self.page_directory.get( LID) # do the actual summing sum += self.ranges[page_range_num].read_column( page_range_num, page_num, offset, column_number) return sum def delete_record(self, key): """ delete all records with the given primary key :param key: int # primary key value of the record to be deleted """ # get the RID of the record RID = self.indexes[self.prim_key_col_num].locate(key) # get the location of the record if RID: page_range_num, page_num, offset = self.page_directory.get(RID[0]) # appending deleted RID to rid stack self.rid_stack.append([RID[0], page_range_num, page_num, offset]) # lazy delete the record self.ranges[page_range_num].delete_record(page_num, offset) # modify the number of records in the table self.num_records -= 1 def save_table(self, directory_name): """ saves table data and page range data to files :param directory_name: string # name of db directory """ # write table data to file sys.setrecursionlimit(RECURSION_LIMIT) with open(os.path.expanduser(directory_name + self.name + '/table'), 'wb+') as output: pickle.dump(self, output, pickle.HIGHEST_PROTOCOL) bp.close() def __merge(self, original_page_range): # beginning merge on this page range original_page_range.merge = True # create a copy of the page range copy_page_range = original_page_range # dictionary of RIDs we've seen so far has_seen = [] # iterate through all tail pages except last, check for last update to record for tail_num in range(original_page_range.last_tail_page + 1, -1, -2): iterate_offset = PAGE_SIZE for i in range(int(RECORDS_PER_PAGE)): iterate_offset -= 8 # return user data and rid column query_cols = [0, 1, 0, 0, 0] + ([1] * self.num_columns) record = original_page_range.read_record( [[original_page_range.my_index, tail_num, iterate_offset]], query_cols)[0] base_rid = record.columns[0] if not (base_rid in has_seen): has_seen.append(base_rid) _, page_num, offset = self.page_directory[base_rid] # TODO: clean this logic up? # ignore meta data columns and copy just the user data over for n, column in enumerate(copy_page_range.columns): if n > 4: column.update_value(page_num, offset, record.columns[n - 4]) # TODO - double check that correct # edit base record's meta data columns accordingly copy_page_range.columns[ SCHEMA_ENCODING_COLUMN].update_value( page_num, offset, 0) # if have seen all rids in base pages, break since found all latest updates if len(has_seen) == self.rid: break # update last tail page copy_page_range.last_tail_page = copy_page_range.columns[ 0].last_page - 1 # update tps of copy of page range copy_page_range.tps = copy_page_range.read_column( copy_page_range.my_index, copy_page_range.columns[0].last_page - 1, PAGE_SIZE - 8, RID_COLUMN) # update range and get new index to update page directory self.ranges.append(copy_page_range) range_index = len(self.ranges) - 1 # update page directory to point to new merged pages for rid in has_seen: _, page_num, offset = self.page_directory[rid] values = [range_index, page_num, offset] self.page_directory[rid] = values # finished merging this page range original_page_range.merge = False # reset update counter to trigger merge self.ranges[range_index].update_count = 0 # update merge count self.ranges[range_index].num_merges += 1
class NastyConfict(Base, TestCase): def setUp(self): self.t = OOBTree() # This tests a problem that cropped up while trying to write # testBucketSplitConflict (below): conflict resolution wasn't # working at all in non-trivial cases. Symptoms varied from # strange complaints about pickling (despite that the test isn't # doing any *directly*), thru SystemErrors from Python and # AssertionErrors inside the BTree code. def testResolutionBlowsUp(self): b = self.t for i in range(0, 200, 4): b[i] = i # bucket 0 has 15 values: 0, 4 .. 56 # bucket 1 has 15 values: 60, 64 .. 116 # bucket 2 has 20 values: 120, 124 .. 196 state = b.__getstate__() # Looks like: ((bucket0, 60, bucket1, 120, bucket2), firstbucket) # If these fail, the *preconditions* for running the test aren't # satisfied -- the test itself hasn't been run yet. self.assertEqual(len(state), 2) self.assertEqual(len(state[0]), 5) self.assertEqual(state[0][1], 60) self.assertEqual(state[0][3], 120) # Invoke conflict resolution by committing a transaction. self.openDB() r1 = self.db.open().root() r1["t"] = self.t transaction.commit() r2 = self.db.open().root() copy = r2["t"] # Make sure all of copy is loaded. list(copy.values()) self.assertEqual(self.t._p_serial, copy._p_serial) self.t.update({1: 2, 2: 3}) transaction.commit() copy.update({3: 4}) transaction.commit() # if this doesn't blow up list(copy.values()) # and this doesn't either, then fine def testBucketSplitConflict(self): # Tests that a bucket split is viewed as a conflict. # It's (almost necessarily) a white-box test, and sensitive to # implementation details. b = self.t for i in range(0, 200, 4): b[i] = i # bucket 0 has 15 values: 0, 4 .. 56 # bucket 1 has 15 values: 60, 64 .. 116 # bucket 2 has 20 values: 120, 124 .. 196 state = b.__getstate__() # Looks like: ((bucket0, 60, bucket1, 120, bucket2), firstbucket) # If these fail, the *preconditions* for running the test aren't # satisfied -- the test itself hasn't been run yet. self.assertEqual(len(state), 2) self.assertEqual(len(state[0]), 5) self.assertEqual(state[0][1], 60) self.assertEqual(state[0][3], 120) # Invoke conflict resolution by committing a transaction. self.openDB() tm1 = transaction.TransactionManager() r1 = self.db.open(transaction_manager=tm1).root() r1["t"] = self.t tm1.commit() tm2 = transaction.TransactionManager() r2 = self.db.open(transaction_manager=tm2).root() copy = r2["t"] # Make sure all of copy is loaded. list(copy.values()) self.assertEqual(self.t._p_serial, copy._p_serial) # In one transaction, add 16 new keys to bucket1, to force a bucket # split. b = self.t numtoadd = 16 candidate = 60 while numtoadd: if not b.has_key(candidate): b[candidate] = candidate numtoadd -= 1 candidate += 1 # bucket 0 has 15 values: 0, 4 .. 56 # bucket 1 has 15 values: 60, 61 .. 74 # bucket 2 has 16 values: [75, 76 .. 81] + [84, 88 ..116] # bucket 3 has 20 values: 120, 124 .. 196 state = b.__getstate__() # Looks like: ((b0, 60, b1, 75, b2, 120, b3), firstbucket) # The next block is still verifying preconditions. self.assertEqual(len(state), 2) self.assertEqual(len(state[0]), 7) self.assertEqual(state[0][1], 60) self.assertEqual(state[0][3], 75) self.assertEqual(state[0][5], 120) tm1.commit() # In the other transaction, add 3 values near the tail end of bucket1. # This doesn't cause a split. b = copy for i in range(112, 116): b[i] = i # bucket 0 has 15 values: 0, 4 .. 56 # bucket 1 has 18 values: 60, 64 .. 112, 113, 114, 115, 116 # bucket 2 has 20 values: 120, 124 .. 196 state = b.__getstate__() # Looks like: ((bucket0, 60, bucket1, 120, bucket2), firstbucket) # The next block is still verifying preconditions. self.assertEqual(len(state), 2) self.assertEqual(len(state[0]), 5) self.assertEqual(state[0][1], 60) self.assertEqual(state[0][3], 120) self.assertRaises(ConflictError, tm2.commit) def testEmptyBucketConflict(self): # Tests that an emptied bucket *created by* conflict resolution is # viewed as a conflict: conflict resolution doesn't have enough # info to unlink the empty bucket from the BTree correctly. b = self.t for i in range(0, 200, 4): b[i] = i # bucket 0 has 15 values: 0, 4 .. 56 # bucket 1 has 15 values: 60, 64 .. 116 # bucket 2 has 20 values: 120, 124 .. 196 state = b.__getstate__() # Looks like: ((bucket0, 60, bucket1, 120, bucket2), firstbucket) # If these fail, the *preconditions* for running the test aren't # satisfied -- the test itself hasn't been run yet. self.assertEqual(len(state), 2) self.assertEqual(len(state[0]), 5) self.assertEqual(state[0][1], 60) self.assertEqual(state[0][3], 120) # Invoke conflict resolution by committing a transaction. self.openDB() tm1 = transaction.TransactionManager() r1 = self.db.open(transaction_manager=tm1).root() r1["t"] = self.t tm1.commit() tm2 = transaction.TransactionManager() r2 = self.db.open(transaction_manager=tm2).root() copy = r2["t"] # Make sure all of copy is loaded. list(copy.values()) self.assertEqual(self.t._p_serial, copy._p_serial) # In one transaction, delete half of bucket 1. b = self.t for k in 60, 64, 68, 72, 76, 80, 84, 88: del b[k] # bucket 0 has 15 values: 0, 4 .. 56 # bucket 1 has 7 values: 92, 96, 100, 104, 108, 112, 116 # bucket 2 has 20 values: 120, 124 .. 196 state = b.__getstate__() # Looks like: ((bucket0, 60, bucket1, 120, bucket2), firstbucket) # The next block is still verifying preconditions. self.assertEqual(len(state), 2) self.assertEqual(len(state[0]), 5) self.assertEqual(state[0][1], 60) self.assertEqual(state[0][3], 120) tm1.commit() # In the other transaction, delete the other half of bucket 1. b = copy for k in 92, 96, 100, 104, 108, 112, 116: del b[k] # bucket 0 has 15 values: 0, 4 .. 56 # bucket 1 has 8 values: 60, 64, 68, 72, 76, 80, 84, 88 # bucket 2 has 20 values: 120, 124 .. 196 state = b.__getstate__() # Looks like: ((bucket0, 60, bucket1, 120, bucket2), firstbucket) # The next block is still verifying preconditions. self.assertEqual(len(state), 2) self.assertEqual(len(state[0]), 5) self.assertEqual(state[0][1], 60) self.assertEqual(state[0][3], 120) # Conflict resolution empties bucket1 entirely. This used to # create an "insane" BTree (a legit BTree cannot contain an empty # bucket -- it contains NULL pointers the BTree code doesn't # expect, and segfaults result). self.assertRaises(ConflictError, tm2.commit) def testEmptyBucketNoConflict(self): # Tests that a plain empty bucket (on input) is not viewed as a # conflict. b = self.t for i in range(0, 200, 4): b[i] = i # bucket 0 has 15 values: 0, 4 .. 56 # bucket 1 has 15 values: 60, 64 .. 116 # bucket 2 has 20 values: 120, 124 .. 196 state = b.__getstate__() # Looks like: ((bucket0, 60, bucket1, 120, bucket2), firstbucket) # If these fail, the *preconditions* for running the test aren't # satisfied -- the test itself hasn't been run yet. self.assertEqual(len(state), 2) self.assertEqual(len(state[0]), 5) self.assertEqual(state[0][1], 60) self.assertEqual(state[0][3], 120) # Invoke conflict resolution by committing a transaction. self.openDB() r1 = self.db.open().root() r1["t"] = self.t transaction.commit() r2 = self.db.open().root() copy = r2["t"] # Make sure all of copy is loaded. list(copy.values()) self.assertEqual(self.t._p_serial, copy._p_serial) # In one transaction, just add a key. b = self.t b[1] = 1 # bucket 0 has 16 values: [0, 1] + [4, 8 .. 56] # bucket 1 has 15 values: 60, 64 .. 116 # bucket 2 has 20 values: 120, 124 .. 196 state = b.__getstate__() # Looks like: ((bucket0, 60, bucket1, 120, bucket2), firstbucket) # The next block is still verifying preconditions. self.assertEqual(len(state), 2) self.assertEqual(len(state[0]), 5) self.assertEqual(state[0][1], 60) self.assertEqual(state[0][3], 120) transaction.commit() # In the other transaction, delete bucket 2. b = copy for k in range(120, 200, 4): del b[k] # bucket 0 has 15 values: 0, 4 .. 56 # bucket 1 has 15 values: 60, 64 .. 116 state = b.__getstate__() # Looks like: ((bucket0, 60, bucket1), firstbucket) # The next block is still verifying preconditions. self.assertEqual(len(state), 2) self.assertEqual(len(state[0]), 3) self.assertEqual(state[0][1], 60) # This shouldn't create a ConflictError. transaction.commit() # And the resulting BTree shouldn't have internal damage. b._check() # The snaky control flow in _bucket__p_resolveConflict ended up trying # to decref a NULL pointer if conflict resolution was fed 3 empty # buckets. http://collector.zope.org/Zope/553 def testThreeEmptyBucketsNoSegfault(self): self.openDB() tm1 = transaction.TransactionManager() r1 = self.db.open(transaction_manager=tm1).root() self.assertEqual(len(self.t), 0) r1["t"] = b = self.t # an empty tree tm1.commit() tm2 = transaction.TransactionManager() r2 = self.db.open(transaction_manager=tm2).root() copy = r2["t"] # Make sure all of copy is loaded. list(copy.values()) # In one transaction, add and delete a key. b[2] = 2 del b[2] tm1.commit() # In the other transaction, also add and delete a key. b = copy b[1] = 1 del b[1] # If the commit() segfaults, the C code is still wrong for this case. self.assertRaises(ConflictError, tm2.commit) def testCantResolveBTreeConflict(self): # Test that a conflict involving two different changes to # an internal BTree node is unresolvable. An internal node # only changes when there are enough additions or deletions # to a child bucket that the bucket is split or removed. # It's (almost necessarily) a white-box test, and sensitive to # implementation details. b = self.t for i in range(0, 200, 4): b[i] = i # bucket 0 has 15 values: 0, 4 .. 56 # bucket 1 has 15 values: 60, 64 .. 116 # bucket 2 has 20 values: 120, 124 .. 196 state = b.__getstate__() # Looks like: ((bucket0, 60, bucket1, 120, bucket2), firstbucket) # If these fail, the *preconditions* for running the test aren't # satisfied -- the test itself hasn't been run yet. self.assertEqual(len(state), 2) self.assertEqual(len(state[0]), 5) self.assertEqual(state[0][1], 60) self.assertEqual(state[0][3], 120) # Set up database connections to provoke conflict. self.openDB() tm1 = transaction.TransactionManager() r1 = self.db.open(transaction_manager=tm1).root() r1["t"] = self.t tm1.commit() tm2 = transaction.TransactionManager() r2 = self.db.open(transaction_manager=tm2).root() copy = r2["t"] # Make sure all of copy is loaded. list(copy.values()) self.assertEqual(self.t._p_serial, copy._p_serial) # Now one transaction should add enough keys to cause a split, # and another should remove all the keys in one bucket. for k in range(200, 300, 4): self.t[k] = k tm1.commit() for k in range(0, 60, 4): del copy[k] try: tm2.commit() except ConflictError, detail: self.assert_(str(detail).startswith('database conflict error')) else:
class Subscriptions(SimpleItem): security = ClassSecurityInfo() title = "Meeting registrations" def __init__(self, id): """ """ super(SimpleItem, self).__init__(id) self.id = id self._signups = OOBTree() self._account_subscriptions = OOBTree() security.declarePublic('getMeeting') def getMeeting(self): return self.aq_parent.aq_parent def _validate_signup(self, form): """ """ formdata = {} formerrors = {} keys = ('first_name', 'last_name', 'email', 'organization', 'phone') formdata = dict((key, form.get(key, '')) for key in keys) for key in formdata: if formdata[key] == '': formerrors[key] = 'This field is mandatory' if formerrors == {}: if formdata['email'].count('@') != 1: formerrors['email'] = ('An email address must contain ' 'a single @') if formerrors == {}: formerrors = None return formdata, formerrors def _add_signup(self, formdata): """ """ meeting = self.getMeeting() key = random_key() name = formdata['first_name'] + ' ' + formdata['last_name'] email = formdata['email'] organization = formdata['organization'] phone = formdata['phone'] signup = SignUp(key, name, email, organization, phone) self._signups.insert(key, signup) if meeting.auto_register: self._accept_signup(key) email_sender = self.getMeeting().getEmailSender() email_sender.send_signup_email(signup) if self.REQUEST.AUTHENTICATED_USER.getUserName() == 'Anonymous User': self.REQUEST.SESSION['nymt-current-key'] = key security.declareProtected(view, 'signup') def signup(self, REQUEST): """ """ meeting = self.getMeeting() if not meeting.allow_register: return REQUEST.RESPONSE.redirect(self.absolute_url() + '/subscription_not_allowed') if REQUEST.get('add_users'): return self.subscribe_accounts(REQUEST) if REQUEST.get('add_signup'): formdata, formerrors = self._validate_signup(REQUEST.form) # check Captcha/reCaptcha if not (self.checkPermissionSkipCaptcha() or REQUEST.SESSION.get('captcha_passed')): recaptcha_response = REQUEST.form.get('g-recaptcha-response', '') captcha_validator = self.validateCaptcha(recaptcha_response, REQUEST) if captcha_validator: if formerrors is None: formerrors = {} formerrors['captcha'] = captcha_validator else: REQUEST.SESSION['captcha_passed'] = True if formerrors is not None: return self.getFormsTool().getContent( {'here': self, 'formdata': formdata, 'formerrors': formerrors}, 'naaya.content.meeting.subscription_signup') else: self._add_signup(formdata) if self.getMeeting().survey_required: REQUEST.RESPONSE.redirect( self.getMeeting().absolute_url()) else: REQUEST.RESPONSE.redirect(self.absolute_url() + '/signup_successful') # check Captcha/reCaptcha also for searching users captcha_validator = None if (REQUEST.get('search_user') or REQUEST.get('search_user_with_role')): if not (self.checkPermissionSkipCaptcha() or REQUEST.SESSION.get('captcha_passed')): recaptcha_response = REQUEST.form.get('g-recaptcha-response', '') captcha_validator = self.validateCaptcha(recaptcha_response, REQUEST) if not captcha_validator: REQUEST.SESSION['captcha_passed'] = True return self.getFormsTool().getContent( {'here': self, 'captcha_errors': captcha_validator}, 'naaya.content.meeting.subscription_signup') security.declareProtected(view, 'signup_successful') def signup_successful(self, REQUEST): """ """ return self.getFormsTool().getContent( {'here': self}, 'naaya.content.meeting.subscription_signup_successful') security.declareProtected(view, 'subscribe') def subscribe(self, REQUEST): """ """ meeting = self.getMeeting() if not meeting.allow_register: return REQUEST.RESPONSE.redirect(self.absolute_url() + '/subscription_not_allowed') return self.getFormsTool().getContent( {'here': self}, 'naaya.content.meeting.subscription_subscribe') def getSignups(self): """ """ if not self.checkPermissionParticipateInMeeting(): raise Unauthorized return self._signups.itervalues() security.declareProtected(PERMISSION_ADMIN_MEETING, 'getSignup') def getSignup(self, key): """ """ return self._signups.get(key, None) def index_html(self, REQUEST): """ """ if not self.checkPermissionParticipateInMeeting(): raise Unauthorized return self.getFormsTool().getContent( {'here': self}, 'naaya.content.meeting.subscription_index') def _accept_signup(self, key): """ """ meeting = self.getMeeting() meeting.getParticipants()._set_attendee(key, PARTICIPANT_ROLE) signup = self._signups[key] signup.accepted = 'accepted' email_sender = meeting.getEmailSender() email_sender.send_signup_accepted_email(signup) def _reject_signup(self, key): """ """ meeting = self.getMeeting() signup = self._signups[key] signup.accepted = 'rejected' participants = meeting.getParticipants() # delete the 'reimbursed' status participants.setAttendeeInfo([key], 'reimbursed', False) if key in participants._get_attendees(): participants._del_attendee(key) email_sender = meeting.getEmailSender() email_sender.send_signup_rejected_email(signup) def _delete_signup(self, key): """ """ meeting = self.getMeeting() signup = self._signups.pop(key, None) if signup is None: return participants = meeting.getParticipants() if key in participants._get_attendees(): participants._del_attendee(key) email_sender = meeting.getEmailSender() email_sender.send_signup_rejected_email(signup) def _is_signup(self, key): """ """ return key in self._signups def _is_accepted_signup(self, key): """ """ return self._is_signup(key) and \ self._signups[key].accepted == 'accepted' def _is_pending_signup(self, key): """ """ return self._is_signup(key) and \ self._signups[key].accepted == 'new' def manageSubscriptions(self, REQUEST): """ """ if not (self.checkPermissionAdminMeeting() or self.nfp_for_country()): raise Unauthorized uids = REQUEST.form.get('uids', []) assert isinstance(uids, list) for uid in uids: if 'accept' in REQUEST.form: if self._is_signup(uid): self._accept_signup(uid) else: self._accept_account_subscription(uid) elif 'reject' in REQUEST.form: if self._is_signup(uid): self._reject_signup(uid) else: self._reject_account_subscription(uid) elif 'delete' in REQUEST.form: if not self.checkPermissionAdminMeeting(): raise Unauthorized if self._is_signup(uid): self._delete_signup(uid) else: self._delete_account_subscription(uid) if 'set_representative' in REQUEST.form: self.setRepresentatives(REQUEST) elif 'unset_representative' in REQUEST.form: self.setRepresentatives(REQUEST, remove=True) elif 'set_reimbursement' in REQUEST.form: self.setReimbursement(REQUEST) elif 'unset_reimbursement' in REQUEST.form: self.setReimbursement(REQUEST, remove=True) elif 'save_changes' in REQUEST.form: self.save_changes(REQUEST) return REQUEST.RESPONSE.redirect(self.absolute_url()) security.declarePublic('welcome') def welcome(self, REQUEST, came_from=None): """ """ if 'logout' in REQUEST.form: REQUEST.SESSION['nymt-current-key'] = None return REQUEST.RESPONSE.redirect(self.getMeeting().absolute_url()) key = REQUEST.get('key', None) signup = self.getSignup(key) if self._is_signup(key): REQUEST.SESSION['nymt-current-key'] = key if came_from: return REQUEST.RESPONSE.redirect(came_from) else: return REQUEST.RESPONSE.redirect( self.getMeeting().absolute_url()) return self.getFormsTool().getContent( {'here': self, 'signup': signup}, 'naaya.content.meeting.subscription_welcome') def _add_account_subscription(self, uid, accept=False): """ """ # If the subscription already exists or the user is alread signed up # skip the whole thing if self._is_account_subscription(uid): return key = uid.replace('signup:', '') if self._is_signup(key): return site = self.getSite() meeting = self.getMeeting() name = getUserFullName(site, uid) # If for any reason we still don't have a name, at least use UID if not name: name = uid email = getUserEmail(site, uid) organization = getUserOrganization(site, uid) if not organization: organization = self.get_survey_answer(uid, 'w_organization') if not organization: organization = self.get_survey_answer(uid, 'w_organisation') phone = getUserPhoneNumber(site, uid) if not phone: phone = self.get_survey_answer(uid, 'w_telephone') if not phone: phone = self.get_survey_answer(uid, 'w_phone') account_subscription = AccountSubscription(uid, name, email, organization, phone) self._account_subscriptions.insert(uid, account_subscription) if meeting.auto_register or accept: self._accept_account_subscription(uid) email_sender = self.getMeeting().getEmailSender() email_sender.send_account_subscription_email(account_subscription) security.declareProtected(PERMISSION_ADMIN_MEETING, 'update_account_subscription') def update_account_subscription(self, uid): """ """ site = self.getSite() name = getUserFullName(site, uid) email = getUserEmail(site, uid) organization = getUserOrganization(site, uid) phone = getUserPhoneNumber(site, uid) account_subscription = AccountSubscription(uid, name, email, organization, phone) self._account_subscriptions.update({uid: account_subscription}) security.declareProtected(view, 'subscribe_accounts') def subscribe_accounts(self, REQUEST): """ """ meeting = self.getMeeting() if not meeting.allow_register: return REQUEST.RESPONSE.redirect(self.absolute_url() + '/subscription_not_allowed') # check Captcha/reCaptcha also for searching users if not (self.checkPermissionSkipCaptcha() or REQUEST.SESSION.get('captcha_passed')): recaptcha_response = REQUEST.form.get('g-recaptcha-response', '') captcha_validator = self.validateCaptcha(recaptcha_response, REQUEST) if captcha_validator: return self.getFormsTool().getContent( {'here': self, 'captcha_errors': captcha_validator}, 'naaya.content.meeting.subscription_signup') else: REQUEST.SESSION['captcha_passed'] = True uids = REQUEST.form.get('uids', []) assert isinstance(uids, list) for uid in uids: self._add_account_subscription(uid) return REQUEST.RESPONSE.redirect( self.absolute_url() + '/subscribe_account_successful?uids='+','.join(uids)) security.declareProtected(view, 'subscribe_my_account') def subscribe_my_account(self, REQUEST): """ """ meeting = self.getMeeting() if not meeting.allow_register: return REQUEST.RESPONSE.redirect(self.absolute_url() + '/subscription_not_allowed') self._add_account_subscription(REQUEST.AUTHENTICATED_USER.getId()) if self.survey_required: site = self.getSite() path = str(self.survey_pointer) survey_ob = site.unrestrictedTraverse(path, None) if survey_ob is not None and \ survey_ob.meta_type == 'Naaya Mega Survey': answers = survey_ob.getAnswers() respondents = [a.respondent for a in answers] current_user = REQUEST.AUTHENTICATED_USER.getUserName() if current_user not in respondents: self.setSessionInfoTrans( 'Registration successfully sent for approval. ' 'Please also respond to the following questionaire.') return REQUEST.RESPONSE.redirect( '%s/%s' % (self.getSite().absolute_url(), self.survey_pointer)) return REQUEST.RESPONSE.redirect(self.absolute_url() + '/subscribe_account_successful') security.declareProtected(view, 'subscribe_account_successful') def subscribe_account_successful(self, REQUEST): """ """ return self.getFormsTool().getContent( {'here': self}, 'naaya.content.meeting.subscription_subscribe_successful') def getAccountSubscriptions(self): """ """ if not self.checkPermissionParticipateInMeeting(): raise Unauthorized return self._account_subscriptions.itervalues() def getSubscriptions(self): """ """ if not self.checkPermissionParticipateInMeeting(): raise Unauthorized subscriptions = (list(self._signups.itervalues()) + list(self._account_subscriptions.itervalues())) statuses = {'new': 0, 'accepted': 1, 'rejected': 2 } return sorted(subscriptions, key=lambda x: statuses.get(x.accepted)) security.declareProtected(PERMISSION_ADMIN_MEETING, 'getAccountSubscription') def getAccountSubscription(self, uid): """ """ return self._account_subscriptions.get(uid, None) def _is_account_subscription(self, uid): """ """ return uid in self._account_subscriptions and \ self._account_subscriptions[uid].accepted == 'accepted' def _accept_account_subscription(self, uid): """ """ meeting = self.getMeeting() meeting.getParticipants()._set_attendee(uid, PARTICIPANT_ROLE) account_subscription = self._account_subscriptions[uid] account_subscription.accepted = 'accepted' email_sender = meeting.getEmailSender() email_sender.send_account_subscription_accepted_email( account_subscription) def _reject_account_subscription(self, uid): """ """ meeting = self.getMeeting() account_subscription = self._account_subscriptions[uid] account_subscription.accepted = 'rejected' participants = meeting.getParticipants() # remove the 'reimbursed' status participants.setAttendeeInfo([uid], 'reimbursed', False) if uid in participants._get_attendees(): participants._del_attendee(uid) email_sender = meeting.getEmailSender() email_sender.send_account_subscription_rejected_email( account_subscription) def _delete_account_subscription(self, uid): """ """ meeting = self.getMeeting() account_subscription = self._account_subscriptions.pop(uid, None) if account_subscription is None: return participants = meeting.getParticipants() if uid in participants._get_attendees(): participants._del_attendee(uid) email_sender = meeting.getEmailSender() email_sender.send_account_subscription_rejected_email( account_subscription) security.declareProtected(view, 'subscription_not_allowed') def subscription_not_allowed(self, REQUEST): """ """ return self.getFormsTool().getContent( {'here': self}, 'naaya.content.meeting.subscription_not_allowed')
class NastyConfict(Base, TestCase): def setUp(self): self.t = OOBTree() # This tests a problem that cropped up while trying to write # testBucketSplitConflict (below): conflict resolution wasn't # working at all in non-trivial cases. Symptoms varied from # strange complaints about pickling (despite that the test isn't # doing any *directly*), thru SystemErrors from Python and # AssertionErrors inside the BTree code. def testResolutionBlowsUp(self): b = self.t for i in range(0, 200, 4): b[i] = i # bucket 0 has 15 values: 0, 4 .. 56 # bucket 1 has 15 values: 60, 64 .. 116 # bucket 2 has 20 values: 120, 124 .. 196 state = b.__getstate__() # Looks like: ((bucket0, 60, bucket1, 120, bucket2), firstbucket) # If these fail, the *preconditions* for running the test aren't # satisfied -- the test itself hasn't been run yet. self.assertEqual(len(state), 2) self.assertEqual(len(state[0]), 5) self.assertEqual(state[0][1], 60) self.assertEqual(state[0][3], 120) # Invoke conflict resolution by committing a transaction. self.openDB() r1 = self.db.open().root() r1["t"] = self.t transaction.commit() r2 = self.db.open().root() copy = r2["t"] # Make sure all of copy is loaded. list(copy.values()) self.assertEqual(self.t._p_serial, copy._p_serial) self.t.update({1:2, 2:3}) transaction.commit() copy.update({3:4}) transaction.commit() # if this doesn't blow up list(copy.values()) # and this doesn't either, then fine def testBucketSplitConflict(self): # Tests that a bucket split is viewed as a conflict. # It's (almost necessarily) a white-box test, and sensitive to # implementation details. b = self.t for i in range(0, 200, 4): b[i] = i # bucket 0 has 15 values: 0, 4 .. 56 # bucket 1 has 15 values: 60, 64 .. 116 # bucket 2 has 20 values: 120, 124 .. 196 state = b.__getstate__() # Looks like: ((bucket0, 60, bucket1, 120, bucket2), firstbucket) # If these fail, the *preconditions* for running the test aren't # satisfied -- the test itself hasn't been run yet. self.assertEqual(len(state), 2) self.assertEqual(len(state[0]), 5) self.assertEqual(state[0][1], 60) self.assertEqual(state[0][3], 120) # Invoke conflict resolution by committing a transaction. self.openDB() r1 = self.db.open().root() r1["t"] = self.t transaction.commit() r2 = self.db.open(synch=False).root() copy = r2["t"] # Make sure all of copy is loaded. list(copy.values()) self.assertEqual(self.t._p_serial, copy._p_serial) # In one transaction, add 16 new keys to bucket1, to force a bucket # split. b = self.t numtoadd = 16 candidate = 60 while numtoadd: if not b.has_key(candidate): b[candidate] = candidate numtoadd -= 1 candidate += 1 # bucket 0 has 15 values: 0, 4 .. 56 # bucket 1 has 15 values: 60, 61 .. 74 # bucket 2 has 16 values: [75, 76 .. 81] + [84, 88 ..116] # bucket 3 has 20 values: 120, 124 .. 196 state = b.__getstate__() # Looks like: ((b0, 60, b1, 75, b2, 120, b3), firstbucket) # The next block is still verifying preconditions. self.assertEqual(len(state) , 2) self.assertEqual(len(state[0]), 7) self.assertEqual(state[0][1], 60) self.assertEqual(state[0][3], 75) self.assertEqual(state[0][5], 120) transaction.commit() # In the other transaction, add 3 values near the tail end of bucket1. # This doesn't cause a split. b = copy for i in range(112, 116): b[i] = i # bucket 0 has 15 values: 0, 4 .. 56 # bucket 1 has 18 values: 60, 64 .. 112, 113, 114, 115, 116 # bucket 2 has 20 values: 120, 124 .. 196 state = b.__getstate__() # Looks like: ((bucket0, 60, bucket1, 120, bucket2), firstbucket) # The next block is still verifying preconditions. self.assertEqual(len(state), 2) self.assertEqual(len(state[0]), 5) self.assertEqual(state[0][1], 60) self.assertEqual(state[0][3], 120) self.assertRaises(ConflictError, transaction.commit) transaction.abort() # horrible things happen w/o this def testEmptyBucketConflict(self): # Tests that an emptied bucket *created by* conflict resolution is # viewed as a conflict: conflict resolution doesn't have enough # info to unlink the empty bucket from the BTree correctly. b = self.t for i in range(0, 200, 4): b[i] = i # bucket 0 has 15 values: 0, 4 .. 56 # bucket 1 has 15 values: 60, 64 .. 116 # bucket 2 has 20 values: 120, 124 .. 196 state = b.__getstate__() # Looks like: ((bucket0, 60, bucket1, 120, bucket2), firstbucket) # If these fail, the *preconditions* for running the test aren't # satisfied -- the test itself hasn't been run yet. self.assertEqual(len(state), 2) self.assertEqual(len(state[0]), 5) self.assertEqual(state[0][1], 60) self.assertEqual(state[0][3], 120) # Invoke conflict resolution by committing a transaction. self.openDB() r1 = self.db.open().root() r1["t"] = self.t transaction.commit() r2 = self.db.open(synch=False).root() copy = r2["t"] # Make sure all of copy is loaded. list(copy.values()) self.assertEqual(self.t._p_serial, copy._p_serial) # In one transaction, delete half of bucket 1. b = self.t for k in 60, 64, 68, 72, 76, 80, 84, 88: del b[k] # bucket 0 has 15 values: 0, 4 .. 56 # bucket 1 has 7 values: 92, 96, 100, 104, 108, 112, 116 # bucket 2 has 20 values: 120, 124 .. 196 state = b.__getstate__() # Looks like: ((bucket0, 60, bucket1, 120, bucket2), firstbucket) # The next block is still verifying preconditions. self.assertEqual(len(state) , 2) self.assertEqual(len(state[0]), 5) self.assertEqual(state[0][1], 60) self.assertEqual(state[0][3], 120) transaction.commit() # In the other transaction, delete the other half of bucket 1. b = copy for k in 92, 96, 100, 104, 108, 112, 116: del b[k] # bucket 0 has 15 values: 0, 4 .. 56 # bucket 1 has 8 values: 60, 64, 68, 72, 76, 80, 84, 88 # bucket 2 has 20 values: 120, 124 .. 196 state = b.__getstate__() # Looks like: ((bucket0, 60, bucket1, 120, bucket2), firstbucket) # The next block is still verifying preconditions. self.assertEqual(len(state), 2) self.assertEqual(len(state[0]), 5) self.assertEqual(state[0][1], 60) self.assertEqual(state[0][3], 120) # Conflict resolution empties bucket1 entirely. This used to # create an "insane" BTree (a legit BTree cannot contain an empty # bucket -- it contains NULL pointers the BTree code doesn't # expect, and segfaults result). self.assertRaises(ConflictError, transaction.commit) transaction.abort() # horrible things happen w/o this def testEmptyBucketNoConflict(self): # Tests that a plain empty bucket (on input) is not viewed as a # conflict. b = self.t for i in range(0, 200, 4): b[i] = i # bucket 0 has 15 values: 0, 4 .. 56 # bucket 1 has 15 values: 60, 64 .. 116 # bucket 2 has 20 values: 120, 124 .. 196 state = b.__getstate__() # Looks like: ((bucket0, 60, bucket1, 120, bucket2), firstbucket) # If these fail, the *preconditions* for running the test aren't # satisfied -- the test itself hasn't been run yet. self.assertEqual(len(state), 2) self.assertEqual(len(state[0]), 5) self.assertEqual(state[0][1], 60) self.assertEqual(state[0][3], 120) # Invoke conflict resolution by committing a transaction. self.openDB() r1 = self.db.open().root() r1["t"] = self.t transaction.commit() r2 = self.db.open().root() copy = r2["t"] # Make sure all of copy is loaded. list(copy.values()) self.assertEqual(self.t._p_serial, copy._p_serial) # In one transaction, just add a key. b = self.t b[1] = 1 # bucket 0 has 16 values: [0, 1] + [4, 8 .. 56] # bucket 1 has 15 values: 60, 64 .. 116 # bucket 2 has 20 values: 120, 124 .. 196 state = b.__getstate__() # Looks like: ((bucket0, 60, bucket1, 120, bucket2), firstbucket) # The next block is still verifying preconditions. self.assertEqual(len(state), 2) self.assertEqual(len(state[0]), 5) self.assertEqual(state[0][1], 60) self.assertEqual(state[0][3], 120) transaction.commit() # In the other transaction, delete bucket 2. b = copy for k in range(120, 200, 4): del b[k] # bucket 0 has 15 values: 0, 4 .. 56 # bucket 1 has 15 values: 60, 64 .. 116 state = b.__getstate__() # Looks like: ((bucket0, 60, bucket1), firstbucket) # The next block is still verifying preconditions. self.assertEqual(len(state), 2) self.assertEqual(len(state[0]), 3) self.assertEqual(state[0][1], 60) # This shouldn't create a ConflictError. transaction.commit() # And the resulting BTree shouldn't have internal damage. b._check() # The snaky control flow in _bucket__p_resolveConflict ended up trying # to decref a NULL pointer if conflict resolution was fed 3 empty # buckets. http://collector.zope.org/Zope/553 def testThreeEmptyBucketsNoSegfault(self): self.openDB() r1 = self.db.open().root() self.assertEqual(len(self.t), 0) r1["t"] = b = self.t # an empty tree transaction.commit() r2 = self.db.open(synch=False).root() copy = r2["t"] # Make sure all of copy is loaded. list(copy.values()) # In one transaction, add and delete a key. b[2] = 2 del b[2] transaction.commit() # In the other transaction, also add and delete a key. b = copy b[1] = 1 del b[1] # If the commit() segfaults, the C code is still wrong for this case. self.assertRaises(ConflictError, transaction.commit) transaction.abort() def testCantResolveBTreeConflict(self): # Test that a conflict involving two different changes to # an internal BTree node is unresolvable. An internal node # only changes when there are enough additions or deletions # to a child bucket that the bucket is split or removed. # It's (almost necessarily) a white-box test, and sensitive to # implementation details. b = self.t for i in range(0, 200, 4): b[i] = i # bucket 0 has 15 values: 0, 4 .. 56 # bucket 1 has 15 values: 60, 64 .. 116 # bucket 2 has 20 values: 120, 124 .. 196 state = b.__getstate__() # Looks like: ((bucket0, 60, bucket1, 120, bucket2), firstbucket) # If these fail, the *preconditions* for running the test aren't # satisfied -- the test itself hasn't been run yet. self.assertEqual(len(state), 2) self.assertEqual(len(state[0]), 5) self.assertEqual(state[0][1], 60) self.assertEqual(state[0][3], 120) # Set up database connections to provoke conflict. self.openDB() r1 = self.db.open().root() r1["t"] = self.t transaction.commit() r2 = self.db.open(synch=False).root() copy = r2["t"] # Make sure all of copy is loaded. list(copy.values()) self.assertEqual(self.t._p_serial, copy._p_serial) # Now one transaction should add enough keys to cause a split, # and another should remove all the keys in one bucket. for k in range(200, 300, 4): self.t[k] = k transaction.commit() for k in range(0, 60, 4): del copy[k] try: transaction.commit() except ConflictError, detail: self.assert_(str(detail).startswith('database conflict error')) transaction.abort() else:
def doForwardPass(T, fitnessPathHDD, pC0E0, pC1E1, pE0, pE1, finalTime, batchSize): # start the clock startTime = time.clock() # the current working directory is the folder for the respective parameter combination # a plotting folder exists # add a state distribution folder set_global_Policy({}) stateDistPath = 'plotting/StateDistribution' fitnessPath = 'fitness' if not os.path.exists(stateDistPath): os.makedirs(stateDistPath) pC0E0, pC1E1, pE0, pE1 = float(pC0E0), float(pC1E1), float(pE0), float(pE1) tValues = np.arange(1,T-1, 1) """ This is the initialization - here we basically initialize a state distribution vector for t equals 0 and determine how the population starts """ print "start initialization" # make one folder per time step if not os.path.exists(os.path.join(stateDistPath, str(1))): os.makedirs(os.path.join(stateDistPath, str(1))) # what follows now should always be just one file in the directory # however I will make it general enough in case future models will deal with a larger state space for batchPstar in os.listdir('fitness/%s' % 1): # directly navigating to the corresponding time step folder currBatchPstar = {} # either a pickled file or a shelve if batchPstar.endswith('.p'): currBatchPstar.update(pickle.load(open("fitness/1/%s" % batchPstar, 'rb'))) else: # must be a shelve currBatchPstar.update(shelve.open("fitness/1/%s" % batchPstar)) currLen = batchPstar.replace('TF', '').replace('.p', '') # now the actual initialization intialization(currBatchPstar, pC0E0, pC1E1, pE0, pE1, finalTime, stateDistPath, currLen) del currBatchPstar print 'finished initialization' """ Here, the actual calculation of the state distribution matrix will begin - this will constitute a forward pass - the resulting P matrices are a one-to-one mapping from policy states, if a state from the policy is not in the in the P matrix, it's state distribution can be assumed to be zero """ # initialize a global dicts for P and policy star so that all parallel workers have access to it for t in tValues: global currBatchPstar currBatchPstar = {} global P P = {} #print "currently preparing time step: " + str(t) if t > 17: fitnessPath = fitnessPathHDD if not os.path.exists(os.path.join(stateDistPath, str(t + 1))): os.makedirs(os.path.join(stateDistPath, str(t + 1))) # make the respective path a variable here batchPstarList = [batchPstar for batchPstar in os.listdir(os.path.join(fitnessPath, '%s' % t))] batchPstarListSorted = [batchPstar for batchPstar in sorted(batchPstarList, key=lambda x: int(x.replace('TF', '').replace('.p', '')))] down() widgets = ['Time step %s:' % t, Percentage(), ' ', Bar(marker=('-'), left='[', right=']'), ' ', ETA(), ' '] pbarTimeStep = ProgressBar(widgets=widgets, maxval=int(len(batchPstarListSorted))).start() for batchPstar in pbarTimeStep(batchPstarListSorted): # directly navigating to the corresponding time step folder up() localP = OOBTree() localBatchPstar = OOBTree() currLen = batchPstar.replace('TF', '').replace('.p', '') # first load the respective policy # either a pickled file or a shelve if batchPstar.endswith('.p'): localBatchPstar.update(pickle.load(open(os.path.join(fitnessPath, "%s/%s" % (t, batchPstar)), 'rb'))) else: # must be a shelve localBatchPstar.update(shelve.open(os.path.join(fitnessPath, "%s/%s" % (t, batchPstar)))) # free up memory by deleting the local copies localBatchPstarCompressed = compressDict(localBatchPstar) del localBatchPstar set_global_Policy(localBatchPstarCompressed) del localBatchPstarCompressed if os.path.exists(os.path.join(stateDistPath, "%s/P%s" % (t, currLen))): localP.update(shelve.open(os.path.join(stateDistPath, "%s/P%s" % (t, currLen)))) else: print "No such file" + str(os.path.join(stateDistPath, "%s/P%s" % (t, currLen))) set_global_P(localP) del localP """ Implement the parallelization - this part will perform the actual calculation of the state distribution matrix - for each state extract the relevant statistics from both the state distribution vector and the optimal policy - next use these informaton to do the forward updating - it should be possible to distribute that across workers - should be inspired by the parallelization of the terminla fitness calculation? """ # now iterae through the optimal policy # structure: startCondition = int(currLen) - batchSize # prepare the respective pDicts # these will be the resultng P-values that I will need to fill identTimesTwo = startCondition * 2 + batchSize identTimesTwoPlusOne = identTimesTwo + batchSize cutOff1 = startCondition + (int(batchSize)/2 - 1) cutOff2 = cutOff1+ 1 print "Currently working on batch: %s" % currLen # this quantity is only dependent on t survivalProb = 1 - float(finalTime[t + 1]) # what is the maximim identifier? maxIdent,lowestIdent = findMaxIdentifier(currBatchPstar) subBatches = [] # this step is in place to decide whether we want to initalize two outcome files or one if maxIdent >= (batchSize / 2): subBatches.append(identTimesTwo) subBatches.append(identTimesTwoPlusOne) else: subBatches.append(identTimesTwo) for elem in subBatches: finalResults = OOBTree() if elem == identTimesTwo: parallelStates = [currState for currState in sorted(currBatchPstar.keys(), key=lambda x: int(x.split(';')[0])) if int(currState.split(";")[0]) <= cutOff1] else: parallelStates = [currState for currState in sorted(currBatchPstar.keys(), key=lambda x: int(x.split(';')[0])) if int(currState.split(";")[0]) >= cutOff2] parallelStatesChunks = chunks(parallelStates,25000) # usually this should be the batchsize parameter del parallelStates for chunkStates in parallelStatesChunks: # setting up the pool pool = Pool(32) results = pool.map(func_star, itertools.izip(chunkStates, itertools.repeat(survivalProb))) pool.close() pool.join() for tempResult in results: finalResults.update(tempResult) del results # safe the results myShelve = shelve.open(os.path.join(stateDistPath, "%s/P%s" % (t + 1, elem))) myShelve.update(finalResults) myShelve.close() del finalResults del parallelStatesChunks time.sleep(0.005) del currBatchPstar del P print "Elapsed time for thr forward pass: " + str(time.clock() - startTime)
class ZODBContinuousIncreasingIdGenerator(IdGenerator): """ Create some Ids with the zodb storage """ zope.interface.implements(interfaces.IIdGenerator) # CMF Type Definition meta_type = 'ERP5 ZODB Continous Increasing Id Generator' portal_type = 'ZODB Continous Increasing Id Generator' add_permission = Permissions.AddPortalContent # Declarative security security = ClassSecurityInfo() security.declareObjectProtected(Permissions.AccessContentsInformation) def _generateNewId(self, id_group, id_count=1, default=None): """ Return the new_id from the last_id of the zodb Use int to store the last_id, use also a persistant mapping for to be persistent. """ if id_group in (None, 'None'): raise ValueError, '%s is not a valid group Id.' % (repr(id_group), ) if default is None: default = 0 last_id_dict = getattr(self, 'last_id_dict', None) if last_id_dict is None: # If the dictionary not exist initialize generator self.initializeGenerator() last_id_dict = self.last_id_dict # Retrieve the last id and increment new_id = last_id_dict.get(id_group, default - 1) + id_count # Store the new_id in the dictionary last_id_dict[id_group] = new_id return new_id security.declareProtected(Permissions.AccessContentsInformation, 'generateNewId') def generateNewId(self, id_group=None, default=None): """ Generate the next id in the sequence of ids of a particular group """ return self._generateNewId(id_group=id_group, default=default) security.declareProtected(Permissions.AccessContentsInformation, 'generateNewIdList') def generateNewIdList(self, id_group=None, id_count=1, default=None): """ Generate a list of next ids in the sequence of ids of a particular group """ new_id = 1 + self._generateNewId(id_group=id_group, id_count=id_count, default=default) return range(new_id - id_count, new_id) security.declareProtected(Permissions.AccessContentsInformation, 'initializeGenerator') def initializeGenerator(self): """ Initialize generator. This is mostly used when a new ERP5 site is created. Some generators will need to do some initialization like prepare some data in ZODB """ LOG('initialize ZODB Generator', INFO, 'Id Generator: %s' % (self,)) if getattr(self, 'last_id_dict', None) is None: self.last_id_dict = OOBTree() # XXX compatiblity code below, dump the old dictionnaries portal_ids = getattr(self, 'portal_ids', None) # Dump the dict_ids dictionary if getattr(portal_ids, 'dict_ids', None) is not None: for id_group, last_id in portal_ids.dict_ids.items(): if not isinstance(id_group, str): id_group = repr(id_group) if self.last_id_dict.has_key(id_group) and \ self.last_id_dict[id_group] > last_id: continue self.last_id_dict[id_group] = last_id security.declareProtected(Permissions.AccessContentsInformation, 'clearGenerator') def clearGenerator(self): """ Clear generators data. This can be usefull when working on a development instance or in some other rare cases. This will loose data and must be use with caution This can be incompatible with some particular generator implementation, in this case a particular error will be raised (to be determined and added here) """ # Remove dictionary self.last_id_dict = OOBTree() security.declareProtected(Permissions.ModifyPortalContent, 'exportGeneratorIdDict') def exportGeneratorIdDict(self): """ Export last id values in a dictionnary in the form { group_id : last_id } """ return dict(self.last_id_dict) security.declareProtected(Permissions.ModifyPortalContent, 'importGeneratorIdDict') def importGeneratorIdDict(self, id_dict, clear=False): """ Import data, this is usefull if we want to replace a generator by another one. """ if clear: self.clearGenerator() if not isinstance(id_dict, dict): raise TypeError, 'the argument given is not a dictionary' for value in id_dict.values(): if not isinstance(value, int): raise TypeError, 'the value given in dictionary is not a integer' self.last_id_dict.update(id_dict) security.declareProtected(Permissions.ModifyPortalContent, 'rebuildGeneratorIdDict') def rebuildGeneratorIdDict(self): """ Rebuild generator id dict. In fact, export it, clear it and import it into new dict. This is mostly intendted to use when we are migrating the id dict structure. """ id_dict = self.exportGeneratorIdDict() self.importGeneratorIdDict(id_dict=id_dict, clear=True)