class Portlet(Persistent):
__name__ = None
__parent__ = None
type_name = u"Portlet"
type_title = _(u"Portlet")
type_description = _(u"A mini view rendered ")
portlet_type = u""
add_permission = "Add %s" % type_name
def __init__(self, portlet_type, **kw):
self.uid = unicode(uuid4())
self.portlet_type = portlet_type
self.__settings__ = OOBTree()
settings = kw.pop('settings', {})
self.settings = settings
self.__dict__.update(**kw)
super(Portlet, self).__init__()
@property
def title(self):
return self.settings.get('title',
getattr(self.portlet_adapter, 'title', u''))
@property
def description(self):
return self.settings.get(
'description', getattr(self.portlet_adapter, 'description', u''))
@property
def settings(self):
return self.__settings__
@settings.setter
def settings(self, value):
self.__settings__.clear()
self.__settings__.update(value)
@property
def schema_factory(self):
return self.portlet_adapter.schema_factory
@property
def portlet_adapter(self):
reg = get_current_registry()
return reg.getAdapter(self, IPortletType, name=self.portlet_type)
@property
def slot(self):
try:
return self.__parent__.slot
except AttributeError:
pass
def render(self, context, request, view, **kw):
try:
return self.portlet_adapter.render(context, request, view, **kw)
except ComponentLookupError:
_logger.error("portlet %r not found for context %r" %
(self.portlet_type, context))
return ""
def index(table_name, attr, data_structure):
if data_structure == "Btree":
tree = OOBTree()
T = tables[table_name]
indexes = np.arange(len(T.table))
header = T.header
header_idx = {k: v for v, k in enumerate(header)}
attr_idx = header_idx[attr]
T_table = T.table
for i in range(len(T_table)):
value = T_table[i][attr_idx]
if tree.has_key(value):
current_idxs = tree[value]
current_idxs.append(i)
tree.update({value: current_idxs})
else:
tree.update({value: [i]})
tables[table_name].index = tree
tables[table_name].index_att = attr
else:
dict = {}
T = tables[table_name]
indexes = np.arange(len(T.table))
header = T.header
header_idx = {k: v for v, k in enumerate(header)}
attr_idx = header_idx[attr]
T_table = T.table
for i in range(len(T_table)):
value = T_table[i][attr_idx]
dict.setdefault(value, []).append(i)
tables[table_name].index = dict
tables[table_name].index_att = attr
def pushCondition(database, table_schema, tables, conditions):
newDatabase = {}
for table in tables:
temp = database[table]
inCondition = False
for c in conditions:
if c[0] == table:
condition = c
inCondition = True
break
if inCondition and len(condition) < 6: #not two var
newTree = OOBTree()
for key in temp:
if condition[1] == 0: #key column
value = key
else:
value = temp[key][condition[1] - 1]
if evaluateCondition(value, condition[3], condition[2]):
newTree.update({key: temp[key]})
newDatabase[table] = newTree
else:
newDatabase[table] = temp
return newDatabase
def loadDatabase(database, table):
#load the data into a btree (assuming that the first column is the primary key)
file = open("database_files/" + table + ".csv", "r")
t = OOBTree()
for line in file:
data = line[:-1].split(',')
fixed_data = []
i = 0
while i < len(data):
if data[i][0] != '"' and data[i][0] != "'":
fixed_data.append(float(data[i]))
elif (data[i][0] == "'" or data[i][0]
== '"') and (data[i][-1] == "'" or data[i][-1] == '"'):
fixed_data.append(data[i][1:-1])
else:
j = i + 1
temp = data[i][1:]
while j < len(data):
temp += "," + data[j]
if data[j][-1] == "'" or data[j][-1] == '"':
fixed_data.append(temp[:-1])
break
j += 1
i = j
i += 1
t.update({fixed_data[0]: fixed_data[1:]})
database[table] = t
file.close()
class Portlet(Persistent):
__name__ = None
__parent__ = None
type_name = u"Portlet"
type_title = _(u"Portlet")
type_description = _(u"A mini view rendered ")
portlet_type = u""
add_permission = "Add %s" % type_name
def __init__(self, portlet_type, **kw):
self.uid = unicode(uuid4())
self.portlet_type = portlet_type
self.__settings__ = OOBTree()
settings = kw.pop('settings', {})
self.settings = settings
self.__dict__.update(**kw)
super(Portlet, self).__init__()
@property
def title(self):
return self.settings.get('title', getattr(self.portlet_adapter, 'title', u''))
@property
def description(self):
return self.settings.get('description', getattr(self.portlet_adapter, 'description', u''))
@property
def settings(self):
return self.__settings__
@settings.setter
def settings(self, value):
self.__settings__.clear()
self.__settings__.update(value)
@property
def schema_factory(self):
return self.portlet_adapter.schema_factory
@property
def portlet_adapter(self):
reg = get_current_registry()
return reg.getAdapter(self, IPortletType, name = self.portlet_type)
@property
def slot(self):
try:
return self.__parent__.slot
except AttributeError:
pass
def render(self, context, request, view, **kw):
try:
return self.portlet_adapter.render(context, request, view, **kw)
except ComponentLookupError:
_logger.error("portlet %r not found for context %r" % (self.portlet_type, context))
return ""
def setupNewSubsystemsFromMovedFileInfo(loggingMixin, mocker):
""" Setup for testing create new runs and subsystems based on moved file information. """
# Ensure that processing clasess doesn't actually create any files or folders...
mocker.patch("overwatch.processing.processingClasses.os.makedirs")
# Set the relevant subsystems
# We want the TPC to use HLT subsystem files.
subsystems = ["EMC", "HLT", "TPC"]
processRuns.processingParameters["subsystemList"] = subsystems
runDir = "Run123"
hltMode = "C"
runDict = {
runDir: {
"hltMode":
hltMode,
"HLT": [
"HLThists.2015_11_24_18_05_10.root",
"HLThists.2015_11_24_18_09_12.root"
],
"EMC": [
"EMChists.2015_11_24_18_05_12.root",
"EMChists.2015_11_24_18_09_14.root"
],
}
}
# Same as the runDict above, but with an additional set of files.
additionalRunDict = {
runDir: {
"hltMode":
hltMode,
"HLT": [
"HLThists.2015_11_24_19_05_10.root",
"HLThists.2015_11_24_19_09_12.root"
],
"EMC": [
"EMChists.2015_11_24_19_05_11.root",
"EMChists.2015_11_24_19_09_13.root"
],
}
}
# Ensure we don't create random directories
mocker.patch("overwatch.processing.processingClasses.os.path.exists")
# Create the runs container
runs = OOBTree()
runs.update({
runDir:
processingClasses.runContainer(runDir=runDir,
fileMode=True,
hltMode=hltMode)
})
# Add some subsystems
runs[runDir].subsystems
return runs, runDir, runDict, additionalRunDict, subsystems
def compressDict(originalDict):
newDict = {}
for key in originalDict:
(_, a, b, _) = originalDict[key]
newDict[key] = (a, b)
t = OOBTree()
t.update(newDict)
del newDict
return t
def unionResultSets(sets):
""" perform union of ResultSets """
docids = None
words = OOBTree()
sets = list(sets)
sets.sort(lambda s1,s2: cmp(len(s1.docIds()),len(s2.docIds())))
for set in sets:
docids = IntUnion(docids, set.docIds())
words.update(set.words())
return ResultSet(docids, words)
def create_index_btree():
"""
Build b-tree for efficient index file search
"""
index_path = 'Index/'
index_file_names = sorted(listdir(index_path))
t = OOBTree()
# Create a dictionary where there are file names on key and index on values
name_id_dict = {k: v for v, k in enumerate(index_file_names)}
t.update(name_id_dict)
# Build tree
return t
def add_index(self, attr, idex_name):
if attr[0] not in self.uniqueattr.keys():
raise Exception(
'ERROR: The attr is not unique and cannot create index')
# If unique:
if attr[0] not in self.index:
self.index[attr[0]] = idex_name
# Get pairs {v1:p1, v2:p2,...}
nodes = self.uniqueattr[attr[0]]
# Create a b tree
t = OOBTree()
t.update(nodes)
self.BTree[idex_name] = t
return t
def test_create_tree(self):
array = list(range(self.complexity))
random.shuffle(array)
st = time.clock()
d = {i: -i for i in array}
elapse1 = time.clock() - st
t = OOBTree()
st = time.clock()
t.update(d)
elapse2 = time.clock() - st
print("creating a dict using %.6f sec, a tree %.6f sec" % (elapse1, elapse2))
self.assertLess(elapse1, elapse2)
def testFailedJoin(self):
"""
Tests if the join works correctly.
"""
b_tree = OOBTree()
b_tree.update({1: "Monkey D. Luffy", 2: "Roronoa Zoro", 3: "Nami"})
failed_counter = 0
key = 10
data = {"from": "East Blue"}
(mod_data, mod_tree,
failed_counter) = self.processing.join(b_tree, key, data,
failed_counter)
self.assertEqual(mod_data, {"from": "East Blue"})
self.assertEqual(len(mod_tree), 3)
self.assertEqual(failed_counter, 1)
def BTreeJoin(orderedPK=False, orderedFK=False):
#Arquivos que usaremos no JOIN
PKFile = ""
FKFile = ""
if (orderedPK):
PKFile = tg.PKOrderedFileName
else:
PKFile = tg.PKUnorderedFileName
if (orderedFK):
FKFile = tg.FKOrderedFileName
else:
FKFile = tg.FKUnorderedFileName
#Carregando o indice da BTree
btree = OOBTree()
lineCounter = 0
with open(FKFile) as f:
for line in f:
l = line.split(tg.fieldSeparator)
btree.update({l[1]: lineCounter})
lineCounter += 1
# checking the size
print("Btree size: " + str(len(btree)))
lineSize = tg.registrySize + 2
start = dt.datetime.now()
matches = 0
with open(FKFile) as right:
with open(PKFile) as left:
for line in left:
leftRegistry = line.split(tg.fieldSeparator)
position = btree[leftRegistry[0]] * lineSize
right.seek(position)
rightRegistry = right.read(tg.registrySize).split(
tg.fieldSeparator)
if leftRegistry[0] == rightRegistry[1]:
matches += 1
joined = leftRegistry + rightRegistry
else:
print("This should never happen!")
end = dt.datetime.now()
print("Total joined registers: " + str(matches))
print("Time taken: " + str((end - start).total_seconds()) + "s")
def test_create_tree(self):
array = list(range(self.complexity))
random.shuffle(array)
st = time.clock()
d = {i: -i for i in array}
elapse1 = time.clock() - st
t = OOBTree()
st = time.clock()
t.update(d)
elapse2 = time.clock() - st
print("creating a dict using %.6f sec, a tree %.6f sec" %
(elapse1, elapse2))
self.assertLess(elapse1, elapse2)
def file_to_tree(root_guarda, file):
'''
Função responsável por, a partir de um arquivo, criar uma árvore B obedecendo os padrões de arquivos e diretórios
:param root_guarda : caminho da pasta do programa guarda onde está o arquivo
:param file : arquivo a ser lido
'''
tree = OOBTree()
file = open(root_guarda+file,"r")
for line in file:
file_hash = line.split("\t")
if file_hash[1] == "dir\n":
tree.update({file_hash[0]:file_to_tree(root_guarda, file_hash[0]+".txt")})
continue
else:
tree.update({file_hash[0]:str(file_hash[1])[:-1]})
file.close()
return tree
def test_getitem(self):
array = list(range(self.complexity))
random.shuffle(array)
d = {i: -i for i in array}
t = OOBTree()
t.update(d)
st = time.clock()
for k in array:
d[k]
elapse1 = time.clock() - st
st = time.clock()
for k in array:
t[k]
elapse2 = time.clock() - st
print("get item from a dict using %.6f sec, a tree %.6f sec" % (elapse1, elapse2))
self.assertLess(elapse1, elapse2)
class IX_IndexHandle:
def __init__(self, fileName, indexNo):
self.filename = fileName + '.' + str(indexNo)
self.indexNo = indexNo
self.fileName = fileName
with open(self.filename, "r", encoding="utf-8") as f:
tree_dict = json.load(f)
keys = list(map(int, tree_dict.keys()))
values = list(tree_dict.values())
new_tree_dict = dict(zip(keys, values))
self.tree = Tree()
self.tree.update(new_tree_dict)
pass
def __del__(self):
pass
def InsertEntry(self, record, rid):
key = record[self.indexNo]
l = self.tree.get(key)
pageNum = rid.getPageNum()
slotNum = rid.getSlotNum()
if l:
l.append((pageNum, slotNum))
else:
self.tree[key] = [(pageNum, slotNum)]
def DeleteEntry(self, record, rid):
key = record[self.indexNo]
l = self.tree.get(key)
pageNum = rid.getPageNum()
slotNum = rid.getSlotNum()
if l:
l.remove((pageNum, slotNum))
else:
return False
def ForcePages(self):
with open(self.filename, "w", encoding="utf-8") as f:
tree_dict = dict(self.tree.items())
json.dump(tree_dict, f)
def test_range_seach(self):
"""
"""
array = list(range(self.complexity))
random.shuffle(array)
d = {i: -i for i in array}
t = OOBTree()
t.update(d)
# === dict ===
st = time.clock()
result1 = list()
for k, v in d.items():
if (self.lower <= k <= self.upper):
result1.append(v)
elapse1 = time.clock() - st
# === bisearch ===
sorted_keys = list(d.keys()) # sort the list of keys
sorted_keys.sort()
st = time.clock()
lower_ind = bisect.bisect_left(sorted_keys,
self.lower) # find the min index
upper_ind = bisect.bisect_right(sorted_keys,
self.upper) - 1 # find the max index
result2 = list() # fetch item
for ind in range(lower_ind, upper_ind + 1):
result2.append(d[sorted_keys[ind]])
elapse2 = time.clock() - st
# === tree ===
st = time.clock()
result3 = t.values(min=self.lower,
max=self.upper,
excludemin=False,
excludemax=False)
elapse3 = time.clock() - st
print("results are:", result1, result2, list(result3))
print("dict method = %.6f, bisearch = %.6f, tree = %.6f" %
(elapse1, elapse2, elapse3))
self.assertGreater(elapse1, elapse2)
self.assertGreater(elapse2, elapse3)
def test_getitem(self):
array = list(range(self.complexity))
random.shuffle(array)
d = {i: -i for i in array}
t = OOBTree()
t.update(d)
st = time.clock()
for k in array:
d[k]
elapse1 = time.clock() - st
st = time.clock()
for k in array:
t[k]
elapse2 = time.clock() - st
print("get item from a dict using %.6f sec, a tree %.6f sec" %
(elapse1, elapse2))
self.assertLess(elapse1, elapse2)
def buildTreeForAttr(dictAttr, fileName, attrIndx, file_path, return_tree):
"""
This funciton accepts a dctionary dictAttr, build an OOBtree, store the value in OOBTree and then
dump the Btree object into a file with name defined as fileName + _Attr_ + attrIndx + _.tree
Args:
dictAttr: a dictionary that store the (attribute, offset set) pair
fileName: csvfile name without '.csv'
attrIndx: index of attribute
file_path: path to store the btree file
Returns:
None
"""
sys.setrecursionlimit(100000)
t = OOBTree()
t.update(dictAttr)
os.makedirs(file_path, exist_ok=True)
with open(file_path + '/' + fileName + '_Attr_' + str(attrIndx) + '_.tree',
"wb") as f:
cPickle.dump(t, f)
if return_tree:
return t
def createBTree(collectionName, column):
"""Creates a BTree for the given collection based on specific column
Parameters
----------
collectionName : str
The name of the collection we want to create BTree for
column : str
Column we use to create BTree
"""
# Creating a defaultdict to prevent KeyError and if a key is not present, it is intialized to empty array
hashmap = defaultdict(lambda: [])
collection = allCollections[collectionName]
for row in range(len(collection)):
hashmap[collection[row][column]].append(row)
# Intializing the empty BTree
tree = OOBTree()
# To update the tree we need hash of all values which is calculated above
tree.update(hashmap)
bTreedKeys[collectionName + '_' + column] = tree
print('Created a BTree on ' + column + ' for collection ' + collectionName)
class Root(Content, LocalRolesMixin, DCMetadataMixin, ContextACLMixin):
type_name = "Root"
type_title = _("Site root")
add_permission = "Add %s" % type_name
search_visible = False
is_permanent = True
def __init__(self, data=None, **kwargs):
self.catalog = Catalog()
self.document_map = DocumentMap()
populate_catalog(self.catalog)
self.__site_settings__ = OOBTree()
super(Root, self).__init__(data=data, **kwargs)
@property
def site_settings(self):
return getattr(self, "__site_settings__", {})
@site_settings.setter
def site_settings(self, value):
self.__site_settings__.clear()
self.__site_settings__.update(value)
class Root(Content, LocalRolesMixin, DCMetadataMixin, ContextACLMixin):
type_name = u"Root"
type_title = _(u"Site root")
add_permission = "Add %s" % type_name
search_visible = False
is_permanent = True
def __init__(self, data=None, **kwargs):
self.catalog = Catalog()
self.document_map = DocumentMap()
populate_catalog(self.catalog)
self.__site_settings__ = OOBTree()
super(Root, self).__init__(data=data, **kwargs)
@property
def site_settings(self):
return getattr(self, '__site_settings__', {})
@site_settings.setter
def site_settings(self, value):
self.__site_settings__.clear()
self.__site_settings__.update(value)
def Btree(items, dict, btree_dict, col_dict):
var = re.split(r'[\s\,\(\)]+', items)
while '' in var:
var.remove('')
ori_table = var[1]
index_col = var[2]
table_col = ori_table + '_' + index_col
col_dict[table_col] = 'Btree'
col = find_col(dict[ori_table], index_col)
dict = {}
for index in range(0, len(col)):
if col[index] not in dict.keys():
res = []
res.append(index)
dict[col[index]] = res
else:
res = dict[col[index]]
res.append(index)
dict[col[index]] = res
t = OOBTree()
t.update(dict)
btree_dict[table_col] = t
return 1
def create_hash_dir(name, directory, method, key, root_guarda):
dir_tree = OOBTree()
path = directory+name+"/"
for i in os.listdir(path):
if i == ".guarda":
continue
if method == '--hash':
hasher = hashlib.sha1()
elif method == '--hmac':
hasher = hmac.new(key.encode(), digestmod=hashlib.sha1)
if (os.path.isfile(path+i)):
with open(path+i, 'rb') as node:
buf = None
buf = node.read()
hasher.update(buf)
dir_tree.update({i:hasher.hexdigest()})
continue
if (os.path.isdir(path+i)):
dir_tree.update({i:create_hash_dir(i, path, method, key, root_guarda)})
return dir_tree
def index(self):
for table_def in table_data.table_data.values():
print ("Looking for Indexes on " + table_def["name"]) + "..."
table_data.indexes[table_def["name"]] = {}
for column_def in table_def["columns"].values():
if column_def["index"]:
print "Indexing table " + table_def["name"] + " column " + column_def["name"]
index_tree = OOBTree()
table_data.indexes[table_def["name"]][column_def["name"]] = index_tree
i = 0
for data_file in listdir(table_def["data_store"]):
with open(table_def["data_store"] + data_file) as indexing_data:
page_index = 0
table_reader = csv.reader(indexing_data, delimiter=",", quotechar="|")
for row in table_reader:
index_data = row[column_def["position"]]
if not index_data in index_tree:
index_tree.update({index_data : [(data_file, page_index)]})
else:
index_tree[index_data].append((data_file, page_index))
page_index += 1
i += 1
print "Indexed " + str(i) + " files..."
def create_index(self, field_to_index: str, index_type="btree") -> None:
index_of_field = self.get_index_of_field(field_to_index)
with open(f"{DB_ROOT}/{self.name}.csv", "r") as csv_file:
csv_reader = csv.reader(csv_file)
index_values = {}
next(csv_reader)
for record in csv_reader:
if record:
index_values[record[index_of_field]] = {
"path": "{DB_ROOT}/{self.name}.csv",
"line": csv_reader.line_num
}
if index_type == "btree":
btree_index = OOBTree()
btree_index.update(index_values)
self.btree_index_info[field_to_index] = btree_index
if index_type == "hash":
self.hash_index_info[field_to_index] = index_values
else:
raise ValueError("No such index type")
def Btree(self, condition, return_=False):
'''
(i) Create a btree with an index on an attribute from a table that user specified.
(ii) Inputs: An attribute from a table
(iii) Outputs: output a btree that is indexed with the attribute specified.
(iv) There is a global effect that a btree is created
'''
condition = condition.replace(' ', '')
condition = re.split('[,\(\)\n\s]', condition)
unwanted = ['', 'Btree']
condition = [e for e in condition if e not in unwanted]
table = eval(condition[0])
btree = OOBTree()
for i, value in enumerate(table[condition[1]]):
if value in btree:
btree.update({value: btree[value] + [i]})
else:
btree.update({value: [i]})
self.flag = 'btree'
self.index['btree'] = condition[1]
self.btree = btree
if return_:
return btree
def test_range_seach(self):
"""
"""
array = list(range(self.complexity))
random.shuffle(array)
d = {i: -i for i in array}
t = OOBTree()
t.update(d)
# === dict ===
st = time.clock()
result1 = list()
for k, v in d.items():
if (self.lower <= k <= self.upper):
result1.append(v)
elapse1 = time.clock() - st
# === bisearch ===
sorted_keys = list(d.keys()) # sort the list of keys
sorted_keys.sort()
st = time.clock()
lower_ind = bisect.bisect_left(sorted_keys, self.lower) # find the min index
upper_ind = bisect.bisect_right(sorted_keys, self.upper) - 1 # find the max index
result2 = list() # fetch item
for ind in range(lower_ind, upper_ind+1):
result2.append(d[sorted_keys[ind]])
elapse2 = time.clock() - st
# === tree ===
st = time.clock()
result3 = t.values(min=self.lower, max=self.upper, excludemin=False, excludemax=False)
elapse3 = time.clock() - st
print("results are:", result1, result2, list(result3))
print("dict method = %.6f, bisearch = %.6f, tree = %.6f" % (elapse1, elapse2, elapse3))
self.assertGreater(elapse1, elapse2)
self.assertGreater(elapse2, elapse3)
def init(method, directory,out, key):
path = None
try:
path_guarda = str(directory) +".guarda/"
#se já existir programa guarda : apaga
if os.path.isdir(path_guarda):
shutil.rmtree(path_guarda)
os.makedirs(path_guarda)
guarda = open(path_guarda +"guarda.txt", "w+")
guarda.close()
guarda_tree = OOBTree()
for i in os.listdir(directory):
if i == ".guarda" or i == ".tmp":
continue
if method == '--hash':
hasher = hashlib.sha1()
elif method == '--hmac':
hasher = hmac.new(key.encode(), digestmod=hashlib.sha1)
if (os.path.isfile(directory+i)):
with open(directory+i, 'rb') as node:
buf = None
buf = node.read()
hasher.update(buf)
guarda_tree.update({i:hasher.hexdigest()})
continue
if (os.path.isdir(directory+i)):
guarda_tree.update({i:create_hash_dir(i, directory, method, key, directory+".guarda/")})
tree_to_file("guarda", directory, directory+".guarda/", guarda_tree)
except OSError as error:
print(error)
class Subscriptions(SimpleItem):
security = ClassSecurityInfo()
title = "Meeting registrations"
def __init__(self, id):
""" """
super(SimpleItem, self).__init__(id)
self.id = id
self._signups = OOBTree()
self._account_subscriptions = OOBTree()
security.declarePublic('getMeeting')
def getMeeting(self):
return self.aq_parent.aq_parent
def _validate_signup(self, form):
""" """
formdata = {}
formerrors = {}
keys = ('first_name', 'last_name', 'email', 'organization', 'phone')
formdata = dict( (key, form.get(key, '')) for key in keys )
for key in formdata:
if formdata[key] == '':
formerrors[key] = 'This field is mandatory'
if formerrors == {}:
if formdata['email'].count('@') != 1:
formerrors['email'] = 'An email address must contain a single @'
if formerrors == {}:
formerrors = None
return formdata, formerrors
def _add_signup(self, formdata):
""" """
meeting = self.getMeeting()
key = random_key()
name = formdata['first_name'] + ' ' + formdata['last_name']
email = formdata['email']
organization = formdata['organization']
phone = formdata['phone']
signup = SignUp(key, name, email, organization, phone)
self._signups.insert(key, signup)
if meeting.auto_register:
self._accept_signup(key)
email_sender = self.getMeeting().getEmailSender()
email_sender.send_signup_email(signup)
security.declareProtected(view, 'signup')
def signup(self, REQUEST):
""" """
meeting = self.getMeeting()
if not meeting.allow_register:
return REQUEST.RESPONSE.redirect(self.absolute_url() + '/subscription_not_allowed')
if REQUEST.REQUEST_METHOD == 'GET':
return self.getFormsTool().getContent({'here': self},
'naaya.content.meeting.subscription_signup')
if REQUEST.REQUEST_METHOD == 'POST':
formdata, formerrors = self._validate_signup(REQUEST.form)
#check Captcha/reCaptcha
if not self.checkPermissionSkipCaptcha():
contact_word = REQUEST.form.get('contact_word', '')
captcha_validator = self.validateCaptcha(contact_word, REQUEST)
if captcha_validator:
if formerrors is None:
formerrors = {}
formerrors['captcha'] = captcha_validator
if formerrors is not None:
return self.getFormsTool().getContent({'here': self,
'formdata': formdata,
'formerrors': formerrors},
'naaya.content.meeting.subscription_signup')
else:
self._add_signup(formdata)
REQUEST.RESPONSE.redirect(self.absolute_url() + '/signup_successful')
security.declareProtected(view, 'signup_successful')
def signup_successful(self, REQUEST):
""" """
return self.getFormsTool().getContent({'here': self}, 'naaya.content.meeting.subscription_signup_successful')
security.declareProtected(view, 'subscribe')
def subscribe(self, REQUEST):
""" """
meeting = self.getMeeting()
if not meeting.allow_register:
return REQUEST.RESPONSE.redirect(self.absolute_url() + '/subscription_not_allowed')
return self.getFormsTool().getContent({'here': self}, 'naaya.content.meeting.subscription_subscribe')
security.declareProtected(PERMISSION_ADMIN_MEETING, 'getSignups')
def getSignups(self):
""" """
return self._signups.itervalues()
security.declareProtected(PERMISSION_ADMIN_MEETING, 'getSignup')
def getSignup(self, key):
""" """
return self._signups.get(key, None)
security.declareProtected(PERMISSION_ADMIN_MEETING, 'index_html')
def index_html(self, REQUEST):
""" """
return self.getFormsTool().getContent({'here': self}, 'naaya.content.meeting.subscription_index')
def _accept_signup(self, key):
""" """
meeting = self.getMeeting()
meeting.getParticipants()._set_attendee(key, PARTICIPANT_ROLE)
signup = self._signups[key]
signup.accepted = 'accepted'
email_sender = meeting.getEmailSender()
result = email_sender.send_signup_accepted_email(signup)
def _reject_signup(self, key):
""" """
meeting = self.getMeeting()
signup = self._signups[key]
signup.accepted = 'rejected'
participants = meeting.getParticipants()
if key in participants._get_attendees():
participants._del_attendee(key)
email_sender = meeting.getEmailSender()
result = email_sender.send_signup_rejected_email(signup)
def _delete_signup(self, key):
""" """
meeting = self.getMeeting()
signup = self._signups.pop(key, None)
if signup is None:
return
participants = meeting.getParticipants()
if key in participants._get_attendees():
participants._del_attendee(key)
email_sender = meeting.getEmailSender()
result = email_sender.send_signup_rejected_email(signup)
def _is_signup(self, key):
""" """
return self._signups.has_key(key) and self._signups[key].accepted == 'accepted'
security.declareProtected(PERMISSION_ADMIN_MEETING, 'manageSignups')
def manageSignups(self, REQUEST):
""" """
keys = REQUEST.form.get('keys', [])
assert isinstance(keys, list)
if 'accept' in REQUEST.form:
for key in keys:
self._accept_signup(key)
elif 'reject' in REQUEST.form:
for key in keys:
self._reject_signup(key)
elif 'delete' in REQUEST.form:
for key in keys:
self._delete_signup(key)
return REQUEST.RESPONSE.redirect(self.absolute_url())
security.declarePublic('welcome')
def welcome(self, REQUEST):
""" """
if 'logout' in REQUEST.form:
REQUEST.SESSION['nymt-current-key'] = None
return REQUEST.RESPONSE.redirect(self.getMeeting().absolute_url())
key = REQUEST.get('key', None)
signup = self.getSignup(key)
if self._is_signup(key):
REQUEST.SESSION['nymt-current-key'] = key
return self.getFormsTool().getContent({'here': self,
'signup': signup},
'naaya.content.meeting.subscription_welcome')
def _add_account_subscription(self, uid):
""" """
site = self.getSite()
meeting = self.getMeeting()
name = getUserFullName(site, uid)
email = getUserEmail(site, uid)
organization = getUserOrganization(site, uid)
if not organization:
organization = self.get_survey_answer(uid, 'w_organization')
if not organization:
organization = self.get_survey_answer(uid, 'w_organisation')
phone = getUserPhoneNumber(site, uid)
if not phone:
phone = self.get_survey_answer(uid, 'w_telephone')
if not phone:
phone = self.get_survey_answer(uid, 'w_phone')
account_subscription = AccountSubscription(uid, name, email, organization, phone)
self._account_subscriptions.insert(uid, account_subscription)
if meeting.auto_register:
self._accept_account_subscription(uid)
email_sender = self.getMeeting().getEmailSender()
email_sender.send_account_subscription_email(account_subscription)
security.declareProtected(PERMISSION_ADMIN_MEETING, 'update_account_subscription')
def update_account_subscription(self, uid):
""" """
site = self.getSite()
name = getUserFullName(site, uid)
email = getUserEmail(site, uid)
organization = getUserOrganization(site, uid)
phone = getUserPhoneNumber(site, uid)
account_subscription = AccountSubscription(uid, name, email, organization, phone)
self._account_subscriptions.update({uid: account_subscription})
security.declareProtected(view, 'subscribe_account')
def subscribe_account(self, REQUEST):
""" """
meeting = self.getMeeting()
if not meeting.allow_register:
return REQUEST.RESPONSE.redirect(self.absolute_url() + '/subscription_not_allowed')
self._add_account_subscription(REQUEST.AUTHENTICATED_USER.getId())
if self.survey_required:
site = self.getSite()
path = str(self.survey_pointer)
survey_ob = site.unrestrictedTraverse(path, None)
if survey_ob is not None and survey_ob.meta_type == 'Naaya Mega Survey':
answers = survey_ob.getAnswers()
respondents = [a.respondent for a in answers]
current_user = REQUEST.AUTHENTICATED_USER.getUserName()
if current_user not in respondents:
self.setSessionInfoTrans(
'Registration successfully sent for approval. '
'Please also respond to the following questionaire.')
return REQUEST.RESPONSE.redirect('%s/%s' % (self.getSite().absolute_url(), self.survey_pointer))
return REQUEST.RESPONSE.redirect(self.absolute_url() + '/subscribe_account_successful')
security.declareProtected(view, 'subscribe_account_successful')
def subscribe_account_successful(self, REQUEST):
""" """
return self.getFormsTool().getContent({'here': self}, 'naaya.content.meeting.subscription_subscribe_successful')
security.declareProtected(PERMISSION_ADMIN_MEETING, 'getAccountSubscriptions')
def getAccountSubscriptions(self):
""" """
return self._account_subscriptions.itervalues()
security.declareProtected(PERMISSION_ADMIN_MEETING, 'getAccountSubscription')
def getAccountSubscription(self, uid):
""" """
return self._account_subscriptions.get(uid, None)
def _is_account_subscription(self, uid):
""" """
return self._account_subscriptions.has_key(uid) and self._account_subscriptions[uid].accepted == 'accepted'
security.declareProtected(PERMISSION_ADMIN_MEETING, 'manageSignups')
def manageAccountSubscriptions(self, REQUEST):
""" """
uids = REQUEST.form.get('uids', [])
assert isinstance(uids, list)
if 'accept' in REQUEST.form:
for uid in uids:
self._accept_account_subscription(uid)
elif 'reject' in REQUEST.form:
for uid in uids:
self._reject_account_subscription(uid)
elif 'delete' in REQUEST.form:
for uid in uids:
self._delete_account_subscription(uid)
return REQUEST.RESPONSE.redirect(self.absolute_url())
def _accept_account_subscription(self, uid):
""" """
meeting = self.getMeeting()
meeting.getParticipants()._set_attendee(uid, PARTICIPANT_ROLE)
account_subscription = self._account_subscriptions[uid]
account_subscription.accepted = 'accepted'
email_sender = meeting.getEmailSender()
result = email_sender.send_account_subscription_accepted_email(account_subscription)
def _reject_account_subscription(self, uid):
""" """
meeting = self.getMeeting()
account_subscription = self._account_subscriptions[uid]
account_subscription.accepted = 'rejected'
participants = meeting.getParticipants()
if uid in participants._get_attendees():
participants._del_attendee(uid)
email_sender = meeting.getEmailSender()
result = email_sender.send_account_subscription_rejected_email(account_subscription)
def _delete_account_subscription(self, uid):
""" """
meeting = self.getMeeting()
account_subscription = self._account_subscriptions.pop(uid, None)
if account_subscription is None:
return
participants = meeting.getParticipants()
if uid in participants._get_attendees():
participants._del_attendee(uid)
email_sender = meeting.getEmailSender()
result = email_sender.send_account_subscription_rejected_email(account_subscription)
security.declareProtected(view, 'subscription_not_allowed')
def subscription_not_allowed(self, REQUEST):
""" """
return self.getFormsTool().getContent({'here': self}, 'naaya.content.meeting.subscription_not_allowed')
class Index:
"""
Indexes the specified column of the specified table to speed up select queries
This data structure is usually a B-Tree
"""
def __init__(self):
# the column this index operates on
self.column_number = 0
# the actual tree that does the work associated with indexing
self.index = OOBTree()
# the table this index operates on
self.table = None
def locate(self, value):
"""
returns the location of all records with the given value
:param value: int # the value of the key for which we need the RID
"""
return self.index.get(value, False)
def create_index(self, table, column_number):
"""
Create index on specific column
:param table: table object # the table the index will operate on
:param column_number: int # the column number in the table the index will operate on
:return: index object # the created index object
"""
self.table = table
self.column_number = column_number
page = 0
offset = 0
# for each page range in the table
for page_range in self.table.ranges:
# for each record in the page range
for i in range(page_range.num_of_records):
# get the key for this record relative to the chosen column
key = page_range.columns[self.column_number].pages[page].read(
offset)
# get the RID for this record
RID = page_range.columns[RID_COLUMN].pages[page].read(offset)
# increment the offset
offset += 8
# if reached the end of the page increment page and reset offset
if offset == PAGE_SIZE:
page += 1
offset = 0
self.add_index_item(key, RID)
return self
def add_index_item(self, key, RID):
"""
Add to the index the key value and RID pair
:param key: int # the key to store this RID at
:param RID: int # the RID of the key
"""
# if there are RID/s at this key get them, else false
rids = self.index.get(key, False)
# if there are rids at this key already append the new RID
if rids:
self.index.update({key: rids.append(RID)})
return
# otherwise add key and RID
self.index.update({key: [RID]})
def drop_index(self, table, column_number):
"""
Drop index of specific column
"""
self.table.indexes[self.column_number] = None
self.index = None
self.table = None
self.column_number = 0
class ZODBContinuousIncreasingIdGenerator(IdGenerator):
"""
Create some Ids with the zodb storage
"""
zope.interface.implements(interfaces.IIdGenerator)
# CMF Type Definition
meta_type = 'ERP5 ZODB Continous Increasing Id Generator'
portal_type = 'ZODB Continous Increasing Id Generator'
add_permission = Permissions.AddPortalContent
# Declarative security
security = ClassSecurityInfo()
security.declareObjectProtected(Permissions.AccessContentsInformation)
def _generateNewId(self, id_group, id_count=1, default=None, poison=False):
"""
Return the new_id from the last_id of the zodb
Use int to store the last_id, use also a persistant mapping for to be
persistent.
"""
if id_group in (None, 'None'):
raise ValueError, '%s is not a valid group Id.' % (
repr(id_group), )
if default is None:
default = 0
last_id_dict = getattr(self, 'last_id_dict', None)
if last_id_dict is None:
# If the dictionary not exist initialize generator
self.initializeGenerator()
last_id_dict = self.last_id_dict
# Retrieve the last id and increment
new_id = last_id_dict.get(id_group, default - 1) + id_count
# Store the new_id in the dictionary
last_id_dict[id_group] = None if poison else new_id
return new_id
security.declareProtected(Permissions.AccessContentsInformation,
'generateNewId')
def generateNewId(self, id_group=None, default=None, poison=False):
"""
Generate the next id in the sequence of ids of a particular group
"""
return self._generateNewId(id_group=id_group,
default=default,
poison=poison)
security.declareProtected(Permissions.AccessContentsInformation,
'generateNewIdList')
def generateNewIdList(self,
id_group=None,
id_count=1,
default=None,
poison=False):
"""
Generate a list of next ids in the sequence of ids of a particular group
"""
new_id = 1 + self._generateNewId(id_group=id_group,
id_count=id_count,
default=default,
poison=poison)
return range(new_id - id_count, new_id)
security.declareProtected(Permissions.ModifyPortalContent,
'initializeGenerator')
def initializeGenerator(self):
"""
Initialize generator. This is mostly used when a new ERP5 site
is created. Some generators will need to do some initialization like
prepare some data in ZODB
"""
LOG('initialize ZODB Generator', INFO, 'Id Generator: %s' % (self, ))
if getattr(self, 'last_id_dict', None) is None:
self.last_id_dict = OOBTree()
# XXX compatiblity code below, dump the old dictionnaries
portal_ids = getattr(self, 'portal_ids', None)
# Dump the dict_ids dictionary
if getattr(portal_ids, 'dict_ids', None) is not None:
for id_group, last_id in portal_ids.dict_ids.items():
if not isinstance(id_group, str):
id_group = repr(id_group)
if self.last_id_dict.has_key(id_group) and \
self.last_id_dict[id_group] > last_id:
continue
self.last_id_dict[id_group] = last_id
security.declareProtected(Permissions.ModifyPortalContent,
'clearGenerator')
def clearGenerator(self):
"""
Clear generators data. This can be usefull when working on a
development instance or in some other rare cases. This will
loose data and must be use with caution
This can be incompatible with some particular generator implementation,
in this case a particular error will be raised (to be determined and
added here)
"""
# Remove dictionary
self.last_id_dict = OOBTree()
security.declareProtected(Permissions.ModifyPortalContent,
'exportGeneratorIdDict')
def exportGeneratorIdDict(self):
"""
Export last id values in a dictionnary in the form { group_id : last_id }
"""
return dict(self.last_id_dict)
security.declareProtected(Permissions.ModifyPortalContent,
'importGeneratorIdDict')
def importGeneratorIdDict(self, id_dict, clear=False):
"""
Import data, this is usefull if we want to replace a generator by
another one.
"""
if clear:
self.clearGenerator()
if not isinstance(id_dict, dict):
raise TypeError, 'the argument given is not a dictionary'
for value in id_dict.values():
if not isinstance(value, (int, long)):
raise TypeError, 'the value given in dictionary is not a integer'
self.last_id_dict.update(id_dict)
security.declareProtected(Permissions.ModifyPortalContent,
'rebuildGeneratorIdDict')
def rebuildGeneratorIdDict(self):
"""
Rebuild generator id dict.
In fact, export it, clear it and import it into new dict.
This is mostly intendted to use when we are migrating the id dict
structure.
"""
id_dict = self.exportGeneratorIdDict()
self.importGeneratorIdDict(id_dict=id_dict, clear=True)
print "Joined " + str(joinnum) + " lines."
#1d. BTree Join (Index-Based Join)
#We'll be using BTrees.OOBTree module. The library file is included in the project folder
#to install it, do "pip install BTrees"
#this method is very similar to the method studied before, but it uses the BTree data structure
#instead of hash tables
print "Calculating BTree Join time"
start = datetime.datetime.now()
btree = OOBTree()
position = 1
right = open(right_un)
for line in right:
r_data = line.split(delimiter)
btree.update({r_data[1]: position})
position += 1
joinnum = 0
left = open(left_un)
for line in left:
l_data = line.split(delimiter)
position = btree[l_data[0]]
r_data = linecache.getline(right_un, position).split(delimiter)
if l_data[0] == r_data[1]:
joinnum += 1
end = datetime.datetime.now()
print "BTree Join Time: " + str((end - start).total_seconds()) + "sec."
print "Joined " + str(joinnum) + " lines."
print "All done!"
class SiteData(Persistent):
""" Utility used to store and access site use data/logs
"""
implements(ISiteData)
def __init__(self):
self.user_data = OOBTree()
self.log_data = OOBTree()
def store_data(self, data):
""" Extracted and store the data in OOBTtrees
"""
# extract csv files from received zip file
zipped_file = StringIO()
zipped_file.write(data[2:]) # [2:] removes leading '\r\n'
zipped_file.seek(0)
zf = zipfile.ZipFile(zipped_file, mode='r')
users = StringIO()
logs = StringIO()
esheets = StringIO()
for filename in zf.namelist():
# store user data in persistent storage
if filename == 'users.csv':
users.write(zf.read(filename))
users.seek(0)
user_data = users.getvalue().splitlines()
# as a reference from export-user-profiles view:
# fieldnames=['username', 'fullname', 'email',
# 'teacher_mobile_number', 'school',
# 'province', 'EMIS',
# 'school_contact_number',
# 'school_email', 'qualification',
# 'years_teaching','last_login_time', 'uuid'
# sort data by province, school and teacher
user_data.sort(key= lambda line: ( line.split(",")[5],
line.split(",")[4],
line.split(",")[1]))
# place data in an organised dictionary
for user in user_data:
username, fullname, email, mobile, school_name, \
province_name, EMIS, school_contact_number, school_email, \
qualification, years_teaching, last_login_time, \
teacher_uuid = user.split(',')
if self.user_data.get(province_name) is None:
# province entry did not yet exist, initialise to {}
self.user_data.update({province_name:{}})
school_dict = self.user_data.get(province_name)
try:
x = school_dict[school_name]
# if this school exists, we do not set it to blank
except KeyError:
# school entry did not yet exist, initialise to {}
school_dict.update({school_name: {}})
teacher_dict = school_dict[school_name]
try:
x = teacher_dict[teacher_uuid]
except KeyError:
teacher_dict.update({teacher_uuid: {}})
teacher_data_dict = teacher_dict[teacher_uuid]
try:
x = teacher_data_dict['evaluationsheets']
except KeyError:
teacher_data_dict.update({'evaluationsheets': {}})
e_list = teacher_data_dict['evaluationsheets']
teacher_data = { 'username' : username,
'fullname' : fullname,
'email' : email,
'mobile' : mobile,
'qualification' : qualification,
'years_teaching' : years_teaching,
'last_login_time' : last_login_time,
'evaluationsheets' : e_list,
}
teacher_dict.update({teacher_uuid: teacher_data})
school_dict.update({school_name: teacher_dict})
self.user_data.update({province_name:school_dict})
# store log data in persistent storage
if filename == 'logs.csv':
logs.write(zf.read(filename))
logs.seek(0)
log_data = logs.getvalue().splitlines()
# as reference from exportloggedrequests
# fieldnames=['time', 'path', 'username','province','school'],
# delete all log values that do not contain a province or school
# entry
log_data_clean = []
for entry in range(len(log_data)):
time, path, username, province_name, school_name = \
log_data[entry].split(',')
if province_name != '':
log_data_clean.append(log_data[entry])
# sort data by province, school and time
log_data_clean.sort(key= lambda line: ( line.split(",")[3],
line.split(",")[4],
line.split(",")[0]))
# place data in an organised dictionary
for entry in range(len(log_data_clean)):
time, path, username, province_name, school_name = \
log_data_clean[entry].split(',')
if self.log_data.get(province_name) is None:
# province entry did not yet exist, initialise to {}
self.log_data.update({province_name:{}})
school_dict = self.log_data.get(province_name)
try:
x = school_dict[school_name]
# if this school exists, we do not set it to blank
except KeyError:
# school entry did not yet exist, initialise to {}
school_dict.update({school_name: {}})
date_dict = school_dict[school_name]
date_uuid = time[:10]
try:
x = date_dict[date_uuid]
# if this date exists, we do not set it to blank
except KeyError:
# date entry did not yet exist, initialise to {}
date_dict.update({date_uuid: []})
date_data = date_dict[date_uuid] # get existing list of
# paths
# append another path to existing list of paths
date_data.append(path)
date_dict.update({date_uuid: date_data})
school_dict.update({school_name: date_dict})
self.log_data.update({province_name:school_dict})
# 2nd pass to extract data out of evaluationsheets and store in
# persistent structure (requires users.csv to have been parsed already)
for filename in zf.namelist():
if filename == 'evaluation_sheets.csv':
esheets.write(zf.read(filename))
esheets.seek(0)
esheet_data = esheets.getvalue().splitlines()
# as a reference from export-evaluationsheets view:
# fieldnames=['assessment', 'assessment_date',
# 'classlist','learner','learner_uid',
# 'activity_number','rating','school',
# 'province','uuid','esheet_uid']
# sort data by uuid, assessment, learner, activity_number
esheet_data.sort(key= lambda line: ( line.split(",")[9],
line.split(",")[0],
line.split(",")[3],
line.split(",")[5]))
# place data in the same organised dictionary as user data
for esheet in esheet_data:
assessment, assessment_date, classlist, learner, \
learner_uid, activity_number, rating, school_name, \
province_name, teacher_uuid, esheet_uid = esheet.split(',')
school_dict = self.user_data.get(province_name)
teacher_dict = school_dict[school_name]
teacher_data_dict = teacher_dict[teacher_uuid]
try:
x = teacher_data_dict['evaluationsheets']
except KeyError:
teacher_data_dict.update({'evaluationsheets': {}})
e_list = teacher_data_dict['evaluationsheets']
evalsheet_key = assessment + '_' + classlist + '_' +\
esheet_uid
try:
x = e_list[evalsheet_key]
except KeyError:
e_list.update({evalsheet_key: {}})
assessment_obj = e_list[evalsheet_key]
try:
x = assessment_obj['learners']
except KeyError:
assessment_obj.update({'learners': {}})
l_list = assessment_obj['learners']
try:
x = l_list[learner_uid]
except KeyError:
l_list.update({learner_uid: {}})
learner_obj = l_list[learner_uid]
try:
x = learner_obj['activity_data']
except KeyError:
learner_obj.update({'activity_data': []})
activities = learner_obj['activity_data']
activity = { 'activity_number' : activity_number,
'rating' : rating
}
activities.append(activity)
learner_obj.update({'activity_data': activities})
learner_obj.update({'learner_name': learner})
l_list.update({learner_uid: learner_obj})
assessment_obj.update({'learners': l_list})
assessment_obj.update({'assessment_date': assessment_date})
assessment_obj.update({'assessment': assessment})
assessment_obj.update({'classlist': classlist})
e_list.update({evalsheet_key: assessment_obj})
teacher_data_dict.update({'evaluationsheets': e_list})
teacher_dict.update({teacher_uuid: teacher_data_dict})
school_dict.update({school_name: teacher_dict})
self.user_data.update({province_name:school_dict})
zf.close()
return
def insert(self,row,b_id):
self.row_num.append(row)
self.business_id.append(b_id)
datas = pandas.read_csv(path+"review-1m.csv")
for i in range (datas.shape[0]):
print(i)
row = i
b_id = datas.iloc[row]['business_id']
key = datas.iloc[row]['useful']
if t.has_key(key):
c = t.__getitem__(key)
c.insert(row,b_id)
t.update({key:c})
else:
c = content()
c.insert(row,b_id)
t.update({key:c})
funny_file = open(r'../test/useful.pkl', 'wb')
pickle.dump(t, funny_file)
funny_file.close()
#c = content(1)
#c.insert(1,100,200)
#c.insert(1,200,300)
#print(c.key,c.row_num,c.business_id)
#
#t.update({1:c})
#c = t.__getitem__(1)
def migrate_PersistentDict(dict):
result = OOBTree()
result.update(dict)
return result
def new_register(self, userids):
reg = OOBTree()
reg.update({'userids': frozenset(userids), 'time': utcnow()})
self.registers[self.get_next_key()] = reg
class Table:
def __init__(self, name, num_columns, key):
"""
The actual table holding the records
:param name: string # Table name
:param num_columns: int # Number of user Columns: all columns are integer
:param key: int # Index of primary key column
"""
# name of the table
self.name = name
# column number of the primary key column
self.prim_key_col_num = key + NUMBER_OF_META_COLUMNS
# the number user columns
self.num_columns = num_columns
# the total number of columns
self.number_of_columns = self.num_columns + NUMBER_OF_META_COLUMNS
# accepts a record's RID and returns page_range_index, page_number and offset
self.page_directory = OOBTree()
# a list containing the page ranges for the table
self.ranges = [
PageRange(self.number_of_columns, self.prim_key_col_num, 0,
os.path.expanduser("~/ECS165/" + self.name))
]
# the number of records in the tale
self.num_records = 0
# highest used rid number
self.rid = 0
# lowest used lid number
self.lid = LID_MAX
# a list of indexes for the table
self.indexes = make_indexes(self.number_of_columns,
self.prim_key_col_num,
table=self)
# name of directory that table is in
self.directory_name = "~/ECS165/"
# stack of deleted, available RIDs
self.rid_stack = []
def get_rid_value(self):
"""
Manage the RIDs of the table
:return: int # an integer value to represent the next RID
"""
self.rid += 1
return self.rid
def get_lid_value(self):
"""
Manage the LIDs of the table
:return: int # an integer value to represent the next LID
"""
self.lid -= 1
return self.lid
def add_record(self, columns):
"""
Add a record to the table
:param columns: [] # the column values of the record
"""
# TODO: Check to see if primary key already exists in table
# TA said this error check isn't necessary
# check if can grab old, deleted rid from rid_stack
if len(self.rid_stack) > 0:
RID, page_range_index, page_num, offset = self.rid_stack.pop(0)
columns = [0, RID, int(time() * 1000000), 0, 0] + columns
self.ranges[page_range_index].update_base_record(
page_num, offset, columns)
else:
# get new rid and add meta-data columns
RID = self.get_rid_value()
columns = [0, RID, int(time() * 1000000), 0, 0] + columns
# get a hold of the last page range
page_range = self.ranges[-1]
#page_range_index = len(self.ranges) - 1
# if it is full
if not page_range.has_capacity():
# create a new one and append it
page_range = PageRange(
num_of_columns=self.number_of_columns,
primary_key_column=self.prim_key_col_num,
page_range_number=page_range.my_index + 1,
directory_name=os.path.expanduser("~/ECS165/" + self.name))
self.ranges.append(page_range)
# write record to page range and return page number and offset of record
page_num, offset = page_range.add_base_record(columns)
page_range_index = len(self.ranges) - 1
# increment the number of records
self.num_records += 1
# update page directory
self.page_directory.update({RID: [page_range_index, page_num, offset]})
# update primary key index
self.indexes[self.prim_key_col_num].add_index_item(
columns[self.prim_key_col_num], RID)
def read_record(self, key, column_number, query_columns):
"""
Read the record from the table
:param key: int # the value to select records based on
:param column_number: int # the column number to match keys on
:param query_columns: [] # a list of Nones and 1s defining which column values to return
:return: [] # a list of records matching the arguments
"""
# a list to hold the matching records
records = []
# if no index exists for this column, create it
index = self.indexes[column_number]
if not index:
self.indexes[column_number] = index = Index().create_index(
table=self, column_number=column_number)
# get the matching rids
rids = index.locate(value=key)
# if there are no matching rids return an empty list
if not rids:
return records
# for each matching record
for RID in rids:
# get the location in the table
page_range_num, page_num, offset = self.page_directory.get(RID)
# TODO - double check that logic is correct
# check to see if the record has been updated
LID = self.ranges[page_range_num].read_column(
page_range_num, page_num, offset, INDIRECTION_COLUMN)
tps = self.ranges[page_range_num].tps
# if a merge hasn't occurred, check for an update
if self.ranges[page_range_num].num_merges == 0 and LID != 0:
# get the updated records location
page_range_num, page_num, offset = self.page_directory.get(LID)
# if a merge has occurred, check TPS and LID values
elif self.ranges[page_range_num].num_merges > 0 and 0 < LID < tps:
# get the updated records location
page_range_num, page_num, offset = self.page_directory.get(LID)
# get the record
record = self.ranges[page_range_num].read_record(
[[page_range_num, page_num, offset]], query_columns)
# append the record to records
records = records + record
# return the records
return records
def update_record(self, key, columns):
"""
Add a tail record for a specific record
:param key: int # the primary key value for finding the record
:param columns: [] # a list of the new values to be added
"""
# create an LID for the tail record
LID = self.get_lid_value()
# get the RID of the base record
RID = self.indexes[self.prim_key_col_num].locate(value=key)
# get the location of the base record
page_range_num, page_num, offset = self.page_directory.get(RID[0])
# get current schema encoding
schema_encoding = self.ranges[page_range_num].read_column(
page_range_num, page_num, offset, SCHEMA_ENCODING_COLUMN)
# get the new schema encoding by ORing the new one with the existing one
new_schema_encoding = schema_encoding | get_schema_encoding(columns)
# if there is already a tail record get it's LID
indirection_value = self.ranges[page_range_num].read_column(
page_range_num, page_num, offset, INDIRECTION_COLUMN)
# update the base record with the new indirection value and schema encoding
self.ranges[page_range_num].update_schema_indirection(
new_schema_encoding, LID, page_num, offset)
# if there was originally an indirection value in the base record
if indirection_value:
# find it
_, page_num, offset = self.page_directory.get(
indirection_value, [0, page_num, offset])
# get the base or tail record
# TODO: improve efficiency by only getting record values we need
record = \
self.ranges[page_range_num].read_record([[page_range_num, page_num, offset]], [1] * self.number_of_columns)[0]
columns = [
indirection_value, LID,
int(time() * 1000000), new_schema_encoding, RID[0]
] + list(columns)
# for every column, if we have a new value save it, otherwise use old value
for i in range(NUMBER_OF_META_COLUMNS, len(columns)):
if columns[i] is None:
columns[i] = record.columns[i]
# add tail record
page_num, offset = self.ranges[page_range_num].add_tail_record(columns)
# update page directory
self.page_directory.update({LID: [page_range_num, page_num, offset]})
# if we've reached update threshold, start merge
if self.ranges[page_range_num].check_threshold():
merge_thread = Thread(target=self.__merge,
args=(self.ranges[page_range_num], ))
merge_thread.start()
def sum_records(self, start_range, end_range, column_number):
"""
Sum all the records of the given column from the starting rid to the ending rid
:param start_range: int # the key of the first record to accumulate
:param end_range: int # the rid of the last record to accumulate(inclusive)
:param column_number: int # the column to accumulate on
:return: int # the outcome of summing all the records
"""
sum = 0
# TODO: TA said this check isn't necessary
# if start_range >= end_range:
# return 0
# for each possible key within the given range
for key_in_range in range(start_range, end_range + 1):
# get the list of RIDs that match
rids = self.indexes[self.prim_key_col_num].locate(key_in_range)
# if no matching RIDs break from this iteration of the loop
if not rids:
continue
# otherwise for every matching RID
for RID in rids:
# get the location of the record and check to see if there has been an update
page_range_num, page_num, offset = self.page_directory.get(RID)
LID = self.ranges[page_range_num].read_column(
page_range_num, page_num, offset, INDIRECTION_COLUMN)
# if an update get the location of the latest update
if LID:
page_range_num, page_num, offset = self.page_directory.get(
LID)
# do the actual summing
sum += self.ranges[page_range_num].read_column(
page_range_num, page_num, offset, column_number)
return sum
def delete_record(self, key):
"""
delete all records with the given primary key
:param key: int # primary key value of the record to be deleted
"""
# get the RID of the record
RID = self.indexes[self.prim_key_col_num].locate(key)
# get the location of the record
if RID:
page_range_num, page_num, offset = self.page_directory.get(RID[0])
# appending deleted RID to rid stack
self.rid_stack.append([RID[0], page_range_num, page_num, offset])
# lazy delete the record
self.ranges[page_range_num].delete_record(page_num, offset)
# modify the number of records in the table
self.num_records -= 1
def save_table(self, directory_name):
"""
saves table data and page range data to files
:param directory_name: string # name of db directory
"""
# write table data to file
sys.setrecursionlimit(RECURSION_LIMIT)
with open(os.path.expanduser(directory_name + self.name + '/table'),
'wb+') as output:
pickle.dump(self, output, pickle.HIGHEST_PROTOCOL)
bp.close()
def __merge(self, original_page_range):
# beginning merge on this page range
original_page_range.merge = True
# create a copy of the page range
copy_page_range = original_page_range
# dictionary of RIDs we've seen so far
has_seen = []
# iterate through all tail pages except last, check for last update to record
for tail_num in range(original_page_range.last_tail_page + 1, -1, -2):
iterate_offset = PAGE_SIZE
for i in range(int(RECORDS_PER_PAGE)):
iterate_offset -= 8
# return user data and rid column
query_cols = [0, 1, 0, 0, 0] + ([1] * self.num_columns)
record = original_page_range.read_record(
[[original_page_range.my_index, tail_num, iterate_offset]],
query_cols)[0]
base_rid = record.columns[0]
if not (base_rid in has_seen):
has_seen.append(base_rid)
_, page_num, offset = self.page_directory[base_rid]
# TODO: clean this logic up?
# ignore meta data columns and copy just the user data over
for n, column in enumerate(copy_page_range.columns):
if n > 4:
column.update_value(page_num, offset,
record.columns[n - 4])
# TODO - double check that correct
# edit base record's meta data columns accordingly
copy_page_range.columns[
SCHEMA_ENCODING_COLUMN].update_value(
page_num, offset, 0)
# if have seen all rids in base pages, break since found all latest updates
if len(has_seen) == self.rid:
break
# update last tail page
copy_page_range.last_tail_page = copy_page_range.columns[
0].last_page - 1
# update tps of copy of page range
copy_page_range.tps = copy_page_range.read_column(
copy_page_range.my_index, copy_page_range.columns[0].last_page - 1,
PAGE_SIZE - 8, RID_COLUMN)
# update range and get new index to update page directory
self.ranges.append(copy_page_range)
range_index = len(self.ranges) - 1
# update page directory to point to new merged pages
for rid in has_seen:
_, page_num, offset = self.page_directory[rid]
values = [range_index, page_num, offset]
self.page_directory[rid] = values
# finished merging this page range
original_page_range.merge = False
# reset update counter to trigger merge
self.ranges[range_index].update_count = 0
# update merge count
self.ranges[range_index].num_merges += 1
class NastyConfict(Base, TestCase):
def setUp(self):
self.t = OOBTree()
# This tests a problem that cropped up while trying to write
# testBucketSplitConflict (below): conflict resolution wasn't
# working at all in non-trivial cases. Symptoms varied from
# strange complaints about pickling (despite that the test isn't
# doing any *directly*), thru SystemErrors from Python and
# AssertionErrors inside the BTree code.
def testResolutionBlowsUp(self):
b = self.t
for i in range(0, 200, 4):
b[i] = i
# bucket 0 has 15 values: 0, 4 .. 56
# bucket 1 has 15 values: 60, 64 .. 116
# bucket 2 has 20 values: 120, 124 .. 196
state = b.__getstate__()
# Looks like: ((bucket0, 60, bucket1, 120, bucket2), firstbucket)
# If these fail, the *preconditions* for running the test aren't
# satisfied -- the test itself hasn't been run yet.
self.assertEqual(len(state), 2)
self.assertEqual(len(state[0]), 5)
self.assertEqual(state[0][1], 60)
self.assertEqual(state[0][3], 120)
# Invoke conflict resolution by committing a transaction.
self.openDB()
r1 = self.db.open().root()
r1["t"] = self.t
transaction.commit()
r2 = self.db.open().root()
copy = r2["t"]
# Make sure all of copy is loaded.
list(copy.values())
self.assertEqual(self.t._p_serial, copy._p_serial)
self.t.update({1: 2, 2: 3})
transaction.commit()
copy.update({3: 4})
transaction.commit() # if this doesn't blow up
list(copy.values()) # and this doesn't either, then fine
def testBucketSplitConflict(self):
# Tests that a bucket split is viewed as a conflict.
# It's (almost necessarily) a white-box test, and sensitive to
# implementation details.
b = self.t
for i in range(0, 200, 4):
b[i] = i
# bucket 0 has 15 values: 0, 4 .. 56
# bucket 1 has 15 values: 60, 64 .. 116
# bucket 2 has 20 values: 120, 124 .. 196
state = b.__getstate__()
# Looks like: ((bucket0, 60, bucket1, 120, bucket2), firstbucket)
# If these fail, the *preconditions* for running the test aren't
# satisfied -- the test itself hasn't been run yet.
self.assertEqual(len(state), 2)
self.assertEqual(len(state[0]), 5)
self.assertEqual(state[0][1], 60)
self.assertEqual(state[0][3], 120)
# Invoke conflict resolution by committing a transaction.
self.openDB()
tm1 = transaction.TransactionManager()
r1 = self.db.open(transaction_manager=tm1).root()
r1["t"] = self.t
tm1.commit()
tm2 = transaction.TransactionManager()
r2 = self.db.open(transaction_manager=tm2).root()
copy = r2["t"]
# Make sure all of copy is loaded.
list(copy.values())
self.assertEqual(self.t._p_serial, copy._p_serial)
# In one transaction, add 16 new keys to bucket1, to force a bucket
# split.
b = self.t
numtoadd = 16
candidate = 60
while numtoadd:
if not b.has_key(candidate):
b[candidate] = candidate
numtoadd -= 1
candidate += 1
# bucket 0 has 15 values: 0, 4 .. 56
# bucket 1 has 15 values: 60, 61 .. 74
# bucket 2 has 16 values: [75, 76 .. 81] + [84, 88 ..116]
# bucket 3 has 20 values: 120, 124 .. 196
state = b.__getstate__()
# Looks like: ((b0, 60, b1, 75, b2, 120, b3), firstbucket)
# The next block is still verifying preconditions.
self.assertEqual(len(state), 2)
self.assertEqual(len(state[0]), 7)
self.assertEqual(state[0][1], 60)
self.assertEqual(state[0][3], 75)
self.assertEqual(state[0][5], 120)
tm1.commit()
# In the other transaction, add 3 values near the tail end of bucket1.
# This doesn't cause a split.
b = copy
for i in range(112, 116):
b[i] = i
# bucket 0 has 15 values: 0, 4 .. 56
# bucket 1 has 18 values: 60, 64 .. 112, 113, 114, 115, 116
# bucket 2 has 20 values: 120, 124 .. 196
state = b.__getstate__()
# Looks like: ((bucket0, 60, bucket1, 120, bucket2), firstbucket)
# The next block is still verifying preconditions.
self.assertEqual(len(state), 2)
self.assertEqual(len(state[0]), 5)
self.assertEqual(state[0][1], 60)
self.assertEqual(state[0][3], 120)
self.assertRaises(ConflictError, tm2.commit)
def testEmptyBucketConflict(self):
# Tests that an emptied bucket *created by* conflict resolution is
# viewed as a conflict: conflict resolution doesn't have enough
# info to unlink the empty bucket from the BTree correctly.
b = self.t
for i in range(0, 200, 4):
b[i] = i
# bucket 0 has 15 values: 0, 4 .. 56
# bucket 1 has 15 values: 60, 64 .. 116
# bucket 2 has 20 values: 120, 124 .. 196
state = b.__getstate__()
# Looks like: ((bucket0, 60, bucket1, 120, bucket2), firstbucket)
# If these fail, the *preconditions* for running the test aren't
# satisfied -- the test itself hasn't been run yet.
self.assertEqual(len(state), 2)
self.assertEqual(len(state[0]), 5)
self.assertEqual(state[0][1], 60)
self.assertEqual(state[0][3], 120)
# Invoke conflict resolution by committing a transaction.
self.openDB()
tm1 = transaction.TransactionManager()
r1 = self.db.open(transaction_manager=tm1).root()
r1["t"] = self.t
tm1.commit()
tm2 = transaction.TransactionManager()
r2 = self.db.open(transaction_manager=tm2).root()
copy = r2["t"]
# Make sure all of copy is loaded.
list(copy.values())
self.assertEqual(self.t._p_serial, copy._p_serial)
# In one transaction, delete half of bucket 1.
b = self.t
for k in 60, 64, 68, 72, 76, 80, 84, 88:
del b[k]
# bucket 0 has 15 values: 0, 4 .. 56
# bucket 1 has 7 values: 92, 96, 100, 104, 108, 112, 116
# bucket 2 has 20 values: 120, 124 .. 196
state = b.__getstate__()
# Looks like: ((bucket0, 60, bucket1, 120, bucket2), firstbucket)
# The next block is still verifying preconditions.
self.assertEqual(len(state), 2)
self.assertEqual(len(state[0]), 5)
self.assertEqual(state[0][1], 60)
self.assertEqual(state[0][3], 120)
tm1.commit()
# In the other transaction, delete the other half of bucket 1.
b = copy
for k in 92, 96, 100, 104, 108, 112, 116:
del b[k]
# bucket 0 has 15 values: 0, 4 .. 56
# bucket 1 has 8 values: 60, 64, 68, 72, 76, 80, 84, 88
# bucket 2 has 20 values: 120, 124 .. 196
state = b.__getstate__()
# Looks like: ((bucket0, 60, bucket1, 120, bucket2), firstbucket)
# The next block is still verifying preconditions.
self.assertEqual(len(state), 2)
self.assertEqual(len(state[0]), 5)
self.assertEqual(state[0][1], 60)
self.assertEqual(state[0][3], 120)
# Conflict resolution empties bucket1 entirely. This used to
# create an "insane" BTree (a legit BTree cannot contain an empty
# bucket -- it contains NULL pointers the BTree code doesn't
# expect, and segfaults result).
self.assertRaises(ConflictError, tm2.commit)
def testEmptyBucketNoConflict(self):
# Tests that a plain empty bucket (on input) is not viewed as a
# conflict.
b = self.t
for i in range(0, 200, 4):
b[i] = i
# bucket 0 has 15 values: 0, 4 .. 56
# bucket 1 has 15 values: 60, 64 .. 116
# bucket 2 has 20 values: 120, 124 .. 196
state = b.__getstate__()
# Looks like: ((bucket0, 60, bucket1, 120, bucket2), firstbucket)
# If these fail, the *preconditions* for running the test aren't
# satisfied -- the test itself hasn't been run yet.
self.assertEqual(len(state), 2)
self.assertEqual(len(state[0]), 5)
self.assertEqual(state[0][1], 60)
self.assertEqual(state[0][3], 120)
# Invoke conflict resolution by committing a transaction.
self.openDB()
r1 = self.db.open().root()
r1["t"] = self.t
transaction.commit()
r2 = self.db.open().root()
copy = r2["t"]
# Make sure all of copy is loaded.
list(copy.values())
self.assertEqual(self.t._p_serial, copy._p_serial)
# In one transaction, just add a key.
b = self.t
b[1] = 1
# bucket 0 has 16 values: [0, 1] + [4, 8 .. 56]
# bucket 1 has 15 values: 60, 64 .. 116
# bucket 2 has 20 values: 120, 124 .. 196
state = b.__getstate__()
# Looks like: ((bucket0, 60, bucket1, 120, bucket2), firstbucket)
# The next block is still verifying preconditions.
self.assertEqual(len(state), 2)
self.assertEqual(len(state[0]), 5)
self.assertEqual(state[0][1], 60)
self.assertEqual(state[0][3], 120)
transaction.commit()
# In the other transaction, delete bucket 2.
b = copy
for k in range(120, 200, 4):
del b[k]
# bucket 0 has 15 values: 0, 4 .. 56
# bucket 1 has 15 values: 60, 64 .. 116
state = b.__getstate__()
# Looks like: ((bucket0, 60, bucket1), firstbucket)
# The next block is still verifying preconditions.
self.assertEqual(len(state), 2)
self.assertEqual(len(state[0]), 3)
self.assertEqual(state[0][1], 60)
# This shouldn't create a ConflictError.
transaction.commit()
# And the resulting BTree shouldn't have internal damage.
b._check()
# The snaky control flow in _bucket__p_resolveConflict ended up trying
# to decref a NULL pointer if conflict resolution was fed 3 empty
# buckets. http://collector.zope.org/Zope/553
def testThreeEmptyBucketsNoSegfault(self):
self.openDB()
tm1 = transaction.TransactionManager()
r1 = self.db.open(transaction_manager=tm1).root()
self.assertEqual(len(self.t), 0)
r1["t"] = b = self.t # an empty tree
tm1.commit()
tm2 = transaction.TransactionManager()
r2 = self.db.open(transaction_manager=tm2).root()
copy = r2["t"]
# Make sure all of copy is loaded.
list(copy.values())
# In one transaction, add and delete a key.
b[2] = 2
del b[2]
tm1.commit()
# In the other transaction, also add and delete a key.
b = copy
b[1] = 1
del b[1]
# If the commit() segfaults, the C code is still wrong for this case.
self.assertRaises(ConflictError, tm2.commit)
def testCantResolveBTreeConflict(self):
# Test that a conflict involving two different changes to
# an internal BTree node is unresolvable. An internal node
# only changes when there are enough additions or deletions
# to a child bucket that the bucket is split or removed.
# It's (almost necessarily) a white-box test, and sensitive to
# implementation details.
b = self.t
for i in range(0, 200, 4):
b[i] = i
# bucket 0 has 15 values: 0, 4 .. 56
# bucket 1 has 15 values: 60, 64 .. 116
# bucket 2 has 20 values: 120, 124 .. 196
state = b.__getstate__()
# Looks like: ((bucket0, 60, bucket1, 120, bucket2), firstbucket)
# If these fail, the *preconditions* for running the test aren't
# satisfied -- the test itself hasn't been run yet.
self.assertEqual(len(state), 2)
self.assertEqual(len(state[0]), 5)
self.assertEqual(state[0][1], 60)
self.assertEqual(state[0][3], 120)
# Set up database connections to provoke conflict.
self.openDB()
tm1 = transaction.TransactionManager()
r1 = self.db.open(transaction_manager=tm1).root()
r1["t"] = self.t
tm1.commit()
tm2 = transaction.TransactionManager()
r2 = self.db.open(transaction_manager=tm2).root()
copy = r2["t"]
# Make sure all of copy is loaded.
list(copy.values())
self.assertEqual(self.t._p_serial, copy._p_serial)
# Now one transaction should add enough keys to cause a split,
# and another should remove all the keys in one bucket.
for k in range(200, 300, 4):
self.t[k] = k
tm1.commit()
for k in range(0, 60, 4):
del copy[k]
try:
tm2.commit()
except ConflictError, detail:
self.assert_(str(detail).startswith('database conflict error'))
else:
class Subscriptions(SimpleItem):
security = ClassSecurityInfo()
title = "Meeting registrations"
def __init__(self, id):
""" """
super(SimpleItem, self).__init__(id)
self.id = id
self._signups = OOBTree()
self._account_subscriptions = OOBTree()
security.declarePublic('getMeeting')
def getMeeting(self):
return self.aq_parent.aq_parent
def _validate_signup(self, form):
""" """
formdata = {}
formerrors = {}
keys = ('first_name', 'last_name', 'email', 'organization', 'phone')
formdata = dict((key, form.get(key, '')) for key in keys)
for key in formdata:
if formdata[key] == '':
formerrors[key] = 'This field is mandatory'
if formerrors == {}:
if formdata['email'].count('@') != 1:
formerrors['email'] = ('An email address must contain '
'a single @')
if formerrors == {}:
formerrors = None
return formdata, formerrors
def _add_signup(self, formdata):
""" """
meeting = self.getMeeting()
key = random_key()
name = formdata['first_name'] + ' ' + formdata['last_name']
email = formdata['email']
organization = formdata['organization']
phone = formdata['phone']
signup = SignUp(key, name, email, organization, phone)
self._signups.insert(key, signup)
if meeting.auto_register:
self._accept_signup(key)
email_sender = self.getMeeting().getEmailSender()
email_sender.send_signup_email(signup)
if self.REQUEST.AUTHENTICATED_USER.getUserName() == 'Anonymous User':
self.REQUEST.SESSION['nymt-current-key'] = key
security.declareProtected(view, 'signup')
def signup(self, REQUEST):
""" """
meeting = self.getMeeting()
if not meeting.allow_register:
return REQUEST.RESPONSE.redirect(self.absolute_url() +
'/subscription_not_allowed')
if REQUEST.get('add_users'):
return self.subscribe_accounts(REQUEST)
if REQUEST.get('add_signup'):
formdata, formerrors = self._validate_signup(REQUEST.form)
# check Captcha/reCaptcha
if not (self.checkPermissionSkipCaptcha() or
REQUEST.SESSION.get('captcha_passed')):
recaptcha_response = REQUEST.form.get('g-recaptcha-response',
'')
captcha_validator = self.validateCaptcha(recaptcha_response,
REQUEST)
if captcha_validator:
if formerrors is None:
formerrors = {}
formerrors['captcha'] = captcha_validator
else:
REQUEST.SESSION['captcha_passed'] = True
if formerrors is not None:
return self.getFormsTool().getContent(
{'here': self,
'formdata': formdata,
'formerrors': formerrors},
'naaya.content.meeting.subscription_signup')
else:
self._add_signup(formdata)
if self.getMeeting().survey_required:
REQUEST.RESPONSE.redirect(
self.getMeeting().absolute_url())
else:
REQUEST.RESPONSE.redirect(self.absolute_url() +
'/signup_successful')
# check Captcha/reCaptcha also for searching users
captcha_validator = None
if (REQUEST.get('search_user') or
REQUEST.get('search_user_with_role')):
if not (self.checkPermissionSkipCaptcha() or
REQUEST.SESSION.get('captcha_passed')):
recaptcha_response = REQUEST.form.get('g-recaptcha-response',
'')
captcha_validator = self.validateCaptcha(recaptcha_response,
REQUEST)
if not captcha_validator:
REQUEST.SESSION['captcha_passed'] = True
return self.getFormsTool().getContent(
{'here': self, 'captcha_errors': captcha_validator},
'naaya.content.meeting.subscription_signup')
security.declareProtected(view, 'signup_successful')
def signup_successful(self, REQUEST):
""" """
return self.getFormsTool().getContent(
{'here': self},
'naaya.content.meeting.subscription_signup_successful')
security.declareProtected(view, 'subscribe')
def subscribe(self, REQUEST):
""" """
meeting = self.getMeeting()
if not meeting.allow_register:
return REQUEST.RESPONSE.redirect(self.absolute_url() +
'/subscription_not_allowed')
return self.getFormsTool().getContent(
{'here': self}, 'naaya.content.meeting.subscription_subscribe')
def getSignups(self):
""" """
if not self.checkPermissionParticipateInMeeting():
raise Unauthorized
return self._signups.itervalues()
security.declareProtected(PERMISSION_ADMIN_MEETING, 'getSignup')
def getSignup(self, key):
""" """
return self._signups.get(key, None)
def index_html(self, REQUEST):
""" """
if not self.checkPermissionParticipateInMeeting():
raise Unauthorized
return self.getFormsTool().getContent(
{'here': self}, 'naaya.content.meeting.subscription_index')
def _accept_signup(self, key):
""" """
meeting = self.getMeeting()
meeting.getParticipants()._set_attendee(key, PARTICIPANT_ROLE)
signup = self._signups[key]
signup.accepted = 'accepted'
email_sender = meeting.getEmailSender()
email_sender.send_signup_accepted_email(signup)
def _reject_signup(self, key):
""" """
meeting = self.getMeeting()
signup = self._signups[key]
signup.accepted = 'rejected'
participants = meeting.getParticipants()
# delete the 'reimbursed' status
participants.setAttendeeInfo([key], 'reimbursed', False)
if key in participants._get_attendees():
participants._del_attendee(key)
email_sender = meeting.getEmailSender()
email_sender.send_signup_rejected_email(signup)
def _delete_signup(self, key):
""" """
meeting = self.getMeeting()
signup = self._signups.pop(key, None)
if signup is None:
return
participants = meeting.getParticipants()
if key in participants._get_attendees():
participants._del_attendee(key)
email_sender = meeting.getEmailSender()
email_sender.send_signup_rejected_email(signup)
def _is_signup(self, key):
""" """
return key in self._signups
def _is_accepted_signup(self, key):
""" """
return self._is_signup(key) and \
self._signups[key].accepted == 'accepted'
def _is_pending_signup(self, key):
""" """
return self._is_signup(key) and \
self._signups[key].accepted == 'new'
def manageSubscriptions(self, REQUEST):
""" """
if not (self.checkPermissionAdminMeeting() or self.nfp_for_country()):
raise Unauthorized
uids = REQUEST.form.get('uids', [])
assert isinstance(uids, list)
for uid in uids:
if 'accept' in REQUEST.form:
if self._is_signup(uid):
self._accept_signup(uid)
else:
self._accept_account_subscription(uid)
elif 'reject' in REQUEST.form:
if self._is_signup(uid):
self._reject_signup(uid)
else:
self._reject_account_subscription(uid)
elif 'delete' in REQUEST.form:
if not self.checkPermissionAdminMeeting():
raise Unauthorized
if self._is_signup(uid):
self._delete_signup(uid)
else:
self._delete_account_subscription(uid)
if 'set_representative' in REQUEST.form:
self.setRepresentatives(REQUEST)
elif 'unset_representative' in REQUEST.form:
self.setRepresentatives(REQUEST, remove=True)
elif 'set_reimbursement' in REQUEST.form:
self.setReimbursement(REQUEST)
elif 'unset_reimbursement' in REQUEST.form:
self.setReimbursement(REQUEST, remove=True)
elif 'save_changes' in REQUEST.form:
self.save_changes(REQUEST)
return REQUEST.RESPONSE.redirect(self.absolute_url())
security.declarePublic('welcome')
def welcome(self, REQUEST, came_from=None):
""" """
if 'logout' in REQUEST.form:
REQUEST.SESSION['nymt-current-key'] = None
return REQUEST.RESPONSE.redirect(self.getMeeting().absolute_url())
key = REQUEST.get('key', None)
signup = self.getSignup(key)
if self._is_signup(key):
REQUEST.SESSION['nymt-current-key'] = key
if came_from:
return REQUEST.RESPONSE.redirect(came_from)
else:
return REQUEST.RESPONSE.redirect(
self.getMeeting().absolute_url())
return self.getFormsTool().getContent(
{'here': self,
'signup': signup},
'naaya.content.meeting.subscription_welcome')
def _add_account_subscription(self, uid, accept=False):
""" """
# If the subscription already exists or the user is alread signed up
# skip the whole thing
if self._is_account_subscription(uid):
return
key = uid.replace('signup:', '')
if self._is_signup(key):
return
site = self.getSite()
meeting = self.getMeeting()
name = getUserFullName(site, uid)
# If for any reason we still don't have a name, at least use UID
if not name:
name = uid
email = getUserEmail(site, uid)
organization = getUserOrganization(site, uid)
if not organization:
organization = self.get_survey_answer(uid, 'w_organization')
if not organization:
organization = self.get_survey_answer(uid, 'w_organisation')
phone = getUserPhoneNumber(site, uid)
if not phone:
phone = self.get_survey_answer(uid, 'w_telephone')
if not phone:
phone = self.get_survey_answer(uid, 'w_phone')
account_subscription = AccountSubscription(uid, name, email,
organization, phone)
self._account_subscriptions.insert(uid, account_subscription)
if meeting.auto_register or accept:
self._accept_account_subscription(uid)
email_sender = self.getMeeting().getEmailSender()
email_sender.send_account_subscription_email(account_subscription)
security.declareProtected(PERMISSION_ADMIN_MEETING,
'update_account_subscription')
def update_account_subscription(self, uid):
""" """
site = self.getSite()
name = getUserFullName(site, uid)
email = getUserEmail(site, uid)
organization = getUserOrganization(site, uid)
phone = getUserPhoneNumber(site, uid)
account_subscription = AccountSubscription(uid, name, email,
organization, phone)
self._account_subscriptions.update({uid: account_subscription})
security.declareProtected(view, 'subscribe_accounts')
def subscribe_accounts(self, REQUEST):
""" """
meeting = self.getMeeting()
if not meeting.allow_register:
return REQUEST.RESPONSE.redirect(self.absolute_url() +
'/subscription_not_allowed')
# check Captcha/reCaptcha also for searching users
if not (self.checkPermissionSkipCaptcha() or
REQUEST.SESSION.get('captcha_passed')):
recaptcha_response = REQUEST.form.get('g-recaptcha-response', '')
captcha_validator = self.validateCaptcha(recaptcha_response,
REQUEST)
if captcha_validator:
return self.getFormsTool().getContent(
{'here': self, 'captcha_errors': captcha_validator},
'naaya.content.meeting.subscription_signup')
else:
REQUEST.SESSION['captcha_passed'] = True
uids = REQUEST.form.get('uids', [])
assert isinstance(uids, list)
for uid in uids:
self._add_account_subscription(uid)
return REQUEST.RESPONSE.redirect(
self.absolute_url() +
'/subscribe_account_successful?uids='+','.join(uids))
security.declareProtected(view, 'subscribe_my_account')
def subscribe_my_account(self, REQUEST):
""" """
meeting = self.getMeeting()
if not meeting.allow_register:
return REQUEST.RESPONSE.redirect(self.absolute_url() +
'/subscription_not_allowed')
self._add_account_subscription(REQUEST.AUTHENTICATED_USER.getId())
if self.survey_required:
site = self.getSite()
path = str(self.survey_pointer)
survey_ob = site.unrestrictedTraverse(path, None)
if survey_ob is not None and \
survey_ob.meta_type == 'Naaya Mega Survey':
answers = survey_ob.getAnswers()
respondents = [a.respondent for a in answers]
current_user = REQUEST.AUTHENTICATED_USER.getUserName()
if current_user not in respondents:
self.setSessionInfoTrans(
'Registration successfully sent for approval. '
'Please also respond to the following questionaire.')
return REQUEST.RESPONSE.redirect(
'%s/%s' % (self.getSite().absolute_url(),
self.survey_pointer))
return REQUEST.RESPONSE.redirect(self.absolute_url() +
'/subscribe_account_successful')
security.declareProtected(view, 'subscribe_account_successful')
def subscribe_account_successful(self, REQUEST):
""" """
return self.getFormsTool().getContent(
{'here': self},
'naaya.content.meeting.subscription_subscribe_successful')
def getAccountSubscriptions(self):
""" """
if not self.checkPermissionParticipateInMeeting():
raise Unauthorized
return self._account_subscriptions.itervalues()
def getSubscriptions(self):
""" """
if not self.checkPermissionParticipateInMeeting():
raise Unauthorized
subscriptions = (list(self._signups.itervalues()) +
list(self._account_subscriptions.itervalues()))
statuses = {'new': 0,
'accepted': 1,
'rejected': 2
}
return sorted(subscriptions, key=lambda x: statuses.get(x.accepted))
security.declareProtected(PERMISSION_ADMIN_MEETING,
'getAccountSubscription')
def getAccountSubscription(self, uid):
""" """
return self._account_subscriptions.get(uid, None)
def _is_account_subscription(self, uid):
""" """
return uid in self._account_subscriptions and \
self._account_subscriptions[uid].accepted == 'accepted'
def _accept_account_subscription(self, uid):
""" """
meeting = self.getMeeting()
meeting.getParticipants()._set_attendee(uid, PARTICIPANT_ROLE)
account_subscription = self._account_subscriptions[uid]
account_subscription.accepted = 'accepted'
email_sender = meeting.getEmailSender()
email_sender.send_account_subscription_accepted_email(
account_subscription)
def _reject_account_subscription(self, uid):
""" """
meeting = self.getMeeting()
account_subscription = self._account_subscriptions[uid]
account_subscription.accepted = 'rejected'
participants = meeting.getParticipants()
# remove the 'reimbursed' status
participants.setAttendeeInfo([uid], 'reimbursed', False)
if uid in participants._get_attendees():
participants._del_attendee(uid)
email_sender = meeting.getEmailSender()
email_sender.send_account_subscription_rejected_email(
account_subscription)
def _delete_account_subscription(self, uid):
""" """
meeting = self.getMeeting()
account_subscription = self._account_subscriptions.pop(uid, None)
if account_subscription is None:
return
participants = meeting.getParticipants()
if uid in participants._get_attendees():
participants._del_attendee(uid)
email_sender = meeting.getEmailSender()
email_sender.send_account_subscription_rejected_email(
account_subscription)
security.declareProtected(view, 'subscription_not_allowed')
def subscription_not_allowed(self, REQUEST):
""" """
return self.getFormsTool().getContent(
{'here': self}, 'naaya.content.meeting.subscription_not_allowed')
class NastyConfict(Base, TestCase):
def setUp(self):
self.t = OOBTree()
# This tests a problem that cropped up while trying to write
# testBucketSplitConflict (below): conflict resolution wasn't
# working at all in non-trivial cases. Symptoms varied from
# strange complaints about pickling (despite that the test isn't
# doing any *directly*), thru SystemErrors from Python and
# AssertionErrors inside the BTree code.
def testResolutionBlowsUp(self):
b = self.t
for i in range(0, 200, 4):
b[i] = i
# bucket 0 has 15 values: 0, 4 .. 56
# bucket 1 has 15 values: 60, 64 .. 116
# bucket 2 has 20 values: 120, 124 .. 196
state = b.__getstate__()
# Looks like: ((bucket0, 60, bucket1, 120, bucket2), firstbucket)
# If these fail, the *preconditions* for running the test aren't
# satisfied -- the test itself hasn't been run yet.
self.assertEqual(len(state), 2)
self.assertEqual(len(state[0]), 5)
self.assertEqual(state[0][1], 60)
self.assertEqual(state[0][3], 120)
# Invoke conflict resolution by committing a transaction.
self.openDB()
r1 = self.db.open().root()
r1["t"] = self.t
transaction.commit()
r2 = self.db.open().root()
copy = r2["t"]
# Make sure all of copy is loaded.
list(copy.values())
self.assertEqual(self.t._p_serial, copy._p_serial)
self.t.update({1:2, 2:3})
transaction.commit()
copy.update({3:4})
transaction.commit() # if this doesn't blow up
list(copy.values()) # and this doesn't either, then fine
def testBucketSplitConflict(self):
# Tests that a bucket split is viewed as a conflict.
# It's (almost necessarily) a white-box test, and sensitive to
# implementation details.
b = self.t
for i in range(0, 200, 4):
b[i] = i
# bucket 0 has 15 values: 0, 4 .. 56
# bucket 1 has 15 values: 60, 64 .. 116
# bucket 2 has 20 values: 120, 124 .. 196
state = b.__getstate__()
# Looks like: ((bucket0, 60, bucket1, 120, bucket2), firstbucket)
# If these fail, the *preconditions* for running the test aren't
# satisfied -- the test itself hasn't been run yet.
self.assertEqual(len(state), 2)
self.assertEqual(len(state[0]), 5)
self.assertEqual(state[0][1], 60)
self.assertEqual(state[0][3], 120)
# Invoke conflict resolution by committing a transaction.
self.openDB()
r1 = self.db.open().root()
r1["t"] = self.t
transaction.commit()
r2 = self.db.open(synch=False).root()
copy = r2["t"]
# Make sure all of copy is loaded.
list(copy.values())
self.assertEqual(self.t._p_serial, copy._p_serial)
# In one transaction, add 16 new keys to bucket1, to force a bucket
# split.
b = self.t
numtoadd = 16
candidate = 60
while numtoadd:
if not b.has_key(candidate):
b[candidate] = candidate
numtoadd -= 1
candidate += 1
# bucket 0 has 15 values: 0, 4 .. 56
# bucket 1 has 15 values: 60, 61 .. 74
# bucket 2 has 16 values: [75, 76 .. 81] + [84, 88 ..116]
# bucket 3 has 20 values: 120, 124 .. 196
state = b.__getstate__()
# Looks like: ((b0, 60, b1, 75, b2, 120, b3), firstbucket)
# The next block is still verifying preconditions.
self.assertEqual(len(state) , 2)
self.assertEqual(len(state[0]), 7)
self.assertEqual(state[0][1], 60)
self.assertEqual(state[0][3], 75)
self.assertEqual(state[0][5], 120)
transaction.commit()
# In the other transaction, add 3 values near the tail end of bucket1.
# This doesn't cause a split.
b = copy
for i in range(112, 116):
b[i] = i
# bucket 0 has 15 values: 0, 4 .. 56
# bucket 1 has 18 values: 60, 64 .. 112, 113, 114, 115, 116
# bucket 2 has 20 values: 120, 124 .. 196
state = b.__getstate__()
# Looks like: ((bucket0, 60, bucket1, 120, bucket2), firstbucket)
# The next block is still verifying preconditions.
self.assertEqual(len(state), 2)
self.assertEqual(len(state[0]), 5)
self.assertEqual(state[0][1], 60)
self.assertEqual(state[0][3], 120)
self.assertRaises(ConflictError, transaction.commit)
transaction.abort() # horrible things happen w/o this
def testEmptyBucketConflict(self):
# Tests that an emptied bucket *created by* conflict resolution is
# viewed as a conflict: conflict resolution doesn't have enough
# info to unlink the empty bucket from the BTree correctly.
b = self.t
for i in range(0, 200, 4):
b[i] = i
# bucket 0 has 15 values: 0, 4 .. 56
# bucket 1 has 15 values: 60, 64 .. 116
# bucket 2 has 20 values: 120, 124 .. 196
state = b.__getstate__()
# Looks like: ((bucket0, 60, bucket1, 120, bucket2), firstbucket)
# If these fail, the *preconditions* for running the test aren't
# satisfied -- the test itself hasn't been run yet.
self.assertEqual(len(state), 2)
self.assertEqual(len(state[0]), 5)
self.assertEqual(state[0][1], 60)
self.assertEqual(state[0][3], 120)
# Invoke conflict resolution by committing a transaction.
self.openDB()
r1 = self.db.open().root()
r1["t"] = self.t
transaction.commit()
r2 = self.db.open(synch=False).root()
copy = r2["t"]
# Make sure all of copy is loaded.
list(copy.values())
self.assertEqual(self.t._p_serial, copy._p_serial)
# In one transaction, delete half of bucket 1.
b = self.t
for k in 60, 64, 68, 72, 76, 80, 84, 88:
del b[k]
# bucket 0 has 15 values: 0, 4 .. 56
# bucket 1 has 7 values: 92, 96, 100, 104, 108, 112, 116
# bucket 2 has 20 values: 120, 124 .. 196
state = b.__getstate__()
# Looks like: ((bucket0, 60, bucket1, 120, bucket2), firstbucket)
# The next block is still verifying preconditions.
self.assertEqual(len(state) , 2)
self.assertEqual(len(state[0]), 5)
self.assertEqual(state[0][1], 60)
self.assertEqual(state[0][3], 120)
transaction.commit()
# In the other transaction, delete the other half of bucket 1.
b = copy
for k in 92, 96, 100, 104, 108, 112, 116:
del b[k]
# bucket 0 has 15 values: 0, 4 .. 56
# bucket 1 has 8 values: 60, 64, 68, 72, 76, 80, 84, 88
# bucket 2 has 20 values: 120, 124 .. 196
state = b.__getstate__()
# Looks like: ((bucket0, 60, bucket1, 120, bucket2), firstbucket)
# The next block is still verifying preconditions.
self.assertEqual(len(state), 2)
self.assertEqual(len(state[0]), 5)
self.assertEqual(state[0][1], 60)
self.assertEqual(state[0][3], 120)
# Conflict resolution empties bucket1 entirely. This used to
# create an "insane" BTree (a legit BTree cannot contain an empty
# bucket -- it contains NULL pointers the BTree code doesn't
# expect, and segfaults result).
self.assertRaises(ConflictError, transaction.commit)
transaction.abort() # horrible things happen w/o this
def testEmptyBucketNoConflict(self):
# Tests that a plain empty bucket (on input) is not viewed as a
# conflict.
b = self.t
for i in range(0, 200, 4):
b[i] = i
# bucket 0 has 15 values: 0, 4 .. 56
# bucket 1 has 15 values: 60, 64 .. 116
# bucket 2 has 20 values: 120, 124 .. 196
state = b.__getstate__()
# Looks like: ((bucket0, 60, bucket1, 120, bucket2), firstbucket)
# If these fail, the *preconditions* for running the test aren't
# satisfied -- the test itself hasn't been run yet.
self.assertEqual(len(state), 2)
self.assertEqual(len(state[0]), 5)
self.assertEqual(state[0][1], 60)
self.assertEqual(state[0][3], 120)
# Invoke conflict resolution by committing a transaction.
self.openDB()
r1 = self.db.open().root()
r1["t"] = self.t
transaction.commit()
r2 = self.db.open().root()
copy = r2["t"]
# Make sure all of copy is loaded.
list(copy.values())
self.assertEqual(self.t._p_serial, copy._p_serial)
# In one transaction, just add a key.
b = self.t
b[1] = 1
# bucket 0 has 16 values: [0, 1] + [4, 8 .. 56]
# bucket 1 has 15 values: 60, 64 .. 116
# bucket 2 has 20 values: 120, 124 .. 196
state = b.__getstate__()
# Looks like: ((bucket0, 60, bucket1, 120, bucket2), firstbucket)
# The next block is still verifying preconditions.
self.assertEqual(len(state), 2)
self.assertEqual(len(state[0]), 5)
self.assertEqual(state[0][1], 60)
self.assertEqual(state[0][3], 120)
transaction.commit()
# In the other transaction, delete bucket 2.
b = copy
for k in range(120, 200, 4):
del b[k]
# bucket 0 has 15 values: 0, 4 .. 56
# bucket 1 has 15 values: 60, 64 .. 116
state = b.__getstate__()
# Looks like: ((bucket0, 60, bucket1), firstbucket)
# The next block is still verifying preconditions.
self.assertEqual(len(state), 2)
self.assertEqual(len(state[0]), 3)
self.assertEqual(state[0][1], 60)
# This shouldn't create a ConflictError.
transaction.commit()
# And the resulting BTree shouldn't have internal damage.
b._check()
# The snaky control flow in _bucket__p_resolveConflict ended up trying
# to decref a NULL pointer if conflict resolution was fed 3 empty
# buckets. http://collector.zope.org/Zope/553
def testThreeEmptyBucketsNoSegfault(self):
self.openDB()
r1 = self.db.open().root()
self.assertEqual(len(self.t), 0)
r1["t"] = b = self.t # an empty tree
transaction.commit()
r2 = self.db.open(synch=False).root()
copy = r2["t"]
# Make sure all of copy is loaded.
list(copy.values())
# In one transaction, add and delete a key.
b[2] = 2
del b[2]
transaction.commit()
# In the other transaction, also add and delete a key.
b = copy
b[1] = 1
del b[1]
# If the commit() segfaults, the C code is still wrong for this case.
self.assertRaises(ConflictError, transaction.commit)
transaction.abort()
def testCantResolveBTreeConflict(self):
# Test that a conflict involving two different changes to
# an internal BTree node is unresolvable. An internal node
# only changes when there are enough additions or deletions
# to a child bucket that the bucket is split or removed.
# It's (almost necessarily) a white-box test, and sensitive to
# implementation details.
b = self.t
for i in range(0, 200, 4):
b[i] = i
# bucket 0 has 15 values: 0, 4 .. 56
# bucket 1 has 15 values: 60, 64 .. 116
# bucket 2 has 20 values: 120, 124 .. 196
state = b.__getstate__()
# Looks like: ((bucket0, 60, bucket1, 120, bucket2), firstbucket)
# If these fail, the *preconditions* for running the test aren't
# satisfied -- the test itself hasn't been run yet.
self.assertEqual(len(state), 2)
self.assertEqual(len(state[0]), 5)
self.assertEqual(state[0][1], 60)
self.assertEqual(state[0][3], 120)
# Set up database connections to provoke conflict.
self.openDB()
r1 = self.db.open().root()
r1["t"] = self.t
transaction.commit()
r2 = self.db.open(synch=False).root()
copy = r2["t"]
# Make sure all of copy is loaded.
list(copy.values())
self.assertEqual(self.t._p_serial, copy._p_serial)
# Now one transaction should add enough keys to cause a split,
# and another should remove all the keys in one bucket.
for k in range(200, 300, 4):
self.t[k] = k
transaction.commit()
for k in range(0, 60, 4):
del copy[k]
try:
transaction.commit()
except ConflictError, detail:
self.assert_(str(detail).startswith('database conflict error'))
transaction.abort()
else:
def doForwardPass(T, fitnessPathHDD, pC0E0, pC1E1, pE0, pE1, finalTime, batchSize):
# start the clock
startTime = time.clock()
# the current working directory is the folder for the respective parameter combination
# a plotting folder exists
# add a state distribution folder
set_global_Policy({})
stateDistPath = 'plotting/StateDistribution'
fitnessPath = 'fitness'
if not os.path.exists(stateDistPath):
os.makedirs(stateDistPath)
pC0E0, pC1E1, pE0, pE1 = float(pC0E0), float(pC1E1), float(pE0), float(pE1)
tValues = np.arange(1,T-1, 1)
"""
This is the initialization
- here we basically initialize a state distribution vector for t equals 0 and determine how the population starts
"""
print "start initialization"
# make one folder per time step
if not os.path.exists(os.path.join(stateDistPath, str(1))):
os.makedirs(os.path.join(stateDistPath, str(1)))
# what follows now should always be just one file in the directory
# however I will make it general enough in case future models will deal with a larger state space
for batchPstar in os.listdir('fitness/%s' % 1): # directly navigating to the corresponding time step folder
currBatchPstar = {}
# either a pickled file or a shelve
if batchPstar.endswith('.p'):
currBatchPstar.update(pickle.load(open("fitness/1/%s" % batchPstar, 'rb')))
else: # must be a shelve
currBatchPstar.update(shelve.open("fitness/1/%s" % batchPstar))
currLen = batchPstar.replace('TF', '').replace('.p', '')
# now the actual initialization
intialization(currBatchPstar, pC0E0, pC1E1, pE0, pE1, finalTime, stateDistPath, currLen)
del currBatchPstar
print 'finished initialization'
"""
Here, the actual calculation of the state distribution matrix will begin
- this will constitute a forward pass
- the resulting P matrices are a one-to-one mapping from policy states, if a state from the policy is not in the
in the P matrix, it's state distribution can be assumed to be zero
"""
# initialize a global dicts for P and policy star so that all parallel workers have access to it
for t in tValues:
global currBatchPstar
currBatchPstar = {}
global P
P = {}
#print "currently preparing time step: " + str(t)
if t > 17:
fitnessPath = fitnessPathHDD
if not os.path.exists(os.path.join(stateDistPath, str(t + 1))):
os.makedirs(os.path.join(stateDistPath, str(t + 1)))
# make the respective path a variable here
batchPstarList = [batchPstar for batchPstar in os.listdir(os.path.join(fitnessPath, '%s' % t))]
batchPstarListSorted = [batchPstar for batchPstar in
sorted(batchPstarList, key=lambda x: int(x.replace('TF', '').replace('.p', '')))]
down()
widgets = ['Time step %s:' % t, Percentage(), ' ',
Bar(marker=('-'), left='[', right=']'), ' ', ETA(), ' ']
pbarTimeStep = ProgressBar(widgets=widgets, maxval=int(len(batchPstarListSorted))).start()
for batchPstar in pbarTimeStep(batchPstarListSorted): # directly navigating to the corresponding time step folder
up()
localP = OOBTree()
localBatchPstar = OOBTree()
currLen = batchPstar.replace('TF', '').replace('.p', '')
# first load the respective policy
# either a pickled file or a shelve
if batchPstar.endswith('.p'):
localBatchPstar.update(pickle.load(open(os.path.join(fitnessPath, "%s/%s" % (t, batchPstar)), 'rb')))
else: # must be a shelve
localBatchPstar.update(shelve.open(os.path.join(fitnessPath, "%s/%s" % (t, batchPstar))))
# free up memory by deleting the local copies
localBatchPstarCompressed = compressDict(localBatchPstar)
del localBatchPstar
set_global_Policy(localBatchPstarCompressed)
del localBatchPstarCompressed
if os.path.exists(os.path.join(stateDistPath, "%s/P%s" % (t, currLen))):
localP.update(shelve.open(os.path.join(stateDistPath, "%s/P%s" % (t, currLen))))
else:
print "No such file" + str(os.path.join(stateDistPath, "%s/P%s" % (t, currLen)))
set_global_P(localP)
del localP
"""
Implement the parallelization
- this part will perform the actual calculation of the state distribution matrix
- for each state extract the relevant statistics from both the state distribution vector and the optimal
policy
- next use these informaton to do the forward updating
- it should be possible to distribute that across workers
- should be inspired by the parallelization of the terminla fitness calculation?
"""
# now iterae through the optimal policy
# structure:
startCondition = int(currLen) - batchSize
# prepare the respective pDicts
# these will be the resultng P-values that I will need to fill
identTimesTwo = startCondition * 2 + batchSize
identTimesTwoPlusOne = identTimesTwo + batchSize
cutOff1 = startCondition + (int(batchSize)/2 - 1)
cutOff2 = cutOff1+ 1
print "Currently working on batch: %s" % currLen
# this quantity is only dependent on t
survivalProb = 1 - float(finalTime[t + 1])
# what is the maximim identifier?
maxIdent,lowestIdent = findMaxIdentifier(currBatchPstar)
subBatches = []
# this step is in place to decide whether we want to initalize two outcome files or one
if maxIdent >= (batchSize / 2):
subBatches.append(identTimesTwo)
subBatches.append(identTimesTwoPlusOne)
else:
subBatches.append(identTimesTwo)
for elem in subBatches:
finalResults = OOBTree()
if elem == identTimesTwo:
parallelStates = [currState for currState in
sorted(currBatchPstar.keys(), key=lambda x: int(x.split(';')[0])) if
int(currState.split(";")[0]) <= cutOff1]
else:
parallelStates = [currState for currState in
sorted(currBatchPstar.keys(),
key=lambda x: int(x.split(';')[0])) if
int(currState.split(";")[0]) >= cutOff2]
parallelStatesChunks = chunks(parallelStates,25000) # usually this should be the batchsize parameter
del parallelStates
for chunkStates in parallelStatesChunks:
# setting up the pool
pool = Pool(32)
results = pool.map(func_star,
itertools.izip(chunkStates, itertools.repeat(survivalProb)))
pool.close()
pool.join()
for tempResult in results:
finalResults.update(tempResult)
del results
# safe the results
myShelve = shelve.open(os.path.join(stateDistPath, "%s/P%s" % (t + 1, elem)))
myShelve.update(finalResults)
myShelve.close()
del finalResults
del parallelStatesChunks
time.sleep(0.005)
del currBatchPstar
del P
print "Elapsed time for thr forward pass: " + str(time.clock() - startTime)
class ZODBContinuousIncreasingIdGenerator(IdGenerator):
"""
Create some Ids with the zodb storage
"""
zope.interface.implements(interfaces.IIdGenerator)
# CMF Type Definition
meta_type = 'ERP5 ZODB Continous Increasing Id Generator'
portal_type = 'ZODB Continous Increasing Id Generator'
add_permission = Permissions.AddPortalContent
# Declarative security
security = ClassSecurityInfo()
security.declareObjectProtected(Permissions.AccessContentsInformation)
def _generateNewId(self, id_group, id_count=1, default=None):
"""
Return the new_id from the last_id of the zodb
Use int to store the last_id, use also a persistant mapping for to be
persistent.
"""
if id_group in (None, 'None'):
raise ValueError, '%s is not a valid group Id.' % (repr(id_group), )
if default is None:
default = 0
last_id_dict = getattr(self, 'last_id_dict', None)
if last_id_dict is None:
# If the dictionary not exist initialize generator
self.initializeGenerator()
last_id_dict = self.last_id_dict
# Retrieve the last id and increment
new_id = last_id_dict.get(id_group, default - 1) + id_count
# Store the new_id in the dictionary
last_id_dict[id_group] = new_id
return new_id
security.declareProtected(Permissions.AccessContentsInformation,
'generateNewId')
def generateNewId(self, id_group=None, default=None):
"""
Generate the next id in the sequence of ids of a particular group
"""
return self._generateNewId(id_group=id_group, default=default)
security.declareProtected(Permissions.AccessContentsInformation,
'generateNewIdList')
def generateNewIdList(self, id_group=None, id_count=1, default=None):
"""
Generate a list of next ids in the sequence of ids of a particular group
"""
new_id = 1 + self._generateNewId(id_group=id_group, id_count=id_count,
default=default)
return range(new_id - id_count, new_id)
security.declareProtected(Permissions.AccessContentsInformation,
'initializeGenerator')
def initializeGenerator(self):
"""
Initialize generator. This is mostly used when a new ERP5 site
is created. Some generators will need to do some initialization like
prepare some data in ZODB
"""
LOG('initialize ZODB Generator', INFO, 'Id Generator: %s' % (self,))
if getattr(self, 'last_id_dict', None) is None:
self.last_id_dict = OOBTree()
# XXX compatiblity code below, dump the old dictionnaries
portal_ids = getattr(self, 'portal_ids', None)
# Dump the dict_ids dictionary
if getattr(portal_ids, 'dict_ids', None) is not None:
for id_group, last_id in portal_ids.dict_ids.items():
if not isinstance(id_group, str):
id_group = repr(id_group)
if self.last_id_dict.has_key(id_group) and \
self.last_id_dict[id_group] > last_id:
continue
self.last_id_dict[id_group] = last_id
security.declareProtected(Permissions.AccessContentsInformation,
'clearGenerator')
def clearGenerator(self):
"""
Clear generators data. This can be usefull when working on a
development instance or in some other rare cases. This will
loose data and must be use with caution
This can be incompatible with some particular generator implementation,
in this case a particular error will be raised (to be determined and
added here)
"""
# Remove dictionary
self.last_id_dict = OOBTree()
security.declareProtected(Permissions.ModifyPortalContent,
'exportGeneratorIdDict')
def exportGeneratorIdDict(self):
"""
Export last id values in a dictionnary in the form { group_id : last_id }
"""
return dict(self.last_id_dict)
security.declareProtected(Permissions.ModifyPortalContent,
'importGeneratorIdDict')
def importGeneratorIdDict(self, id_dict, clear=False):
"""
Import data, this is usefull if we want to replace a generator by
another one.
"""
if clear:
self.clearGenerator()
if not isinstance(id_dict, dict):
raise TypeError, 'the argument given is not a dictionary'
for value in id_dict.values():
if not isinstance(value, int):
raise TypeError, 'the value given in dictionary is not a integer'
self.last_id_dict.update(id_dict)
security.declareProtected(Permissions.ModifyPortalContent,
'rebuildGeneratorIdDict')
def rebuildGeneratorIdDict(self):
"""
Rebuild generator id dict.
In fact, export it, clear it and import it into new dict.
This is mostly intendted to use when we are migrating the id dict
structure.
"""
id_dict = self.exportGeneratorIdDict()
self.importGeneratorIdDict(id_dict=id_dict, clear=True)
