def initDocListTypes(doc, usr):
'''When posting a doc with listType attributes/fields, they must be initialized with provided data and validated and saved back to the doc for further handling, ie, validation, etc.'''
attrNams = doc.keys()
# loop through all doc keys
for attrNam in attrNams:
attrVal = doc[attrNam]
# is it a list
if type(attrVal) == list:
attrValList = attrVal
# loop through all the elements in the list
for attrValListOffset in range(0, len(attrValList)):
attrValListItem = attrValList[attrValListOffset]
attr_c = attrValListItem['_c']
modelClass = _cs[attr_c]['modelClass']
# init a class model for this item
model = modelClass()
# set model attribute values
for k,v in attrValListItem.iteritems(): setattr(model, k, v)
# generate a next eId
resp = nextEId(doc, attrNam)
doc = resp['doc']
model.eId = resp['eId']
# generate dNam
if hasattr(model, 'vNam') and 'dNam' in model._fields and not model.dNam:
model.dNam = model.vNam
# generate dNamS
if 'dNamS' in model._fields and hasattr(model, 'vNamS') and not model.dNamS:
model.dNamS = model.vNamS
# remove all empty fields
attrValListItemClean = doc_remove_empty_keys(to_python(model, allow_none=True))
# validate
errors = validate(modelClass, attrValListItemClean)
# TODO: handle and test errors
if errors:
total_errors += errors['count']
post_errors.append(errors)
continue
# logit update
attrValListItemClean = logit(usr, attrValListItemClean, method='post')
# save cleaned listType item back to doc
doc[attrNam][attrValListOffset] = attrValListItemClean
# now all listType items are clean, validated, and logged
return doc
def load_data(db, es, json_filepath):
'''Bulk load from json into corresponding collections.
Each item is expected to contain '_c' which represents the model class and the collection it belongs to.
Validation rules are tested for each doc.
'''
docs_to_insert = []
# the data will either come from MongoVue or mongoexport
# mongoVue delimits each doc record with },{
# mongoexport delimits each doc record with linefeed
# let's find out which one it is by reading in file
data = open(json_filepath).read()
if re.search(r"\},\s*\{", data):
# each doc is delimited with },{ like from MongoVue copy to clipboard
clean_data = preparse_json_doc(data)
# strip beginning {
clean_data = re.sub(r"^(\s*\[\s*)*\s*\{\s*", "", clean_data)
# strip ending
clean_data = re.sub(r"\s*\}\s*(\s*\])*$", "", clean_data)
docs = re.split(r"\},\s*\{", clean_data)
for doc in docs:
doc = doc.replace('\n', '')
doc = "{" + doc + "}"
doc = json.loads(doc, object_hook = mongo_json_object_hook)
docs_to_insert.append(doc)
else:
# assume mongoexport with each doc on one line
file = open(json_filepath)
lines = [re.sub('\n', '', line) for line in filter(lambda a:(a != '\n'), file.readlines())]
for line in lines:
clean_line = preparse_json_doc(line)
doc = json.loads(clean_line, object_hook = mongo_json_object_hook)
docs_to_insert.append(doc)
file.close()
response = {}
status = 200
docs = []
load_errors = []
total_errors = 0
total_added = 0
for doc in docs_to_insert:
class_id = doc['_c']
model = getattr(models, class_id)
collection_name = model.meta['collection']
collection = db[collection_name]
errors = {}
# Validate
doc_errors = validate(model, doc)
if doc_errors:
total_errors += doc_errors['count']
load_errors.append(doc_errors)
continue
# init model for this doc
initialized_model = model(**doc)
#log date time user involved with this event
# m.logit(user_id, 'post')
# need to stuff into mongo
doc_validated = initialized_model.to_python()
dumped = bson_json_util.dumps(doc_validated)
doc_info = {}
doc_validated['_id'] = doc['_id']
doc_validated['_id'] = collection.save(doc_validated, safe = True)
# try to load generic display name used for indexing, etc
try:
doc_validated['dNam'] = initialized_model.dNam
except:
pass
docs.append(doc_validated)
ret = es.index(initialized_model.index, es.__dict__['index_name'], initialized_model._c, doc['_id'].__str__())
total_added += 1
response['total_inserted'] = len(docs)
if load_errors:
response['total_invalid'] = len(load_errors)
response['errors'] = load_errors
response['total_errors'] = total_errors
status = 400
else:
response['total_invalid'] = 0
response['docs'] = docs
print "{count} docs loaded from: {json_filepath}!\n".format(count = total_added, json_filepath = json_filepath)
return {'response':response, 'status':status}
def post(self, **kwargs):
'''Insert a doc
newDocTmp: Initialize a temp (tmp) doc if no OID and no data.
cloneDocTmp: Clone to a temp doc if OID and no isTmp flag set.
upsertDocTmp: Update or Insert temp doc to base collection if OID and isTmp is set.
insertDoc: Insert a new doc if no OID and there are more attributes than _c.
'''
db = self.db
response = {}
docs = []
status = 200
usrOID = self.usr['OID']
docs_to_post = kwargs['docs']
post_errors = []
total_errors = 0
for doc in docs_to_post:
errors = {}
doc_info = {}
# required attribute
_c = doc['_c']
# shortcut
doc_keys = doc.keys()
modelClass = getattr(models, _c)
_id = doc['_id'] if '_id' in doc_keys else None
where = {'_id': ObjectId(_id)} if _id else None
attrNam = doc['attrNam'] if 'attrNam' in doc_keys else None
attr_c = doc['attr_c'] if attrNam else None
attrVal = doc['attrVal'] if attrNam else None
collNam = modelClass.meta['collection']
collTmp = db[collNam + '_tmp']
coll = db[collNam]
# if _ id is passed, it directs that a temp doc be initialized and inserted into the appropriate Tmp (temp) collection.
# if an _id IS passed, it directs that the doc passed in be validated and inserted in the base collection and the temp doc in the temp collection be deleted.
# if attrNam, posting a new value to a listtype attribute/field
if attrNam:
for elem in attrVal:
modelClass = getattr(models.embed, attr_c)
model = modelClass()
for k,v in elem.iteritems(): setattr(model, k, v)
resp = nextEId(doc, attrNam)
doc = resp['doc']
# model.eIds = doc['eIds']
model.eId = resp['eId']
if hasattr(model, 'vNam') and 'dNam' in model._fields and not model.dNam:
model.dNam = model.vNam
# generate a slug if slug is empty
if 'slug' in model._fields and 'dNam' in model._fields and not model.slug:
model.slug = slugify(model.dNam, coll)
# set dNamS if used:
# if dNamS is empty, set to value of slug
if 'dNamS' in model._fields and not model.dNamS:
if hasattr(model, 'vNamS'):
model.dNamS = model.vNamS
else:
model.dNamS = model.slug
embedDoc = doc_remove_empty_keys(to_python(model, allow_none=True))
doc_errors = validate(modelClass, embedDoc)
if doc_errors:
total_errors += doc_errors['count']
post_errors.append(doc_errors)
continue
embedDoc['_c'] = attr_c
# logit update
embedDoc = logit(self.usr, embedDoc, method='post')
collTmp.update(where,
{"$push": { attrNam: embedDoc}, "$set": {'eIds': doc['eIds']}}
)
doc_info['doc'] = embedDoc
docs.append(doc_info)
# http://docs.mongodb.org/manual/applications/update/
# { $set: { 'emails.$.eId': 2 }
response['total_inserted'] = len(docs)
if post_errors:
response['total_invalid'] = len(post_errors)
response['errors'] = post_errors
response['total_errors'] = total_errors
status = 400
else:
response['total_invalid'] = 0
response['docs'] = docs
return {'response': response, 'status': status}
# Initialize a temp (tmp) doc if no OID and no data.
elif not '_id' in doc_keys and len(doc_keys) == 1:
newDocTmp = True
useTmpDoc = True
# Clone to a temp doc if OID and no isTmp flag set.
elif '_id' in doc_keys and not 'isTmp' in doc_keys and len(doc_keys) > 2:
cloneDocTmp = True
useTmpDoc = True
# find source doc
# set locked = True
doc_cloned = coll.find_and_modify(
query = {'_id':_id},
update = {"$set": {'locked': True}},
new = True
)
# set cloned doc in tmp collection isTmp = True
doc_cloned['isTmp'] = True
# don't need locked set in tmp doc
del doc_cloned['locked']
# clone/save doc to tmp collection
# TODO properly handle exception
try:
collTmp.insert(doc_cloned)
except:
pass
doc_info['doc'] = doc_cloned
# TODO
# should return a link to object according to good practice
#doc_info['link'] = get_document_link(class_name, id)
docs.append(doc_info)
continue
# Update temp doc to base collection if OID and isTmp is set.
elif '_id' in doc_keys and 'isTmp' in doc_keys and doc['isTmp']:
upsertDocTmp = True
useTmpDoc = False
tmp_doc = collTmp.find_one({'_id': _id})
# unset isTmp
_id = tmp_doc['_id']
tmp_doc['locked'] = False
del tmp_doc['_id']
del tmp_doc['isTmp']
# logit update
tmp_doc = logit(self.usr, tmp_doc)
# update original/source doc
doc = coll.update({'_id': _id}, {"$set": tmp_doc}, upsert=True, safe=True)
# remove tmp doc
collTmp.remove({'_id': _id})
# though not necessary, to be consistant, return full doc
doc = coll.find_one({'_id': _id})
doc_info['doc'] = doc
docs.append(doc_info)
continue
# Insert a new doc if no OID and there are more attributes than _c.
elif not '_id' in doc_keys and len(doc_keys) > 2:
insertDoc = True
useTmpDoc = False
# need to cruz through all doc keys to handle arrays/listtype fields.
# they need to be initialized according to the appropriate model and validated, etc.
doc = initDocListTypes(doc, self.usr)
# init model instance
model = modelClass(**doc)
# set isTemp
model.isTmp = useTmpDoc and 'isTmp' in model._fields
# try to generate dNam
# if there is a vNam class property
# if already has a value, use it
if hasattr(model, 'vNam') and not useTmpDoc and 'dNam' in model._fields and not model.dNam:
model.dNam = model.vNam
# assign dId
if 'dId' in model._fields and not model.dId:
response = nextId(coll)
if response['status'] == 200:
model.dId = response['nextId']
else:
# handle error condition
pass
# generate a slug if:
# not a temp doc and slug is empty
if 'slug' in model._fields and not useTmpDoc and not model.slug:
model.slug = slugify(model.dNam, coll)
# set dNamS if used:
# not a temp doc and dNamS is empty, set to value of slug
if 'dNamS' in model._fields and not useTmpDoc and not model.dNamS:
model.dNamS = model.slug
# do not validate if using temp doc
if not useTmpDoc:
doc_errors = validate(modelClass, doc)
if doc_errors:
total_errors += doc_errors['count']
post_errors.append(doc_errors)
continue
# modelClass stuffed in all available fields
# let's remove all empty fields to keep mongo clean.
doc = to_python(model, allow_none=True)
# do not log if using temp doc
#log date time user involved with this event
if not useTmpDoc:
doc = logit(self.usr, doc, 'post')
doc['_c'] = _c
doc_clean = doc_remove_empty_keys(doc)
# posting an existing temp doc to base collection
if _id:
doc_clean['_id'] = str(_id)
id = str(coll.insert(doc_clean))
# TODO properly handle exception
try:
collTmp.remove({'_id': _id})
except:
pass
# posting initialized temp doc
else:
# TODO properly handle exception
try:
coll.insert(doc_clean)
except:
pass
doc_info['doc'] = doc_clean
# TODO
# should return a link to object according to good practice
#doc_info['link'] = get_document_link(class_name, id)
docs.append(doc_info)
response['total_inserted'] = len(docs)
if post_errors:
response['total_invalid'] = len(post_errors)
response['errors'] = post_errors
response['total_errors'] = total_errors
status = 400
else:
response['total_invalid'] = 0
response['docs'] = docs
return {'response': response, 'status': status}
def post_attr(self, doc, attrNam, attr_c, attrVal, useTmpDoc = True):
'''
doc = base doc to post/add attrVal to (in the attrNam field)
Must contain _c, and _id
usr = user object
coll = collection
attr_c = model class
_id = doc id
attrNam = attribute/fieldname
attrVal = value to post/add
'''
db = self.db
errors = {}
doc_info = {}
response = {}
status = 200
post_errors = []
total_errors = 0
doc_id = doc['_id']
usrOID = self.usr['OID']
where = {'_id': doc_id}
doc_c = doc['_c']
docModel = _cs[doc_c]['modelClass']()
collNam = docModel.meta['collection']
coll = db[collNam + '_tmp'] if useTmpDoc else db[collNam]
attrModel = _cs[attr_c]['modelClass']()
for k,v in attrVal.iteritems(): setattr(attrModel, k, v)
# set the next element id eId
resp = nextEId(doc, attrNam)
doc = resp['doc']
attrModel.eId = resp['eId']
if hasattr(attrModel, 'vNam') and 'dNam' in attrModel._fields and not attrModel.dNam:
attrModel.dNam = attrModel.vNam
# set dNamS if used:
# if dNamS is empty, set to value of slug
if hasattr(attrModel, 'vNam') and 'dNamS' in attrModel._fields and not attrModel.dNamS:
attrModel.dNamS = attrModel.vNamS
attrValClean = doc_remove_empty_keys(to_python(attrModel, allow_none=True))
attrErrors = validate(_cs[attr_c]['modelClass'], attrValClean)
if attrErrors:
response['total_invalid'] = 1
response['errors'] = attrErrors
response['total_errors'] = 1
return {'response': response, 'status': 400}
# logit update
attrValClean = logit(self.usr, attrValClean, method='post')
resp = coll.update(where,
{"$push": { attrNam: attrValClean}, "$set": {'eIds': doc['eIds']}}
)
response['update_response'] = resp
return {'response': response, 'status': 200}
def post_embedded(collection, id, embedded):
# get collection name for this model
col = models[collection].meta['collection']
_cls = models[collection].meta['_c']
user_objectId = ObjectId("50468de92558713d84b03fd7")
response = {}
status = 200
docs = []
# Retrieve base doc to insert new embedded docs
base_doc = mongo.db[col].find_one({"_id":ObjectId(id)})
base_model = models[col](**base_doc)
#log date time user involved with this event
base_model.logit(user_objectId, 'patch')
# grab data and parse
js = json.loads(request.data)
# what is the doc class for these docs to embed
field_name = js['field_name']
_cls = js['_c']
# docs to insert
docs_to_post = js['docs']
# if a dict, then stuff it into a list
if type(docs_to_post) == dict: docs_to_post = [docs_to_post]
post_errors = []
total_errors = 0
for doc in docs_to_post:
# Validate
doc_errors = validate(models[field_name], doc)
if doc_errors:
total_errors += doc_errors['count']
post_errors.append(doc_errors)
continue
doc_model = models[field_name](**doc)
doc_info = {}
doc_info['doc'] = doc_model.to_python()
docs.append(doc_model)
response['total_inserted'] = len(docs)
if post_errors:
response['total_invalid'] = len(post_errors)
response['errors'] = post_errors
response['total_errors'] = total_errors
status = 400
else:
response['total_invalid'] = 0
for doc in docs:
base_model.emails.insert(1, doc)
js = base_model.to_python()
doc = mongo.db[col].update({"_id":ObjectId(id)}, {"$set": js})
response[collection] = docs
return prep_response(response, status = status)
def patch_embedded(collection, id, embedded):
# get collection name for this model
col = models[collection].meta['collection']
_cls = models[collection].meta['_c']
user_objectId = ObjectId("50468de92558713d84b03fd7")
response = {}
status = 200
docs = []
# grab data and parse
js = json.loads(request.data)
# what is the doc class for these docs to embed
field_name = js['field_name']
_cls = js['_c']
# Retrieve base doc to insert new embedded docs
doc_to_patch = mongo.db[col].find_one({"_id":ObjectId(id)})
if not doc_to_patch:
return 'error'
base_model = models[col](**doc_to_patch)
#log date time user involved with this event
base_model.logit(user_objectId, 'patch')
# docs to insert
original_values = js['original_values']
key_field = original_values['key_field']
replacement_values = js['replacement_values']
# need to get the position of original embedded doc to update
embedded_pos = [i for i,x in enumerate([x[key_field] for x in doc_to_patch[field_name]]) if x == original_values['value']]
if not embedded_pos:
return 'error: did not find item to replace'
embedded_pos = embedded_pos[0]
elem = doc_to_patch[field_name][embedded_pos]
print 'found', elem
for fld, val in replacement_values.iteritems():
print fld, val
elem[fld] = val
print 'replacement_values', replacement_values
print 'updated', elem
# replace this element with new one
doc_to_patch[field_name][embedded_pos] = elem
doc = mongo.db[col].update({"_id":ObjectId(id)}, {"$set": {"%s" % field_name:doc_to_patch[field_name]}})
post_errors = []
total_errors = 0
# make sure doc_to_path exists
# Validate
doc_errors = validate(models[field_name], replacement_values)
if doc_errors:
total_errors += doc_errors['count']
post_errors.append(doc_errors)
else:
doc_model = models[field_name](**doc)
doc_info = {}
doc_info['doc'] = doc_model.to_python()
docs.append(doc_model)
response['total_inserted'] = len(docs)
if post_errors:
response['total_invalid'] = len(post_errors)
response['errors'] = post_errors
response['total_errors'] = total_errors
status = 400
else:
response['total_invalid'] = 0
for doc in docs:
base_model.emails.insert(1, doc)
js = base_model.to_python()
doc = mongo.db[col].update({"_id":ObjectId(id)}, {"$set": js})
response[collection] = docs
return prep_response(response, status = status)
def post_embedded(collection, id, embedded):
# get collection name for this model
col = models[collection].meta['collection']
_cls = models[collection].meta['_c']
user_objectId = ObjectId("50468de92558713d84b03fd7")
response = {}
status = 200
docs = []
# Retrieve base doc to insert new embedded docs
base_doc = mongo.db[col].find_one({"_id": ObjectId(id)})
base_model = models[col](**base_doc)
#log date time user involved with this event
base_model.logit(user_objectId, 'patch')
# grab data and parse
js = json.loads(request.data)
# what is the doc class for these docs to embed
field_name = js['field_name']
_cls = js['_c']
# docs to insert
docs_to_post = js['docs']
# if a dict, then stuff it into a list
if type(docs_to_post) == dict: docs_to_post = [docs_to_post]
post_errors = []
total_errors = 0
for doc in docs_to_post:
# Validate
doc_errors = validate(models[field_name], doc)
if doc_errors:
total_errors += doc_errors['count']
post_errors.append(doc_errors)
continue
doc_model = models[field_name](**doc)
doc_info = {}
doc_info['doc'] = doc_model.to_python()
docs.append(doc_model)
response['total_inserted'] = len(docs)
if post_errors:
response['total_invalid'] = len(post_errors)
response['errors'] = post_errors
response['total_errors'] = total_errors
status = 400
else:
response['total_invalid'] = 0
for doc in docs:
base_model.emails.insert(1, doc)
js = base_model.to_python()
doc = mongo.db[col].update({"_id": ObjectId(id)}, {"$set": js})
response[collection] = docs
return prep_response(response, status=status)
def patch_embedded(collection, id, embedded):
# get collection name for this model
col = models[collection].meta['collection']
_cls = models[collection].meta['_c']
user_objectId = ObjectId("50468de92558713d84b03fd7")
response = {}
status = 200
docs = []
# grab data and parse
js = json.loads(request.data)
# what is the doc class for these docs to embed
field_name = js['field_name']
_cls = js['_c']
# Retrieve base doc to insert new embedded docs
doc_to_patch = mongo.db[col].find_one({"_id": ObjectId(id)})
if not doc_to_patch:
return 'error'
base_model = models[col](**doc_to_patch)
#log date time user involved with this event
base_model.logit(user_objectId, 'patch')
# docs to insert
original_values = js['original_values']
key_field = original_values['key_field']
replacement_values = js['replacement_values']
# need to get the position of original embedded doc to update
embedded_pos = [
i
for i, x in enumerate([x[key_field] for x in doc_to_patch[field_name]])
if x == original_values['value']
]
if not embedded_pos:
return 'error: did not find item to replace'
embedded_pos = embedded_pos[0]
elem = doc_to_patch[field_name][embedded_pos]
print 'found', elem
for fld, val in replacement_values.iteritems():
print fld, val
elem[fld] = val
print 'replacement_values', replacement_values
print 'updated', elem
# replace this element with new one
doc_to_patch[field_name][embedded_pos] = elem
doc = mongo.db[col].update(
{"_id": ObjectId(id)},
{"$set": {
"%s" % field_name: doc_to_patch[field_name]
}})
post_errors = []
total_errors = 0
# make sure doc_to_path exists
# Validate
doc_errors = validate(models[field_name], replacement_values)
if doc_errors:
total_errors += doc_errors['count']
post_errors.append(doc_errors)
else:
doc_model = models[field_name](**doc)
doc_info = {}
doc_info['doc'] = doc_model.to_python()
docs.append(doc_model)
response['total_inserted'] = len(docs)
if post_errors:
response['total_invalid'] = len(post_errors)
response['errors'] = post_errors
response['total_errors'] = total_errors
status = 400
else:
response['total_invalid'] = 0
for doc in docs:
base_model.emails.insert(1, doc)
js = base_model.to_python()
doc = mongo.db[col].update({"_id": ObjectId(id)}, {"$set": js})
response[collection] = docs
return prep_response(response, status=status)