def __init__(self):
self._serverUrl = config["couchdb.url.dbadmin"]
self._targetName = config["couchdb.db.resourcedata"]
self.documents = deque()
s = couchdb.Server(self._serverUrl)
self._db = s[self._targetName]
self.repl_helper = ResourceDataReplacement()
self.threads = {}
self.max_threads = 50
def __init__(self):
self._serverUrl = config["couchdb.url.dbadmin"]
self._targetName = config["couchdb.db.resourcedata"]
self.documents = []
s = couchdb.Server(self._serverUrl)
self._db = s[self._targetName]
self.repl_helper = ResourceDataReplacement()
class IncomingCopyHandler(BaseChangeHandler):
def __init__(self):
self._serverUrl = config["couchdb.url.dbadmin"]
self._targetName = config["couchdb.db.resourcedata"]
self.documents = []
s = couchdb.Server(self._serverUrl)
self._db = s[self._targetName]
self.repl_helper = ResourceDataReplacement()
def _canHandle(self, change, database):
if (_DOC in change) and (
change[_DOC].get(_DOC_TYPE) == _RESOURCE_DISTRIBUTABLE_TYPE or change[_DOC].get(_DOC_TYPE) == _RESOURCE_TYPE
):
return True
return False
def _handle(self, change, database):
def handleDocument(newDoc):
should_delete = True
try:
# newDoc['node_timestamp'] = h.nowToISO8601Zformat()
ResourceDataModelValidator.set_timestamps(newDoc)
del newDoc["_rev"]
self.repl_helper.handle(newDoc)
# rd = ResourceDataModel(newDoc)
# rd.save(log_exceptions=False)
except SpecValidationException as e:
log.error(newDoc["_id"] + str(e))
except ResourceConflict as ex:
log.exception(ex)
except Exception as ex:
should_delete = False # don't delete something unexpected happend
log.error(ex)
if should_delete:
try:
del database[newDoc["_id"]]
except Exception as ex:
log.error(ex)
self.documents.append(change[_DOC])
if len(self.documents) >= _DOCUMENT_UPDATE_THRESHOLD or len(self.documents) >= database.info()["doc_count"]:
for doc in self.documents:
t = Thread(target=handleDocument, args=(doc,))
t.start()
self.documents = []
class IncomingCopyHandler(BaseChangeHandler):
def __init__(self):
self._serverUrl = config["couchdb.url.dbadmin"]
self._targetName = config["couchdb.db.resourcedata"]
self.documents = deque()
s = couchdb.Server(self._serverUrl)
self._db = s[self._targetName]
self.repl_helper = ResourceDataReplacement()
self.threads = {}
self.max_threads = 50
def _canHandle(self, change, database):
if ((_DOC in change) and \
(change[_DOC].get(_DOC_TYPE) == _RESOURCE_DISTRIBUTABLE_TYPE or \
change[_DOC].get(_DOC_TYPE) == _RESOURCE_TYPE)):
return True
return False
def _handle(self, change, database):
def threadName(doc):
return "T-" + doc["_id"]
def handleDocument(newDoc):
should_delete = True
try:
# newDoc['node_timestamp'] = h.nowToISO8601Zformat()
ResourceDataModelValidator.set_timestamps(newDoc)
del newDoc["_rev"]
self.repl_helper.handle(newDoc)
# rd = ResourceDataModel(newDoc)
# rd.save(log_exceptions=False)
except SpecValidationException as e:
log.error("SpecValidationException: %s, %s", newDoc['_id'],
str(e))
except couchdb.ResourceConflict as rc:
log.error("Document conflicts", exc_info=1)
except Exception as ex:
should_delete = False # don't delete something unexpected happend
log.error("Unable to save %s", newDoc['_id'], exc_info=ex)
if should_delete:
try:
del database[newDoc['_id']]
except Exception as ex:
log.error("Error when deleting", exc_info=ex)
try:
del self.threads[threadName(newDoc)]
except:
pass
self.documents.append(change[_DOC])
if len(self.documents) >= _DOCUMENT_UPDATE_THRESHOLD or len(
self.documents) >= database.info()['doc_count']:
while len(self.documents) > 0:
doc = self.documents.popleft()
tname = threadName(doc)
t = Thread(target=handleDocument, name=tname, args=(doc, ))
self.threads[tname] = t
t.start()
while len(self.threads) > self.max_threads:
time.sleep(.1)
def isRunning(self):
return len(self.threads) > 0
def threadCount(self):
return len(self.threads)
class PublishController(BaseController):
"""REST Controller styled on the Atom Publishing Protocol"""
# To properly map this controller, ensure your config/routing.py
# file has a resource setup:
# map.resource('publish', 'publisher')
__ERROR = 'error'
__OK = "OK"
__DOCUMENT_RESULTS = 'document_results'
__DOCUMENTS = 'documents'
repl_helper = ResourceDataReplacement()
@oauth.authorize("oauth-sign",
_service_doc(True),
roles=None,
mapper=signing.lrsignature_mapper,
post_cond=_no_abort)
@bauth.authorize("oauth-sign",
_service_doc(),
roles=None,
pre_cond=_continue_if_missing_oauth,
realm="Learning Registry")
def create(self, *args, **kwargs):
results = {self.__OK: True}
error_message = None
try:
data = json.loads(request.body)
doc_limit = _service_doc()()['service_data']['doc_limit']
if not self.__DOCUMENTS in data.keys():
# Comply with LR-RQST-009 'Missing documents in POST'
results[
self.__ERROR] = "Missing field 'documents' in post body"
elif len(data[self.__DOCUMENTS]) < 1:
# Comply with LR-API-PUBLISH-001 'List of documents is empty'
results[self.__ERROR] = "List of documents is empty"
elif len(data[self.__DOCUMENTS]) > doc_limit:
error_message = "number of posted docs {0} exceeds doc limit: {1}".format(
len(data['documents']), str(doc_limit))
log.debug(error_message)
results[self.__ERROR] = error_message
else:
results[self.__DOCUMENT_RESULTS] = map(
lambda doc: signing.sign_doc(
doc, cb=self._publish, session_key="oauth-sign"),
data[self.__DOCUMENTS])
except Exception as ex:
log.exception(ex)
results[self.__ERROR] = str(ex)
if results.has_key(self.__ERROR):
results[self.__OK] = False
return json.dumps(results)
def _isResourceDataFilteredOut(self, resourceData):
if (LRNode.filterDescription is None
or LRNode.filterDescription.filter is None
or LRNode.filterDescription.custom_filter == True):
#Do custom the filter I supposed ... for now just resturn false.
return [False, None]
matchResult = False
envelopFilter = ""
for f in LRNode.filterDescription.filter:
log.info("\n" + str(f) + "\n")
# Ckeck if jsonObject object has the key if it has search
# for the regular expression in the filter otherwise keep looking
key = f['filter_key']
resourceValue = None
for k in resourceData.keys():
if re.search(key, k) is not None:
resourceValue = str(resourceData[k])
break
if resourceValue is None:
continue
value = f['filter_value']
log.info("\n" + str(key) + ", " + str(value) + ", " +
resourceValue + "\n")
if (re.search(value, resourceValue) is not None):
matchResult = True
envelopFilter = "Filter '"+str(f)+"' for key '"+str(key)+\
"' matches '"+resourceValue+"'"
break
#Check if what matching means base on the include_exclude
# True: the filters describe what documents to accept all others
# are rejected
# False: the filters describe what documents to reject
# all others are accepted
if LRNode.filterDescription.include_exclude is None or \
LRNode.filterDescription.include_exclude == True:
if matchResult == False:
return True, "\nDocument failed to match filter: \n"+\
pprint.pformat(LRNode.filterDescription.specData, indent=4)+"\n"
else:
if matchResult == True:
return True, "\nExcluded by filter: \n" + envelopFilter
return [False, None]
def _publish(self, resourceData):
if isinstance(resourceData, unicode):
resourceData = json.loads(resourceData)
result = {self.__OK: True}
try:
# Set the envelop data timestaps.
resourceData = ResourceDataModelValidator.set_timestamps(
resourceData)
#Check if the envelop get filtered out
isFilteredOut, reason = self._isResourceDataFilteredOut(
resourceData)
if isFilteredOut:
result[self.__ERROR] = reason
else:
resourceData[
"publishing_node"] = LRNode.nodeDescription.node_id
result = self.repl_helper.handle(resourceData)
# ResourceDataModelValidator.save(resourceData)
result[ResourceDataModelValidator.DOC_ID] = resourceData[
ResourceDataModelValidator.DOC_ID]
except SpecValidationException as ex:
log.exception(ex)
result[self.__ERROR] = "\n" + pprint.pformat(str(ex), indent=4)
except Exception as ex:
log.exception(ex)
result[self.__ERROR] = str(ex)
if result.has_key(self.__ERROR):
result[self.__OK] = False
return result
class IncomingCopyHandler(BaseChangeHandler):
def __init__(self):
self._serverUrl = config["couchdb.url.dbadmin"]
self._targetName = config["couchdb.db.resourcedata"]
self.documents = deque()
s = couchdb.Server(self._serverUrl)
self._db = s[self._targetName]
self.repl_helper = ResourceDataReplacement()
self.threads = {}
self.max_threads = 50
def _canHandle(self, change, database):
if ((_DOC in change) and \
(change[_DOC].get(_DOC_TYPE) == _RESOURCE_DISTRIBUTABLE_TYPE or \
change[_DOC].get(_DOC_TYPE) == _RESOURCE_TYPE)):
return True
return False
def _handle(self, change, database):
def threadName(doc):
return "T-"+doc["_id"]
def handleDocument(newDoc):
should_delete = True
try:
# newDoc['node_timestamp'] = h.nowToISO8601Zformat()
ResourceDataModelValidator.set_timestamps(newDoc)
del newDoc["_rev"]
self.repl_helper.handle(newDoc)
# rd = ResourceDataModel(newDoc)
# rd.save(log_exceptions=False)
except SpecValidationException as e:
log.error("SpecValidationException: %s, %s",newDoc['_id'],str(e))
except couchdb.ResourceConflict as rc:
log.error("Document conflicts", exc_info=1)
except Exception as ex:
should_delete = False # don't delete something unexpected happend
log.error("Unable to save %s", newDoc['_id'], exc_info=ex)
if should_delete:
try:
del database[newDoc['_id']]
except Exception as ex:
log.error("Error when deleting", exc_info=ex)
try:
del self.threads[threadName(newDoc)]
except:
pass
self.documents.append(change[_DOC])
if len(self.documents) >= _DOCUMENT_UPDATE_THRESHOLD or len(self.documents) >= database.info()['doc_count']:
while len(self.documents) > 0:
doc = self.documents.popleft()
tname = threadName(doc)
t = Thread(target=handleDocument, name=tname, args=(doc,))
self.threads[tname] = t
t.start()
while len(self.threads) > self.max_threads:
time.sleep(.1)
def isRunning(self):
return len(self.threads) > 0
def threadCount(self):
return len(self.threads)