class IncomingCopyHandler(BaseChangeHandler): def __init__(self): self._serverUrl = config["couchdb.url.dbadmin"] self._targetName = config["couchdb.db.resourcedata"] self.documents = [] s = couchdb.Server(self._serverUrl) self._db = s[self._targetName] self.repl_helper = ResourceDataReplacement() def _canHandle(self, change, database): if (_DOC in change) and ( change[_DOC].get(_DOC_TYPE) == _RESOURCE_DISTRIBUTABLE_TYPE or change[_DOC].get(_DOC_TYPE) == _RESOURCE_TYPE ): return True return False def _handle(self, change, database): def handleDocument(newDoc): should_delete = True try: # newDoc['node_timestamp'] = h.nowToISO8601Zformat() ResourceDataModelValidator.set_timestamps(newDoc) del newDoc["_rev"] self.repl_helper.handle(newDoc) # rd = ResourceDataModel(newDoc) # rd.save(log_exceptions=False) except SpecValidationException as e: log.error(newDoc["_id"] + str(e)) except ResourceConflict as ex: log.exception(ex) except Exception as ex: should_delete = False # don't delete something unexpected happend log.error(ex) if should_delete: try: del database[newDoc["_id"]] except Exception as ex: log.error(ex) self.documents.append(change[_DOC]) if len(self.documents) >= _DOCUMENT_UPDATE_THRESHOLD or len(self.documents) >= database.info()["doc_count"]: for doc in self.documents: t = Thread(target=handleDocument, args=(doc,)) t.start() self.documents = []
class IncomingCopyHandler(BaseChangeHandler): def __init__(self): self._serverUrl = config["couchdb.url.dbadmin"] self._targetName = config["couchdb.db.resourcedata"] self.documents = deque() s = couchdb.Server(self._serverUrl) self._db = s[self._targetName] self.repl_helper = ResourceDataReplacement() self.threads = {} self.max_threads = 50 def _canHandle(self, change, database): if ((_DOC in change) and \ (change[_DOC].get(_DOC_TYPE) == _RESOURCE_DISTRIBUTABLE_TYPE or \ change[_DOC].get(_DOC_TYPE) == _RESOURCE_TYPE)): return True return False def _handle(self, change, database): def threadName(doc): return "T-" + doc["_id"] def handleDocument(newDoc): should_delete = True try: # newDoc['node_timestamp'] = h.nowToISO8601Zformat() ResourceDataModelValidator.set_timestamps(newDoc) del newDoc["_rev"] self.repl_helper.handle(newDoc) # rd = ResourceDataModel(newDoc) # rd.save(log_exceptions=False) except SpecValidationException as e: log.error("SpecValidationException: %s, %s", newDoc['_id'], str(e)) except couchdb.ResourceConflict as rc: log.error("Document conflicts", exc_info=1) except Exception as ex: should_delete = False # don't delete something unexpected happend log.error("Unable to save %s", newDoc['_id'], exc_info=ex) if should_delete: try: del database[newDoc['_id']] except Exception as ex: log.error("Error when deleting", exc_info=ex) try: del self.threads[threadName(newDoc)] except: pass self.documents.append(change[_DOC]) if len(self.documents) >= _DOCUMENT_UPDATE_THRESHOLD or len( self.documents) >= database.info()['doc_count']: while len(self.documents) > 0: doc = self.documents.popleft() tname = threadName(doc) t = Thread(target=handleDocument, name=tname, args=(doc, )) self.threads[tname] = t t.start() while len(self.threads) > self.max_threads: time.sleep(.1) def isRunning(self): return len(self.threads) > 0 def threadCount(self): return len(self.threads)
class IncomingCopyHandler(BaseChangeHandler): def __init__(self): self._serverUrl = config["couchdb.url.dbadmin"] self._targetName = config["couchdb.db.resourcedata"] self.documents = deque() s = couchdb.Server(self._serverUrl) self._db = s[self._targetName] self.repl_helper = ResourceDataReplacement() self.threads = {} self.max_threads = 50 def _canHandle(self, change, database): if ((_DOC in change) and \ (change[_DOC].get(_DOC_TYPE) == _RESOURCE_DISTRIBUTABLE_TYPE or \ change[_DOC].get(_DOC_TYPE) == _RESOURCE_TYPE)): return True return False def _handle(self, change, database): def threadName(doc): return "T-"+doc["_id"] def handleDocument(newDoc): should_delete = True try: # newDoc['node_timestamp'] = h.nowToISO8601Zformat() ResourceDataModelValidator.set_timestamps(newDoc) del newDoc["_rev"] self.repl_helper.handle(newDoc) # rd = ResourceDataModel(newDoc) # rd.save(log_exceptions=False) except SpecValidationException as e: log.error("SpecValidationException: %s, %s",newDoc['_id'],str(e)) except couchdb.ResourceConflict as rc: log.error("Document conflicts", exc_info=1) except Exception as ex: should_delete = False # don't delete something unexpected happend log.error("Unable to save %s", newDoc['_id'], exc_info=ex) if should_delete: try: del database[newDoc['_id']] except Exception as ex: log.error("Error when deleting", exc_info=ex) try: del self.threads[threadName(newDoc)] except: pass self.documents.append(change[_DOC]) if len(self.documents) >= _DOCUMENT_UPDATE_THRESHOLD or len(self.documents) >= database.info()['doc_count']: while len(self.documents) > 0: doc = self.documents.popleft() tname = threadName(doc) t = Thread(target=handleDocument, name=tname, args=(doc,)) self.threads[tname] = t t.start() while len(self.threads) > self.max_threads: time.sleep(.1) def isRunning(self): return len(self.threads) > 0 def threadCount(self): return len(self.threads)