def update(self, es_connection: Elasticsearch, document: Document = None, *args): """ Update the client_document in Elasticsearch. This should be override by the Document Dao :param es_connection: :param document: :return: doc_status, doc_meta """ return document.update(using=es_connection)
def update(self, **fields): return Document.update(self, **fields)
def merge_generated_parameters(params, idx, hash): """ :param params: :param paramsfile: :param idx: :return: """ layer_id = "%s_%s" % (hash, socket.gethostname()) es = Elasticsearch( current_app.config.get("ELASTIC").split(","), verify_certs=False, timeout=120 ) #See if the hash exists try: doc = Document.get(id=layer_id, using=es, index=".datashader_layers") except NotFoundError: doc = None if not doc: #if not, create the hash in the db but only if it does not already exist try: doc = Document(_id=layer_id, creating_host=socket.gethostname(), creating_pid=os.getpid(), creating_timestamp=datetime.now(), generated_params=None, params=params) doc.save(using=es, index=".datashader_layers", op_type="create", skip_empty=False) current_app.logger.debug("Created Hash document") except ConflictError: current_app.logger.debug("Hash document now exists, continuing") #re-fetch to get sequence number correct doc = Document.get(id=layer_id, using=es, index=".datashader_layers") #Check for generator timeouts: if doc.to_dict().get("generated_params", {}).get("generation_start_time") and \ datetime.now() > datetime.strptime(doc.to_dict().get("generated_params", {}).get("generation_start_time"),"%Y-%m-%dT%H:%M:%S.%f")+timedelta(seconds=5*60): #Something caused the worker generating the params to time out so clear that entry try: doc.update(using=es, index=".datashader_layers", retry_on_conflict=0, refresh=True, \ generated_params=None) except ConflictError: current_app.logger.debug("Abandoned resetting parameters due to conflict, other process has completed.") #Loop-check if the generated params are in missing/in-process/complete timeout_at = datetime.now()+timedelta(seconds=45) while doc.to_dict().get("generated_params", {}).get("complete", False) == False: if datetime.now() > timeout_at: current_app.logger.info("Hit timeout waiting for generated parameters to be placed into database") break #If missing, mark them as in generation if not doc.to_dict().get("generated_params", None): #Mark them as being generated but do so with concurrenty control #https://www.elastic.co/guide/en/elasticsearch/reference/current/optimistic-concurrency-control.html current_app.logger.info("Discovering generated parameters") generated_params = dict() generated_params["complete"] = False generated_params["generation_start_time"] = datetime.now() generated_params["generating_host"] = socket.gethostname() generated_params["generating_pid"] = os.getpid() try: doc.update(using=es, index=".datashader_layers", retry_on_conflict=0, refresh=True, \ generated_params=generated_params) except ConflictError: current_app.logger.debug("Abandoned generating parameters due to conflict, will wait for other process to complete.") break #Generate and save off parameters current_app.logger.warn("Discovering generated params") generated_params.update(generate_global_params(params, idx)) generated_params["generation_complete_time"] = datetime.now() generated_params["complete"] = True #Store off generated params doc.update(using=es, index=".datashader_layers", retry_on_conflict=0, refresh=True, \ generated_params=generated_params) break else: time.sleep(1) doc = Document.get(id=layer_id, using=es, index=".datashader_layers") #We now have params so use them params["generated_params"] = doc.to_dict().get("generated_params") return params