def file_complete_event(tr): """ Note when a variable is complete, we know for sure that all variable's files are fetched, because a variable is atomic, i.e. it is not possible to retrieve a subset of variable's files (this is true because you can't select a subset of the files of a variable with the search-API (search-API temporal n spatial filters are at variable level without the possibility to ask a subset of the variable's files)) but a dataset can be marked as complete even if it contains only a subset of variables included in this dataset (but still all variables that have been discovered for this dataset must be complete) """ sdlog.log("SYDEVENT-001","'file_complete_event' triggered (%s)"%tr.file_functional_id,event_triggered_log_level) if sdconfig.is_event_enabled(sdconst.EVENT_FILE_COMPLETE,tr.project): event=Event(name=sdconst.EVENT_FILE_COMPLETE) event.project=tr.project event.model=tr.model event.dataset_pattern=tr.dataset.local_path event.variable=tr.variable event.filename_pattern=tr.filename event.crea_date=sdtime.now() event.priority=sdconst.DEFAULT_PRIORITY sdeventdao.add_event(event,commit=True) # update dataset (all except 'latest' flag) tr.dataset.status=sddatasetflag.compute_dataset_status(tr.dataset) tr.dataset.last_done_transfer_date=tr.end_date sddatasetdao.update_dataset(tr.dataset) if sdvariable.is_variable_complete(tr.dataset.dataset_id,tr.variable): variable_complete_event(tr.project,tr.model,tr.dataset,tr.variable) # trigger 'variable complete' event
def loadTableInMemory(tablename,indexname): global _in_memory_conn sdlog.log("SDDATABA-INF001","loading '%s' table"%tablename) # create a database in memory if _in_memory_conn is None: _in_memory_conn = sqlite3.connect(":memory:") # attach persistent DB _in_memory_conn.execute("ATTACH '%s' AS persistentdb"%get_db_name()) # drop table if already exists in memory _in_memory_conn.execute("drop table if exists main.'%s'"%tablename) # copy table from persistent DB to memory _in_memory_conn.execute("create table main.'%s' as select * from persistentdb.[%s]"%(tablename,tablename)) # create index _in_memory_conn.execute("create index if not exists main.'%s' on '%s' (file)"%(indexname,tablename)) # commit _in_memory_conn.commit() # detach persistent DB _in_memory_conn.execute("detach persistentdb") _in_memory_conn.row_factory = sqlite3.Row
def variable_complete_event(project,model,dataset,variable): sdlog.log("SYDEVENT-002","'variable_complete_event' triggered (%s,%s)"%(dataset.dataset_functional_id,variable),event_triggered_log_level) # cascade 1 if dataset.status==sdconst.DATASET_STATUS_COMPLETE: dataset_complete_event(project,model,dataset) # trigger 'dataset complete' event # cascade 2 if project=='CMIP5': assert '/output/' not in dataset.path (ds_path_output1,ds_path_output2)=sdproduct.get_output12_dataset_paths(dataset.path) if sddatasetdao.exists_dataset(path=ds_path_output1) and sddatasetdao.exists_dataset(path=ds_path_output2): d1=sddatasetdao.get_dataset(path=ds_path_output1) d2=sddatasetdao.get_dataset(path=ds_path_output2) if sdvariable.is_variable_complete(d1.dataset_id,variable) and sdvariable.is_variable_complete(d2.dataset_id,variable): dataset_pattern=sdproduct.replace_output12_product_with_wildcard(dataset.local_path) variable_complete_output12_event(project,model,dataset_pattern,variable) # trigger event (cross dataset event) else: # we also trigger the 'variable_complete_output12_event' event if the variable is over one product only (because if only one product, then output12 event is also true) dataset_pattern=sdproduct.replace_output12_product_with_wildcard(dataset.local_path) variable_complete_output12_event(project,model,dataset_pattern,variable) # trigger event (cross dataset event)
def output12_dataset_latest_event(project,model,dataset_pattern,commit=True): # this event means output12 dataset has been granted latest sdlog.log("SYDEVENT-009","'output12_dataset_latest_event' triggered (%s)"%dataset_pattern,event_triggered_log_level) # not used """
def file_complete_event(tr): """ Note when a variable is complete, we know for sure that all variable's files are fetched, because a variable is atomic, i.e. it is not possible to retrieve a subset of variable's files (this is true because you can't select a subset of the files of a variable with the search-API (search-API temporal n spatial filters are at variable level without the possibility to ask a subset of the variable's files)) but a dataset can be marked as complete even if it contains only a subset of variables included in this dataset (but still all variables that have been discovered for this dataset must be complete) """ sdlog.log("SYDEVENT-001", "'file_complete_event' triggered (%s)" % tr.file_functional_id, event_triggered_log_level) if sdconfig.is_event_enabled(sdconst.EVENT_FILE_COMPLETE, tr.project): event = Event(name=sdconst.EVENT_FILE_COMPLETE) event.project = tr.project event.model = tr.model event.dataset_pattern = tr.dataset.local_path event.variable = tr.variable event.filename_pattern = tr.filename event.crea_date = sdtime.now() event.priority = sdconst.DEFAULT_PRIORITY sdeventdao.add_event(event, commit=True) # update dataset (all except 'latest' flag) tr.dataset.status = sddatasetflag.compute_dataset_status(tr.dataset) tr.dataset.last_done_transfer_date = tr.end_date sddatasetdao.update_dataset(tr.dataset) if sdvariable.is_variable_complete(tr.dataset.dataset_id, tr.variable): variable_complete_event( tr.project, tr.model, tr.dataset, tr.variable) # trigger 'variable complete' event
def latest_dataset_complete_event(project, model, dataset_pattern, commit=True): # this event means latest dataset has been completed (beware: no 'latest switch' event here: was latest before and still is) sdlog.log( "SYDEVENT-045", "'latest_dataset_complete_event' triggered (%s)" % dataset_pattern, event_triggered_log_level) if project == 'CMIP5': # CMIP5 use output12 special event return if project in sdconst.PROJECT_WITH_ONE_VARIABLE_PER_DATASET: # CORDEX and CMIP6 use only variable level event return event = Event(name=sdconst.EVENT_LATEST_DATASET_COMPLETE) event.project = project event.model = model event.dataset_pattern = dataset_pattern event.variable = '' event.filename_pattern = '' event.crea_date = sdtime.now() event.priority = sdconst.DEFAULT_PRIORITY sdeventdao.add_event(event, commit=commit)
def non_latest_dataset_complete_output12_event(project, model, dataset_pattern, commit=True): # this event means one non-latest dataset has been completed (i.e. was not latest before and still isn't) sdlog.log( "SYDEVENT-007", "'non_latest_dataset_complete_output12_event' triggered (%s)" % dataset_pattern, event_triggered_log_level) # not used for now """ event=Event(name=sdconst.EVENT_OUTPUT12_NON_LATEST_DATASET_COMPLETE) event.project=project event.model=model event.dataset_pattern=dataset_pattern event.variable='' event.filename_pattern='' event.crea_date=sdtime.now() event.priority=sdconst.DEFAULT_PRIORITY sdeventdao.add_event(event,commit=commit) """ pass
def dataset_latest_event(project,model,dataset_path,commit=True): # this event means one dataset has been granted latest (i.e. was not latest before and now is) sdlog.log("SYDEVENT-008","'dataset_latest_event' triggered (%s)"%dataset_path,event_triggered_log_level) # not used for now """ event=Event(name=sdconst.EVENT_DATASET_LATEST) event.project=project event.model=model event.dataset_pattern=dataset_pattern event.variable='' event.filename_pattern='' event.crea_date=sdtime.now() event.priority=sdconst.DEFAULT_PRIORITY sdeventdao.add_event(event,commit=commit) """ # cascade if project=='CMIP5': assert '/output/' not in dataset_path (ds_path_output1,ds_path_output2)=sdproduct.get_output12_dataset_paths(dataset_path) if sddatasetdao.exists_dataset(path=ds_path_output1) and sddatasetdao.exists_dataset(path=ds_path_output2): d1=sddatasetdao.get_dataset(path=ds_path_output1) d2=sddatasetdao.get_dataset(path=ds_path_output2) if d1.latest and d2.latest: dataset_pattern=sdproduct.replace_output12_product_with_wildcard(dataset_path) output12_dataset_latest_event(project,model,dataset_pattern,commit=commit) # trigger event else: dataset_pattern=sdproduct.replace_output12_product_with_wildcard(dataset_path) output12_dataset_latest_event(project,model,dataset_pattern,commit=commit) # trigger event
def add_selection(us): # compute selection checksum from scratch l__file_checksum=computechecksum(us.getselectionfilefullpath()) if not existsSelection(us): # add selection in database if missing us.setChecksum(l__file_checksum) us.setStatus(sdconst.SELECTION_STATUS_NEW) us.setFullScan(True) insertSelection(us) # warning: this modify us object (set PK) else: # selection already in database from_db_us=fetchselection(us.getFilename()) # retrieve us from DB us.setSelectionID(from_db_us.getSelectionID()) # copy DB id # check if same checksums if l__file_checksum==from_db_us.getChecksum(): # same checksum # retrieve status us.setStatus(from_db_us.getStatus()) us.setChecksum(from_db_us.getChecksum()) if us.getStatus()==sdconst.SELECTION_STATUS_NORMAL: # nothing to do here (let (a) and (b) decide if we need fullscan) pass elif us.getStatus()==sdconst.SELECTION_STATUS_MODIFIED: us.setFullScan(True) elif us.getStatus()==sdconst.SELECTION_STATUS_NEW: us.setFullScan(True) else: raise SDException("SYNDATSEL-ERR071","unknown status") else: # same checksum # checksum differ sdlog.log("SYNDATSEL-INF197","%s selection has been modified (marked for fullscan)"%us.getFilename()) us.setChecksum(l__file_checksum) # update checksum us.setStatus(sdconst.SELECTION_STATUS_MODIFIED) # update status updateselection(us) # add selection in selection list # TODO _selections[us.getFilename()]=us
def dataset_complete_event(project,model,dataset,commit=True): sdlog.log("SYDEVENT-004","'dataset_complete_event' triggered (%s)"%dataset.dataset_functional_id,event_triggered_log_level) if project=='CMIP5': (ds_path_output1,ds_path_output2)=sdproduct.get_output12_dataset_paths(dataset.path) if sddatasetdao.exists_dataset(path=ds_path_output1) and sddatasetdao.exists_dataset(path=ds_path_output2): d1=sddatasetdao.get_dataset(path=ds_path_output1) d2=sddatasetdao.get_dataset(path=ds_path_output2) if d1.status==sdconst.DATASET_STATUS_COMPLETE and d2.status==sdconst.DATASET_STATUS_COMPLETE: dataset_pattern=sdproduct.replace_output12_product_with_wildcard(dataset.local_path) dataset_complete_output12_event(project,model,dataset_pattern,commit=commit) if d1.latest and d2.latest: latest_dataset_complete_output12_event(project,model,dataset_pattern,commit=commit) elif not d1.latest and not d2.latest: non_latest_dataset_complete_output12_event(project,model,dataset_pattern,commit=commit) else: sdlog.warning("SYDEVENT-032","Event not triggered as one product is latest while the other product is not") # TODO: is this the right way to handle this case ? else: dataset_pattern=sdproduct.replace_output12_product_with_wildcard(dataset.local_path) dataset_complete_output12_event(project,model,dataset_pattern,commit=commit) if dataset.latest: latest_dataset_complete_output12_event(project,model,dataset_pattern,commit=commit) else: non_latest_dataset_complete_output12_event(project,model,dataset_pattern,commit=commit) # <<<--- 'latest' flag management related code begin # store current 'latest' flag state old_latest=dataset.latest # TODO: check if we we switch latest flag independently for each product (meaning output1 latest can be 1 while output2 latest is 0) # tag4342342 # compute new 'latest' flag if not old_latest: # old state is not latest sddatasetflag.update_latest_flag(dataset) # warning: this method modifies the dataset in memory (and in database too) else: # nothing to do concerning the 'latest' flag as the current dataset is already the latest # (the latest flag can only be switched off (i.e. to False) by *other* datasets versions, not by himself !!!) pass # store new 'latest' flag state new_latest=dataset.latest # --->>> 'latest' flag management related code end # cascade 2 if (not old_latest) and new_latest: dataset_latest_event(project,model,dataset.path,commit=commit) # trigger 'dataset_latest' event
def variable_complete_event(project, model, dataset, variable, commit=True): sdlog.log( "SYDEVENT-002", "'variable_complete_event' triggered (%s,%s)" % (dataset.dataset_functional_id, variable), event_triggered_log_level) if sdconfig.is_event_enabled(sdconst.EVENT_VARIABLE_COMPLETE, project): event = Event(name=sdconst.EVENT_VARIABLE_COMPLETE) event.project = project event.model = model event.dataset_pattern = dataset.local_path event.variable = variable event.filename_pattern = '' event.crea_date = sdtime.now() event.priority = sdconst.DEFAULT_PRIORITY sdeventdao.add_event(event, commit=commit) # cascade 1 (trigger dataset event) if dataset.status == sdconst.DATASET_STATUS_COMPLETE: dataset_complete_event(project, model, dataset) # trigger 'dataset complete' event # cascade 2 (trigger variable output12 event) if project == 'CMIP5': if '/output/' in dataset.path: return (ds_path_output1, ds_path_output2) = sdproduct.get_output12_dataset_paths(dataset.path) if sddatasetdao.exists_dataset( path=ds_path_output1) and sddatasetdao.exists_dataset( path=ds_path_output2): d1 = sddatasetdao.get_dataset(path=ds_path_output1) d2 = sddatasetdao.get_dataset(path=ds_path_output2) if sdvariable.is_variable_complete( d1.dataset_id, variable) and sdvariable.is_variable_complete( d2.dataset_id, variable): dataset_pattern = sdproduct.replace_output12_product_with_wildcard( dataset.local_path) variable_complete_output12_event( project, model, dataset_pattern, variable) # trigger event (cross dataset event) else: # we also trigger the 'variable_complete_output12_event' event if the variable is over one product only (because if only one product, then output12 event is also true) dataset_pattern = sdproduct.replace_output12_product_with_wildcard( dataset.local_path) variable_complete_output12_event( project, model, dataset_pattern, variable) # trigger event (cross dataset event)
def variable_complete_output12_event(project,model,dataset_pattern,variable,commit=True): sdlog.log("SYDEVENT-003","'variable_complete_output12_event' triggered (%s,%s)"%(dataset_pattern,variable),event_triggered_log_level) event=Event(name=sdconst.EVENT_OUTPUT12_VARIABLE_COMPLETE) event.project=project event.model=model event.dataset_pattern=dataset_pattern event.variable=variable event.filename_pattern='' event.crea_date=sdtime.now() event.priority=sdconst.DEFAULT_PRIORITY sdeventdao.add_event(event,commit=commit)
def output12_dataset_latest_event(project, model, dataset_pattern, commit=True): # this event means output12 dataset has been granted latest sdlog.log( "SYDEVENT-009", "'output12_dataset_latest_event' triggered (%s)" % dataset_pattern, event_triggered_log_level) # not used """
def latest_output12_dataset_complete_event(project,model,dataset_pattern,commit=True): # this event means latest output12 dataset has been completed (beware: no 'latest switch' event here: was latest before and still is) sdlog.log("SYDEVENT-006","'latest_output12_dataset_complete_event' triggered (%s)"%dataset_pattern,event_triggered_log_level) event=Event(name=sdconst.EVENT_OUTPUT12_LATEST_DATASET_COMPLETE) event.project=project event.model=model event.dataset_pattern=dataset_pattern event.variable='' event.filename_pattern='' event.crea_date=sdtime.now() event.priority=sdconst.DEFAULT_PRIORITY sdeventdao.add_event(event,commit=commit)
def dataset_latest_event(project, model, dataset_path, commit=True): # this event means one dataset has been granted latest (i.e. was not latest before and now is) sdlog.log("SYDEVENT-008", "'dataset_latest_event' triggered (%s)" % dataset_path, event_triggered_log_level) # not used for now """ event=Event(name=sdconst.EVENT_DATASET_LATEST) event.project=project event.model=model event.dataset_pattern=dataset_pattern event.variable='' event.filename_pattern='' event.crea_date=sdtime.now() event.priority=sdconst.DEFAULT_PRIORITY sdeventdao.add_event(event,commit=commit) """ # cascade if project == 'CMIP5': if '/output/' in dataset_path: return (ds_path_output1, ds_path_output2) = sdproduct.get_output12_dataset_paths(dataset_path) if sddatasetdao.exists_dataset( path=ds_path_output1) and sddatasetdao.exists_dataset( path=ds_path_output2): d1 = sddatasetdao.get_dataset(path=ds_path_output1) d2 = sddatasetdao.get_dataset(path=ds_path_output2) if d1.latest and d2.latest: dataset_pattern = sdproduct.replace_output12_product_with_wildcard( dataset_path) output12_dataset_latest_event(project, model, dataset_pattern, commit=commit) # trigger event else: dataset_pattern = sdproduct.replace_output12_product_with_wildcard( dataset_path) output12_dataset_latest_event(project, model, dataset_pattern, commit=commit) # trigger event
def dataset_complete_output12_event(project,model,dataset_pattern,commit=True): sdlog.log("SYDEVENT-005","'dataset_complete_output12_event' triggered (%s)"%dataset_pattern,event_triggered_log_level) # not used """ event=Event(name=sdconst.EVENT_OUTPUT12_DATASET_COMPLETE) event.project=project event.model=model event.dataset_pattern=dataset_pattern event.variable='' event.filename_pattern='' event.crea_date=sdtime.now() event.priority=sdconst.DEFAULT_PRIORITY sdeventdao.add_event(event,commit=commit) """ pass
def variable_complete_output12_event(project, model, dataset_pattern, variable, commit=True): sdlog.log( "SYDEVENT-003", "'variable_complete_output12_event' triggered (%s,%s)" % (dataset_pattern, variable), event_triggered_log_level) event = Event(name=sdconst.EVENT_OUTPUT12_VARIABLE_COMPLETE) event.project = project event.model = model event.dataset_pattern = dataset_pattern event.variable = variable event.filename_pattern = '' event.crea_date = sdtime.now() event.priority = sdconst.DEFAULT_PRIORITY sdeventdao.add_event(event, commit=commit)
def non_latest_dataset_complete_output12_event(project,model,dataset_pattern,commit=True): # this event means one non-latest dataset has been completed (i.e. was not latest before and still isn't) sdlog.log("SYDEVENT-007","'non_latest_dataset_complete_output12_event' triggered (%s)"%dataset_pattern,event_triggered_log_level) # not used for now """ event=Event(name=sdconst.EVENT_OUTPUT12_NON_LATEST_DATASET_COMPLETE) event.project=project event.model=model event.dataset_pattern=dataset_pattern event.variable='' event.filename_pattern='' event.crea_date=sdtime.now() event.priority=sdconst.DEFAULT_PRIORITY sdeventdao.add_event(event,commit=commit) """ pass
def latest_output12_dataset_complete_event(project, model, dataset_pattern, commit=True): # this event means latest output12 dataset has been completed (beware: no 'latest switch' event here: was latest before and still is) sdlog.log( "SYDEVENT-006", "'latest_output12_dataset_complete_event' triggered (%s)" % dataset_pattern, event_triggered_log_level) event = Event(name=sdconst.EVENT_OUTPUT12_LATEST_DATASET_COMPLETE) event.project = project event.model = model event.dataset_pattern = dataset_pattern event.variable = '' event.filename_pattern = '' event.crea_date = sdtime.now() event.priority = sdconst.DEFAULT_PRIORITY sdeventdao.add_event(event, commit=commit)
def file_complete_event(tr): """ Note when a variable is complete, we know for sure that all variable's files are fetched, because a variable is atomic, i.e. it is not possible to retrieve a subset of variable's files (this is true because you can't select a subset of the files of a variable with the search-API (search-API temporal n spatial filters are at variable level without the possibility to ask a subset of the variable's files)) but a dataset can be marked as complete even if it contains only a subset of variables included in this dataset (but still all variables that have been discovered for this dataset must be complete) """ sdlog.log("SYDEVENT-001","'file_complete_event' triggered (%s)"%tr.file_functional_id,event_triggered_log_level) # update dataset (all except 'latest' flag) tr.dataset.status=sddatasetflag.compute_dataset_status(tr.dataset) tr.dataset.last_done_transfer_date=tr.end_date sddatasetdao.update_dataset(tr.dataset) if sdvariable.is_variable_complete(tr.dataset.dataset_id,tr.variable): variable_complete_event(tr.project,tr.model,tr.dataset,tr.variable) # trigger 'variable complete' event
def dataset_latest_event(project,model,dataset_path,commit=True): # this event means one dataset has been granted latest (i.e. was not latest before and now is) sdlog.log("SYDEVENT-008","'dataset_latest_event' triggered (%s)"%dataset_path,event_triggered_log_level) # cascade if project=='CMIP5': assert '/output/' not in dataset_path (ds_path_output1,ds_path_output2)=sdproduct.get_output12_dataset_paths(dataset_path) if sddatasetdao.exists_dataset(path=ds_path_output1) and sddatasetdao.exists_dataset(path=ds_path_output2): d1=sddatasetdao.get_dataset(path=ds_path_output1) d2=sddatasetdao.get_dataset(path=ds_path_output2) if d1.latest and d2.latest: dataset_pattern=sdproduct.replace_output12_product_with_wildcard(dataset_path) dataset_latest_output12_event(project,model,dataset_pattern,commit=commit) # trigger event else: dataset_pattern=sdproduct.replace_output12_product_with_wildcard(dataset_path) dataset_latest_output12_event(project,model,dataset_pattern,commit=commit) # trigger event
def dataset_complete_output12_event(project, model, dataset_pattern, commit=True): sdlog.log( "SYDEVENT-005", "'dataset_complete_output12_event' triggered (%s)" % dataset_pattern, event_triggered_log_level) # not used """ event=Event(name=sdconst.EVENT_OUTPUT12_DATASET_COMPLETE) event.project=project event.model=model event.dataset_pattern=dataset_pattern event.variable='' event.filename_pattern='' event.crea_date=sdtime.now() event.priority=sdconst.DEFAULT_PRIORITY sdeventdao.add_event(event,commit=commit) """ pass
def variable_complete_event(project,model,dataset,variable,commit=True): sdlog.log("SYDEVENT-002","'variable_complete_event' triggered (%s,%s)"%(dataset.dataset_functional_id,variable),event_triggered_log_level) if sdconfig.is_event_enabled(sdconst.EVENT_VARIABLE_COMPLETE,project): event=Event(name=sdconst.EVENT_VARIABLE_COMPLETE) event.project=project event.model=model event.dataset_pattern=dataset.local_path event.variable=variable event.filename_pattern='' event.crea_date=sdtime.now() event.priority=sdconst.DEFAULT_PRIORITY sdeventdao.add_event(event,commit=commit) # cascade 1 (trigger dataset event) if dataset.status==sdconst.DATASET_STATUS_COMPLETE: dataset_complete_event(project,model,dataset) # trigger 'dataset complete' event # cascade 2 (trigger variable output12 event) if project=='CMIP5': assert '/output/' not in dataset.path (ds_path_output1,ds_path_output2)=sdproduct.get_output12_dataset_paths(dataset.path) if sddatasetdao.exists_dataset(path=ds_path_output1) and sddatasetdao.exists_dataset(path=ds_path_output2): d1=sddatasetdao.get_dataset(path=ds_path_output1) d2=sddatasetdao.get_dataset(path=ds_path_output2) if sdvariable.is_variable_complete(d1.dataset_id,variable) and sdvariable.is_variable_complete(d2.dataset_id,variable): dataset_pattern=sdproduct.replace_output12_product_with_wildcard(dataset.local_path) variable_complete_output12_event(project,model,dataset_pattern,variable) # trigger event (cross dataset event) else: # we also trigger the 'variable_complete_output12_event' event if the variable is over one product only (because if only one product, then output12 event is also true) dataset_pattern=sdproduct.replace_output12_product_with_wildcard(dataset.local_path) variable_complete_output12_event(project,model,dataset_pattern,variable) # trigger event (cross dataset event)
def latest_dataset_complete_event(project,model,dataset_pattern,commit=True): # this event means latest dataset has been completed (beware: no 'latest switch' event here: was latest before and still is) sdlog.log("SYDEVENT-045","'latest_dataset_complete_event' triggered (%s)"%dataset_pattern,event_triggered_log_level) if project=='CMIP5': # CMIP5 use output12 special event return if project in sdconst.PROJECT_WITH_ONE_VARIABLE_PER_DATASET: # CORDEX and CMIP6 use only variable level event return event=Event(name=sdconst.EVENT_LATEST_DATASET_COMPLETE) event.project=project event.model=model event.dataset_pattern=dataset_pattern event.variable='' event.filename_pattern='' event.crea_date=sdtime.now() event.priority=sdconst.DEFAULT_PRIORITY sdeventdao.add_event(event,commit=commit)
def dataset_latest_output12_event(project,model,dataset_pattern,commit=True): sdlog.log("SYDEVENT-009","'dataset_latest_output12_event' triggered (%s)"%dataset_pattern,event_triggered_log_level) # not used """
def dataset_complete_event(project,model,dataset,commit=True): sdlog.log("SYDEVENT-004","'dataset_complete_event' triggered (%s)"%dataset.dataset_functional_id,event_triggered_log_level) # not used for now """ event=Event(name=sdconst.EVENT_DATASET_COMPLETE) event.project=project event.model=model event.dataset_pattern=dataset_pattern event.variable='' event.filename_pattern='' event.crea_date=sdtime.now() event.priority=sdconst.DEFAULT_PRIORITY sdeventdao.add_event(event,commit=commit) """ # <<<--- 'latest' flag management related code begin # store current 'latest' flag state old_latest=dataset.latest # TODO: check if we we switch latest flag independently for each product (meaning output1 latest can be 1 while output2 latest is 0) # tag4342342 # compute new 'latest' flag if not old_latest: # old state is not latest sddatasetflag.update_latest_flag(dataset) # warning: this method modifies the dataset object in memory (and in database too) else: # nothing to do concerning the 'latest' flag as the current dataset is already the latest # (the latest flag can only be switched off (i.e. to False) by *other* datasets versions, not by himself !!!) pass # store new 'latest' flag state new_latest=dataset.latest # --->>> 'latest' flag management related code end # cascade 1 (trigger dataset latest switch event) if (not old_latest) and new_latest: # latest flag has been switched from false to true dataset_latest_event(project,model,dataset.path,commit=commit) # trigger 'dataset_latest' event # cascade 2 (trigger latest dataset complete event) if dataset.latest: latest_dataset_complete_event(project,model,dataset.local_path,commit=commit) else: non_latest_dataset_complete_event(project,model,dataset.local_path,commit=commit) # cascade 3 (trigger output12 dataset complete event) if project=='CMIP5': (ds_path_output1,ds_path_output2)=sdproduct.get_output12_dataset_paths(dataset.path) if sddatasetdao.exists_dataset(path=ds_path_output1) and sddatasetdao.exists_dataset(path=ds_path_output2): d1=sddatasetdao.get_dataset(path=ds_path_output1) d2=sddatasetdao.get_dataset(path=ds_path_output2) if d1.status==sdconst.DATASET_STATUS_COMPLETE and d2.status==sdconst.DATASET_STATUS_COMPLETE: dataset_pattern=sdproduct.replace_output12_product_with_wildcard(dataset.local_path) dataset_complete_output12_event(project,model,dataset_pattern,commit=commit) else: # only one product exists for this dataset # not sure if this code is required. # basically, it says that if only one product is present (output1 or output2) # then the 'output12' is considered ready to be triggered # (i.e. output12 does not require output1 and output2 to be present, # it only require that if there are, they must both be complete) # dataset_pattern=sdproduct.replace_output12_product_with_wildcard(dataset.local_path) dataset_complete_output12_event(project,model,dataset_pattern,commit=commit) # cascade 4 (trigger latest output12 dataset complete event) if project=='CMIP5': (ds_path_output1,ds_path_output2)=sdproduct.get_output12_dataset_paths(dataset.path) if sddatasetdao.exists_dataset(path=ds_path_output1) and sddatasetdao.exists_dataset(path=ds_path_output2): d1=sddatasetdao.get_dataset(path=ds_path_output1) d2=sddatasetdao.get_dataset(path=ds_path_output2) if d1.status==sdconst.DATASET_STATUS_COMPLETE and d2.status==sdconst.DATASET_STATUS_COMPLETE: if d1.latest and d2.latest: latest_output12_dataset_complete_event(project,model,dataset_pattern,commit=commit) elif not d1.latest and not d2.latest: non_latest_dataset_complete_output12_event(project,model,dataset_pattern,commit=commit) else: sdlog.warning("SYDEVENT-032","Event not triggered as one product is latest while the other product is not") # TODO: is this the right way to handle this case ? else: # only one product exists for this dataset # not sure if this code is required. # basically, it says that if only one product is present (output1 or output2) # then the 'output12' is considered ready to be triggered # (i.e. output12 does not require output1 and output2 to be present, # it only require that if there are, they must both be complete) # if dataset.latest: latest_output12_dataset_complete_event(project,model,dataset_pattern,commit=commit) else: non_latest_dataset_complete_output12_event(project,model,dataset_pattern,commit=commit)
def dataset_complete_event(project, model, dataset, commit=True): sdlog.log( "SYDEVENT-004", "'dataset_complete_event' triggered (%s)" % dataset.dataset_functional_id, event_triggered_log_level) # not used for now """ event=Event(name=sdconst.EVENT_DATASET_COMPLETE) event.project=project event.model=model event.dataset_pattern=dataset_pattern event.variable='' event.filename_pattern='' event.crea_date=sdtime.now() event.priority=sdconst.DEFAULT_PRIORITY sdeventdao.add_event(event,commit=commit) """ # <<<--- 'latest' flag management related code begin # store current 'latest' flag state old_latest = dataset.latest # TODO: check if we we switch latest flag independently for each product (meaning output1 latest can be 1 while output2 latest is 0) # tag4342342 # compute new 'latest' flag if not old_latest: # old state is not latest sddatasetflag.update_latest_flag( dataset ) # warning: this method modifies the dataset object in memory (and in database too) else: # nothing to do concerning the 'latest' flag as the current dataset is already the latest # (the latest flag can only be switched off (i.e. to False) by *other* datasets versions, not by himself !!!) pass # store new 'latest' flag state new_latest = dataset.latest # --->>> 'latest' flag management related code end # cascade 1 (trigger dataset latest switch event) if (not old_latest) and new_latest: # latest flag has been switched from false to true dataset_latest_event(project, model, dataset.path, commit=commit) # trigger 'dataset_latest' event # cascade 2 (trigger latest dataset complete event) if dataset.latest: latest_dataset_complete_event(project, model, dataset.local_path, commit=commit) else: non_latest_dataset_complete_event(project, model, dataset.local_path, commit=commit) # cascade 3 (trigger output12 dataset complete event) if project == 'CMIP5': (ds_path_output1, ds_path_output2) = sdproduct.get_output12_dataset_paths(dataset.path) if sddatasetdao.exists_dataset( path=ds_path_output1) and sddatasetdao.exists_dataset( path=ds_path_output2): d1 = sddatasetdao.get_dataset(path=ds_path_output1) d2 = sddatasetdao.get_dataset(path=ds_path_output2) if d1.status == sdconst.DATASET_STATUS_COMPLETE and d2.status == sdconst.DATASET_STATUS_COMPLETE: dataset_pattern = sdproduct.replace_output12_product_with_wildcard( dataset.local_path) dataset_complete_output12_event(project, model, dataset_pattern, commit=commit) else: # only one product exists for this dataset # not sure if this code is required. # basically, it says that if only one product is present (output1 or output2) # then the 'output12' is considered ready to be triggered # (i.e. output12 does not require output1 and output2 to be present, # it only require that if there are, they must both be complete) # dataset_pattern = sdproduct.replace_output12_product_with_wildcard( dataset.local_path) dataset_complete_output12_event(project, model, dataset_pattern, commit=commit) # cascade 4 (trigger latest output12 dataset complete event) if project == 'CMIP5': (ds_path_output1, ds_path_output2) = sdproduct.get_output12_dataset_paths(dataset.path) if sddatasetdao.exists_dataset( path=ds_path_output1) and sddatasetdao.exists_dataset( path=ds_path_output2): d1 = sddatasetdao.get_dataset(path=ds_path_output1) d2 = sddatasetdao.get_dataset(path=ds_path_output2) if d1.status == sdconst.DATASET_STATUS_COMPLETE and d2.status == sdconst.DATASET_STATUS_COMPLETE: if d1.latest and d2.latest: latest_output12_dataset_complete_event(project, model, dataset_pattern, commit=commit) elif not d1.latest and not d2.latest: non_latest_dataset_complete_output12_event(project, model, dataset_pattern, commit=commit) else: sdlog.warning( "SYDEVENT-032", "Event not triggered as one product is latest while the other product is not" ) # TODO: is this the right way to handle this case ? else: # only one product exists for this dataset # not sure if this code is required. # basically, it says that if only one product is present (output1 or output2) # then the 'output12' is considered ready to be triggered # (i.e. output12 does not require output1 and output2 to be present, # it only require that if there are, they must both be complete) # if dataset.latest: latest_output12_dataset_complete_event(project, model, dataset_pattern, commit=commit) else: non_latest_dataset_complete_output12_event(project, model, dataset_pattern, commit=commit)