def file_complete_event(tr):
"""
Note
when a variable is complete, we know for sure that all variable's files are fetched,
because a variable is atomic, i.e. it is not possible to retrieve a subset of variable's files
(this is true because you can't select a subset of the files of a
variable with the search-API (search-API temporal n spatial filters are
at variable level without the possibility to ask a subset of the variable's files))
but a dataset can be marked as complete even if it contains only a subset of variables included in this dataset
(but still all variables that have been discovered for this dataset must be complete)
"""
sdlog.log("SYDEVENT-001","'file_complete_event' triggered (%s)"%tr.file_functional_id,event_triggered_log_level)
if sdconfig.is_event_enabled(sdconst.EVENT_FILE_COMPLETE,tr.project):
event=Event(name=sdconst.EVENT_FILE_COMPLETE)
event.project=tr.project
event.model=tr.model
event.dataset_pattern=tr.dataset.local_path
event.variable=tr.variable
event.filename_pattern=tr.filename
event.crea_date=sdtime.now()
event.priority=sdconst.DEFAULT_PRIORITY
sdeventdao.add_event(event,commit=True)
# update dataset (all except 'latest' flag)
tr.dataset.status=sddatasetflag.compute_dataset_status(tr.dataset)
tr.dataset.last_done_transfer_date=tr.end_date
sddatasetdao.update_dataset(tr.dataset)
if sdvariable.is_variable_complete(tr.dataset.dataset_id,tr.variable):
variable_complete_event(tr.project,tr.model,tr.dataset,tr.variable) # trigger 'variable complete' event
def loadTableInMemory(tablename,indexname):
global _in_memory_conn
sdlog.log("SDDATABA-INF001","loading '%s' table"%tablename)
# create a database in memory
if _in_memory_conn is None:
_in_memory_conn = sqlite3.connect(":memory:")
# attach persistent DB
_in_memory_conn.execute("ATTACH '%s' AS persistentdb"%get_db_name())
# drop table if already exists in memory
_in_memory_conn.execute("drop table if exists main.'%s'"%tablename)
# copy table from persistent DB to memory
_in_memory_conn.execute("create table main.'%s' as select * from persistentdb.[%s]"%(tablename,tablename))
# create index
_in_memory_conn.execute("create index if not exists main.'%s' on '%s' (file)"%(indexname,tablename))
# commit
_in_memory_conn.commit()
# detach persistent DB
_in_memory_conn.execute("detach persistentdb")
_in_memory_conn.row_factory = sqlite3.Row
def variable_complete_event(project,model,dataset,variable):
sdlog.log("SYDEVENT-002","'variable_complete_event' triggered (%s,%s)"%(dataset.dataset_functional_id,variable),event_triggered_log_level)
# cascade 1
if dataset.status==sdconst.DATASET_STATUS_COMPLETE:
dataset_complete_event(project,model,dataset) # trigger 'dataset complete' event
# cascade 2
if project=='CMIP5':
assert '/output/' not in dataset.path
(ds_path_output1,ds_path_output2)=sdproduct.get_output12_dataset_paths(dataset.path)
if sddatasetdao.exists_dataset(path=ds_path_output1) and sddatasetdao.exists_dataset(path=ds_path_output2):
d1=sddatasetdao.get_dataset(path=ds_path_output1)
d2=sddatasetdao.get_dataset(path=ds_path_output2)
if sdvariable.is_variable_complete(d1.dataset_id,variable) and sdvariable.is_variable_complete(d2.dataset_id,variable):
dataset_pattern=sdproduct.replace_output12_product_with_wildcard(dataset.local_path)
variable_complete_output12_event(project,model,dataset_pattern,variable) # trigger event (cross dataset event)
else:
# we also trigger the 'variable_complete_output12_event' event if the variable is over one product only (because if only one product, then output12 event is also true)
dataset_pattern=sdproduct.replace_output12_product_with_wildcard(dataset.local_path)
variable_complete_output12_event(project,model,dataset_pattern,variable) # trigger event (cross dataset event)
def output12_dataset_latest_event(project,model,dataset_pattern,commit=True):
# this event means output12 dataset has been granted latest
sdlog.log("SYDEVENT-009","'output12_dataset_latest_event' triggered (%s)"%dataset_pattern,event_triggered_log_level)
# not used
"""
def file_complete_event(tr):
"""
Note
when a variable is complete, we know for sure that all variable's files are fetched,
because a variable is atomic, i.e. it is not possible to retrieve a subset of variable's files
(this is true because you can't select a subset of the files of a
variable with the search-API (search-API temporal n spatial filters are
at variable level without the possibility to ask a subset of the variable's files))
but a dataset can be marked as complete even if it contains only a subset of variables included in this dataset
(but still all variables that have been discovered for this dataset must be complete)
"""
sdlog.log("SYDEVENT-001",
"'file_complete_event' triggered (%s)" % tr.file_functional_id,
event_triggered_log_level)
if sdconfig.is_event_enabled(sdconst.EVENT_FILE_COMPLETE, tr.project):
event = Event(name=sdconst.EVENT_FILE_COMPLETE)
event.project = tr.project
event.model = tr.model
event.dataset_pattern = tr.dataset.local_path
event.variable = tr.variable
event.filename_pattern = tr.filename
event.crea_date = sdtime.now()
event.priority = sdconst.DEFAULT_PRIORITY
sdeventdao.add_event(event, commit=True)
# update dataset (all except 'latest' flag)
tr.dataset.status = sddatasetflag.compute_dataset_status(tr.dataset)
tr.dataset.last_done_transfer_date = tr.end_date
sddatasetdao.update_dataset(tr.dataset)
if sdvariable.is_variable_complete(tr.dataset.dataset_id, tr.variable):
variable_complete_event(
tr.project, tr.model, tr.dataset,
tr.variable) # trigger 'variable complete' event
def latest_dataset_complete_event(project,
model,
dataset_pattern,
commit=True):
# this event means latest dataset has been completed (beware: no 'latest switch' event here: was latest before and still is)
sdlog.log(
"SYDEVENT-045",
"'latest_dataset_complete_event' triggered (%s)" % dataset_pattern,
event_triggered_log_level)
if project == 'CMIP5':
# CMIP5 use output12 special event
return
if project in sdconst.PROJECT_WITH_ONE_VARIABLE_PER_DATASET:
# CORDEX and CMIP6 use only variable level event
return
event = Event(name=sdconst.EVENT_LATEST_DATASET_COMPLETE)
event.project = project
event.model = model
event.dataset_pattern = dataset_pattern
event.variable = ''
event.filename_pattern = ''
event.crea_date = sdtime.now()
event.priority = sdconst.DEFAULT_PRIORITY
sdeventdao.add_event(event, commit=commit)
def non_latest_dataset_complete_output12_event(project,
model,
dataset_pattern,
commit=True):
# this event means one non-latest dataset has been completed (i.e. was not latest before and still isn't)
sdlog.log(
"SYDEVENT-007",
"'non_latest_dataset_complete_output12_event' triggered (%s)" %
dataset_pattern, event_triggered_log_level)
# not used for now
"""
event=Event(name=sdconst.EVENT_OUTPUT12_NON_LATEST_DATASET_COMPLETE)
event.project=project
event.model=model
event.dataset_pattern=dataset_pattern
event.variable=''
event.filename_pattern=''
event.crea_date=sdtime.now()
event.priority=sdconst.DEFAULT_PRIORITY
sdeventdao.add_event(event,commit=commit)
"""
pass
def dataset_latest_event(project,model,dataset_path,commit=True):
# this event means one dataset has been granted latest (i.e. was not latest before and now is)
sdlog.log("SYDEVENT-008","'dataset_latest_event' triggered (%s)"%dataset_path,event_triggered_log_level)
# not used for now
"""
event=Event(name=sdconst.EVENT_DATASET_LATEST)
event.project=project
event.model=model
event.dataset_pattern=dataset_pattern
event.variable=''
event.filename_pattern=''
event.crea_date=sdtime.now()
event.priority=sdconst.DEFAULT_PRIORITY
sdeventdao.add_event(event,commit=commit)
"""
# cascade
if project=='CMIP5':
assert '/output/' not in dataset_path
(ds_path_output1,ds_path_output2)=sdproduct.get_output12_dataset_paths(dataset_path)
if sddatasetdao.exists_dataset(path=ds_path_output1) and sddatasetdao.exists_dataset(path=ds_path_output2):
d1=sddatasetdao.get_dataset(path=ds_path_output1)
d2=sddatasetdao.get_dataset(path=ds_path_output2)
if d1.latest and d2.latest:
dataset_pattern=sdproduct.replace_output12_product_with_wildcard(dataset_path)
output12_dataset_latest_event(project,model,dataset_pattern,commit=commit) # trigger event
else:
dataset_pattern=sdproduct.replace_output12_product_with_wildcard(dataset_path)
output12_dataset_latest_event(project,model,dataset_pattern,commit=commit) # trigger event
def add_selection(us):
# compute selection checksum from scratch
l__file_checksum=computechecksum(us.getselectionfilefullpath())
if not existsSelection(us):
# add selection in database if missing
us.setChecksum(l__file_checksum)
us.setStatus(sdconst.SELECTION_STATUS_NEW)
us.setFullScan(True)
insertSelection(us) # warning: this modify us object (set PK)
else:
# selection already in database
from_db_us=fetchselection(us.getFilename()) # retrieve us from DB
us.setSelectionID(from_db_us.getSelectionID()) # copy DB id
# check if same checksums
if l__file_checksum==from_db_us.getChecksum():
# same checksum
# retrieve status
us.setStatus(from_db_us.getStatus())
us.setChecksum(from_db_us.getChecksum())
if us.getStatus()==sdconst.SELECTION_STATUS_NORMAL:
# nothing to do here (let (a) and (b) decide if we need fullscan)
pass
elif us.getStatus()==sdconst.SELECTION_STATUS_MODIFIED:
us.setFullScan(True)
elif us.getStatus()==sdconst.SELECTION_STATUS_NEW:
us.setFullScan(True)
else:
raise SDException("SYNDATSEL-ERR071","unknown status")
else:
# same checksum
# checksum differ
sdlog.log("SYNDATSEL-INF197","%s selection has been modified (marked for fullscan)"%us.getFilename())
us.setChecksum(l__file_checksum) # update checksum
us.setStatus(sdconst.SELECTION_STATUS_MODIFIED) # update status
updateselection(us)
# add selection in selection list
# TODO
_selections[us.getFilename()]=us
def dataset_complete_event(project,model,dataset,commit=True):
sdlog.log("SYDEVENT-004","'dataset_complete_event' triggered (%s)"%dataset.dataset_functional_id,event_triggered_log_level)
if project=='CMIP5':
(ds_path_output1,ds_path_output2)=sdproduct.get_output12_dataset_paths(dataset.path)
if sddatasetdao.exists_dataset(path=ds_path_output1) and sddatasetdao.exists_dataset(path=ds_path_output2):
d1=sddatasetdao.get_dataset(path=ds_path_output1)
d2=sddatasetdao.get_dataset(path=ds_path_output2)
if d1.status==sdconst.DATASET_STATUS_COMPLETE and d2.status==sdconst.DATASET_STATUS_COMPLETE:
dataset_pattern=sdproduct.replace_output12_product_with_wildcard(dataset.local_path)
dataset_complete_output12_event(project,model,dataset_pattern,commit=commit)
if d1.latest and d2.latest:
latest_dataset_complete_output12_event(project,model,dataset_pattern,commit=commit)
elif not d1.latest and not d2.latest:
non_latest_dataset_complete_output12_event(project,model,dataset_pattern,commit=commit)
else:
sdlog.warning("SYDEVENT-032","Event not triggered as one product is latest while the other product is not") # TODO: is this the right way to handle this case ?
else:
dataset_pattern=sdproduct.replace_output12_product_with_wildcard(dataset.local_path)
dataset_complete_output12_event(project,model,dataset_pattern,commit=commit)
if dataset.latest:
latest_dataset_complete_output12_event(project,model,dataset_pattern,commit=commit)
else:
non_latest_dataset_complete_output12_event(project,model,dataset_pattern,commit=commit)
# <<<--- 'latest' flag management related code begin
# store current 'latest' flag state
old_latest=dataset.latest
# TODO: check if we we switch latest flag independently for each product (meaning output1 latest can be 1 while output2 latest is 0)
# tag4342342
# compute new 'latest' flag
if not old_latest:
# old state is not latest
sddatasetflag.update_latest_flag(dataset) # warning: this method modifies the dataset in memory (and in database too)
else:
# nothing to do concerning the 'latest' flag as the current dataset is already the latest
# (the latest flag can only be switched off (i.e. to False) by *other* datasets versions, not by himself !!!)
pass
# store new 'latest' flag state
new_latest=dataset.latest
# --->>> 'latest' flag management related code end
# cascade 2
if (not old_latest) and new_latest:
dataset_latest_event(project,model,dataset.path,commit=commit) # trigger 'dataset_latest' event
def variable_complete_event(project, model, dataset, variable, commit=True):
sdlog.log(
"SYDEVENT-002", "'variable_complete_event' triggered (%s,%s)" %
(dataset.dataset_functional_id, variable), event_triggered_log_level)
if sdconfig.is_event_enabled(sdconst.EVENT_VARIABLE_COMPLETE, project):
event = Event(name=sdconst.EVENT_VARIABLE_COMPLETE)
event.project = project
event.model = model
event.dataset_pattern = dataset.local_path
event.variable = variable
event.filename_pattern = ''
event.crea_date = sdtime.now()
event.priority = sdconst.DEFAULT_PRIORITY
sdeventdao.add_event(event, commit=commit)
# cascade 1 (trigger dataset event)
if dataset.status == sdconst.DATASET_STATUS_COMPLETE:
dataset_complete_event(project, model,
dataset) # trigger 'dataset complete' event
# cascade 2 (trigger variable output12 event)
if project == 'CMIP5':
if '/output/' in dataset.path:
return
(ds_path_output1,
ds_path_output2) = sdproduct.get_output12_dataset_paths(dataset.path)
if sddatasetdao.exists_dataset(
path=ds_path_output1) and sddatasetdao.exists_dataset(
path=ds_path_output2):
d1 = sddatasetdao.get_dataset(path=ds_path_output1)
d2 = sddatasetdao.get_dataset(path=ds_path_output2)
if sdvariable.is_variable_complete(
d1.dataset_id,
variable) and sdvariable.is_variable_complete(
d2.dataset_id, variable):
dataset_pattern = sdproduct.replace_output12_product_with_wildcard(
dataset.local_path)
variable_complete_output12_event(
project, model, dataset_pattern,
variable) # trigger event (cross dataset event)
else:
# we also trigger the 'variable_complete_output12_event' event if the variable is over one product only (because if only one product, then output12 event is also true)
dataset_pattern = sdproduct.replace_output12_product_with_wildcard(
dataset.local_path)
variable_complete_output12_event(
project, model, dataset_pattern,
variable) # trigger event (cross dataset event)
def variable_complete_output12_event(project,model,dataset_pattern,variable,commit=True):
sdlog.log("SYDEVENT-003","'variable_complete_output12_event' triggered (%s,%s)"%(dataset_pattern,variable),event_triggered_log_level)
event=Event(name=sdconst.EVENT_OUTPUT12_VARIABLE_COMPLETE)
event.project=project
event.model=model
event.dataset_pattern=dataset_pattern
event.variable=variable
event.filename_pattern=''
event.crea_date=sdtime.now()
event.priority=sdconst.DEFAULT_PRIORITY
sdeventdao.add_event(event,commit=commit)
def output12_dataset_latest_event(project,
model,
dataset_pattern,
commit=True):
# this event means output12 dataset has been granted latest
sdlog.log(
"SYDEVENT-009",
"'output12_dataset_latest_event' triggered (%s)" % dataset_pattern,
event_triggered_log_level)
# not used
"""
def latest_output12_dataset_complete_event(project,model,dataset_pattern,commit=True):
# this event means latest output12 dataset has been completed (beware: no 'latest switch' event here: was latest before and still is)
sdlog.log("SYDEVENT-006","'latest_output12_dataset_complete_event' triggered (%s)"%dataset_pattern,event_triggered_log_level)
event=Event(name=sdconst.EVENT_OUTPUT12_LATEST_DATASET_COMPLETE)
event.project=project
event.model=model
event.dataset_pattern=dataset_pattern
event.variable=''
event.filename_pattern=''
event.crea_date=sdtime.now()
event.priority=sdconst.DEFAULT_PRIORITY
sdeventdao.add_event(event,commit=commit)
def dataset_latest_event(project, model, dataset_path, commit=True):
# this event means one dataset has been granted latest (i.e. was not latest before and now is)
sdlog.log("SYDEVENT-008",
"'dataset_latest_event' triggered (%s)" % dataset_path,
event_triggered_log_level)
# not used for now
"""
event=Event(name=sdconst.EVENT_DATASET_LATEST)
event.project=project
event.model=model
event.dataset_pattern=dataset_pattern
event.variable=''
event.filename_pattern=''
event.crea_date=sdtime.now()
event.priority=sdconst.DEFAULT_PRIORITY
sdeventdao.add_event(event,commit=commit)
"""
# cascade
if project == 'CMIP5':
if '/output/' in dataset_path:
return
(ds_path_output1,
ds_path_output2) = sdproduct.get_output12_dataset_paths(dataset_path)
if sddatasetdao.exists_dataset(
path=ds_path_output1) and sddatasetdao.exists_dataset(
path=ds_path_output2):
d1 = sddatasetdao.get_dataset(path=ds_path_output1)
d2 = sddatasetdao.get_dataset(path=ds_path_output2)
if d1.latest and d2.latest:
dataset_pattern = sdproduct.replace_output12_product_with_wildcard(
dataset_path)
output12_dataset_latest_event(project,
model,
dataset_pattern,
commit=commit) # trigger event
else:
dataset_pattern = sdproduct.replace_output12_product_with_wildcard(
dataset_path)
output12_dataset_latest_event(project,
model,
dataset_pattern,
commit=commit) # trigger event
def dataset_complete_output12_event(project,model,dataset_pattern,commit=True):
sdlog.log("SYDEVENT-005","'dataset_complete_output12_event' triggered (%s)"%dataset_pattern,event_triggered_log_level)
# not used
"""
event=Event(name=sdconst.EVENT_OUTPUT12_DATASET_COMPLETE)
event.project=project
event.model=model
event.dataset_pattern=dataset_pattern
event.variable=''
event.filename_pattern=''
event.crea_date=sdtime.now()
event.priority=sdconst.DEFAULT_PRIORITY
sdeventdao.add_event(event,commit=commit)
"""
pass
def variable_complete_output12_event(project,
model,
dataset_pattern,
variable,
commit=True):
sdlog.log(
"SYDEVENT-003",
"'variable_complete_output12_event' triggered (%s,%s)" %
(dataset_pattern, variable), event_triggered_log_level)
event = Event(name=sdconst.EVENT_OUTPUT12_VARIABLE_COMPLETE)
event.project = project
event.model = model
event.dataset_pattern = dataset_pattern
event.variable = variable
event.filename_pattern = ''
event.crea_date = sdtime.now()
event.priority = sdconst.DEFAULT_PRIORITY
sdeventdao.add_event(event, commit=commit)
def non_latest_dataset_complete_output12_event(project,model,dataset_pattern,commit=True):
# this event means one non-latest dataset has been completed (i.e. was not latest before and still isn't)
sdlog.log("SYDEVENT-007","'non_latest_dataset_complete_output12_event' triggered (%s)"%dataset_pattern,event_triggered_log_level)
# not used for now
"""
event=Event(name=sdconst.EVENT_OUTPUT12_NON_LATEST_DATASET_COMPLETE)
event.project=project
event.model=model
event.dataset_pattern=dataset_pattern
event.variable=''
event.filename_pattern=''
event.crea_date=sdtime.now()
event.priority=sdconst.DEFAULT_PRIORITY
sdeventdao.add_event(event,commit=commit)
"""
pass
def latest_output12_dataset_complete_event(project,
model,
dataset_pattern,
commit=True):
# this event means latest output12 dataset has been completed (beware: no 'latest switch' event here: was latest before and still is)
sdlog.log(
"SYDEVENT-006",
"'latest_output12_dataset_complete_event' triggered (%s)" %
dataset_pattern, event_triggered_log_level)
event = Event(name=sdconst.EVENT_OUTPUT12_LATEST_DATASET_COMPLETE)
event.project = project
event.model = model
event.dataset_pattern = dataset_pattern
event.variable = ''
event.filename_pattern = ''
event.crea_date = sdtime.now()
event.priority = sdconst.DEFAULT_PRIORITY
sdeventdao.add_event(event, commit=commit)
def file_complete_event(tr):
"""
Note
when a variable is complete, we know for sure that all variable's files are fetched,
because a variable is atomic, i.e. it is not possible to retrieve a subset of variable's files
(this is true because you can't select a subset of the files of a
variable with the search-API (search-API temporal n spatial filters are
at variable level without the possibility to ask a subset of the variable's files))
but a dataset can be marked as complete even if it contains only a subset of variables included in this dataset
(but still all variables that have been discovered for this dataset must be complete)
"""
sdlog.log("SYDEVENT-001","'file_complete_event' triggered (%s)"%tr.file_functional_id,event_triggered_log_level)
# update dataset (all except 'latest' flag)
tr.dataset.status=sddatasetflag.compute_dataset_status(tr.dataset)
tr.dataset.last_done_transfer_date=tr.end_date
sddatasetdao.update_dataset(tr.dataset)
if sdvariable.is_variable_complete(tr.dataset.dataset_id,tr.variable):
variable_complete_event(tr.project,tr.model,tr.dataset,tr.variable) # trigger 'variable complete' event
def dataset_latest_event(project,model,dataset_path,commit=True):
# this event means one dataset has been granted latest (i.e. was not latest before and now is)
sdlog.log("SYDEVENT-008","'dataset_latest_event' triggered (%s)"%dataset_path,event_triggered_log_level)
# cascade
if project=='CMIP5':
assert '/output/' not in dataset_path
(ds_path_output1,ds_path_output2)=sdproduct.get_output12_dataset_paths(dataset_path)
if sddatasetdao.exists_dataset(path=ds_path_output1) and sddatasetdao.exists_dataset(path=ds_path_output2):
d1=sddatasetdao.get_dataset(path=ds_path_output1)
d2=sddatasetdao.get_dataset(path=ds_path_output2)
if d1.latest and d2.latest:
dataset_pattern=sdproduct.replace_output12_product_with_wildcard(dataset_path)
dataset_latest_output12_event(project,model,dataset_pattern,commit=commit) # trigger event
else:
dataset_pattern=sdproduct.replace_output12_product_with_wildcard(dataset_path)
dataset_latest_output12_event(project,model,dataset_pattern,commit=commit) # trigger event
def dataset_complete_output12_event(project,
model,
dataset_pattern,
commit=True):
sdlog.log(
"SYDEVENT-005",
"'dataset_complete_output12_event' triggered (%s)" % dataset_pattern,
event_triggered_log_level)
# not used
"""
event=Event(name=sdconst.EVENT_OUTPUT12_DATASET_COMPLETE)
event.project=project
event.model=model
event.dataset_pattern=dataset_pattern
event.variable=''
event.filename_pattern=''
event.crea_date=sdtime.now()
event.priority=sdconst.DEFAULT_PRIORITY
sdeventdao.add_event(event,commit=commit)
"""
pass
def variable_complete_event(project,model,dataset,variable,commit=True):
sdlog.log("SYDEVENT-002","'variable_complete_event' triggered (%s,%s)"%(dataset.dataset_functional_id,variable),event_triggered_log_level)
if sdconfig.is_event_enabled(sdconst.EVENT_VARIABLE_COMPLETE,project):
event=Event(name=sdconst.EVENT_VARIABLE_COMPLETE)
event.project=project
event.model=model
event.dataset_pattern=dataset.local_path
event.variable=variable
event.filename_pattern=''
event.crea_date=sdtime.now()
event.priority=sdconst.DEFAULT_PRIORITY
sdeventdao.add_event(event,commit=commit)
# cascade 1 (trigger dataset event)
if dataset.status==sdconst.DATASET_STATUS_COMPLETE:
dataset_complete_event(project,model,dataset) # trigger 'dataset complete' event
# cascade 2 (trigger variable output12 event)
if project=='CMIP5':
assert '/output/' not in dataset.path
(ds_path_output1,ds_path_output2)=sdproduct.get_output12_dataset_paths(dataset.path)
if sddatasetdao.exists_dataset(path=ds_path_output1) and sddatasetdao.exists_dataset(path=ds_path_output2):
d1=sddatasetdao.get_dataset(path=ds_path_output1)
d2=sddatasetdao.get_dataset(path=ds_path_output2)
if sdvariable.is_variable_complete(d1.dataset_id,variable) and sdvariable.is_variable_complete(d2.dataset_id,variable):
dataset_pattern=sdproduct.replace_output12_product_with_wildcard(dataset.local_path)
variable_complete_output12_event(project,model,dataset_pattern,variable) # trigger event (cross dataset event)
else:
# we also trigger the 'variable_complete_output12_event' event if the variable is over one product only (because if only one product, then output12 event is also true)
dataset_pattern=sdproduct.replace_output12_product_with_wildcard(dataset.local_path)
variable_complete_output12_event(project,model,dataset_pattern,variable) # trigger event (cross dataset event)
def latest_dataset_complete_event(project,model,dataset_pattern,commit=True):
# this event means latest dataset has been completed (beware: no 'latest switch' event here: was latest before and still is)
sdlog.log("SYDEVENT-045","'latest_dataset_complete_event' triggered (%s)"%dataset_pattern,event_triggered_log_level)
if project=='CMIP5':
# CMIP5 use output12 special event
return
if project in sdconst.PROJECT_WITH_ONE_VARIABLE_PER_DATASET:
# CORDEX and CMIP6 use only variable level event
return
event=Event(name=sdconst.EVENT_LATEST_DATASET_COMPLETE)
event.project=project
event.model=model
event.dataset_pattern=dataset_pattern
event.variable=''
event.filename_pattern=''
event.crea_date=sdtime.now()
event.priority=sdconst.DEFAULT_PRIORITY
sdeventdao.add_event(event,commit=commit)
def dataset_latest_output12_event(project,model,dataset_pattern,commit=True):
sdlog.log("SYDEVENT-009","'dataset_latest_output12_event' triggered (%s)"%dataset_pattern,event_triggered_log_level)
# not used
"""
def dataset_complete_event(project,model,dataset,commit=True):
sdlog.log("SYDEVENT-004","'dataset_complete_event' triggered (%s)"%dataset.dataset_functional_id,event_triggered_log_level)
# not used for now
"""
event=Event(name=sdconst.EVENT_DATASET_COMPLETE)
event.project=project
event.model=model
event.dataset_pattern=dataset_pattern
event.variable=''
event.filename_pattern=''
event.crea_date=sdtime.now()
event.priority=sdconst.DEFAULT_PRIORITY
sdeventdao.add_event(event,commit=commit)
"""
# <<<--- 'latest' flag management related code begin
# store current 'latest' flag state
old_latest=dataset.latest
# TODO: check if we we switch latest flag independently for each product (meaning output1 latest can be 1 while output2 latest is 0)
# tag4342342
# compute new 'latest' flag
if not old_latest:
# old state is not latest
sddatasetflag.update_latest_flag(dataset) # warning: this method modifies the dataset object in memory (and in database too)
else:
# nothing to do concerning the 'latest' flag as the current dataset is already the latest
# (the latest flag can only be switched off (i.e. to False) by *other* datasets versions, not by himself !!!)
pass
# store new 'latest' flag state
new_latest=dataset.latest
# --->>> 'latest' flag management related code end
# cascade 1 (trigger dataset latest switch event)
if (not old_latest) and new_latest:
# latest flag has been switched from false to true
dataset_latest_event(project,model,dataset.path,commit=commit) # trigger 'dataset_latest' event
# cascade 2 (trigger latest dataset complete event)
if dataset.latest:
latest_dataset_complete_event(project,model,dataset.local_path,commit=commit)
else:
non_latest_dataset_complete_event(project,model,dataset.local_path,commit=commit)
# cascade 3 (trigger output12 dataset complete event)
if project=='CMIP5':
(ds_path_output1,ds_path_output2)=sdproduct.get_output12_dataset_paths(dataset.path)
if sddatasetdao.exists_dataset(path=ds_path_output1) and sddatasetdao.exists_dataset(path=ds_path_output2):
d1=sddatasetdao.get_dataset(path=ds_path_output1)
d2=sddatasetdao.get_dataset(path=ds_path_output2)
if d1.status==sdconst.DATASET_STATUS_COMPLETE and d2.status==sdconst.DATASET_STATUS_COMPLETE:
dataset_pattern=sdproduct.replace_output12_product_with_wildcard(dataset.local_path)
dataset_complete_output12_event(project,model,dataset_pattern,commit=commit)
else:
# only one product exists for this dataset
# not sure if this code is required.
# basically, it says that if only one product is present (output1 or output2)
# then the 'output12' is considered ready to be triggered
# (i.e. output12 does not require output1 and output2 to be present,
# it only require that if there are, they must both be complete)
#
dataset_pattern=sdproduct.replace_output12_product_with_wildcard(dataset.local_path)
dataset_complete_output12_event(project,model,dataset_pattern,commit=commit)
# cascade 4 (trigger latest output12 dataset complete event)
if project=='CMIP5':
(ds_path_output1,ds_path_output2)=sdproduct.get_output12_dataset_paths(dataset.path)
if sddatasetdao.exists_dataset(path=ds_path_output1) and sddatasetdao.exists_dataset(path=ds_path_output2):
d1=sddatasetdao.get_dataset(path=ds_path_output1)
d2=sddatasetdao.get_dataset(path=ds_path_output2)
if d1.status==sdconst.DATASET_STATUS_COMPLETE and d2.status==sdconst.DATASET_STATUS_COMPLETE:
if d1.latest and d2.latest:
latest_output12_dataset_complete_event(project,model,dataset_pattern,commit=commit)
elif not d1.latest and not d2.latest:
non_latest_dataset_complete_output12_event(project,model,dataset_pattern,commit=commit)
else:
sdlog.warning("SYDEVENT-032","Event not triggered as one product is latest while the other product is not") # TODO: is this the right way to handle this case ?
else:
# only one product exists for this dataset
# not sure if this code is required.
# basically, it says that if only one product is present (output1 or output2)
# then the 'output12' is considered ready to be triggered
# (i.e. output12 does not require output1 and output2 to be present,
# it only require that if there are, they must both be complete)
#
if dataset.latest:
latest_output12_dataset_complete_event(project,model,dataset_pattern,commit=commit)
else:
non_latest_dataset_complete_output12_event(project,model,dataset_pattern,commit=commit)
def dataset_complete_event(project, model, dataset, commit=True):
sdlog.log(
"SYDEVENT-004", "'dataset_complete_event' triggered (%s)" %
dataset.dataset_functional_id, event_triggered_log_level)
# not used for now
"""
event=Event(name=sdconst.EVENT_DATASET_COMPLETE)
event.project=project
event.model=model
event.dataset_pattern=dataset_pattern
event.variable=''
event.filename_pattern=''
event.crea_date=sdtime.now()
event.priority=sdconst.DEFAULT_PRIORITY
sdeventdao.add_event(event,commit=commit)
"""
# <<<--- 'latest' flag management related code begin
# store current 'latest' flag state
old_latest = dataset.latest
# TODO: check if we we switch latest flag independently for each product (meaning output1 latest can be 1 while output2 latest is 0)
# tag4342342
# compute new 'latest' flag
if not old_latest:
# old state is not latest
sddatasetflag.update_latest_flag(
dataset
) # warning: this method modifies the dataset object in memory (and in database too)
else:
# nothing to do concerning the 'latest' flag as the current dataset is already the latest
# (the latest flag can only be switched off (i.e. to False) by *other* datasets versions, not by himself !!!)
pass
# store new 'latest' flag state
new_latest = dataset.latest
# --->>> 'latest' flag management related code end
# cascade 1 (trigger dataset latest switch event)
if (not old_latest) and new_latest:
# latest flag has been switched from false to true
dataset_latest_event(project, model, dataset.path,
commit=commit) # trigger 'dataset_latest' event
# cascade 2 (trigger latest dataset complete event)
if dataset.latest:
latest_dataset_complete_event(project,
model,
dataset.local_path,
commit=commit)
else:
non_latest_dataset_complete_event(project,
model,
dataset.local_path,
commit=commit)
# cascade 3 (trigger output12 dataset complete event)
if project == 'CMIP5':
(ds_path_output1,
ds_path_output2) = sdproduct.get_output12_dataset_paths(dataset.path)
if sddatasetdao.exists_dataset(
path=ds_path_output1) and sddatasetdao.exists_dataset(
path=ds_path_output2):
d1 = sddatasetdao.get_dataset(path=ds_path_output1)
d2 = sddatasetdao.get_dataset(path=ds_path_output2)
if d1.status == sdconst.DATASET_STATUS_COMPLETE and d2.status == sdconst.DATASET_STATUS_COMPLETE:
dataset_pattern = sdproduct.replace_output12_product_with_wildcard(
dataset.local_path)
dataset_complete_output12_event(project,
model,
dataset_pattern,
commit=commit)
else:
# only one product exists for this dataset
# not sure if this code is required.
# basically, it says that if only one product is present (output1 or output2)
# then the 'output12' is considered ready to be triggered
# (i.e. output12 does not require output1 and output2 to be present,
# it only require that if there are, they must both be complete)
#
dataset_pattern = sdproduct.replace_output12_product_with_wildcard(
dataset.local_path)
dataset_complete_output12_event(project,
model,
dataset_pattern,
commit=commit)
# cascade 4 (trigger latest output12 dataset complete event)
if project == 'CMIP5':
(ds_path_output1,
ds_path_output2) = sdproduct.get_output12_dataset_paths(dataset.path)
if sddatasetdao.exists_dataset(
path=ds_path_output1) and sddatasetdao.exists_dataset(
path=ds_path_output2):
d1 = sddatasetdao.get_dataset(path=ds_path_output1)
d2 = sddatasetdao.get_dataset(path=ds_path_output2)
if d1.status == sdconst.DATASET_STATUS_COMPLETE and d2.status == sdconst.DATASET_STATUS_COMPLETE:
if d1.latest and d2.latest:
latest_output12_dataset_complete_event(project,
model,
dataset_pattern,
commit=commit)
elif not d1.latest and not d2.latest:
non_latest_dataset_complete_output12_event(project,
model,
dataset_pattern,
commit=commit)
else:
sdlog.warning(
"SYDEVENT-032",
"Event not triggered as one product is latest while the other product is not"
) # TODO: is this the right way to handle this case ?
else:
# only one product exists for this dataset
# not sure if this code is required.
# basically, it says that if only one product is present (output1 or output2)
# then the 'output12' is considered ready to be triggered
# (i.e. output12 does not require output1 and output2 to be present,
# it only require that if there are, they must both be complete)
#
if dataset.latest:
latest_output12_dataset_complete_event(project,
model,
dataset_pattern,
commit=commit)
else:
non_latest_dataset_complete_output12_event(project,
model,
dataset_pattern,
commit=commit)