def file_complete_event(tr):
"""
Note
when a variable is complete, we know for sure that all variable's files are fetched,
because a variable is atomic, i.e. it is not possible to retrieve a subset of variable's files
(this is true because you can't select a subset of the files of a
variable with the search-API (search-API temporal n spatial filters are
at variable level without the possibility to ask a subset of the variable's files))
but a dataset can be marked as complete even if it contains only a subset of variables included in this dataset
(but still all variables that have been discovered for this dataset must be complete)
"""
sdlog.log("SYDEVENT-001",
"'file_complete_event' triggered (%s)" % tr.file_functional_id,
event_triggered_log_level)
if sdconfig.is_event_enabled(sdconst.EVENT_FILE_COMPLETE, tr.project):
event = Event(name=sdconst.EVENT_FILE_COMPLETE)
event.project = tr.project
event.model = tr.model
event.dataset_pattern = tr.dataset.local_path
event.variable = tr.variable
event.filename_pattern = tr.filename
event.crea_date = sdtime.now()
event.priority = sdconst.DEFAULT_PRIORITY
sdeventdao.add_event(event, commit=True)
# update dataset (all except 'latest' flag)
tr.dataset.status = sddatasetflag.compute_dataset_status(tr.dataset)
tr.dataset.last_done_transfer_date = tr.end_date
sddatasetdao.update_dataset(tr.dataset)
if sdvariable.is_variable_complete(tr.dataset.dataset_id, tr.variable):
variable_complete_event(
tr.project, tr.model, tr.dataset,
tr.variable) # trigger 'variable complete' event
def latest_dataset_complete_event(project,
model,
dataset_pattern,
commit=True):
# this event means latest dataset has been completed (beware: no 'latest switch' event here: was latest before and still is)
sdlog.log(
"SYDEVENT-045",
"'latest_dataset_complete_event' triggered (%s)" % dataset_pattern,
event_triggered_log_level)
if project == 'CMIP5':
# CMIP5 use output12 special event
return
if project in sdconst.PROJECT_WITH_ONE_VARIABLE_PER_DATASET:
# CORDEX and CMIP6 use only variable level event
return
event = Event(name=sdconst.EVENT_LATEST_DATASET_COMPLETE)
event.project = project
event.model = model
event.dataset_pattern = dataset_pattern
event.variable = ''
event.filename_pattern = ''
event.crea_date = sdtime.now()
event.priority = sdconst.DEFAULT_PRIORITY
sdeventdao.add_event(event, commit=commit)
def file_complete_event(tr):
"""
Note
when a variable is complete, we know for sure that all variable's files are fetched,
because a variable is atomic, i.e. it is not possible to retrieve a subset of variable's files
(this is true because you can't select a subset of the files of a
variable with the search-API (search-API temporal n spatial filters are
at variable level without the possibility to ask a subset of the variable's files))
but a dataset can be marked as complete even if it contains only a subset of variables included in this dataset
(but still all variables that have been discovered for this dataset must be complete)
"""
sdlog.log("SYDEVENT-001","'file_complete_event' triggered (%s)"%tr.file_functional_id,event_triggered_log_level)
if sdconfig.is_event_enabled(sdconst.EVENT_FILE_COMPLETE,tr.project):
event=Event(name=sdconst.EVENT_FILE_COMPLETE)
event.project=tr.project
event.model=tr.model
event.dataset_pattern=tr.dataset.local_path
event.variable=tr.variable
event.filename_pattern=tr.filename
event.crea_date=sdtime.now()
event.priority=sdconst.DEFAULT_PRIORITY
sdeventdao.add_event(event,commit=True)
# update dataset (all except 'latest' flag)
tr.dataset.status=sddatasetflag.compute_dataset_status(tr.dataset)
tr.dataset.last_done_transfer_date=tr.end_date
sddatasetdao.update_dataset(tr.dataset)
if sdvariable.is_variable_complete(tr.dataset.dataset_id,tr.variable):
variable_complete_event(tr.project,tr.model,tr.dataset,tr.variable) # trigger 'variable complete' event
def submit(
order_name,
project,
model,
dataset,
variable='',
filename='',
commit=True
): # TODO: replace single quote with None and move 'None2SingleQuote' processing inside Event object (and add comment about why we use single quote instead of None in event table !!!)
event_name = order_name
dataset_pattern = sdproduct.replace_output12_product_with_wildcard(dataset)
filename_pattern = filename
sdlog.info(
"SDPPORDE-001",
"'%s' triggered (%s,%s)" % (event_name, dataset_pattern, variable))
event = Event(name=event_name)
event.project = project
event.model = model
event.dataset_pattern = dataset_pattern
event.variable = variable
event.filename_pattern = filename_pattern
event.crea_date = sdtime.now()
event.priority = sdconst.DEFAULT_PRIORITY
sdeventdao.add_event(event, commit=commit)
def variable_complete_event(project, model, dataset, variable, commit=True):
sdlog.log(
"SYDEVENT-002", "'variable_complete_event' triggered (%s,%s)" %
(dataset.dataset_functional_id, variable), event_triggered_log_level)
if sdconfig.is_event_enabled(sdconst.EVENT_VARIABLE_COMPLETE, project):
event = Event(name=sdconst.EVENT_VARIABLE_COMPLETE)
event.project = project
event.model = model
event.dataset_pattern = dataset.local_path
event.variable = variable
event.filename_pattern = ''
event.crea_date = sdtime.now()
event.priority = sdconst.DEFAULT_PRIORITY
sdeventdao.add_event(event, commit=commit)
# cascade 1 (trigger dataset event)
if dataset.status == sdconst.DATASET_STATUS_COMPLETE:
dataset_complete_event(project, model,
dataset) # trigger 'dataset complete' event
# cascade 2 (trigger variable output12 event)
if project == 'CMIP5':
if '/output/' in dataset.path:
return
(ds_path_output1,
ds_path_output2) = sdproduct.get_output12_dataset_paths(dataset.path)
if sddatasetdao.exists_dataset(
path=ds_path_output1) and sddatasetdao.exists_dataset(
path=ds_path_output2):
d1 = sddatasetdao.get_dataset(path=ds_path_output1)
d2 = sddatasetdao.get_dataset(path=ds_path_output2)
if sdvariable.is_variable_complete(
d1.dataset_id,
variable) and sdvariable.is_variable_complete(
d2.dataset_id, variable):
dataset_pattern = sdproduct.replace_output12_product_with_wildcard(
dataset.local_path)
variable_complete_output12_event(
project, model, dataset_pattern,
variable) # trigger event (cross dataset event)
else:
# we also trigger the 'variable_complete_output12_event' event if the variable is over one product only (because if only one product, then output12 event is also true)
dataset_pattern = sdproduct.replace_output12_product_with_wildcard(
dataset.local_path)
variable_complete_output12_event(
project, model, dataset_pattern,
variable) # trigger event (cross dataset event)
def variable_complete_output12_event(project,model,dataset_pattern,variable,commit=True):
sdlog.info("SYDEVENT-003","'variable_complete_output12_event' triggered (%s,%s)"%(dataset_pattern,variable))
event=Event(name=sdconst.EVENT_OUTPUT12_VARIABLE_COMPLETE)
event.project=project
event.model=model
event.dataset_pattern=dataset_pattern
event.variable=variable
event.filename_pattern=''
event.crea_date=sdtime.now()
event.priority=sdconst.DEFAULT_PRIORITY
sdeventdao.add_event(event,commit=commit)
def latest_dataset_complete_output12_event(project,model,dataset_pattern,commit=True):
# this event means one latest dataset has been completed (i.e. was latest before and still is)
sdlog.info("SYDEVENT-006","'latest_dataset_complete_output12_event' triggered (%s)"%dataset_pattern)
event=Event(name=sdconst.EVENT_OUTPUT12_LATEST_DATASET_COMPLETE)
event.project=project
event.model=model
event.dataset_pattern=dataset_pattern
event.variable=''
event.filename_pattern=''
event.crea_date=sdtime.now()
event.priority=sdconst.DEFAULT_PRIORITY
sdeventdao.add_event(event,commit=commit)
def variable_complete_output12_event(project,
model,
dataset_pattern,
variable,
commit=True):
sdlog.log(
"SYDEVENT-003",
"'variable_complete_output12_event' triggered (%s,%s)" %
(dataset_pattern, variable), event_triggered_log_level)
event = Event(name=sdconst.EVENT_OUTPUT12_VARIABLE_COMPLETE)
event.project = project
event.model = model
event.dataset_pattern = dataset_pattern
event.variable = variable
event.filename_pattern = ''
event.crea_date = sdtime.now()
event.priority = sdconst.DEFAULT_PRIORITY
sdeventdao.add_event(event, commit=commit)
def submit(order_name,project,model,dataset,variable='',filename='',commit=True): # TODO: replace single quote with None and move 'None2SingleQuote' processing inside Event object (and add comment about why we use single quote instead of None in event table !!!)
event_name=order_name
dataset_pattern=sdproduct.replace_output12_product_with_wildcard(dataset)
filename_pattern=filename
sdlog.info("SDPPORDE-001","'%s' triggered (%s,%s)"%(event_name,dataset_pattern,variable))
event=Event(name=event_name)
event.project=project
event.model=model
event.dataset_pattern=dataset_pattern
event.variable=variable
event.filename_pattern=filename_pattern
event.crea_date=sdtime.now()
event.priority=sdconst.DEFAULT_PRIORITY
sdeventdao.add_event(event,commit=commit)
def latest_output12_dataset_complete_event(project,
model,
dataset_pattern,
commit=True):
# this event means latest output12 dataset has been completed (beware: no 'latest switch' event here: was latest before and still is)
sdlog.log(
"SYDEVENT-006",
"'latest_output12_dataset_complete_event' triggered (%s)" %
dataset_pattern, event_triggered_log_level)
event = Event(name=sdconst.EVENT_OUTPUT12_LATEST_DATASET_COMPLETE)
event.project = project
event.model = model
event.dataset_pattern = dataset_pattern
event.variable = ''
event.filename_pattern = ''
event.crea_date = sdtime.now()
event.priority = sdconst.DEFAULT_PRIORITY
sdeventdao.add_event(event, commit=commit)
def variable_complete_event(project,model,dataset,variable,commit=True):
sdlog.log("SYDEVENT-002","'variable_complete_event' triggered (%s,%s)"%(dataset.dataset_functional_id,variable),event_triggered_log_level)
if sdconfig.is_event_enabled(sdconst.EVENT_VARIABLE_COMPLETE,project):
event=Event(name=sdconst.EVENT_VARIABLE_COMPLETE)
event.project=project
event.model=model
event.dataset_pattern=dataset.local_path
event.variable=variable
event.filename_pattern=''
event.crea_date=sdtime.now()
event.priority=sdconst.DEFAULT_PRIORITY
sdeventdao.add_event(event,commit=commit)
# cascade 1 (trigger dataset event)
if dataset.status==sdconst.DATASET_STATUS_COMPLETE:
dataset_complete_event(project,model,dataset) # trigger 'dataset complete' event
# cascade 2 (trigger variable output12 event)
if project=='CMIP5':
assert '/output/' not in dataset.path
(ds_path_output1,ds_path_output2)=sdproduct.get_output12_dataset_paths(dataset.path)
if sddatasetdao.exists_dataset(path=ds_path_output1) and sddatasetdao.exists_dataset(path=ds_path_output2):
d1=sddatasetdao.get_dataset(path=ds_path_output1)
d2=sddatasetdao.get_dataset(path=ds_path_output2)
if sdvariable.is_variable_complete(d1.dataset_id,variable) and sdvariable.is_variable_complete(d2.dataset_id,variable):
dataset_pattern=sdproduct.replace_output12_product_with_wildcard(dataset.local_path)
variable_complete_output12_event(project,model,dataset_pattern,variable) # trigger event (cross dataset event)
else:
# we also trigger the 'variable_complete_output12_event' event if the variable is over one product only (because if only one product, then output12 event is also true)
dataset_pattern=sdproduct.replace_output12_product_with_wildcard(dataset.local_path)
variable_complete_output12_event(project,model,dataset_pattern,variable) # trigger event (cross dataset event)
def latest_dataset_complete_event(project,model,dataset_pattern,commit=True):
# this event means latest dataset has been completed (beware: no 'latest switch' event here: was latest before and still is)
sdlog.log("SYDEVENT-045","'latest_dataset_complete_event' triggered (%s)"%dataset_pattern,event_triggered_log_level)
if project=='CMIP5':
# CMIP5 use output12 special event
return
if project in sdconst.PROJECT_WITH_ONE_VARIABLE_PER_DATASET:
# CORDEX and CMIP6 use only variable level event
return
event=Event(name=sdconst.EVENT_LATEST_DATASET_COMPLETE)
event.project=project
event.model=model
event.dataset_pattern=dataset_pattern
event.variable=''
event.filename_pattern=''
event.crea_date=sdtime.now()
event.priority=sdconst.DEFAULT_PRIORITY
sdeventdao.add_event(event,commit=commit)
