def file_complete_event(tr): """ Note when a variable is complete, we know for sure that all variable's files are fetched, because a variable is atomic, i.e. it is not possible to retrieve a subset of variable's files (this is true because you can't select a subset of the files of a variable with the search-API (search-API temporal n spatial filters are at variable level without the possibility to ask a subset of the variable's files)) but a dataset can be marked as complete even if it contains only a subset of variables included in this dataset (but still all variables that have been discovered for this dataset must be complete) """ sdlog.log("SYDEVENT-001", "'file_complete_event' triggered (%s)" % tr.file_functional_id, event_triggered_log_level) if sdconfig.is_event_enabled(sdconst.EVENT_FILE_COMPLETE, tr.project): event = Event(name=sdconst.EVENT_FILE_COMPLETE) event.project = tr.project event.model = tr.model event.dataset_pattern = tr.dataset.local_path event.variable = tr.variable event.filename_pattern = tr.filename event.crea_date = sdtime.now() event.priority = sdconst.DEFAULT_PRIORITY sdeventdao.add_event(event, commit=True) # update dataset (all except 'latest' flag) tr.dataset.status = sddatasetflag.compute_dataset_status(tr.dataset) tr.dataset.last_done_transfer_date = tr.end_date sddatasetdao.update_dataset(tr.dataset) if sdvariable.is_variable_complete(tr.dataset.dataset_id, tr.variable): variable_complete_event( tr.project, tr.model, tr.dataset, tr.variable) # trigger 'variable complete' event
def latest_dataset_complete_event(project, model, dataset_pattern, commit=True): # this event means latest dataset has been completed (beware: no 'latest switch' event here: was latest before and still is) sdlog.log( "SYDEVENT-045", "'latest_dataset_complete_event' triggered (%s)" % dataset_pattern, event_triggered_log_level) if project == 'CMIP5': # CMIP5 use output12 special event return if project in sdconst.PROJECT_WITH_ONE_VARIABLE_PER_DATASET: # CORDEX and CMIP6 use only variable level event return event = Event(name=sdconst.EVENT_LATEST_DATASET_COMPLETE) event.project = project event.model = model event.dataset_pattern = dataset_pattern event.variable = '' event.filename_pattern = '' event.crea_date = sdtime.now() event.priority = sdconst.DEFAULT_PRIORITY sdeventdao.add_event(event, commit=commit)
def file_complete_event(tr): """ Note when a variable is complete, we know for sure that all variable's files are fetched, because a variable is atomic, i.e. it is not possible to retrieve a subset of variable's files (this is true because you can't select a subset of the files of a variable with the search-API (search-API temporal n spatial filters are at variable level without the possibility to ask a subset of the variable's files)) but a dataset can be marked as complete even if it contains only a subset of variables included in this dataset (but still all variables that have been discovered for this dataset must be complete) """ sdlog.log("SYDEVENT-001","'file_complete_event' triggered (%s)"%tr.file_functional_id,event_triggered_log_level) if sdconfig.is_event_enabled(sdconst.EVENT_FILE_COMPLETE,tr.project): event=Event(name=sdconst.EVENT_FILE_COMPLETE) event.project=tr.project event.model=tr.model event.dataset_pattern=tr.dataset.local_path event.variable=tr.variable event.filename_pattern=tr.filename event.crea_date=sdtime.now() event.priority=sdconst.DEFAULT_PRIORITY sdeventdao.add_event(event,commit=True) # update dataset (all except 'latest' flag) tr.dataset.status=sddatasetflag.compute_dataset_status(tr.dataset) tr.dataset.last_done_transfer_date=tr.end_date sddatasetdao.update_dataset(tr.dataset) if sdvariable.is_variable_complete(tr.dataset.dataset_id,tr.variable): variable_complete_event(tr.project,tr.model,tr.dataset,tr.variable) # trigger 'variable complete' event
def submit( order_name, project, model, dataset, variable='', filename='', commit=True ): # TODO: replace single quote with None and move 'None2SingleQuote' processing inside Event object (and add comment about why we use single quote instead of None in event table !!!) event_name = order_name dataset_pattern = sdproduct.replace_output12_product_with_wildcard(dataset) filename_pattern = filename sdlog.info( "SDPPORDE-001", "'%s' triggered (%s,%s)" % (event_name, dataset_pattern, variable)) event = Event(name=event_name) event.project = project event.model = model event.dataset_pattern = dataset_pattern event.variable = variable event.filename_pattern = filename_pattern event.crea_date = sdtime.now() event.priority = sdconst.DEFAULT_PRIORITY sdeventdao.add_event(event, commit=commit)
def variable_complete_event(project, model, dataset, variable, commit=True): sdlog.log( "SYDEVENT-002", "'variable_complete_event' triggered (%s,%s)" % (dataset.dataset_functional_id, variable), event_triggered_log_level) if sdconfig.is_event_enabled(sdconst.EVENT_VARIABLE_COMPLETE, project): event = Event(name=sdconst.EVENT_VARIABLE_COMPLETE) event.project = project event.model = model event.dataset_pattern = dataset.local_path event.variable = variable event.filename_pattern = '' event.crea_date = sdtime.now() event.priority = sdconst.DEFAULT_PRIORITY sdeventdao.add_event(event, commit=commit) # cascade 1 (trigger dataset event) if dataset.status == sdconst.DATASET_STATUS_COMPLETE: dataset_complete_event(project, model, dataset) # trigger 'dataset complete' event # cascade 2 (trigger variable output12 event) if project == 'CMIP5': if '/output/' in dataset.path: return (ds_path_output1, ds_path_output2) = sdproduct.get_output12_dataset_paths(dataset.path) if sddatasetdao.exists_dataset( path=ds_path_output1) and sddatasetdao.exists_dataset( path=ds_path_output2): d1 = sddatasetdao.get_dataset(path=ds_path_output1) d2 = sddatasetdao.get_dataset(path=ds_path_output2) if sdvariable.is_variable_complete( d1.dataset_id, variable) and sdvariable.is_variable_complete( d2.dataset_id, variable): dataset_pattern = sdproduct.replace_output12_product_with_wildcard( dataset.local_path) variable_complete_output12_event( project, model, dataset_pattern, variable) # trigger event (cross dataset event) else: # we also trigger the 'variable_complete_output12_event' event if the variable is over one product only (because if only one product, then output12 event is also true) dataset_pattern = sdproduct.replace_output12_product_with_wildcard( dataset.local_path) variable_complete_output12_event( project, model, dataset_pattern, variable) # trigger event (cross dataset event)
def variable_complete_output12_event(project,model,dataset_pattern,variable,commit=True): sdlog.info("SYDEVENT-003","'variable_complete_output12_event' triggered (%s,%s)"%(dataset_pattern,variable)) event=Event(name=sdconst.EVENT_OUTPUT12_VARIABLE_COMPLETE) event.project=project event.model=model event.dataset_pattern=dataset_pattern event.variable=variable event.filename_pattern='' event.crea_date=sdtime.now() event.priority=sdconst.DEFAULT_PRIORITY sdeventdao.add_event(event,commit=commit)
def latest_dataset_complete_output12_event(project,model,dataset_pattern,commit=True): # this event means one latest dataset has been completed (i.e. was latest before and still is) sdlog.info("SYDEVENT-006","'latest_dataset_complete_output12_event' triggered (%s)"%dataset_pattern) event=Event(name=sdconst.EVENT_OUTPUT12_LATEST_DATASET_COMPLETE) event.project=project event.model=model event.dataset_pattern=dataset_pattern event.variable='' event.filename_pattern='' event.crea_date=sdtime.now() event.priority=sdconst.DEFAULT_PRIORITY sdeventdao.add_event(event,commit=commit)
def variable_complete_output12_event(project, model, dataset_pattern, variable, commit=True): sdlog.log( "SYDEVENT-003", "'variable_complete_output12_event' triggered (%s,%s)" % (dataset_pattern, variable), event_triggered_log_level) event = Event(name=sdconst.EVENT_OUTPUT12_VARIABLE_COMPLETE) event.project = project event.model = model event.dataset_pattern = dataset_pattern event.variable = variable event.filename_pattern = '' event.crea_date = sdtime.now() event.priority = sdconst.DEFAULT_PRIORITY sdeventdao.add_event(event, commit=commit)
def submit(order_name,project,model,dataset,variable='',filename='',commit=True): # TODO: replace single quote with None and move 'None2SingleQuote' processing inside Event object (and add comment about why we use single quote instead of None in event table !!!) event_name=order_name dataset_pattern=sdproduct.replace_output12_product_with_wildcard(dataset) filename_pattern=filename sdlog.info("SDPPORDE-001","'%s' triggered (%s,%s)"%(event_name,dataset_pattern,variable)) event=Event(name=event_name) event.project=project event.model=model event.dataset_pattern=dataset_pattern event.variable=variable event.filename_pattern=filename_pattern event.crea_date=sdtime.now() event.priority=sdconst.DEFAULT_PRIORITY sdeventdao.add_event(event,commit=commit)
def latest_output12_dataset_complete_event(project, model, dataset_pattern, commit=True): # this event means latest output12 dataset has been completed (beware: no 'latest switch' event here: was latest before and still is) sdlog.log( "SYDEVENT-006", "'latest_output12_dataset_complete_event' triggered (%s)" % dataset_pattern, event_triggered_log_level) event = Event(name=sdconst.EVENT_OUTPUT12_LATEST_DATASET_COMPLETE) event.project = project event.model = model event.dataset_pattern = dataset_pattern event.variable = '' event.filename_pattern = '' event.crea_date = sdtime.now() event.priority = sdconst.DEFAULT_PRIORITY sdeventdao.add_event(event, commit=commit)
def variable_complete_event(project,model,dataset,variable,commit=True): sdlog.log("SYDEVENT-002","'variable_complete_event' triggered (%s,%s)"%(dataset.dataset_functional_id,variable),event_triggered_log_level) if sdconfig.is_event_enabled(sdconst.EVENT_VARIABLE_COMPLETE,project): event=Event(name=sdconst.EVENT_VARIABLE_COMPLETE) event.project=project event.model=model event.dataset_pattern=dataset.local_path event.variable=variable event.filename_pattern='' event.crea_date=sdtime.now() event.priority=sdconst.DEFAULT_PRIORITY sdeventdao.add_event(event,commit=commit) # cascade 1 (trigger dataset event) if dataset.status==sdconst.DATASET_STATUS_COMPLETE: dataset_complete_event(project,model,dataset) # trigger 'dataset complete' event # cascade 2 (trigger variable output12 event) if project=='CMIP5': assert '/output/' not in dataset.path (ds_path_output1,ds_path_output2)=sdproduct.get_output12_dataset_paths(dataset.path) if sddatasetdao.exists_dataset(path=ds_path_output1) and sddatasetdao.exists_dataset(path=ds_path_output2): d1=sddatasetdao.get_dataset(path=ds_path_output1) d2=sddatasetdao.get_dataset(path=ds_path_output2) if sdvariable.is_variable_complete(d1.dataset_id,variable) and sdvariable.is_variable_complete(d2.dataset_id,variable): dataset_pattern=sdproduct.replace_output12_product_with_wildcard(dataset.local_path) variable_complete_output12_event(project,model,dataset_pattern,variable) # trigger event (cross dataset event) else: # we also trigger the 'variable_complete_output12_event' event if the variable is over one product only (because if only one product, then output12 event is also true) dataset_pattern=sdproduct.replace_output12_product_with_wildcard(dataset.local_path) variable_complete_output12_event(project,model,dataset_pattern,variable) # trigger event (cross dataset event)
def latest_dataset_complete_event(project,model,dataset_pattern,commit=True): # this event means latest dataset has been completed (beware: no 'latest switch' event here: was latest before and still is) sdlog.log("SYDEVENT-045","'latest_dataset_complete_event' triggered (%s)"%dataset_pattern,event_triggered_log_level) if project=='CMIP5': # CMIP5 use output12 special event return if project in sdconst.PROJECT_WITH_ONE_VARIABLE_PER_DATASET: # CORDEX and CMIP6 use only variable level event return event=Event(name=sdconst.EVENT_LATEST_DATASET_COMPLETE) event.project=project event.model=model event.dataset_pattern=dataset_pattern event.variable='' event.filename_pattern='' event.crea_date=sdtime.now() event.priority=sdconst.DEFAULT_PRIORITY sdeventdao.add_event(event,commit=commit)