def run_query(self, *args, **kwargs):
logger.warn('fake run_query in %s with %s, %s', self, args, kwargs)
query_out = QueryOutput()
status = self.decide_status()
if status == "submitted":
# set done to submitted?
query_out.set_done(message="job submitted mock",
debug_message="no message really",
job_status='submitted',
comment="mock comment",
warning="mock warning")
elif status == "done":
# set done to submitted?
query_out.set_done(message="job done mock",
debug_message="no message really",
job_status='done',
comment="mock comment",
warning="mock warning")
elif status == "failed":
# set done to submitted?
query_out.set_failed(message="job failed mock",
debug_message="no message really",
job_status='failed',
comment="mock comment",
warning="mock warning")
else:
NotImplementedError
return None, query_out
def test_has_input_products(self, instrument, logger=None):
query_out = QueryOutput()
message = 'OK'
debug_message = 'OK'
query_out.set_done(message=message, debug_message=str(debug_message))
return query_out, [1]
def test_communication(self,
instrument: Instrument = None,
query_type='Real',
logger=None,
config=None,
sentry_client=None) -> QueryOutput:
query_out = QueryOutput()
query_out.set_done(message="mock ok message!",
debug_message="mock ok debug_message")
return query_out
def run_query(self, call_back_url=None, run_asynch=False, logger=None, param_dict=None,):
res = None
message = ''
debug_message = ''
query_out = QueryOutput()
if param_dict is None:
param_dict = self.param_dict
try:
logger.info('--Spiacs disp--')
logger.info('call_back_url %s', call_back_url)
logger.info('data_server_url %s', self.data_server_url)
logger.info('*** run_asynch %s', run_asynch)
res = self._run(self.data_server_url, param_dict)
# DONE
query_out.set_done(message=message, debug_message=str(
debug_message), job_status='done')
# job.set_done()
except SpiacsAnalysisException as e:
run_query_message = f'Spiacs AnalysisException in run_query: {e}'
debug_message = e.debug_message
query_out.set_failed('run query ',
message='run query message=%s' % run_query_message,
logger=logger,
excep=e,
job_status='failed',
e_message=run_query_message,
debug_message=debug_message)
raise SpiacsException(message=run_query_message,
debug_message=debug_message)
except Exception as e:
logger.exception('Spiacs UnknownException: %s', e)
run_query_message = 'Spiacs UnknownException in run_query'
query_out.set_failed('run query ',
message='run query message=%s' % run_query_message,
logger=logger,
excep=e,
job_status='failed',
e_message=run_query_message,
debug_message=e)
raise SpiacsUnknownException(
message=run_query_message, debug_message=e)
return res, query_out
def test_communication(self, max_trial=120, sleep_s=1, logger=None):
logger.info('--> start test connection')
query_out = QueryOutput()
message = 'connection OK'
debug_message = ''
query_out.set_done(message=message, debug_message=str(debug_message))
logger.info('--> end test busy')
return query_out
def test_communication(self, max_trial=120, sleep_s=1, logger=None):
print('--> start test connection')
query_out = QueryOutput()
message = 'connection OK'
debug_message = ''
busy_exception = False
connection_status_message = 'OK'
query_out.set_done(message='connection OK',
debug_message=str(debug_message))
print('--> end test busy')
return query_out
def run_query(self, *args, **kwargs):
logger.warn('fake run_query in %s with %s, %s', self, args, kwargs)
query_out = QueryOutput()
p_value = self.instrument.get_par_by_name('p').value
if p_value == -1:
query_out.set_done(message="job failed mock",
debug_message="no message really",
job_status='failed',
comment="mock comment",
warning="mock warning")
elif 0 <= p_value < 4:
query_out.set_done(message="job submitted mock",
debug_message="no message really",
job_status='submitted',
comment="mock comment",
warning="mock warning")
else:
query_out.set_done(message="job done mock",
debug_message="no message really",
job_status='done',
comment="mock comment",
warning="mock warning")
return None, query_out
def run_query(self,
call_back_url,
run_asynch=True,
logger=None,
target=None,
modules=None,
assume=None):
print(self, "run_query", call_back_url, target, modules, assume)
if target is None:
target = self.target
if modules is None:
modules = self.modules
if assume is None:
assume = self.assume
res = None
#status = 0
message = ''
debug_message = ''
backend_comment = ''
backend_warning = ''
query_out = QueryOutput()
try:
logger.setLevel(logging.ERROR)
print('--osa disp--')
print('call_back_url', call_back_url)
print('*** run_asynch', run_asynch)
res = dc.RemoteDDA(self.data_server_url,
self.data_server_cache).query(
target=target,
modules=modules,
assume=assume,
inject=self.inject,
prompt_delegate=run_asynch,
callback=call_back_url)
backend_comment, backend_warning = self.get_comments(res)
print('--> url for call_back', call_back_url)
print("--> cached object in", res, res.ddcache_root_local)
#DONE
query_out.set_done(message=message,
debug_message=str(debug_message),
job_status='done',
comment=backend_comment,
warning=backend_warning)
#job.set_done()
except dc.AnalysisException as e:
run_query_message = 'AnalysisException'
debug_message = self.get_exceptions_message(e)
# we have to add the exception to the message
try:
my_dict = literal_eval(debug_message)
except:
my_dict = debug_message
if 'exception' in debug_message:
print('debug_message', type(debug_message), debug_message)
#print(my_dict)
if hasattr(my_dict, 'keys'):
if 'exception' in my_dict.keys():
run_query_message = run_query_message + ':%s' % my_dict[
'exception']
elif 'exception' in str(my_dict):
run_query_message = run_query_message + ':%s' % str(
my_dict)
query_out.set_failed('run query ',
message='run query message=%s' %
run_query_message,
logger=logger,
excep=e,
job_status='failed',
e_message=run_query_message,
debug_message=debug_message)
raise DDAException(message=run_query_message,
debug_message=debug_message)
except dc.WorkerException as e:
run_query_message = 'WorkerException'
debug_message = self.get_exceptions_message(e)
#FAILED
query_out.set_failed('run query ',
message='run query message=%s' %
run_query_message,
logger=logger,
excep=e,
job_status='failed',
e_message=run_query_message,
debug_message=debug_message)
raise DDAException(message=run_query_message,
debug_message=debug_message)
except dc.AnalysisDelegatedException as e:
# DONE DELEGATION
backend_comment, backend_warning = self.get_comments(res)
query_out.set_done(message=message,
debug_message=str(debug_message),
job_status='submitted',
comment=backend_comment,
warning=backend_warning)
except Exception as e:
run_query_message = 'DDAUnknownException in run_query: ' + repr(e)
query_out.set_failed('run query ',
message='run query message=%s' %
run_query_message,
logger=logger,
excep=e,
job_status='failed',
e_message=run_query_message,
debug_message='')
raise DDAUnknownException(message=run_query_message)
return res, query_out
def test_has_input_products(self, instrument, logger=None):
print('--> start has input_products')
RA = instrument.get_par_by_name('RA').value
DEC = instrument.get_par_by_name('DEC').value
radius = instrument.get_par_by_name('radius').value
scw_list = instrument.get_par_by_name('scw_list').value
use_max_pointings = instrument.get_par_by_name('max_pointings').value
osa_version = instrument.get_par_by_name('osa_version').value
query_out = QueryOutput()
message = ''
debug_message = ''
has_input_products_message = ''
prod_list = []
if scw_list is not None and scw_list != []:
prod_list = scw_list
else:
T1_iso = instrument.get_par_by_name('T1')._astropy_time.isot
T2_iso = instrument.get_par_by_name('T2')._astropy_time.isot
print('input', RA, DEC, T1_iso, T2_iso)
if self._test_products_with_astroquery:
# note that this just might introduce discrepancy, since it is not the exact workflow used by the backend
from astroquery.heasarc import Heasarc
import astroquery
heasarc = Heasarc()
with astroquery.heasarc.Conf.server.set_temp(
'https://www.isdc.unige.ch/browse/w3query.pl'):
T_i = heasarc.query_region(
SkyCoord(RA, DEC, unit="deg"),
mission='integral_rev3_scw',
resultmax=1000000, # all ppo
radius="200 deg",
cache=False,
time=T1_iso.replace("T", " ") + " .. " +
T2_iso.replace("T", " "),
fields='All')
else:
target = "ReportScWList"
modules = ['git://rangequery/staging-1-3']
scwlist_assumption = OsaDispatcher.get_scwlist_assumption(
None, T1_iso, T2_iso, RA, DEC, radius, use_max_pointings)
assume = [
"rangequery.ReportScWList(input_scwlist=%s)" %
scwlist_assumption[0], scwlist_assumption[1]
]
remote = dc.RemoteDDA(self.data_server_url,
self.data_server_cache)
try:
product = remote.query(target=target,
modules=modules,
assume=assume,
sync=True)
#DONE
query_out.set_done(message=message,
debug_message=str(debug_message))
prod_list = getattr(product, 'scwidlist', None)
if prod_list is None:
raise RuntimeError(f"product.prod_list is None")
if len(prod_list) < 1:
run_query_message = 'scwlist empty'
debug_message = ''
query_out.set_failed('test has input prods',
message=run_query_message,
logger=logger,
job_status='failed',
e_message=run_query_message,
debug_message='')
raise DDAException(message='scwlist empty',
debug_message='')
except dc.WorkerException as e:
run_query_message = 'WorkerException'
debug_message = self.get_exceptions_message(e)
# FAILED
query_out.set_failed('test has input prods',
message='has input_products=%s' %
run_query_message,
logger=logger,
excep=e,
job_status='failed',
e_message=run_query_message,
debug_message=debug_message)
raise DDAException('WorkerException', debug_message)
except dc.AnalysisException as e:
run_query_message = 'AnalysisException'
debug_message = self.get_exceptions_message(e)
query_out.set_failed('test has input prods',
message='run query message=%s' %
run_query_message,
logger=logger,
excep=e,
job_status='failed',
e_message=run_query_message,
debug_message=debug_message)
raise DDAException(message=run_query_message,
debug_message=debug_message)
except Exception as e:
run_query_message = 'DDAUnknownException in test has input prods: ' + repr(
e)
query_out.set_failed('test has input prods ',
message='run query message=%s' %
run_query_message,
logger=logger,
excep=e,
job_status='failed',
e_message=run_query_message,
debug_message='')
raise DDAUnknownException()
return query_out, prod_list
def test_communication(self, max_trial=120, sleep_s=1, logger=None):
print('--> start test connection to', self.data_server_url)
remote = dc.RemoteDDA(self.data_server_url, self.data_server_cache)
query_out = QueryOutput()
message = ''
debug_message = ''
connection_status_message = ''
busy_exception = False
time.sleep(sleep_s)
for i in range(max_trial):
time.sleep(sleep_s)
try:
r = remote.poke()
print('remote poke ok at trial', i)
#DONE
query_out.set_done(message=message,
debug_message=str(debug_message))
busy_exception = False
connection_status_message = 'OK'
break
except dc.WorkerException as e:
connection_status_message = self.get_exception_status_message(
e)
query_out.set_query_exception(e,
'test connection',
message='connection_status=%s' %
connection_status_message,
logger=logger)
busy_exception = True
print('remote poke not ok, trial', i,
connection_status_message)
except Exception as e:
connection_status_message = self.get_exception_status_message(
e)
query_out.set_query_exception(e,
'test connection',
message='connection_status=%s' %
connection_status_message,
logger=logger)
busy_exception = True
print('remote poke not ok, trial', i,
connection_status_message)
if busy_exception == True:
try:
r = remote.poke()
except Exception as e:
connection_status_message = self.get_exception_status_message(
e)
# FAILED
#query_out.set_failed('test connection',
# message='connection_status=%s' % connection_status_message,
# logger=logger,
# excep=e)
run_query_message = 'Connection Error'
debug_message = self.get_exceptions_message(e)
# FAILED
query_out.set_failed('test connection',
message='connection_status=%s' %
connection_status_message,
logger=logger,
excep=e,
e_message=run_query_message,
debug_message=debug_message)
raise DDAException('Connection Error', debug_message)
if connection_status_message == 'busy' or busy_exception == True:
print('server is busy')
# FAILED
#query_out.set_failed('test busy',
# message='connection_status=%s'%connection_status_message,
# logger=logger,
# excep=e)
query_out.set_failed('test busy',
message='connection_status=%s' %
connection_status_message,
logger=logger,
excep=e,
e_message='data server busy',
debug_message='data server busy')
raise DDAException('Connection Error', debug_message)
print('--> end test busy')
return query_out
def run_query(
self,
call_back_url=None,
run_asynch=False,
logger=None,
param_dict=None,
):
res = None
# status = 0
message = ''
debug_message = ''
query_out = QueryOutput()
if param_dict is None:
param_dict = self.param_dict
try:
simple_logger.logger.setLevel(logging.ERROR)
print('--Spiacs disp--')
print('call_back_url', call_back_url)
print('data_server_url', self.data_server_url)
print('*** run_asynch', run_asynch)
res = self._run(self.data_server_url, param_dict)
#res =self._run_test()
#DONE
query_out.set_done(message=message,
debug_message=str(debug_message),
job_status='done')
#job.set_done()
except SpiacsAnalysisException as e:
run_query_message = 'Spiacs AnalysisException in run_query'
debug_message = e.debug_message
query_out.set_failed('run query ',
message='run query message=%s' %
run_query_message,
logger=logger,
excep=e,
job_status='failed',
e_message=run_query_message,
debug_message=debug_message)
raise SpiacsException(message=run_query_message,
debug_message=debug_message)
except Exception as e:
run_query_message = 'Spiacs UnknownException in run_query'
query_out.set_failed('run query ',
message='run query message=%s' %
run_query_message,
logger=logger,
excep=e,
job_status='failed',
e_message=run_query_message,
debug_message=e)
raise SpiacsUnknownException(message=run_query_message,
debug_message=e)
return res, query_out
def test_has_input_products(self, instrument: Instrument,
logger) -> Tuple[QueryOutput, list]:
query_out = QueryOutput()
query_out.set_done(message="mock ok message!",
debug_message="mock ok debug_message")
return query_out, []
def run_query(self,call_back_url,run_asynch=True,logger=None,target=None,modules=None,assume=None):
if target is None:
target=self.target
if modules is None:
modules=self.modules
if assume is None:
assume=self.assume
res = None
#status = 0
message = ''
debug_message = ''
query_out = QueryOutput()
try:
simple_logger.logger.setLevel(logging.ERROR)
print('--osa disp--')
print('call_back_url',call_back_url)
print('*** run_asynch', run_asynch)
res= dc.RemoteDDOSA(self.data_server_url, self.dataserver_cache).query(target=target,
modules=modules,
assume=assume,
inject=self.inject,
prompt_delegate = run_asynch,
callback = call_back_url)
print ('--> url for call_back',call_back_url)
print("--> cached object in", res,res.ddcache_root_local)
#DONE
query_out.set_done(message=message, debug_message=str(debug_message),job_status='done')
#job.set_done()
except dc.AnalysisException as e:
run_query_message = 'AnalysisException'
debug_message=self.get_exceptions_message(e)
query_out.set_failed('run query ',
message='run query message=%s' % run_query_message,
logger=logger,
excep=e,
job_status='failed',
e_message=run_query_message,
debug_message=debug_message)
raise DDOSAException(message=run_query_message,debug_message=debug_message)
except dc.WorkerException as e:
run_query_message = 'WorkerException'
debug_message = self.get_exceptions_message(e)
#FAILED
query_out.set_failed('run query ',
message='run query message=%s' % run_query_message,
logger=logger,
excep=e,
job_status='failed',
e_message=run_query_message,
debug_message=debug_message)
raise DDOSAException(message=run_query_message, debug_message=debug_message)
except dc.AnalysisDelegatedException as e:
# DONE DELEGATION
query_out.set_done(message=message, debug_message=str(debug_message), job_status='submitted')
except Exception as e:
run_query_message = 'DDOSAUnknownException in run_query'
query_out.set_failed('run query ',
message='run query message=%s' %run_query_message,
logger=logger,
excep=e,
job_status='failed',
e_message=run_query_message,
debug_message='')
raise DDOSAUnknownException(message=run_query_message)
return res,query_out
def test_has_input_products(self,instrument,logger=None):
print('--> start has input_products')
RA = instrument.get_par_by_name('RA').value
DEC = instrument.get_par_by_name('DEC').value
radius = instrument.get_par_by_name('radius').value
scw_list = instrument.get_par_by_name('scw_list').value
query_out = QueryOutput()
message = ''
debug_message = ''
has_input_products_message=''
prod_list=[]
if scw_list is not None and scw_list != []:
prod_list=scw_list
else:
T1_iso = instrument.get_par_by_name('T1')._astropy_time.isot
T2_iso = instrument.get_par_by_name('T2')._astropy_time.isot
print ('input',RA,DEC,T1_iso,T2_iso)
target = "ReportScWList"
modules = ['git://rangequery']
assume = ['rangequery.ReportScWList(input_scwlist=rangequery.TimeDirectionScWList)',
'rangequery.TimeDirectionScWList(\
use_coordinates=dict(RA=%(RA)s,DEC=%(DEC)s,radius=%(radius)s),\
use_timespan=dict(T1="%(T1)s",T2="%(T2)s"),\
use_max_pointings=50)' % (dict(RA=RA, DEC=DEC, radius=radius, T1=T1_iso, T2=T2_iso))]
remote = dc.RemoteDDOSA(self.data_server_url, self.dataserver_cache)
try:
product = remote.query(target=target,modules=modules,assume=assume)
#DONE
query_out.set_done(message=message, debug_message=str(debug_message))
prod_list= product.scwidlist
print ('ciccio scwlist for T1,T2',T1_iso,T2_iso,scw_list)
if len(prod_list)<1:
run_query_message = 'scwlist empty'
debug_message = ''
# FAILED
query_out.set_failed('test has input prods',
message=run_query_message,
logger=logger,
e_message=run_query_message,
debug_message='')
raise DDOSAException('scwlist empty', '')
except dc.WorkerException as e:
run_query_message = 'WorkerException'
debug_message = self.get_exceptions_message(e)
# FAILED
query_out.set_failed('test has input prods',
message='has input_products=%s' % run_query_message,
logger=logger,
excep=e,
e_message=run_query_message,
debug_message=debug_message)
raise DDOSAException('WorkerException', debug_message)
except dc.AnalysisException as e:
run_query_message = 'AnalysisException'
debug_message = self.get_exceptions_message(e)
query_out.set_failed('test has input prods',
message='run query message=%s' % run_query_message,
logger=logger,
excep=e,
job_status='failed',
e_message=run_query_message,
debug_message=debug_message)
raise DDOSAException(message=run_query_message, debug_message=debug_message)
except Exception as e:
run_query_message = 'DDOSAUnknownException in test has input prods'
query_out.set_failed('test has input prods ',
message='run query message=%s' % run_query_message,
logger=logger,
excep=e,
job_status='failed',
e_message=run_query_message,
debug_message='')
raise DDOSAUnknownException()
return query_out,prod_list