def wait_jobdone(self, jobId): jobStatus = CasJobs.getJobStatus(jobId)['Status'] while jobStatus < 2: time.sleep(10) jobStatus = CasJobs.getJobStatus(jobId)['Status'] if jobStatus in [2, 3, 4]: raise RuntimeError("Job is canceled or failed") print("Job {} is finished".format(jobId)) return jobStatus
def download_query(self, query, table_name, save_to='data', context='MyDB'): """Perform a query based on the context given""" if context != 'MyDB': jobid = cj.submitJob(query, context=context) cj.waitForJob(jobid, verbose=True) job_status = cj.getJobStatus(jobid) if job_status['Success'] != 5: raise CasJobsError( 'Error Performing the query, {}'.format(job_status)) # Now the results are safely saved in myDB # You can go ahead and download by using a session self.download_csv(table_name) self._download_from_scidrive(table_name, save_to)
def test_CasJobs_getJobStatus(self): jobId = CasJobs.submitJob(sql=CasJobs_TestQuery, context=CasJobs_TestDatabase) jobDescription = CasJobs.getJobStatus(jobId) self.assertEqual(jobDescription["JobID"], jobId)
# In[ ]: # drop or delete table in MyDB database context df = CasJobs.executeQuery(sql="DROP TABLE " + CasJobs_TestTableName1, context="MyDB", format="pandas") print(df) # In[ ]: #get job status jobId = CasJobs.submitJob(sql=CasJobs_TestQuery, context=CasJobs_TestDatabase) jobDescription = CasJobs.getJobStatus(jobId) print(jobId) print(jobDescription) # In[ ]: #cancel a job jobId = CasJobs.submitJob(sql=CasJobs_TestQuery, context=CasJobs_TestDatabase) jobDescription = CasJobs.cancelJob(jobId=jobId) print(jobId) print(jobDescription) # In[ ]:
verylongquery += 'u, g, r, i, z, err_u, err_g, err_r, err_i, err_z, petror90_r \n' verylongquery += 'into mydb.' + bigtablename + '\n' verylongquery += 'from galaxy\n' verylongquery += 'where clean = 1' print('Submitting query:\n', verylongquery) print('\n') thisjobid = CasJobs.submitJob(sql=verylongquery, context=this_context) print('Job submitted with jobId = ', thisjobid) print('\n') waited = CasJobs.waitForJob( jobId=thisjobid) # waited is a dummy variable; just print wait msg jobDescription = CasJobs.getJobStatus(thisjobid) print('\n') print('Information about the job:') #pprint(jobDescription) jobDescriber(jobDescription) # ## Thank you! # # Thanks for reviewing this SciServer example notebook. You can use this notebook as a template to develop your own notebooks, but please do so in a copy rather than in the original example notebook. # As you begin to use any of our SciServer modules in your own notebooks, consult the SciServer scripting documentation at http://www.sciserver.org/docs/sciscript-python/SciServer.html (link opens in a new window). # # If you have questions, please email the SciServer helpdesk at [email protected]. # In[ ]:
def do(self, user='******', password='******', search=1, path_to_model='YSE_DNN_photoZ_model_315.hdf5', debug=True): """ Predicts photometric redshifts from RA and DEC points in SDSS An outline of the algorithem is: first pull from SDSS u,g,r,i,z magnitudes from SDSS; should be able to handle a list/array of RA and DEC place u,g,r,i,z into a vector, append the derived information into the data array predict the information from the model return the predictions in the same order to the user inputs: Ra: list or array of len N, right ascensions of target galaxies in decimal degrees Dec: list or array of len N, declination of target galaxies in decimal degrees search: float, arcmin tolerance to search for the object in SDSS Catalogue path_to_model: str, filepath to saved model for prediction Returns: predictions: array of len N, photometric redshift of input galaxy """ try: nowdate = datetime.datetime.utcnow() - datetime.timedelta(1) from django.db.models import Q #HAS To Remain Here, I dunno why print('Entered the photo_z cron') #save time b/c the other cron jobs print a time for completion transients = Transient.objects.filter( Q(host__isnull=False) & Q(host__photo_z__isnull=True)) my_index = np.array( range(0, len(transients)) ) #dummy index used in DF, then used to create a mapping from matched galaxies back to these hosts transient_dictionary = dict(zip(my_index, transients)) RA = [] DEC = [] for i, T in enumerate(transients): #get rid of fake entries if T.host.ra and T.host.dec: RA.append(T.host.ra) DEC.append(T.host.dec) else: transient_dictionary.pop(i) DF_pre = pd.DataFrame() DF_pre['myindex'] = list(transient_dictionary.keys()) DF_pre['RA'] = RA DF_pre['DEC'] = DEC SciServer.Authentication.login(user, password) try: SciServer.SkyQuery.dropTable('SDSSTable1', datasetName='MyDB') SciServer.SkyQuery.dropTable('SDSSTable2', datasetName='MyDB') except Exception as e: print('tables are not in CAS MyDB, continuing') print( 'if system says table is already in DB, contact Andrew Engel' ) SciServer.CasJobs.uploadPandasDataFrameToTable(DF_pre, 'SDSSTable1', context='MyDB') myquery = 'SELECT s.myindex, p.z, p.zErr, p.nnAvgZ ' myquery += 'INTO MyDB.SDSSTable2 ' myquery += 'FROM MyDB.SDSSTable1 s ' myquery += 'CROSS APPLY dbo.fGetNearestObjEq(s.RA,s.DEC,1.0/60.0) AS nb ' myquery += 'INNER JOIN Photoz p ON p.objID = nb.objID' jobID = CasJobs.submitJob(sql=myquery, context="DR16") waited = SciServer.CasJobs.waitForJob(jobId=jobID) jobDescription = CasJobs.getJobStatus(jobID) jobDescriber(jobDescription) PhotoDF = SciServer.SkyQuery.getTable('SDSSTable2', datasetName='MyDB', top=10) #change this PhotoDF.drop_duplicates( subset='#myindex', inplace=True) #neccessary to patch some bugs PhotoDF.dropna(inplace=True) whats_left = PhotoDF['#myindex'].values point_estimates = PhotoDF['z'].values error = PhotoDF['zErr'].values other_z = PhotoDF['nnAvgZ'] point_estimates[error < -9998] = other_z[ error < -9998] #if there is a bad error, then authors write this is more effective for i, value in enumerate(whats_left): T = transient_dictionary[value] T.host.photo_z = point_estimates[i] T.host.photo_z_err = error[i] #T.host.photo_z_posterior = posterior[i] #Gautham suggested we add it to the host model T.host.photo_z_source = 'SDSS' T.host.save( ) #takes a long time and then my query needs to be reset which is also a long time print('time taken with upload:', datetime.datetime.utcnow() - nowdate) except Exception as e: exc_type, exc_obj, exc_tb = sys.exc_info() print("""Photo-z cron failed with error %s at line number %s""" % (e, exc_tb.tb_lineno)) #html_msg = """Photo-z cron failed with error %s at line number %s"""%(e,exc_tb.tb_lineno) #sendemail(from_addr, user.email, subject, html_msg, # djangoSettings.SMTP_LOGIN, djangoSettings.SMTP_PASSWORD, smtpserver) #IDK = YSE() #why is this suddenly neccessary??? #IDK.do()