def load_output(job): detids = np.genfromtxt(job.detection_file, dtype=None, encoding='ascii') detidstr = str(tuple(detids.tolist())) mydir = Path(os.path.dirname(__file__)) sql = mydir / 'loadphot.sql' with open(sql, 'r') as f: g = f.read().replace('FILENAME', f"'{job.output_file}'") sqlo = job.output_file + '.sql' with open(sqlo, 'w') as f: f.write(g) # do the load cmd = f"psql -h {get_secret('hpss_dbhost')} -p {get_secret('hpss_dbport')} " \ f"-d {get_secret('hpss_dbname')} -U {get_secret('hpss_dbusername')} " \ f"-f {sqlo}" execute(cmd.split()) query = "update detections set alert_ready = 't' where id in %s" % ( detidstr, ) if len(detids) > 0: db.DBSession().execute(query) db.DBSession().commit()
def submit_thumbs(): thumbids = db.DBSession().query(db.models.Thumbnail.id).filter( db.models.Thumbnail.source_id != None, db.models.Thumbnail.public_url == None ) thumbids = [t[0] for t in thumbids] ndt = datetime.datetime.utcnow() nightdate = f'{ndt.year}{ndt.month:02d}{ndt.day:02d}' curdir = os.getcwd() scriptname = Path(f'/global/cscratch1/sd/dgold/zuds/' f'nightly/{nightdate}/{ndt}.thumb.sh'.replace(' ', '_')) scriptname.parent.mkdir(parents=True, exist_ok=True) os.chdir(scriptname.parent) thumbinname = f'{scriptname}'.replace('.sh', '.in') with open(thumbinname, 'w') as f: f.write('\n'.join([str(t) for t in thumbids]) + '\n') jobscript = f"""#!/bin/bash #SBATCH --image=registry.services.nersc.gov/dgold/ztf:latest #SBATCH --volume="/global/homes/d/dgold/lensgrinder/pipeline/:/pipeline;/global/homes/d/dgold:/home/desi;/global/homes/d/dgold/skyportal:/skyportal" #SBATCH -N 1 #SBATCH -C haswell #SBATCH -q realtime #SBATCH --exclusive #SBATCH -J zuds #SBATCH -t 00:60:00 #SBATCH -L SCRATCH #SBATCH -A {get_secret('nersc_account')} HDF5_USE_FILE_LOCKING=FALSE srun -n 64 -c1 --cpu_bind=cores shifter python $HOME/lensgrinder/scripts/dothumb.py {thumbinname} """ with open(scriptname, 'w') as f: f.write(jobscript) cmd = f'sbatch {scriptname}' process = subprocess.Popen( cmd.split(), stdout=subprocess.PIPE, stderr=subprocess.PIPE ) stdout, stderr = process.communicate() print(stdout) if process.returncode != 0: raise RuntimeError( f'Non-zero exit code from sbatch, output was ' f'"{str(stdout)}", "{str(stdout)}".' ) os.chdir(curdir) _ = stdout.strip().split()[-1].decode('ascii')
def get_num_https_ip_port(self): try: sn = db.DBSession() num = sn.session.query(db.AgentServer).filter(db.AgentServer.HTTPS==True).count() return num except Exception as e: print('get_num_https_ip_port:',e) pass
def random_https_ip_port(): '''从数据库中随机获取代理服务器IP端口''' try: sn = db.DBSession() http_ip_port_list = sn.session.query(db.AgentServer).filter(db.AgentServer.HTTPS==True).order_by().all() item = random.choice(http_ip_port_list) return item.IP_PORT except Exception as e: print('random_https_ip_port:',e) pass
def clear_AgentServerSql(self): '''清除数据库中的代理服务器''' sn = db.DBSession() try: res = sn.session.query(db.AgentServer).all() for item in res: sn.session.delete(item) sn.session.commit() return True except: sn.session.rollback() return False
def remove(ip_port): '''移除代理服务器''' sn = db.DBSession() try: item = sn.session.query(db.AgentServer).filter(db.AgentServer.IP_PORT == ip_port).one() sn.session.delete(item) sn.session.commit() sn.session.close() return True except: sn.session.rollback() return False
def start_requests(self): session = db.DBSession() weibo_stars_info_list = session.query( db.WeiboStarsInfo).filter(db.WeiboStarsInfo.weibo_id != 0).all() # Todo: Testing for weibo_stars_info in weibo_stars_info_list[:200]: # for weibo_stars_info in weibo_stars_info_list: weibo_id = weibo_stars_info.weibo_id url = self.weibo_user_home_url.format(weibo_id=weibo_id) yield scrapy.Request(url=url, callback=self.parse_home) session.close()
def get_num_ip_port(url): try: sn = db.DBSession() if url[:5] == 'https': return sn.session.query(db.AgentServer).filter(db.AgentServer.HTTPS==True).count() elif url[:4] == 'http': return sn.session.query(db.AgentServer).filter(db.AgentServer.HTTP==True).count() else: return None except Exception as e: print('get_num_ip_port:',e) pass
def update_AgentServerSource(self): sn = db.DBSession() try: res = sn.session.query(db.AgentServerSource).all() for item in res: sn.session.delete(item) for item in self.AgentsCrawlerList: ass = db.AgentServerSource(SOURCES=item.name,URL=item.url) sn.session.add(ass) sn.session.commit() except Exception as e: print('error:',e) sn.session.rollback()
def insert_bank_crawl_result(self, item): session = db.DBSession() new_bank_crawl_result = db.BankCrawlResult() new_bank_crawl_result.crawl_time = item['crawl_time'] new_bank_crawl_result.bank_name = item['bank_name'] new_bank_crawl_result.source_url = item['source_url'] new_bank_crawl_result.response = item['response'] session.add(new_bank_crawl_result) session.commit() session.close() return
def insert_fund_info(self, item): session = db.DBSession() new_fund_info = db.FundInfo() new_fund_info.fund_name = item['fund_name'] new_fund_info.fund_code = item['fund_code'] new_fund_info.time = item['time'] new_fund_info.IOPV = item['IOPV'] new_fund_info.LJJZ = item['LJJZ'] session.add(new_fund_info) session.commit() session.close() return
def main(): '''''' words = [] wordRaws = 0 except_list = [ 'we', 'you', 'she', 'he', 'am', 'is', 'are', 'was', 'were', 'your', 'may', 'can', 'and', 'or' ] filepath = 'E:\Data\englishexam2' filelist = os.listdir(filepath) for file in filelist: filename = os.path.join(filepath, file) words = words + wordsCollector.words_read(filename) words_filted = wordsCollector.words_filter(words, except_list) print(len(words), len(words_filted)) words_times = wordsCollector.words_count(words_filted) if not os.path.isfile('words.db'): db.db_init() else: pass words = db.Words dbcxn = db.DBSession() #words_insert(words_times, dbcxn, words) # 将分析出来的单词插入数据库 for word, fre in words_times: phonogram, exp = wordsCollector.words_trans(word) #print(phonogram, exp) up_value = [word, fre, exp, phonogram] words_update(up_value, dbcxn, words) # 更新查找到的单词解释 wordRaws += 1 if wordRaws == 1000: #print(wordRaws) dbcxn.commit() wordRaws = 0 else: continue row = words_Query(dbcxn, words) count = 1 wordsToCsv(['单词', '出现次数', '音标', '单词解释']) while True: try: #print(row) wordsToCsv(row) except: break count += 1 dbcxn.close()
def bank_response_to_info(): logger.info('bank_response_to_info()') session = db.DBSession() last_bank_crawl_result_id = 1 last_bank_crawl_result = session.query(db.BankInfo).order_by(db.BankInfo.bank_crawl_result_id.desc()).first() if last_bank_crawl_result is not None: last_bank_crawl_result_id = last_bank_crawl_result.bank_crawl_result_id deal_bank_crawl_result_list = session.query(db.BankCrawlResult)\ .filter(db.BankCrawlResult.id > last_bank_crawl_result_id).all() for deal_bank_crawl_result in deal_bank_crawl_result_list: bank_crawl_response = deal_bank_crawl_result.response bank_crawl_response_json = json.loads(bank_crawl_response) table1_item_list = bank_crawl_response_json['Table1'] rmb_gold_customer_sell = None rmb_gold_update_beijing_time = None for table1_item in table1_item_list: if table1_item['ProdName'] == '人民币账户黄金': rmb_gold_customer_sell = table1_item['CustomerSell'] rmb_gold_update_beijing_time = table1_item['UpdateTime'] break if rmb_gold_customer_sell is None or rmb_gold_update_beijing_time is None: logger.error('Can not find 人民币账户黄金 from bank_crawl_result.response where bank_crawl_result.id = ' + deal_bank_crawl_result.id) continue new_bank_info = db.BankInfo() new_bank_info.bank_crawl_result_id = deal_bank_crawl_result.id new_bank_info.bank_name = deal_bank_crawl_result.bank_name new_bank_info.rmb_gold_customer_sell = decimal.Decimal(rmb_gold_customer_sell) new_bank_info.rmb_gold_update_beijing_time = datetime.strptime(rmb_gold_update_beijing_time, '%Y-%m-%d %H:%M:%S') session.add(new_bank_info) session.commit() session.close() return
def save_to_sql(self,datalist): '''(IP_PORT,ADDR,IS_ANONYMOUS,SURE_TIME)''' sn = db.DBSession() item_dir = {} for item in datalist: try: AgentServerSourceitem = item_dir[item[6]] except: AgentServerSourceitem = sn.session.query(db.AgentServerSource).filter( db.AgentServerSource.SOURCES == item[6]).first() item_dir[item[4]] = AgentServerSourceitem d = db.AgentServer(IP_PORT=item[0], ADDR=item[1], IS_ANONYMOUS=item[2], SURE_TIME=item[3], HTTP=item[4], HTTPS=item[5], SOURCES=AgentServerSourceitem.SOURCES) sn.session.add(d) sn.session.commit() sn.close()
bad = [] while True: # check if the cookies need to be reset tnow = time.time() if tnow - tstart > 86400.: # assume the cookies expire after 1 day tstart = time.time() icookie = ipac_authenticate() idownload_q = db.DBSession().query(db.ScienceImage).outerjoin( db.ZTFFileCopy, db.ScienceImage.id == db.ZTFFileCopy.product_id ).filter( db.ZTFFileCopy.product_id == None, db.ScienceImage.field.in_(ZUDS_FIELDS) ).order_by( db.ScienceImage.id.desc() ).options(db.sa.orm.joinedload(db.ScienceImage.mask_image)) to_download = idownload_q.all() if len(to_download) == 0: # download for other fields time.sleep(30.) # sleep for 3 minutes continue for sci in to_download: for t in ['sci', 'mask']: if sci.id in bad:
def submit_alert_job(): # get the what needs alerts detids = db.DBSession().query(db.Detection.id).outerjoin( db.Alert, db.Alert.detection_id == db.Detection.id).filter( db.Detection.alert_ready == True, db.Alert.id == None).all() detids = [d[0] for d in detids] curdir = os.getcwd() ndt = datetime.datetime.utcnow() nightdate = f'{ndt.year}{ndt.month:02d}{ndt.day:02d}' if len(detids) == 0: raise RuntimeError('No detections to run make alerts for, abandoning ' 'forced photometry and alerting ') scriptname = Path( f'/global/cscratch1/sd/dgold/zuds/nightly/{nightdate}/{ndt}.alert.sh'. replace(' ', '_')) scriptname.parent.mkdir(parents=True, exist_ok=True) os.chdir(scriptname.parent) detinname = f'{scriptname}'.replace('.sh', '.in') with open(detinname, 'w') as f: f.write('\n'.join([str(i) for i in detids]) + '\n') jobscript = f"""#!/bin/bash #SBATCH --image=registry.services.nersc.gov/dgold/ztf:latest #SBATCH --volume="/global/homes/d/dgold/lensgrinder/pipeline/:/pipeline;/global/homes/d/dgold:/home/desi;/global/homes/d/dgold/skyportal:/skyportal" #SBATCH -N 1 #SBATCH -C haswell #SBATCH -q realtime #SBATCH --exclusive #SBATCH -J zuds #SBATCH -t 00:60:00 #SBATCH -L SCRATCH #SBATCH -A {get_secret('nersc_account')} #SBATCH -o {str(scriptname).replace('.sh', '.out')} HDF5_USE_FILE_LOCKING=FALSE srun -n 64 -c1 --cpu_bind=cores shifter python $HOME/lensgrinder/scripts/doalert.py {detinname} """ with open(scriptname, 'w') as f: f.write(jobscript) cmd = f'sbatch {scriptname}' process = subprocess.Popen(cmd.split(), stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = process.communicate() print(stdout) if process.returncode != 0: raise RuntimeError(f'Non-zero exit code from sbatch, output was ' f'"{str(stdout)}", "{str(stdout)}".') os.chdir(curdir) jobid = stdout.strip().split()[-1].decode('ascii') return jobid
db.DBSession().execute(f'UPDATE thumbnails SET modified=now(), photometry_id=photometry.id, ' f'source_id=photometry.source_id from photometry join sources on ' f'photometry.source_id = sources.id join objectswithflux on ' f'sources.id = objectswithflux.source_id where objectswithflux.id in {tuple(bestids)} ' f'and thumbnails.detection_id = objectswithflux.id') db.DBSession().execute( f'''update detections set triggers_alert = 't' from sources s join objectswithflux o on s.id=o.source_id where detections.id = o.id''' ) xmatch([s.id for s in sources]) # need to commit so that sources will be there for forced photometry # jobs running via slurm db.DBSession().commit() if os.getenv('NERSC_HOST') == 'cori': submit_thumbs() else: print('nothing to do') db.DBSession().commit() if __name__ == '__main__': db.DBSession().get_bind().echo=True associate()
def xmatch(source_ids, insert_neighbors=True): sids = '(' + ','.join([f"'{id}'" for id in source_ids]) + ')' q = ''' create temp table rbacc as (select o.source_id, sum(rb.rb_score) as sumrb from detections d join objectswithflux o on d.id = o.id join realbogus rb on rb.detection_id = d.id group by o.source_id); create index on rbacc (source_id); ''' db.DBSession().execute(q) q = ''' update sources set score = rbacc.sumrb from rbacc where sources.id = rbacc.source_id and sources.id in %s; ''' % (sids,) db.DBSession().execute(q) for chunk in ['north', 'south']: tablename = f'dr8_{chunk}_join_neighbors' # update dr8_join insert = f''' insert into {tablename} (select s.id as sid, d.*, rank() over (partition by s.id order by q3c_dist(s.ra, s.dec, d."RA", d."DEC") asc), q3c_dist(s.ra, s.dec, d."RA", d."DEC") * 3600 sep from sources s join dr8_%s d on q3c_join(s.ra, s.dec, d."RA", d."DEC", 30./3600.) where s.id in %s) ''' % (chunk, sids) # this should take about 40 minutes if insert_neighbors: db.DBSession().execute(insert) q = ''' update sources set score = -1, altdata = ('{"rejected": "matched to GAIA dr8"}')::jsonb from %s d where d.sid = sources.id and d.sep < 1.5 and d."PARALLAX" > 0 and sources.id in %s; ''' % (tablename, sids) db.DBSession().execute(q) q = ''' update sources set score = -1, altdata = ('{"rejected": "matched to dr8 masked source ID"}')::jsonb from %s d where d.sid = sources.id and d.sep < 2 and (d."FRACMASKED_G" > 0.2 OR d."FRACMASKED_R" > 0.2 OR d."FRACMASKED_Z" > 0.2) and sources.id in %s; ''' % (tablename, sids) db.DBSession().execute(q) q = ''' update sources set score = -1, altdata = ('{"rejected": "matched to hits ID ' || h.id::text || '"}')::jsonb from %s d join hits h on q3c_join(d."RA", d."DEC", h.ra, h.dec, 0.0002777 * 1.5) where d.sid = sources.id and sources.id in %s and d.sep < 1.5; ''' % (tablename, sids,) db.DBSession().execute(q) q = ''' update sources set score = -1, altdata = ('{"rejected": "matched to MQ ID ' || m.id::text || '"}')::jsonb from %s d join milliquas_v6 m on q3c_join(d."RA", d."DEC", m.ra, m.dec, 0.0002777 * 1.5) where d.sid = sources.id and sources.id in %s and d.sep < 1.5; ''' % (tablename, sids,) db.DBSession().execute(q) q = ''' update sources set score = -1, altdata = ('{"rejected": "right on top of (< 1 arcsec) DR8 PSF "}')::jsonb from %s d where d.sid = sources.id and (d."TYPE" = 'PSF') and sep < 1 and sources.id in %s; ''' % (tablename, sids,) db.DBSession().execute(q) q = ''' update sources set neighbor_info = to_json(d) from %s d where d.sid = sources.id and d.rank = 1 and sources.id in %s ''' % (tablename, sids,) db.DBSession().execute(q) for chunk in ['north', 'south']: tablename = f'dr8_{chunk}_join_neighbors' q = ''' update sources set redshift = (case when d.z_spec = -99 then d.z_phot_median else d.z_spec end) from %s d where d.rank = 1 and d.sid = sources.id and sources.id in %s; ''' % (tablename, sids,) db.DBSession().execute(q) q = ''' update sources set score = -1, altdata = ('{"rejected": "rejected for having z_dr8 < 0.0001"}')::jsonb where sources.redshift <= 0.0001 or sources.redshift is null and sources.id in %s;''' % (sids,) db.DBSession().execute(q) q = ''' create temp table detection_times as (select d.id, s.obsjd - 2400000.5 as mjd from detections d join objectswithflux o on d.id = o.id join singleepochsubtractions se on se.id = o.image_id join scienceimages s on s.id = se.target_image_id where o.source_id is not null) ; insert into detection_times (id, mjd) (select d.id, to_char(c.binright, 'J')::double precision - 2400000.5 from detections d join objectswithflux o on d.id = o.id join multiepochsubtractions m on m.id = o.image_id join sciencecoadds c on c.id = m.target_image_id where o.source_id is not null); create index on detection_times (id, mjd); create index on detection_times (mjd, id); create temp table detection_order as (select s.id as source_id, d.id, rank() over (partition by s.id order by dt.mjd desc) from sources s join objectswithflux o on s.id = o.source_id join detections d on d.id = o.id join detection_times dt on d.id = dt.id ); create index on detection_order (source_id, id, rank); update sources set last_mjd = dt.mjd from detection_order dord join detection_times dt on dord.id = dt.id where sources.id = dord.source_id and dord.rank = 1; ''' db.DBSession().execute(q)
def associate(debug=False): db.DBSession().execute('SET TRANSACTION ISOLATION LEVEL REPEATABLE READ;') # make source / detection association temp table db.DBSession().execute(''' create temp table detsource as (select d.id as detection_id, s.id as source_id from detections d join sources s on q3c_join(s.ra, s.dec, d.ra, d.dec, 0.0002777 * 2)) ; create index on detsource (detection_id, source_id); create index on detsource (source_id, detection_id); ''') r = db.DBSession().execute( '''update objectswithflux set source_id = s.id, modified = now() from detections d join objectswithflux o on d.id = o.id join detsource ds on ds.detection_id = d.id join sources s on ds.source_id = s.id where o.source_id is NULL and o.created_at > now() - interval '7 days' AND objectswithflux.id = d.id returning d.id, s.id''' ) r = list(r) triggers_alert = [row[0] for row in r] print(f'associated {len(r)} detections with existing sources') db.DBSession().execute(''' update sources set ra=dummy.ra, dec=dummy.dec, modified=now() from (select g.id, g.ra, g.dec from (select s.id, d.ra, d.dec, rank() over (partition by o.source_id order by o.flux / o.fluxerr desc) from sources s join objectswithflux o on o.source_id = s.id join detections d on o.id = d.id ) g where rank = 1) dummy where sources.id = dummy.id; ''') q = f'''select d.id, d.ra, d.dec, o.flux / o.fluxerr as snr from detections d join objectswithflux o on d.id = o.id join realbogus rb on rb.detection_id = d.id where o.source_id is NULL and rb.rb_score > {ASSOC_RB_MIN} order by o.id asc''' if debug: q += ' LIMIT 10000' df = pd.DataFrame( list(db.DBSession().execute(q)), columns=['id', 'ra', 'dec', 'snr'] ) df = df.set_index('id') coord = SkyCoord(df['ra'], df['dec'], unit='deg') idx1, idx2, sep, _ = search_around_sky(coord, coord, ASSOC_RADIUS_ARCSEC * u.arcsec) dropdupes = idx1 != idx2 idx1 = idx1[dropdupes] idx2 = idx2[dropdupes] sep = sep[dropdupes] # cluster the detections into sources using DBSCAN clustering = DBSCAN( eps=ASSOC_RADIUS_ARCSEC, min_samples=2, metric='precomputed' ) # construct the sparse pairwise distance matrix nobj = len(df) darcsec = sep.to('arcsec').value distmat = csr_matrix((darcsec, (idx1, idx2)), shape=(nobj, nobj)) clustering.fit(distmat) df['source'] = clustering.labels_ df = df[df['source'] != -1] with db.DBSession().no_autoflush: default_group = db.DBSession().query( db.models.Group ).get(DEFAULT_GROUP) default_instrument = db.DBSession().query( db.models.Instrument ).get(DEFAULT_INSTRUMENT) bestdets = source_bestdet_from_solution(df) # cache d1 and d2 # get thumbnail pics from sqlalchemy.orm import joinedload d1 = db.DBSession().query(db.Detection).filter(db.Detection.id.in_( [int(v) for v in bestdets.values()] )).all() detcache = {d.id: d for d in d1} sourceid_map = {} curval = db.DBSession().execute("select nextval('namenum')").first()[0] stups = [] gtups = [] sources = [] if len(bestdets) > 0: for sourceid in tqdm(bestdets): bestdet = detcache[bestdets[sourceid]] name = publish.get_next_name(num=curval) curval += 1 source = db.models.Source( id=name, groups=[default_group], ra=bestdet.ra, dec=bestdet.dec ) sourceid_map[sourceid] = source sources.append(source) stups.append(f"('{name}', {bestdet.ra}, {bestdet.dec}, now(), now(), 'f', 'f', 'f', 0, 0)") gtups.append(f"('{name}', 1, now(), now())") db.DBSession().execute('INSERT INTO sources (id, ra, dec, created_at, modified, transient, varstar, is_roid, "offset", score) VALUES ' f'{",".join(stups)}') db.DBSession().execute('INSERT INTO group_sources (source_id, group_id, created_at, modified) ' f'VALUES {",".join(gtups)}') pid = [row[0] for row in db.DBSession().execute( 'INSERT INTO photometry (source_id, instrument_id, created_at, modified) ' f'VALUES {",".join(gtups)} RETURNING ID' )] stups = [f"(now(), now(), {p}, '{source.sdss_url}', 'sdss')" for p, source in zip( pid, sources )] dtups = [f"(now(), now(), {p}, '{source.desi_dr8_url}', 'dr8')" for p, source in zip( pid, sources )] db.DBSession().execute( 'insert into thumbnails (created_at, modified, photometry_id, public_url, type) ' f"VALUES {','.join(stups)}") db.DBSession().execute( 'insert into thumbnails (created_at, modified, photometry_id, public_url, type) ' f"VALUES {','.join(dtups)}") db.DBSession().execute(f"select setval('namenum', {curval})") db.DBSession().flush() query = [] for sourceid, group in df.groupby('source'): realid = sourceid_map[sourceid].id dets = group.index.tolist() query.append( f''' update objectswithflux set source_id = '{realid}', modified=now() where objectswithflux.id in {tuple(dets)} ''' ) db.DBSession().execute( ';'.join(query) ) bestids = [int(v) for v in bestdets.values()] db.DBSession().execute(f'UPDATE thumbnails SET modified=now(), photometry_id=photometry.id, ' f'source_id=photometry.source_id from photometry join sources on ' f'photometry.source_id = sources.id join objectswithflux on ' f'sources.id = objectswithflux.source_id where objectswithflux.id in {tuple(bestids)} ' f'and thumbnails.detection_id = objectswithflux.id') db.DBSession().execute( f'''update detections set triggers_alert = 't' from sources s join objectswithflux o on s.id=o.source_id where detections.id = o.id''' ) xmatch([s.id for s in sources]) # need to commit so that sources will be there for forced photometry # jobs running via slurm db.DBSession().commit() if os.getenv('NERSC_HOST') == 'cori': submit_thumbs() else: print('nothing to do') db.DBSession().commit()
def submit_forcephot_chain(): # get the what needs alerts detids = db.DBSession().query(db.Detection.id).outerjoin( db.Alert, db.Alert.detection_id == db.Detection.id).filter( db.Detection.triggers_alert == True, db.sa.or_(db.Detection.alert_ready == None, db.Detection.alert_ready == False), db.Alert.id == None) detids = [d[0] for d in detids] ndt = datetime.datetime.utcnow() nightdate = f'{ndt.year}{ndt.month:02d}{ndt.day:02d}' curdir = os.getcwd() scriptname = Path(f'/global/cscratch1/sd/dgold/zuds/' f'nightly/{nightdate}/{ndt}.phot.sh'.replace(' ', '_')) scriptname.parent.mkdir(parents=True, exist_ok=True) os.chdir(scriptname.parent) image_names = db.DBSession().query( db.SingleEpochSubtraction.basename, db.SingleEpochSubtraction.id).join( db.ReferenceImage, db.SingleEpochSubtraction.reference_image_id == db.ReferenceImage.id).filter( db.ReferenceImage.version == 'zuds5', ).all() image_names = sorted(image_names, key=lambda s: s[0].split('ztf_')[1].split('_')[0], reverse=True) image_names = image_names[:FORCEPHOT_IMAGE_LIMIT] if len(detids) > 0: # only use the latest images random.shuffle(image_names) imginname = f'{scriptname}'.replace('.sh', '.in') photoutname = imginname.replace('.in', '.output') outnames = [] with open(imginname, 'w') as f: for name, idnum in image_names: # name = name[0] g = name.split('_sciimg')[0].split('_') q = g[-1] c = g[-3] b = g[-4] field = g[-5] outnames.append( f'/global/cfs/cdirs/m937/www/data/scratch/{field}/{c}/' f'{q}/{b}/{name} {idnum}') f.write('\n'.join(outnames) + '\n') detinname = f'{scriptname}'.replace('.sh', '.det.in') with open(detinname, 'w') as f: f.write('\n'.join([str(i) for i in detids]) + '\n') jobscript = f"""#!/bin/bash #SBATCH --image=registry.services.nersc.gov/dgold/ztf:latest #SBATCH --volume="/global/homes/d/dgold/lensgrinder/pipeline/:/pipeline;/global/homes/d/dgold:/home/desi;/global/homes/d/dgold/skyportal:/skyportal" #SBATCH -N 13 #SBATCH -C haswell #SBATCH -q realtime #SBATCH --exclusive #SBATCH -J forcephot #SBATCH -t 00:60:00 #SBATCH -L SCRATCH #SBATCH -A {get_secret('nersc_account')} #SBATCH -o {str(scriptname).replace('.sh', '.out')} HDF5_USE_FILE_LOCKING=FALSE srun -n 832 -c1 --cpu_bind=cores shifter python $HOME/lensgrinder/scripts/dophot.py {imginname} {photoutname} """ with open(scriptname, 'w') as f: f.write(jobscript) cmd = f'sbatch {scriptname}' process = subprocess.Popen(cmd.split(), stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = process.communicate() print(stdout) if process.returncode != 0: raise RuntimeError(f'Non-zero exit code from sbatch, output was ' f'"{str(stdout)}", "{str(stdout)}".') jobid = stdout.strip().split()[-1].decode('ascii') os.chdir(curdir) return jobid, detinname, photoutname
print(stdout) if process.returncode != 0: raise RuntimeError(f'Non-zero exit code from sbatch, output was ' f'"{str(stdout)}", "{str(stdout)}".') jobid = stdout.strip().split()[-1].decode('ascii') os.chdir(curdir) return jobid, detinname, photoutname if __name__ == '__main__': while True: db.DBSession().rollback() # look for active jobs active_jobs = db.DBSession().query(db.ForcePhotJob).filter( db.sa.or_( db.ForcePhotJob.status == 'processing', db.ForcePhotJob.status == 'ready_for_loading', )) # get the slurm jobs and their statuses try: job_statuses = get_job_statuses() except RuntimeError as e: exc_info = sys.exc_info() traceback.print_exception(*exc_info)
def get_useful_AgentsServer(self): sn = db.DBSession() res = sn.session.query(db.AgentServer).all() return (item.IP_PORT for item in res)
print(stdout) if process.returncode != 0: raise RuntimeError(f'Non-zero exit code from sbatch, output was ' f'"{str(stdout)}", "{str(stdout)}".') os.chdir(curdir) jobid = stdout.strip().split()[-1].decode('ascii') return jobid if __name__ == '__main__': while True: db.DBSession().rollback() # look for failed jobs and mark them currently_processing = db.DBSession().query( db.Job).filter(db.Job.status == 'processing') alert_processing = db.DBSession().query( db.AlertJob).filter(db.AlertJob.status == 'processing') # get the slurm jobs and their statuses try: job_statuses = get_job_statuses() except RuntimeError as e: exc_info = sys.exc_info() traceback.print_exception(*exc_info)