def import_from_sweeps(db, sdss_tabname, sweep_files, create=False): """ Import an SDSS catalog from a collection of SDSS sweep files. Note: Assumes underlying shared storage for all output table cells (i.e., any worker is able to write to any cell). """ with locking.lock(db.path[0] + "/.__smf-import-lock.lock"): if not db.table_exists(sdss_tabname) and create: # Create the new database sdss_table = db.create_table(sdss_tabname, sdss_table_def) else: sdss_table = db.table(sdss_tabname) t0 = time.time() at = 0; ntot = 0 pool = pool2.Pool() for (file, nloaded) in pool.imap_unordered(sweep_files, import_from_sweeps_aux, (db, sdss_tabname), progress_callback=pool2.progress_pass): at = at + 1 ntot = ntot + nloaded t1 = time.time() time_pass = (t1 - t0) / 60 time_tot = time_pass / at * len(sweep_files) sfile = "..." + file[-67:] if len(file) > 70 else file print(' ===> Imported %-70s [%d/%d, %5.2f%%] +%-6d %9d (%.0f/%.0f min.)' % (sfile, at, len(sweep_files), 100 * float(at) / len(sweep_files), nloaded, ntot, time_pass, time_tot)) del pool
def test_locking(): lock_path = '/tmp/test-lock-%s' % datetime.now().strftime('%Y%m%dT%H%M%S') text = 'we are locked. go away.' # before locking assert locking.locked(lock_path) == False assert locking.unlock(lock_path, text) == 'not locked' # locking assert locking.lock(lock_path, text) == 'ok' # locked assert locking.locked(lock_path) == text assert locking.lock(lock_path, text) == 'locked' assert locking.unlock(lock_path, 'not the right text') == 'miss' # unlocking assert locking.unlock(lock_path, text) == 'ok' # unlocked assert locking.locked(lock_path) == False assert locking.unlock(lock_path, text) == 'not locked' assert not os.path.exists(lock_path)
def test_locking(): lock_path = '/tmp/test-lock-%s' % datetime.now(config.TZ).strftime('%Y%m%dT%H%M%S') text = 'we are locked. go away.' # before locking assert locking.locked(lock_path) == False assert locking.unlock(lock_path, text) == 'not locked' # locking assert locking.lock(lock_path, text) == 'ok' # locked assert locking.locked(lock_path) == text assert locking.lock(lock_path, text) == 'locked' assert locking.unlock(lock_path, 'not the right text') == 'miss' # unlocking assert locking.unlock(lock_path, text) == 'ok' # unlocked assert locking.locked(lock_path) == False assert locking.unlock(lock_path, text) == 'not locked' assert not os.path.exists(lock_path)
def import_from_smf(db, det_tabname, exp_tabname, smf_files, survey, create=False): """ Import a PS1 table from DVO Note: Assumes underlying shared storage for all table cells (i.e., any worker is able to write to any cell). """ with locking.lock(db.path[0] + "/.__smf-import-lock.lock"): if not db.table_exists(det_tabname) and create: # Set up commit hooks exp_table_def['commit_hooks'] = [ ('Updating neighbors', 1, 'lsd.smf', 'make_image_cache', [det_tabname]) ] # Create new tables det_table = db.create_table(det_tabname, det_table_def) exp_table = db.create_table(exp_tabname, exp_table_def) # Set up a one-to-X join relationship between the two tables (join det_table:exp_id->exp_table:exp_id) db.define_default_join(det_tabname, exp_tabname, type = 'indirect', m1 = (det_tabname, "det_id"), m2 = (det_tabname, "exp_id"), _overwrite=create ) else: det_table = db.table(det_tabname) exp_table = db.table(exp_tabname) det_c2f = gen_tab2fits(det_table_def) exp_c2f = gen_tab2fits(exp_table_def) t0 = time.time() at = 0; ntot = 0 pool = pool2.Pool() smf_fns = [] exp_ids = [] for (file, exp_id, smf_fn, nloaded, nin) in pool.imap_unordered(smf_files, import_from_smf_aux, (det_table, exp_table, det_c2f, exp_c2f, survey), progress_callback=pool2.progress_pass): smf_fns.append(smf_fn) exp_ids.append(exp_id) at = at + 1 ntot = ntot + nloaded t1 = time.time() time_pass = (t1 - t0) / 60 time_tot = time_pass / at * len(smf_files) print >>sys.stderr, ' ===> Imported %s [%d/%d, %5.2f%%] +%-6d %9d (%.0f/%.0f min.)' % (file, at, len(smf_files), 100 * float(at) / len(smf_files), nloaded, ntot, time_pass, time_tot) del pool ret = colgroup.ColGroup() ret._EXP = np.array(exp_ids, dtype=np.uint64) ret.smf_fn = np.array(smf_fns, dtype='a40') return ret
def import_from_catalogs(db, det_tabname, exp_tabname, catalog_files, create=False, all=False): """ Import a PTF catalog from a collection of SExtractor catalog files. Note: Assumes underlying shared storage for all output table cells (i.e., any worker is able to write to any cell). """ with locking.lock(db.path[0] + "/.__smf-import-lock.lock"): if not db.table_exists(det_tabname) and create: # Set up commit hooks exp_table_def['commit_hooks'] = [ ('Updating neighbors', 1, 'lsd.smf', 'make_image_cache', [det_tabname]) ] # Create new tables det_table = db.create_table(det_tabname, det_table_def) exp_table = db.create_table(exp_tabname, exp_table_def) # Set up a one-to-X join relationship between the two tables (join det_table:exp_id->exp_table:exp_id) db.define_default_join(det_tabname, exp_tabname, type = 'indirect', m1 = (det_tabname, "det_id"), m2 = (det_tabname, "exp_id") ) else: det_table = db.table(det_tabname) exp_table = db.table(exp_tabname) # MJD of import now = datetime.datetime.now() (djm, j) = sla_caldj(now.year, now.month, now.day) djm -= 55682 t0 = time.time() at = 0; ntot = 0 pool = pool2.Pool() explist_file = open('explist.txt','w') for (file, nloaded, error_type, expID) in pool.imap_unordered(catalog_files, import_from_catalogs_aux, (det_table, exp_table, djm, all), progress_callback=pool2.progress_pass): at = at + 1 ntot = ntot + nloaded t1 = time.time() time_pass = (t1 - t0) / 60 time_tot = time_pass / at * len(catalog_files) # sfile = (file)[-65:] if len(file) > 70 else file sfile = file if error_type == 0: print(' ===> Imported %s [%d/%d, %5.2f%%] +%-6d %9d (%.0f/%.0f min.)' % (sfile, at, len(catalog_files), 100 * float(at) / len(catalog_files), nloaded, ntot, time_pass, time_tot)) explist_file.write('%s\n' % str(expID)) elif error_type == 1: print('%s is missing!' % (sfile)) elif error_type == 2: print('%s has bad data type in exposure database!' % (sfile)) elif error_type == 3: print('%s has bad data type in detection database!' % (sfile)) elif error_type == 4: print('%s has bad WCS transform parameters!' % (sfile)) else: print "Nothing" # show the dirt every 100 ingests # if at % 100. == 0: # gc.collect() # n_collected = gc.collect() # if n_collected > 0: # dump_garbage() del pool explist_file.close()
def import_from_smf(db, det_tabname, exp_tabname, smf_files, survey, create=False): """ Import a PS1 table from DVO Note: Assumes underlying shared storage for all table cells (i.e., any worker is able to write to any cell). """ with locking.lock(db.path[0] + "/.__smf-import-lock.lock"): if not db.table_exists(det_tabname) and create: # Set up commit hooks exp_table_def['commit_hooks'] = [ ('Updating neighbors', 1, 'lsd.smf', 'make_image_cache', [det_tabname]) ] # Create new tables det_table = db.create_table(det_tabname, det_table_def) exp_table = db.create_table(exp_tabname, exp_table_def) # Set up a one-to-X join relationship between the two tables (join det_table:exp_id->exp_table:exp_id) db.define_default_join(det_tabname, exp_tabname, type='indirect', m1=(det_tabname, "det_id"), m2=(det_tabname, "exp_id"), _overwrite=create) else: det_table = db.table(det_tabname) exp_table = db.table(exp_tabname) det_c2f = gen_tab2fits(det_table_def) exp_c2f = gen_tab2fits(exp_table_def) t0 = time.time() at = 0 ntot = 0 pool = pool2.Pool() smf_fns = [] exp_ids = [] for (file, exp_id, smf_fn, nloaded, nin) in pool.imap_unordered( smf_files, import_from_smf_aux, (det_table, exp_table, det_c2f, exp_c2f, survey), progress_callback=pool2.progress_pass): smf_fns.append(smf_fn) exp_ids.append(exp_id) at = at + 1 ntot = ntot + nloaded t1 = time.time() time_pass = (t1 - t0) / 60 time_tot = time_pass / at * len(smf_files) print >> sys.stderr, ' ===> Imported %s [%d/%d, %5.2f%%] +%-6d %9d (%.0f/%.0f min.)' % ( file, at, len(smf_files), 100 * float(at) / len(smf_files), nloaded, ntot, time_pass, time_tot) del pool ret = colgroup.ColGroup() ret._EXP = np.array(exp_ids, dtype=np.uint64) ret.smf_fn = np.array(smf_fns, dtype='a40') return ret