def worker(x): """ Example with context manager """ with locking.lock(lockfn, timeout=None): with open(sumfile, "r+") as fp: n = int(fp.readline().strip()) + 1 fp.seek(0) fp.truncate() fp.write(str(n) + "\n") yield n
def import_from_catalogs(db, det_tabname, exp_tabname, catalog_files, create=False, all=False): """ Import a PTF catalog from a collection of SExtractor catalog files. Note: Assumes underlying shared storage for all output table cells (i.e., any worker is able to write to any cell). """ with locking.lock(db.path[0] + "/.__smf-import-lock.lock"): if not db.table_exists(det_tabname) and create: # Set up commit hooks exp_table_def['commit_hooks'] = [ ('Updating neighbors', 1, 'lsd.smf', 'make_image_cache', [det_tabname]) ] # Create new tables det_table = db.create_table(det_tabname, det_table_def) exp_table = db.create_table(exp_tabname, exp_table_def) # Set up a one-to-X join relationship between the two tables (join det_table:exp_id->exp_table:exp_id) db.define_default_join(det_tabname, exp_tabname, type='indirect', m1=(det_tabname, "det_id"), m2=(det_tabname, "exp_id")) else: det_table = db.table(det_tabname) exp_table = db.table(exp_tabname) # MJD of import now = datetime.datetime.now() djm = astropy.time.Time(datetime.datetime(now.year, now.month, now.day)).mjd djm -= 55682 t0 = time.time() at = 0 ntot = 0 pool = pool2.Pool() explist_file = open('explist.txt', 'w') for (file, nloaded, error_type, expID) in pool.imap_unordered(catalog_files, import_from_catalogs_aux, (det_table, exp_table, djm, all), progress_callback=pool2.progress_pass): at = at + 1 ntot = ntot + nloaded t1 = time.time() time_pass = (t1 - t0) / 60 time_tot = time_pass / at * len(catalog_files) # sfile = (file)[-65:] if len(file) > 70 else file sfile = file if error_type == 0: print( ' ===> Imported %s [%d/%d, %5.2f%%] +%-6d %9d (%.0f/%.0f min.)' % (sfile, at, len(catalog_files), 100 * float(at) / len(catalog_files), nloaded, ntot, time_pass, time_tot)) explist_file.write('%s\n' % str(expID)) elif error_type == 1: print('%s is missing!' % (sfile)) elif error_type == 2: print('%s has bad data type in exposure database!' % (sfile)) elif error_type == 3: print('%s has bad data type in detection database!' % (sfile)) elif error_type == 4: print('%s has bad WCS transform parameters!' % (sfile)) else: print "Nothing" # show the dirt every 100 ingests # if at % 100. == 0: # gc.collect() # n_collected = gc.collect() # if n_collected > 0: # dump_garbage() del pool explist_file.close()
def import_from_catalogs(db, det_tabname, exp_tabname, catalog_files, create=False, all=False): """ Import a PTF catalog from a collection of SExtractor catalog files. Note: Assumes underlying shared storage for all output table cells (i.e., any worker is able to write to any cell). """ with locking.lock(db.path[0] + "/.__smf-import-lock.lock"): if not db.table_exists(det_tabname) and create: # Set up commit hooks exp_table_def["commit_hooks"] = [("Updating neighbors", 1, "lsd.smf", "make_image_cache", [det_tabname])] # Create new tables det_table = db.create_table(det_tabname, det_table_def) exp_table = db.create_table(exp_tabname, exp_table_def) # Set up a one-to-X join relationship between the two tables (join det_table:exp_id->exp_table:exp_id) db.define_default_join( det_tabname, exp_tabname, type="indirect", m1=(det_tabname, "det_id"), m2=(det_tabname, "exp_id") ) else: det_table = db.table(det_tabname) exp_table = db.table(exp_tabname) # MJD of import now = datetime.datetime.now() (djm, j) = sla_caldj(now.year, now.month, now.day) djm -= 55682 t0 = time.time() at = 0 ntot = 0 pool = pool2.Pool() explist_file = open("explist.txt", "w") for (file, nloaded, error_type, expID) in pool.imap_unordered( catalog_files, import_from_catalogs_aux, (det_table, exp_table, djm, all), progress_callback=pool2.progress_pass ): at = at + 1 ntot = ntot + nloaded t1 = time.time() time_pass = (t1 - t0) / 60 time_tot = time_pass / at * len(catalog_files) # sfile = (file)[-65:] if len(file) > 70 else file sfile = file if error_type == 0: print ( " ===> Imported %s [%d/%d, %5.2f%%] +%-6d %9d (%.0f/%.0f min.)" % ( sfile, at, len(catalog_files), 100 * float(at) / len(catalog_files), nloaded, ntot, time_pass, time_tot, ) ) explist_file.write("%s\n" % str(expID)) elif error_type == 1: print ("%s is missing!" % (sfile)) elif error_type == 2: print ("%s has bad data type in exposure database!" % (sfile)) elif error_type == 3: print ("%s has bad data type in detection database!" % (sfile)) elif error_type == 4: print ("%s has bad WCS transform parameters!" % (sfile)) else: print "Nothing" # show the dirt every 100 ingests # if at % 100. == 0: # gc.collect() # n_collected = gc.collect() # if n_collected > 0: # dump_garbage() del pool explist_file.close()