def copy_unconsidered(names):
local_path = '/Users/michele/measure3/2013A-E/genuine_reals_run/tracking/discoveries/'
discovery_files = storage.listdir(DISCOVERIES)
for fn in discovery_files:
for name in names:
if fn.__contains__(name):
print fn
storage.copy(DISCOVERIES+fn, local_path+fn)
return
def get_names(path, blockID):
"""
:param path: the VOSpace path to folder
:param blockID: the block ID, formatted as 'O13AE' etc
:return: a set containing those provisional designations present in the folder
"""
retval = set()
for fn in storage.listdir(path):
name = [f for f in fn.split('.') if f.startswith(blockID)][0]
retval.add(name)
return retval
def get_discoveries(blockID='O13AE'):
retval = []
path = 'vos:OSSOS/measure3/2013A-E/track/submitted/' # rather than discoveries, for longer arcs
# path = 'vos:OSSOS/measure3/2013A-E/track/radec_corrected/'
discoveries = storage.listdir(path)
names = get_names(path, blockID)
print 'Discoveries:', len(names)
for i, kbo in enumerate(names):
try:
arclen, orbit = parse(kbo, discoveries, path=path)
retval.append(orbit)
except:
continue
return retval
def cutout(obj, obj_dir, radius):
cutout_listing = storage.listdir(obj_dir, force=True)
for obs in obj.mpc_observations:
if obs.null_observation:
logging.debug('skipping: {}'.format(obs))
continue
if obs.date > parameters.SURVEY_START:
# can't make postage stamps of earlier linkages
# can't parse for an obs.comment's exposure number if no
# obs.comment exists
try:
parts = storage.frame2expnum(obs.comment.frame)
except Exception as ex:
logging.warning(f"Skipping: {obs}")
logging.debug(f"Failed to map comment.frame to expnum: {ex}")
continue
uri = storage.get_uri(parts['expnum'], version=parts['version'])
sky_coord = obs.coordinate
# Using the WCS rather than the X/Y
# (X/Y can be unreliable over the whole survey)
postage_stamp_filename = f"{obj.provisional_name}_" \
f"{obs.date.mjd:11.5f}_" \
f"{obs.coordinate.ra.degree:09.5f}_" \
f"{obs.coordinate.dec.degeee:09.5f}.fits"
if postage_stamp_filename in cutout_listing:
# skipping existing cutouts
continue
# ast_header = storage._get_sghead(parts['expnum'])
while True:
try:
hdulist = storage.ra_dec_cutout(uri, sky_coord, radius, update_wcs=True)
with open(postage_stamp_filename, 'w') as tmp_file:
hdulist.writeto(tmp_file, overwrite=True, output_verify='fix+ignore')
storage.copy(postage_stamp_filename, obj_dir + "/" + postage_stamp_filename)
os.unlink \
(postage_stamp_filename) # easier not to have them hanging around
except OSError as e: # occasionally the node is not found: report and move on for later cleanup
logging.error("OSError: -> " +str(e))
except Exception as e:
logging.error("Exception: -> " +str(e))
continue
break
def main():
parser = argparse.ArgumentParser(
description='Parse a directory of TNO .ast files and create links in the postage stamp directory '
'that allow retrieval of cutouts of the FITS images associated with the OSSOS detections. '
'Cutouts are defined on the WCS RA/DEC of the object position.')
parser.add_argument("version",
help="The OSSOS data release version these stamps should be assigned to.")
parser.add_argument("--ossin",
action="store",
default="vos:OSSOS/dbaseclone/ast/",
help="The vospace containerNode that clones ossin dbaseclone"
"holding the .ast files of astrometry/photometry measurements.")
parser.add_argument("--blocks", "-b",
action="store",
default=["o3e", "o3o"],
choices=["o3e", "o3o", "O13BL", "Col3N"],
help="Prefixes of object designations to include.")
parser.add_argument("--radius", '-r',
action='store',
default=0.02,
help='Radius (degree) of circle of cutout postage stamp.')
parser.add_argument("--debug", "-d",
action="store_true")
parser.add_argument("--verbose", "-v",
action="store_true")
args = parser.parse_args()
username = raw_input("CADC username: "******"CADC password: ")
if args.debug:
logging.basicConfig(level=logging.DEBUG)
elif args.verbose:
logging.basicConfig(level=logging.INFO)
for fn in storage.listdir(args.ossin)[10:11]: #FIXME: TESTING ONLY
obj = mpc.MPCReader(args.ossin + fn) # let MPCReader's logic determine the provisional name
for block in args.blocks:
if obj.provisional_name.startswith(block):
obj_dir = '{}/{}/{}'.format(storage.POSTAGE_STAMPS, args.version, obj.provisional_name)
if not storage.exists(obj_dir, force=True):
storage.mkdir(obj_dir)
# assert storage.exists(obj_dir, force=True)
cutout(obj, obj_dir, args.radius, username, password)
def ossos_discoveries(no_nt_and_u=True):
"""
Returns a list of orbfit.Orbfit objects with the observations in the Orbfit.observations field.
"""
retval = []
# discovery_files = [n for n in os.listdir(parameters.REAL_KBO_AST_DIR)]# if n.endswith('.mpc') or n.endswith(
# '.ast')]
for filename in storage.listdir(parameters.REAL_KBO_AST_DIR):
# keep out the not-tracked and uncharacterised.
if no_nt_and_u:
if not (filename.__contains__('nt') or filename.__contains__('u')):
print(filename)
observations = mpc.MPCReader(parameters.REAL_KBO_AST_DIR +
filename)
orbit = orbfit.Orbfit(observations.mpc_observations)
retval.append((observations, orbit))
return retval
def plot_ossos_discoveries(ax, blockID='O13AE', date="2013/04/09 08:50:00"):
path = 'vos:OSSOS/measure3/2013A-E/track/discoveries/'
discoveries = storage.listdir(path)
names = get_names(path, blockID)
for kbo in names:
arclen, orbit = parse(kbo, discoveries, path=path)
if date != "2013/04/09 08:50:00": # date is the new moon for a given month
orbit.predict(date.replace('/', '-'))
ra = orbit.coordinate.ra.degrees
dec = orbit.coordinate.dec.degrees
else: # specific date on which discovery was made: use discovery locations
ra = orbit.observations[0].coordinate.ra.degrees
dec = orbit.observations[0].coordinate.dec.degrees
ax.scatter(ra, dec, marker='+', facecolor='k')
ax.annotate(kbo.replace(blockID,''),
(orbit.coordinate.ra.degrees+.05, orbit.coordinate.dec.degrees-0.2),
size='xx-small',
color='k')
return ax
def cutout(obj, obj_dir, radius):
cutout_listing = storage.listdir(obj_dir, force=True)
for obs in obj.mpc_observations:
if obs.null_observation:
logging.debug('skipping: {}'.format(obs))
continue
if obs.date > parameters.SURVEY_START: # can't make postage stamps of earlier linkages
# can't parse for an obs.comment's exposure number if no obs.comment exists
try:
parts = storage.frame2expnum(obs.comment.frame)
except Exception as ex:
logging.error("Skipping: {}\nFailed to map comment.frame to expnum: {}".format(obs, ex))
continue
uri = storage.get_uri(parts['expnum'], version=parts['version'])
sky_coord = obs.coordinate # Using the WCS rather than the X/Y (X/Y can be unreliable over the whole survey)
postage_stamp_filename = "{}_{:11.5f}_{:09.5f}_{:+09.5f}.fits".format(obj.provisional_name,
obs.date.mjd,
obs.coordinate.ra.degree,
obs.coordinate.dec.degree)
if postage_stamp_filename in cutout_listing:
# skipping existing cutouts
continue
# ast_header = storage._get_sghead(parts['expnum'])
while True:
try:
hdulist = storage.ra_dec_cutout(uri, sky_coord, radius, update_wcs=True)
with open(postage_stamp_filename, 'w') as tmp_file:
hdulist.writeto(tmp_file, overwrite=True, output_verify='fix+ignore')
storage.copy(postage_stamp_filename, obj_dir + "/" + postage_stamp_filename)
os.unlink(postage_stamp_filename) # easier not to have them hanging around
except OSError as e: # occasionally the node is not found: report and move on for later cleanup
logging.error("OSError: ->"+str(e))
except Exception as e:
logging.error("Exception: ->"+str(e))
continue
break
provisional_name = [name for name in former_names if (name.startswith('O1') and name.isupper())][0]
outfile.write('{:<12} {:<12}'.format(discovery, provisional_name))
# build the path that the .cands.astrom files would be in.
block = paths[provisional_name[0:5]]
path = '{}/{}/'.format(storage.MEASURE3, block)
if discovery in ['o3e53', 'o3e54', 'o3e55']: # deal with the special case: their .cands.astrom are elsewhere
path = '{}/{}/'.format(storage.MEASURE3, '2013A-E_April9')
if block not in files:
if block in ['2013A-E', '2013A-O']:
# 13AE and 13AO ended up particularly weird. Fortunately, have a local list
with open('/Users/michele/Desktop/{}_mpclist.txt'.format(block), 'r') as infile:
block_files = infile.readlines()
else:
print('starting VOSpace read') # getting file listings from big listings on VOSpace is slooooowwww...
block_files = [name for name in storage.listdir(path) if not name.startswith('fk')]
files[block] = block_files
else:
block_files = files[block]
# not all provisional names have the full Oyearsemesterblock pattern. Because counter.
reals_files = [c for c in block_files if
(c.__contains__(provisional_name) or c.__contains__(provisional_name[-3:]))]
if len(reals_files) > 0: # ensure it's present in this directory
chip = reals_files[0].split('.')[0]
cand_filename = '{}.measure3.cands.astrom'.format(chip)
# now who set the tag?
uri = path + cand_filename
discoverer = storage.vospace.getNode(uri, force=True).props[storage.OSSOS_TAG_URI_BASE + '#done']
outfile.write('{:<12} {:<12}'.format(discoverer, chip))
provisional_name = filter(lambda name: (name.startswith('O1') and name.isupper()), former_names)[0]
outfile.write('{:<12} {:<12}'.format(discovery, provisional_name))
# build the path that the .cands.astrom files would be in.
block = paths[provisional_name[0:5]]
path = '{}/{}/'.format(storage.MEASURE3, block)
if discovery in ['o3e53', 'o3e54', 'o3e55']: # deal with the special case: their .cands.astrom are elsewhere
path = '{}/{}/'.format(storage.MEASURE3, '2013A-E_April9')
if block not in files:
if block in ['2013A-E', '2013A-O']:
# 13AE and 13AO ended up particularly weird. Fortunately, have a local list
with open('/Users/michele/Desktop/{}_mpclist.txt'.format(block), 'r') as infile:
block_files = infile.readlines()
else:
print 'starting VOSpace read' # getting file listings from big listings on VOSpace is slooooowwww...
block_files = filter(lambda name: not name.startswith('fk'), storage.listdir(path))
files[block] = block_files
else:
block_files = files[block]
# not all provisional names have the full Oyearsemesterblock pattern. Because counter.
reals_files = [c for c in block_files if
(c.__contains__(provisional_name) or c.__contains__(provisional_name[-3:]))]
if len(reals_files) > 0: # ensure it's present in this directory
chip = reals_files[0].split('.')[0]
cand_filename = '{}.measure3.cands.astrom'.format(chip)
# now who set the tag?
uri = path + cand_filename
discoverer = storage.vospace.getNode(uri, force=True).props[storage.OSSOS_TAG_URI_BASE + '#done']
outfile.write('{:<12} {:<12}'.format(discoverer, chip))
def listdir(self):
# Don't force because it causes a HUGE performance hit.
logger.debug(f"Getting a listing of {self.directory}")
dir_list = storage.listdir(self.directory, force=False)
logger.debug(f"Got: {dir_list}")
return dir_list
print 'Submitted:', len(submitted)
n_and_sub = []
for dd in discoveries:
if (dd in nailings) and (dd in submitted):
n_and_sub.append(dd)
else:
if (dd in nailings) and (dd not in submitted):
print 'nailing, but not submitted?', dd
if ((dd in submitted) and (dd not in nailings)) and (dd not in initial_submitted):
print 'submitted but not nailed...', dd
# how many nailed files have checkups?
# (don't have to include initial_submitted in these, they don't have nailings or checkups)
for nn in nailings:
if (nn not in arc_extended):
print 'nailed, but arc not extended?', nn
# What's the arc length on the nailed ones?
subs = storage.listdir(SUBMITTED)
arclens = []
for ss in submitted:
arclen, orbit = parse(ss, subs)
arclens.append(arclen)
arclens.sort()
print arclens
print len(arclens)
bins = [0,1,4,20,40,60,90,150]
hist,bin_edges = numpy.histogram(arclens,bins=bins)
print hist
def main():
parser = argparse.ArgumentParser(
description=
'Run SSOIS on a given set of minor planets and return the available images in a particular filter.'
)
parser.add_argument("--filter",
"-f",
action="store",
default='r',
dest="filter",
choices=['r', 'u'],
help="passband: default is r'")
parser.add_argument("--ossin",
action="store",
default="vos:OSSOS/dbaseclone/ast/",
help='vospace dbaseclone containerNode')
parser.add_argument("--dbimages",
action="store",
default="vos:OSSOS/dbimages",
help='vospace dbimages containerNode')
parser.add_argument(
'--type',
default='p',
choices=['o', 'p', 's'],
help="restrict type of image (unprocessed, reduced, calibrated)")
parser.add_argument(
"--output",
"-o",
action="store",
default="/Users/michele//Desktop/band.txt",
help='Location and name of output file containing image IDs.')
args = parser.parse_args()
if args.filter.lower().__contains__('r'):
args.filter = 'r.MP9601' # this is the old (standard) r filter for MegaCam
if args.filter.lower().__contains__('u'):
args.filter = 'u.MP9301'
with open(args.output, 'w') as outfile:
outfile.write(
"{:>10s} {:>10s} {:>2s} {:>5s} {:>5s} {:>16s} {:>16s} {:>16s} {:>12s}\n"
.format("Object", "Image", "Ext", "X", "Y", "RA", "DEC", "time",
"filter"))
for kbo_filename in storage.listdir(args.ossin):
orbit, obs_in_filter = parse_kbo(args.ossin + kbo_filename,
args.filter)
if len(obs_in_filter) > 0:
with open(args.output, 'a') as outfile:
for line in obs_in_filter:
outfile.write(
"{:>10s} {:>10s} {:>2s} {:6.1f} {:6.1f} {:8.16f} {:8.16f} {:>10s} {:>10s}\n"
.format(orbit.name, line['Image'], str(line['Ext']),
line['X'], line['Y'], line['Object_RA'],
line['Object_Dec'],
Time(line['MJD'], format='mjd',
scale='utc'), line['Filter']))
outfile.write('\n')
def getList(name=None):
for filename in storage.listdir(astDir):
if name is None or name in filename:
build(os.path.join(astDir, filename))
'o3e53', 'o3e54', 'o3e55'
]: # deal with the special case: their .cands.astrom are elsewhere
path = '{}/{}/'.format(storage.MEASURE3, '2013A-E_April9')
if block not in files:
if block in ['2013A-E', '2013A-O']:
# 13AE and 13AO ended up particularly weird. Fortunately, have a local list
with open(
'/Users/michele/Desktop/{}_mpclist.txt'.format(block),
'r') as infile:
block_files = infile.readlines()
else:
print(
'starting VOSpace read'
) # getting file listings from big listings on VOSpace is slooooowwww...
block_files = [
name for name in storage.listdir(path)
if not name.startswith('fk')
]
files[block] = block_files
else:
block_files = files[block]
# not all provisional names have the full Oyearsemesterblock pattern. Because counter.
reals_files = [
c for c in block_files
if (c.__contains__(provisional_name)
or c.__contains__(provisional_name[-3:]))
]
if len(reals_files) > 0: # ensure it's present in this directory
chip = reals_files[0].split('.')[0]
def getList(name = None):
for filename in storage.listdir(astDir):
if name is None or name in filename:
build(os.path.join(astDir,filename))
def listdir(self):
# Don't force because it causes a HUGE performance hit.
return storage.listdir(self.directory, force=False)
def listdir(self):
# Don't force because it causes a HUGE performance hit.
return storage.listdir(self.directory, force=False)
