def check_tags(my_expnum, ops_set, my_ccds, dry_run=True):
"""
check the tags for the given expnum/ccd set.
@param ops:
@param my_expnum:
@return:
"""
tags = storage.get_tags(my_expnum)
count = 0
outcount = 0
fails = []
for ccd in my_ccds:
success = True
count += 1
for ops in ops_set:
for fake in ops[0]:
for my_program in ops[1]:
for version in ops[2]:
#print my_expnum, fake, my_program, version, ccd
key = storage.get_process_tag(fake + my_program, ccd, version)
uri = storage.tag_uri(key)
if "Failed to get image" in tags.get(uri, "Failed to get image"):
#print tags.get(uri, None)
fails.append(ccd)
success = False
if success:
outcount += 1
sys.stderr.write("{} {} {:5.1f}%\n".format(outcount, count,100* float(outcount)/count))
#print fails
return set(fails)
def check_tags(my_expnum, ops_set, my_ccds, dry_run=True):
"""
check the tags for the given expnum/ccd set.
@param ops:
@param my_expnum:
@return:
"""
tags = storage.get_tags(my_expnum)
count = 0
success = 0
for ops in ops_set:
for fake in ops[0]:
for my_program in ops[1]:
for version in ops[2]:
for ccd in my_ccds:
count += 1
# print my_expnum, fake, my_program, version, ccd
key = storage.get_process_tag(fake + my_program, ccd,
version)
uri = storage.tag_uri(key)
if tags.get(uri, None) != storage.SUCCESS:
print "{} --> {}".format(uri, tags.get(uri, None))
else:
success += 1
return "{} {} {:5.1f}%".format(success, count,
100 * float(success) / count)
def check_tags(my_expnum, ops_set, my_ccds, dry_run=True):
"""
check the tags for the given expnum/ccd set.
@param ops:
@param my_expnum:
@return:
"""
tags = storage.get_tags(my_expnum)
count = 0
outcount = 0
fails = []
for ccd in my_ccds:
success = True
count += 1
for ops in ops_set:
for fake in ops[0]:
for my_program in ops[1]:
for version in ops[2]:
#print my_expnum, fake, my_program, version, ccd
key = storage.get_process_tag(fake + my_program, ccd,
version)
uri = storage.tag_uri(key)
if "Failed to get image" in tags.get(
uri, "Failed to get image"):
#print tags.get(uri, None)
fails.append(ccd)
success = False
if success:
outcount += 1
sys.stderr.write("{} {} {:5.1f}%\n".format(outcount, count,
100 * float(outcount) / count))
#print fails
return set(fails)
def clean_keys(self, image_id):
# first retrieve and clean off the keys
node = storage.get_tags(image_id, force=True)
unwanted = ['creator', 'date', 'groupread', 'groupwrite', 'ispublic', 'length']
proc_keys = []
for vtag, value in node.items():
if (vtag not in unwanted) and not vtag.__contains__('fwhm') and not vtag.__contains__('zeropoint'):
try:
clean_key = vtag.split("#")[1]
except:
clean_key = vtag
proc_keys.append((clean_key, value))
return proc_keys
def clean_keys(self, image_id):
# first retrieve and clean off the keys
node = storage.get_tags(image_id, force=True)
unwanted = [
'creator', 'date', 'groupread', 'groupwrite', 'ispublic', 'length'
]
proc_keys = []
for vtag, value in list(node.items()):
if (vtag not in unwanted) and not vtag.__contains__(
'fwhm') and not vtag.__contains__('zeropoint'):
try:
clean_key = vtag.split("#")[1]
except:
clean_key = vtag
proc_keys.append((clean_key, value))
return proc_keys
def main(expnum):
commands = ['mkpsf', 'step1', 'step2', 'step3', 'combine']
tags = storage.get_tags(expnum)
for ccd in range(36):
for command in commands:
tag = storage.tag_uri(storage.get_process_tag(command, ccd, 'p'))
if tags.get(tag, None) != storage.SUCCESS:
sys.stderr.write("FAILED: {} {} {}\n".format(
expnum, ccd, command))
continue
tag = storage.tag_uri(
storage.get_process_tag('fk' + command, ccd, 's'))
if tags.get(tag, None) != storage.SUCCESS:
sys.stderr.write("FAILED: {} {} {}\n".format(
expnum, ccd, command))
continue
def main(expnum):
commands= ['mkpsf',
'step1',
'step2',
'step3',
'combine']
tags = storage.get_tags(expnum)
for ccd in range(36):
for command in commands:
tag = storage.tag_uri(storage.get_process_tag(command, ccd, 'p'))
if tags.get(tag, None) != storage.SUCCESS:
sys.stderr.write("FAILED: {} {} {}\n".format(expnum, ccd, command))
continue
tag = storage.tag_uri(storage.get_process_tag('fk'+command, ccd, 's'))
if tags.get(tag, None) != storage.SUCCESS:
sys.stderr.write("FAILED: {} {} {}\n".format(expnum, ccd, command))
continue
def main(commands=['update_header', 'mkpsf', 'step1', 'step2', 'step3', 'combine'],
ccds=range(0,36),
launch=False, triplets_dir=TRIPLETS_NODE,
delete=False):
cmd_order={'LEAD': ['update_header', 'mkpsf', 'step1', 'step2', 'step3', 'combine'],
'OTHER': ['update_header', 'mkpsf', 'step1' ] }
existance = {'mkpsf': ['psf.fits', ],
'step1': ['obj.jmp', 'obj.matt'],
'step2': ['unid.jmp', 'unid.matt'],
'step3': ['cands.jmp', 'cands.matt']}
for line in get_triplets(triplets_dir):
for command in commands:
exp_type = 'LEAD'
for expnum in line[0:3]:
tags = storage.get_tags(expnum)
if command not in cmd_order[exp_type]:
continue
for ccd in ccds:
tag = storage.tag_uri(storage.get_process_tag(command, ccd))
print line[3], command, expnum, ccd, tags.get(tag,None)
if tags.get(tag,None) != storage.SUCCESS:
if launch:
submit(command, expnum, ccd)
if delete:
success=True
for ext in existance[command]:
uri = storage.dbimages_uri(expnum, ccd, version='p', ext=ext)
if not storage.exists(uri):
success=False
print expnum, ccd, success
#for next_command in cmd_order[exp_type][commands.index(command)+1:]:
# storage.set_status(expnum, ccd, next_command, command+" FAILED")
if success:
storage.set_status(expnum, ccd, command, version='p', status='success')
exp_type='OTHER'
