def check_tags(my_expnum, ops_set, my_ccds, dry_run=True): """ check the tags for the given expnum/ccd set. @param ops: @param my_expnum: @return: """ tags = storage.get_tags(my_expnum) count = 0 outcount = 0 fails = [] for ccd in my_ccds: success = True count += 1 for ops in ops_set: for fake in ops[0]: for my_program in ops[1]: for version in ops[2]: #print my_expnum, fake, my_program, version, ccd key = storage.get_process_tag(fake + my_program, ccd, version) uri = storage.tag_uri(key) if "Failed to get image" in tags.get(uri, "Failed to get image"): #print tags.get(uri, None) fails.append(ccd) success = False if success: outcount += 1 sys.stderr.write("{} {} {:5.1f}%\n".format(outcount, count,100* float(outcount)/count)) #print fails return set(fails)
def check_tags(my_expnum, ops_set, my_ccds, dry_run=True): """ check the tags for the given expnum/ccd set. @param ops: @param my_expnum: @return: """ tags = storage.get_tags(my_expnum) count = 0 success = 0 for ops in ops_set: for fake in ops[0]: for my_program in ops[1]: for version in ops[2]: for ccd in my_ccds: count += 1 # print my_expnum, fake, my_program, version, ccd key = storage.get_process_tag(fake + my_program, ccd, version) uri = storage.tag_uri(key) if tags.get(uri, None) != storage.SUCCESS: print "{} --> {}".format(uri, tags.get(uri, None)) else: success += 1 return "{} {} {:5.1f}%".format(success, count, 100 * float(success) / count)
def check_tags(my_expnum, ops_set, my_ccds, dry_run=True): """ check the tags for the given expnum/ccd set. @param ops: @param my_expnum: @return: """ tags = storage.get_tags(my_expnum) count = 0 outcount = 0 fails = [] for ccd in my_ccds: success = True count += 1 for ops in ops_set: for fake in ops[0]: for my_program in ops[1]: for version in ops[2]: #print my_expnum, fake, my_program, version, ccd key = storage.get_process_tag(fake + my_program, ccd, version) uri = storage.tag_uri(key) if "Failed to get image" in tags.get( uri, "Failed to get image"): #print tags.get(uri, None) fails.append(ccd) success = False if success: outcount += 1 sys.stderr.write("{} {} {:5.1f}%\n".format(outcount, count, 100 * float(outcount) / count)) #print fails return set(fails)
def clean_keys(self, image_id): # first retrieve and clean off the keys node = storage.get_tags(image_id, force=True) unwanted = ['creator', 'date', 'groupread', 'groupwrite', 'ispublic', 'length'] proc_keys = [] for vtag, value in node.items(): if (vtag not in unwanted) and not vtag.__contains__('fwhm') and not vtag.__contains__('zeropoint'): try: clean_key = vtag.split("#")[1] except: clean_key = vtag proc_keys.append((clean_key, value)) return proc_keys
def clean_keys(self, image_id): # first retrieve and clean off the keys node = storage.get_tags(image_id, force=True) unwanted = [ 'creator', 'date', 'groupread', 'groupwrite', 'ispublic', 'length' ] proc_keys = [] for vtag, value in list(node.items()): if (vtag not in unwanted) and not vtag.__contains__( 'fwhm') and not vtag.__contains__('zeropoint'): try: clean_key = vtag.split("#")[1] except: clean_key = vtag proc_keys.append((clean_key, value)) return proc_keys
def main(expnum): commands = ['mkpsf', 'step1', 'step2', 'step3', 'combine'] tags = storage.get_tags(expnum) for ccd in range(36): for command in commands: tag = storage.tag_uri(storage.get_process_tag(command, ccd, 'p')) if tags.get(tag, None) != storage.SUCCESS: sys.stderr.write("FAILED: {} {} {}\n".format( expnum, ccd, command)) continue tag = storage.tag_uri( storage.get_process_tag('fk' + command, ccd, 's')) if tags.get(tag, None) != storage.SUCCESS: sys.stderr.write("FAILED: {} {} {}\n".format( expnum, ccd, command)) continue
def main(expnum): commands= ['mkpsf', 'step1', 'step2', 'step3', 'combine'] tags = storage.get_tags(expnum) for ccd in range(36): for command in commands: tag = storage.tag_uri(storage.get_process_tag(command, ccd, 'p')) if tags.get(tag, None) != storage.SUCCESS: sys.stderr.write("FAILED: {} {} {}\n".format(expnum, ccd, command)) continue tag = storage.tag_uri(storage.get_process_tag('fk'+command, ccd, 's')) if tags.get(tag, None) != storage.SUCCESS: sys.stderr.write("FAILED: {} {} {}\n".format(expnum, ccd, command)) continue
def main(commands=['update_header', 'mkpsf', 'step1', 'step2', 'step3', 'combine'], ccds=range(0,36), launch=False, triplets_dir=TRIPLETS_NODE, delete=False): cmd_order={'LEAD': ['update_header', 'mkpsf', 'step1', 'step2', 'step3', 'combine'], 'OTHER': ['update_header', 'mkpsf', 'step1' ] } existance = {'mkpsf': ['psf.fits', ], 'step1': ['obj.jmp', 'obj.matt'], 'step2': ['unid.jmp', 'unid.matt'], 'step3': ['cands.jmp', 'cands.matt']} for line in get_triplets(triplets_dir): for command in commands: exp_type = 'LEAD' for expnum in line[0:3]: tags = storage.get_tags(expnum) if command not in cmd_order[exp_type]: continue for ccd in ccds: tag = storage.tag_uri(storage.get_process_tag(command, ccd)) print line[3], command, expnum, ccd, tags.get(tag,None) if tags.get(tag,None) != storage.SUCCESS: if launch: submit(command, expnum, ccd) if delete: success=True for ext in existance[command]: uri = storage.dbimages_uri(expnum, ccd, version='p', ext=ext) if not storage.exists(uri): success=False print expnum, ccd, success #for next_command in cmd_order[exp_type][commands.index(command)+1:]: # storage.set_status(expnum, ccd, next_command, command+" FAILED") if success: storage.set_status(expnum, ccd, command, version='p', status='success') exp_type='OTHER'