def fix_tags_on_cands_missing_reals(user_id, vos_dir, property): "At the moment this just checks for a single user's missing reals. Easy to generalise it to all users." con = context.get_context(vos_dir) user_progress = [] listing = con.get_listing(tasks.get_suffix('reals')) mpc_listing = con.get_listing('mpc') for filename in listing: if not filename.startswith('fk'): user = storage.get_property(con.get_full_path(filename), property) if (user is not None): # and (user == user_id): # modify 'and' to generalise to all users with work in this directory #realsfile = filename.replace('cands', 'reals') #if not con.exists(realsfile): # print filename, 'no reals file', realsfile # go through the listing of .mpc files and see if any match this reals.astrom is_present = False for mpcfile in [ f for f in mpc_listing if not f.startswith('fk') ]: if mpcfile.startswith(filename): print(filename, user, 'exists!', mpcfile) is_present = True if not is_present: user_progress.append(filename) print(filename, user, 'no mpc file') storage.set_property(con.get_full_path(filename), property, None) print('Fixed files:', len(user_progress)) return
def fix_tags_on_cands_missing_reals(user_id, vos_dir, property): "At the moment this just checks for a single user's missing reals. Easy to generalise it to all users." con = context.get_context(vos_dir) user_progress = [] listing = con.get_listing(tasks.get_suffix('reals')) mpc_listing = con.get_listing('mpc') for filename in listing: if not filename.startswith('fk'): user = storage.get_property(con.get_full_path(filename), property) if (user is not None): # and (user == user_id): # modify 'and' to generalise to all users with work in this directory #realsfile = filename.replace('cands', 'reals') #if not con.exists(realsfile): # print filename, 'no reals file', realsfile # go through the listing of .mpc files and see if any match this reals.astrom is_present = False for mpcfile in [f for f in mpc_listing if not f.startswith('fk')]: if mpcfile.startswith(filename): print filename, user, 'exists!', mpcfile is_present = True if not is_present: user_progress.append(filename) print filename, user, 'no mpc file' storage.set_property(con.get_full_path(filename), property, None) print 'Fixed files:', len(user_progress) return
def _record_index(self, filename, index): if not self.track_partial_results: return processed_indices = self.get_processed_indices(filename) processed_indices.append(index) processed_indices = map(str, processed_indices) storage.set_property(self._get_uri(filename), PROCESSED_INDICES_PROPERTY, VO_INDEX_SEP.join(processed_indices))
def lock(self, filename): uri = self._get_uri(filename) lock_holder = storage.get_property(uri, LOCK_PROPERTY) if lock_holder is None: storage.set_property(uri, LOCK_PROPERTY, self.userid) elif lock_holder == self.userid: # We already had the lock pass else: raise FileLockedException(filename, lock_holder)
def _do_unlock(self, filename): uri = self._get_uri(filename) lock_holder = storage.get_property(uri, LOCK_PROPERTY) if lock_holder is None: # The file isn't actually locked. Probably already cleaned up. pass elif lock_holder == self.userid: # It was us who locked it storage.set_property(uri, LOCK_PROPERTY, None) else: # Can't remove someone else's lock! raise FileLockedException(filename, lock_holder)
def what_locks_remain(directory): con = context.get_context(directory) listing = con.get_listing(tasks.get_suffix('cands')) user_progress = collections.defaultdict(int) for filename in listing: user = storage.get_property(con.get_full_path(filename), LOCK_PROPERTY) if user is not None: user_progress[user] += 1 print filename, 'lock_holder=', user storage.set_property(con.get_full_path(filename), LOCK_PROPERTY, None) for user, num_locked in user_progress.iteritems(): print " %s: %d" % (user, num_locked) return
def what_locks_remain(directory): con = context.get_context(directory) listing = con.get_listing(tasks.get_suffix('cands')) user_progress = collections.defaultdict(int) for filename in listing: user = storage.get_property(con.get_full_path(filename), LOCK_PROPERTY) if user is not None: user_progress[user] += 1 print(filename, 'lock_holder=', user) storage.set_property(con.get_full_path(filename), LOCK_PROPERTY, None) for user, num_locked in user_progress.items(): print(" %s: %d" % (user, num_locked)) return
def update_vos_with_local_files(user_id, vos_dir, dir_to_scan): uploaded_count = 0 donefiles = glob(dir_to_scan + '*.reals.astrom') for fname in donefiles: fn = fname.rsplit('/')[len(fname.rsplit('/')) - 1] vo_reals = vos_dir + fn mv_file = vos_dir + fn.replace('reals', 'cands') # check if file's .cands.astrom equivalent in VOSpace has a #done tag wasdone = storage.get_property(mv_file, 'done') if not wasdone: if not storage.exists(vo_reals): # shouldn't possibly be there but let's just make sure storage.copy(fname, vo_reals) storage.set_property(mv_file, 'done', user_id) # set the .cands.astrom #done tag to the user ID. uploaded_count += 1 else: print fn, wasdone print 'Added unique files:', uploaded_count return
def _record_done(self, filename): storage.set_property(self._get_uri(filename), DONE_PROPERTY, self.userid)
def fix_tags_on_cands_missing_reals(user_id, vos_dir, property): "At the moment this just checks for a single user's missing reals. Easy to generalise it to all users: modify the and" con = context.get_context(vos_dir) user_progress = [] listing = con.get_listing(tasks.get_suffix('cands')) for filename in listing: user = storage.get_property(con.get_full_path(filename), property) if (user is not None) and ( user == user_id): # modify here to generalise to all users with work in this directory user_progress.append(filename) realsfile = filename.replace('cands', 'reals') if not storage.exists(con.get_full_path(realsfile)): print filename, 'no reals file', realsfile storage.set_property(con.get_full_path(filename), property, None) print 'Fixed files:', len(user_progress) return if __name__=='__main__': user_id = '' # set as appropriate vos_dir = storage.MEASURE3 + '/2013A-O/' dir_to_scan = '/Users/michele/Dropbox/OSSOS/measure3/2013A-O/ptsws/' #update_vos_with_local_files(user_id, vos_dir, dir_to_scan) fix_tags_on_cands_missing_reals(user_id, vos_dir, DONE_PROPERTY) # can also be LOCK_PROPERTY