def scan_for_miscreant_files(directory, quarantine_directory, file_of_miscreants):
reals_fks = []
con = context.get_context(directory)
listing = con.listdir()
with open(file_of_miscreants, 'r') as infile:
for line in infile.readlines():
if line.split()[5] != '0':
reals_fks.append(line.split()[3] + '_p' + line.split()[4])
if line.split()[6] != '0':
reals_fks.append('fk_' + line.split()[3] + '_s' + line.split()[4])
for fileID in reals_fks:
# remove any .measure3.reals.astrom if present
reals_file = fileID + '.measure3.reals.astrom'
try:
if con.exists(reals_file): # this isn't catching the error message, no obvious reason why
# anything interesting about this file?
print 'Removing:', reals_file, con.get_file_size(reals_file), \
'done: %s' % storage.get_property(con.get_full_path(reals_file), DONE_PROPERTY), \
'locked: %s' % storage.get_property(con.get_full_path(reals_file), LOCK_PROPERTY)
# con.remove(reals_file)
except:
continue
# obtain all files remaining in the directory listing that contain this chip, just not the removed reals.astrom
fileID_files = [n for n in listing if
n.split('.')[0] == fileID and not n.__contains__('.measure3.reals.astrom')]
print fileID
print fileID_files
# move all remaining matching files to quarantine
for fn in fileID_files:
print directory + fn, '>', quarantine_directory + fn
con.rename(directory + fn, quarantine_directory + fn)
def fix_tags_on_cands_missing_reals(user_id, vos_dir, property):
"At the moment this just checks for a single user's missing reals. Easy to generalise it to all users."
con = context.get_context(vos_dir)
user_progress = []
listing = con.get_listing(tasks.get_suffix('reals'))
mpc_listing = con.get_listing('mpc')
for filename in listing:
if not filename.startswith('fk'):
user = storage.get_property(con.get_full_path(filename), property)
if (user is not None):
# and (user == user_id): # modify 'and' to generalise to all users with work in this directory
#realsfile = filename.replace('cands', 'reals')
#if not con.exists(realsfile):
# print filename, 'no reals file', realsfile
# go through the listing of .mpc files and see if any match this reals.astrom
is_present = False
for mpcfile in [
f for f in mpc_listing if not f.startswith('fk')
]:
if mpcfile.startswith(filename):
print(filename, user, 'exists!', mpcfile)
is_present = True
if not is_present:
user_progress.append(filename)
print(filename, user, 'no mpc file')
storage.set_property(con.get_full_path(filename), property,
None)
print('Fixed files:', len(user_progress))
return
def fix_tags_on_cands_missing_reals(user_id, vos_dir, property):
"At the moment this just checks for a single user's missing reals. Easy to generalise it to all users."
con = context.get_context(vos_dir)
user_progress = []
listing = con.get_listing(tasks.get_suffix('reals'))
mpc_listing = con.get_listing('mpc')
for filename in listing:
if not filename.startswith('fk'):
user = storage.get_property(con.get_full_path(filename), property)
if (user is not None):
# and (user == user_id): # modify 'and' to generalise to all users with work in this directory
#realsfile = filename.replace('cands', 'reals')
#if not con.exists(realsfile):
# print filename, 'no reals file', realsfile
# go through the listing of .mpc files and see if any match this reals.astrom
is_present = False
for mpcfile in [f for f in mpc_listing if not f.startswith('fk')]:
if mpcfile.startswith(filename):
print filename, user, 'exists!', mpcfile
is_present = True
if not is_present:
user_progress.append(filename)
print filename, user, 'no mpc file'
storage.set_property(con.get_full_path(filename), property, None)
print 'Fixed files:', len(user_progress)
return
def get_file_size(self, filename):
length_property = storage.get_property(
self.get_full_path(filename), "length", ossos_base=False)
if length_property is None:
# Fall-back if the length property is not set for some reason
return self._get_file_size_by_downloading(filename)
return int(length_property)
def get_file_size(self, filename):
length_property = storage.get_property(
self.get_full_path(filename), "length", ossos_base=False)
if length_property is None:
# Fall-back if the length property is not set for some reason
return self._get_file_size_by_downloading(filename)
return int(length_property)
def get_processed_indices(self, filename):
if not self.track_partial_results:
return []
uri = self._get_uri(filename)
if not storage.has_property(uri, PROCESSED_INDICES_PROPERTY):
return []
raw_property = storage.get_property(uri, PROCESSED_INDICES_PROPERTY)
return map(int, raw_property.split(VO_INDEX_SEP))
def get_processed_indices(self, filename):
if not self.track_partial_results:
return []
uri = self._get_uri(filename)
if not storage.has_property(uri, PROCESSED_INDICES_PROPERTY):
return []
raw_property = storage.get_property(uri, PROCESSED_INDICES_PROPERTY)
return map(int, raw_property.split(VO_INDEX_SEP))
def lock(self, filename):
uri = self._get_uri(filename)
lock_holder = storage.get_property(uri, LOCK_PROPERTY)
if lock_holder is None:
storage.set_property(uri, LOCK_PROPERTY, self.userid)
elif lock_holder == self.userid:
# We already had the lock
pass
else:
raise FileLockedException(filename, lock_holder)
def lock(self, filename):
uri = self._get_uri(filename)
lock_holder = storage.get_property(uri, LOCK_PROPERTY)
if lock_holder is None:
storage.set_property(uri, LOCK_PROPERTY, self.userid)
elif lock_holder == self.userid:
# We already had the lock
pass
else:
raise FileLockedException(filename, lock_holder)
def ensure_cands_have_matching_reals(directory):
con = context.get_context(directory)
listing = con.get_listing(tasks.get_suffix('cands'))
for filename in listing:
user = storage.get_property(con.get_full_path(filename), DONE_PROPERTY)
if user is not None:
reals_file = filename.replace('cands', 'reals')
if not con.exists(reals_file):
print '.cands.astrom has no matching reals.astrom!', filename, 'done by', user
return
def ensure_cands_have_matching_reals(directory):
con = context.get_context(directory)
listing = con.get_listing(tasks.get_suffix('cands'))
for filename in listing:
user = storage.get_property(con.get_full_path(filename), DONE_PROPERTY)
if user is not None:
reals_file = filename.replace('cands', 'reals')
if not con.exists(reals_file):
print '.cands.astrom has no matching reals.astrom!', filename, 'done by', user
return
def _do_unlock(self, filename):
uri = self._get_uri(filename)
lock_holder = storage.get_property(uri, LOCK_PROPERTY)
if lock_holder is None:
# The file isn't actually locked. Probably already cleaned up.
pass
elif lock_holder == self.userid:
# It was us who locked it
storage.set_property(uri, LOCK_PROPERTY, None)
else:
# Can't remove someone else's lock!
raise FileLockedException(filename, lock_holder)
def _do_unlock(self, filename):
uri = self._get_uri(filename)
lock_holder = storage.get_property(uri, LOCK_PROPERTY)
if lock_holder is None:
# The file isn't actually locked. Probably already cleaned up.
pass
elif lock_holder == self.userid:
# It was us who locked it
storage.set_property(uri, LOCK_PROPERTY, None)
else:
# Can't remove someone else's lock!
raise FileLockedException(filename, lock_holder)
def what_locks_remain(directory):
con = context.get_context(directory)
listing = con.get_listing(tasks.get_suffix('cands'))
user_progress = collections.defaultdict(int)
for filename in listing:
user = storage.get_property(con.get_full_path(filename), LOCK_PROPERTY)
if user is not None:
user_progress[user] += 1
print filename, 'lock_holder=', user
storage.set_property(con.get_full_path(filename), LOCK_PROPERTY, None)
for user, num_locked in user_progress.iteritems():
print " %s: %d" % (user, num_locked)
return
def what_locks_remain(directory):
con = context.get_context(directory)
listing = con.get_listing(tasks.get_suffix('cands'))
user_progress = collections.defaultdict(int)
for filename in listing:
user = storage.get_property(con.get_full_path(filename), LOCK_PROPERTY)
if user is not None:
user_progress[user] += 1
print(filename, 'lock_holder=', user)
storage.set_property(con.get_full_path(filename), LOCK_PROPERTY,
None)
for user, num_locked in user_progress.items():
print(" %s: %d" % (user, num_locked))
return
def print_progress_stats(task, directory):
con = context.get_context(directory)
user_progress = collections.defaultdict(int)
listing = con.get_listing(tasks.get_suffix(task))
for filename in listing:
user = storage.get_property(con.get_full_path(filename), DONE_PROPERTY)
if user is not None:
user_progress[user] += 1
total_processed = sum(user_progress.values())
print "%s: %s: %d of %d processed." % (
directory, task, total_processed, len(listing))
print "---"
for user, num_processed in user_progress.iteritems():
print " %s: %d" % (user, num_processed)
def update_vos_with_local_files(user_id, vos_dir, dir_to_scan):
uploaded_count = 0
donefiles = glob(dir_to_scan + '*.reals.astrom')
for fname in donefiles:
fn = fname.rsplit('/')[len(fname.rsplit('/')) - 1]
vo_reals = vos_dir + fn
mv_file = vos_dir + fn.replace('reals', 'cands')
# check if file's .cands.astrom equivalent in VOSpace has a #done tag
wasdone = storage.get_property(mv_file, 'done')
if not wasdone:
if not storage.exists(vo_reals): # shouldn't possibly be there but let's just make sure
storage.copy(fname, vo_reals)
storage.set_property(mv_file, 'done', user_id) # set the .cands.astrom #done tag to the user ID.
uploaded_count += 1
else:
print fn, wasdone
print 'Added unique files:', uploaded_count
return
def print_progress_stats(task, directory):
con = context.get_context(directory)
user_progress = collections.defaultdict(int)
listing = con.get_listing(tasks.get_suffix(task))
for filename in listing:
user = storage.get_property(con.get_full_path(filename), DONE_PROPERTY)
if user is not None:
user_progress[user] += 1
total_processed = sum(user_progress.values())
total_todo = len(listing)
print datetime.datetime.now()
print "%s: %s: %d of %d processed (%2.1f%%)." % (
directory, task, total_processed, total_todo, (float(total_processed) / float(total_todo)) * 100.)
print "---"
for user, num_processed in user_progress.iteritems():
print " %s: %d" % (user, num_processed)
def print_progress_stats(task, directory):
con = context.get_context(directory)
user_progress = collections.defaultdict(int)
listing = con.get_listing(tasks.get_suffix(task))
for filename in listing:
# if filename.__contains__('p'): # HACK FOR CHECKING P FILES ONLY
user = storage.get_property(con.get_full_path(filename), DONE_PROPERTY)
if user is not None:
user_progress[user] += 1
total_processed = sum(user_progress.values())
total_todo = len([l for l in listing])# if l.__contains__('p')])
print datetime.datetime.now()
print "%s: %s: %d of %d processed (%2.1f%%)." % (
directory, task, total_processed, total_todo, (float(total_processed) / float(total_todo)) * 100.)
print "---"
for user, num_processed in user_progress.iteritems():
print " %s: %d" % (user, num_processed)
def print_progress_stats(task, directory):
con = context.get_context(directory)
user_progress = collections.defaultdict(int)
listing = con.get_listing(tasks.get_suffix(task))
for filename in listing:
# if filename.__contains__('p'): # HACK FOR CHECKING P FILES ONLY
user = storage.get_property(con.get_full_path(filename), DONE_PROPERTY)
if user is not None:
user_progress[user] += 1
total_processed = sum(user_progress.values())
total_todo = len([l for l in listing]) # if l.__contains__('p')])
print(datetime.datetime.now())
print("%s: %s: %d of %d processed (%2.1f%%)." %
(directory, task, total_processed, total_todo,
(float(total_processed) / float(total_todo)) * 100.))
print("---")
for user, num_processed in user_progress.items():
print(" %s: %d" % (user, num_processed))
def get_property(self, filename, property):
return storage.get_property(self.get_uri(filename), property)
def owns_lock(self, filename):
lock_holder = storage.get_property(self._get_uri(filename),
LOCK_PROPERTY)
return lock_holder == self.userid
else:
print fn, wasdone
print 'Added unique files:', uploaded_count
return
def fix_tags_on_cands_missing_reals(user_id, vos_dir, property):
"At the moment this just checks for a single user's missing reals. Easy to generalise it to all users: modify the
and"
con = context.get_context(vos_dir)
user_progress = []
listing = con.get_listing(tasks.get_suffix('cands'))
for filename in listing:
user = storage.get_property(con.get_full_path(filename), property)
if (user is not None) and (
user == user_id): # modify here to generalise to all users with work in this directory
user_progress.append(filename)
realsfile = filename.replace('cands', 'reals')
if not storage.exists(con.get_full_path(realsfile)):
print filename, 'no reals file', realsfile
storage.set_property(con.get_full_path(filename), property, None)
print 'Fixed files:', len(user_progress)
return
if __name__=='__main__':
user_id = '' # set as appropriate
def owns_lock(self, filename):
lock_holder = storage.get_property(self._get_uri(filename),
LOCK_PROPERTY)
return lock_holder == self.userid
def get_property(self, filename, property):
return storage.get_property(self.get_uri(filename), property)
