def test_listdir_for_task(self):
directory = self.get_abs_path("data/testdir")
listing1 = listdir_for_suffix(directory, tasks.get_suffix(tasks.CANDS_TASK))
assert_that(listing1, contains_inanyorder("xxx1.cands.astrom", "xxx2.cands.astrom"))
listing2 = listdir_for_suffix(directory, tasks.get_suffix(tasks.REALS_TASK))
assert_that(listing2, contains_inanyorder("xxx1.reals.astrom", "xxx2.reals.astrom"))
def test_listdir_for_task(self):
directory = self.get_abs_path("data/testdir")
listing1 = listdir_for_suffix(directory,
tasks.get_suffix(tasks.CANDS_TASK))
assert_that(
listing1,
contains_inanyorder("xxx1.cands.astrom", "xxx2.cands.astrom"))
listing2 = listdir_for_suffix(directory,
tasks.get_suffix(tasks.REALS_TASK))
assert_that(
listing2,
contains_inanyorder("xxx1.reals.astrom", "xxx2.reals.astrom"))
def __init__(self, taskname, working_directory):
wx_app = wx.App(False)
debug_mode = config.read("DEBUG")
if debug_mode:
wx.lib.inspection.InspectionTool().Show()
try:
factory = self.task_name_mapping[taskname]()
except KeyError:
raise ValueError("Unknown task: %s" % taskname)
parser = AstromParser()
error_handler = DownloadErrorHandler(self)
downloader = ImageSliceDownloader()
download_manager = AsynchronousImageDownloadManager(
downloader, error_handler)
directory_context = DirectoryContext(working_directory)
progress_manager = ProgressManager(directory_context)
builder = factory.create_workunit_builder(parser, progress_manager)
workunit_provider = WorkUnitProvider(tasks.get_suffix(taskname),
directory_context,
progress_manager, builder)
model = UIModel(workunit_provider, progress_manager, download_manager)
controller = factory.create_controller(model)
self.model = model
self.view = controller.get_view()
wx_app.MainLoop()
def fix_tags_on_cands_missing_reals(user_id, vos_dir, property):
"At the moment this just checks for a single user's missing reals. Easy to generalise it to all users."
con = context.get_context(vos_dir)
user_progress = []
listing = con.get_listing(tasks.get_suffix('reals'))
mpc_listing = con.get_listing('mpc')
for filename in listing:
if not filename.startswith('fk'):
user = storage.get_property(con.get_full_path(filename), property)
if (user is not None):
# and (user == user_id): # modify 'and' to generalise to all users with work in this directory
#realsfile = filename.replace('cands', 'reals')
#if not con.exists(realsfile):
# print filename, 'no reals file', realsfile
# go through the listing of .mpc files and see if any match this reals.astrom
is_present = False
for mpcfile in [
f for f in mpc_listing if not f.startswith('fk')
]:
if mpcfile.startswith(filename):
print(filename, user, 'exists!', mpcfile)
is_present = True
if not is_present:
user_progress.append(filename)
print(filename, user, 'no mpc file')
storage.set_property(con.get_full_path(filename), property,
None)
print('Fixed files:', len(user_progress))
return
def setUp(self):
events.unsub_all()
parser = AstromParser()
directory_manager = LocalDirectoryWorkingContext(
self._get_working_dir())
self.context = directory_manager
progress_manager = LocalProgressManager(directory_manager)
workunit_provider = WorkUnitProvider(
tasks.get_suffix(self._get_task()), directory_manager,
progress_manager,
self._get_workunit_builder(parser, progress_manager))
self.workunit_provider = workunit_provider
self.progress_manager = progress_manager
self.singlet_download_manager = Mock(spec=AsynchronousDownloadManager)
self.triplet_download_manager = Mock(spec=AsynchronousDownloadManager)
self.image_manager = ImageManager(self.singlet_download_manager,
self.triplet_download_manager)
self.model = ValidationModel(self.workunit_provider,
self.image_manager, None)
self.custom_setup()
self.model.start_work()
def fix_tags_on_cands_missing_reals(user_id, vos_dir, property):
"At the moment this just checks for a single user's missing reals. Easy to generalise it to all users."
con = context.get_context(vos_dir)
user_progress = []
listing = con.get_listing(tasks.get_suffix('reals'))
mpc_listing = con.get_listing('mpc')
for filename in listing:
if not filename.startswith('fk'):
user = storage.get_property(con.get_full_path(filename), property)
if (user is not None):
# and (user == user_id): # modify 'and' to generalise to all users with work in this directory
#realsfile = filename.replace('cands', 'reals')
#if not con.exists(realsfile):
# print filename, 'no reals file', realsfile
# go through the listing of .mpc files and see if any match this reals.astrom
is_present = False
for mpcfile in [f for f in mpc_listing if not f.startswith('fk')]:
if mpcfile.startswith(filename):
print filename, user, 'exists!', mpcfile
is_present = True
if not is_present:
user_progress.append(filename)
print filename, user, 'no mpc file'
storage.set_property(con.get_full_path(filename), property, None)
print 'Fixed files:', len(user_progress)
return
def setUp(self):
working_directory = self.get_directory_to_clean()
context = LocalDirectoryWorkingContext(working_directory)
progress_manager = InMemoryProgressManager(context)
parser = AstromParser()
builder = RealsWorkUnitBuilder(parser, context, context,
progress_manager)
undertest = WorkUnitProvider(tasks.get_suffix(tasks.REALS_TASK),
context, progress_manager, builder)
self.progress_manager = progress_manager
self.undertest = undertest
def _create_results_writer(self, full_path, parsed_data):
# NOTE: this import is only here so that we don't load up secondary
# dependencies (like astropy) used in MPCWriter when they are not
# needed (i.e. cands task). This is to help reduce the application
# startup time.
from ossos.mpc import MPCWriter
output_filename = full_path.replace(tasks.get_suffix(tasks.REALS_TASK),
".mpc")
output_filehandle = open(output_filename, "a+b")
return MPCWriter(output_filehandle, auto_flush=False)
def _create_results_writer(self, full_path, parsed_data):
# NOTE: this import is only here so that we don't load up secondary
# dependencies (like astropy) used in MPCWriter when they are not
# needed (i.e. cands task). This is to help reduce the application
# startup time.
from ossos.mpc import MPCWriter
output_filename = full_path.replace(tasks.get_suffix(tasks.REALS_TASK),
".mpc")
output_filehandle = open(output_filename, "a+b")
return MPCWriter(output_filehandle, auto_flush=False)
def setUp(self):
working_directory = self.get_directory_to_clean()
directory_manager = DirectoryContext(working_directory)
progress_manager = InMemoryProgressManager(directory_manager)
parser = AstromParser()
builder = RealsWorkUnitBuilder(parser, progress_manager)
undertest = WorkUnitProvider(tasks.get_suffix(tasks.REALS_TASK),
directory_manager,
progress_manager,
builder)
self.progress_manager = progress_manager
self.undertest = undertest
def ensure_cands_have_matching_reals(directory):
con = context.get_context(directory)
listing = con.get_listing(tasks.get_suffix('cands'))
for filename in listing:
user = storage.get_property(con.get_full_path(filename), DONE_PROPERTY)
if user is not None:
reals_file = filename.replace('cands', 'reals')
if not con.exists(reals_file):
print '.cands.astrom has no matching reals.astrom!', filename, 'done by', user
return
def ensure_cands_have_matching_reals(directory):
con = context.get_context(directory)
listing = con.get_listing(tasks.get_suffix('cands'))
for filename in listing:
user = storage.get_property(con.get_full_path(filename), DONE_PROPERTY)
if user is not None:
reals_file = filename.replace('cands', 'reals')
if not con.exists(reals_file):
print '.cands.astrom has no matching reals.astrom!', filename, 'done by', user
return
def setUp(self):
WxWidgetTestCase.setUp(self)
parser = AstromParser()
context = LocalDirectoryWorkingContext(
self.get_abs_path("data/controller_testdir"))
progress_manager = LocalProgressManager(context)
workunit_provider = WorkUnitProvider(tasks.get_suffix(tasks.REALS_TASK),
context, progress_manager,
RealsWorkUnitBuilder(
parser,
context,
context,
progress_manager))
image_manager = Mock(spec=ImageManager)
self.model = ValidationModel(workunit_provider, image_manager, None)
self.model.start_work()
self.model.get_writer = Mock(return_value=Mock(spec=mpc.MPCWriter))
# We don't actually have any images loaded, so mock this out
source_cutout = Mock(spec=SourceCutout)
source_cutout.pixel_x = 11
source_cutout.pixel_y = 50
self.model.get_current_cutout = Mock(return_value=source_cutout)
x_cen = 10
y_cen = 50
mag = 24.0
self.model.get_current_source_observed_magnitude = Mock(return_value=(x_cen, y_cen, mag))
self.model.is_current_source_adjusted = Mock(return_value=False)
self.model.get_current_fits_header = Mock()
self.name_generator = Mock(spec=ProvisionalNameGenerator)
self.name_generator.generate_name.return_value = TEST_PROVISIONAL_NAME
class TestFactory(object):
def __init__(self, model, name_generator):
self.model = model
self.name_generator = name_generator
def create_controller(self, view):
return ProcessRealsController(self.model, view,
self.name_generator)
self.view = ApplicationView(
TestFactory(self.model, self.name_generator))
self.controller = self.view.controller
self.controller.display_current_image = Mock()
def setUp(self):
WxWidgetTestCase.setUp(self)
parser = AstromParser()
context = LocalDirectoryWorkingContext(
self.get_abs_path("data/controller_testdir"))
progress_manager = LocalProgressManager(context)
workunit_provider = WorkUnitProvider(tasks.get_suffix(tasks.REALS_TASK),
context, progress_manager,
RealsWorkUnitBuilder(
parser,
context,
context,
progress_manager))
image_manager = Mock(spec=ImageManager)
self.model = ValidationModel(workunit_provider, image_manager, None)
self.model.start_work()
self.model.get_writer = Mock(return_value=Mock(spec=mpc.MPCWriter))
# We don't actually have any images loaded, so mock this out
source_cutout = Mock(spec=SourceCutout)
source_cutout.pixel_x = 11
source_cutout.pixel_y = 50
self.model.get_current_cutout = Mock(return_value=source_cutout)
x_cen = 10
y_cen = 50
mag = 24.0
self.model.get_current_source_observed_magnitude = Mock(return_value=(x_cen, y_cen, mag))
self.model.is_current_source_adjusted = Mock(return_value=False)
self.model.get_current_fits_header = Mock()
self.name_generator = Mock(spec=ProvisionalNameGenerator)
self.name_generator.generate_name.return_value = TEST_PROVISIONAL_NAME
class TestFactory(object):
def __init__(self, model, name_generator):
self.model = model
self.name_generator = name_generator
def create_controller(self, view):
return ProcessRealsController(self.model, view,
self.name_generator)
self.view = ApplicationView(
TestFactory(self.model, self.name_generator))
self.controller = self.view.controller
self.controller.display_current_image = Mock()
def setUp(self):
events.unsub_all()
parser = AstromParser()
directory_manager = DirectoryContext(self._get_working_dir())
progress_manager = ProgressManager(directory_manager)
workunit_provider = WorkUnitProvider(tasks.get_suffix(self._get_task()),
directory_manager, progress_manager,
self._get_workunit_builder(parser, progress_manager))
self.workunit_provider = workunit_provider
self.progress_manager = progress_manager
self.download_manager = Mock(spec=AsynchronousImageDownloadManager)
self.model = self.get_model()
def setUp(self):
events.unsub_all()
parser = AstromParser()
directory_manager = DirectoryContext(self._get_working_dir())
progress_manager = ProgressManager(directory_manager)
workunit_provider = WorkUnitProvider(
tasks.get_suffix(self._get_task()), directory_manager,
progress_manager,
self._get_workunit_builder(parser, progress_manager))
self.workunit_provider = workunit_provider
self.progress_manager = progress_manager
self.download_manager = Mock(spec=AsynchronousImageDownloadManager)
self.model = self.get_model()
def what_locks_remain(directory):
con = context.get_context(directory)
listing = con.get_listing(tasks.get_suffix('cands'))
user_progress = collections.defaultdict(int)
for filename in listing:
user = storage.get_property(con.get_full_path(filename), LOCK_PROPERTY)
if user is not None:
user_progress[user] += 1
print filename, 'lock_holder=', user
storage.set_property(con.get_full_path(filename), LOCK_PROPERTY, None)
for user, num_locked in user_progress.iteritems():
print " %s: %d" % (user, num_locked)
return
def setUp(self):
WxWidgetTestCase.setUp(self)
parser = AstromParser()
directory_manager = DirectoryContext(
self.get_abs_path("data/controller_testdir"))
progress_manager = ProgressManager(directory_manager)
workunit_provider = WorkUnitProvider(tasks.get_suffix(tasks.REALS_TASK),
directory_manager, progress_manager,
RealsWorkUnitBuilder(parser, progress_manager))
download_manager = Mock(spec=AsynchronousImageDownloadManager)
self.model = UIModel(workunit_provider, progress_manager, download_manager)
self.name_generator = Mock()
self.controller = ProcessRealsController(self.model, self.name_generator)
def what_locks_remain(directory):
con = context.get_context(directory)
listing = con.get_listing(tasks.get_suffix('cands'))
user_progress = collections.defaultdict(int)
for filename in listing:
user = storage.get_property(con.get_full_path(filename), LOCK_PROPERTY)
if user is not None:
user_progress[user] += 1
print(filename, 'lock_holder=', user)
storage.set_property(con.get_full_path(filename), LOCK_PROPERTY,
None)
for user, num_locked in user_progress.items():
print(" %s: %d" % (user, num_locked))
return
def print_progress_stats(task, directory):
con = context.get_context(directory)
user_progress = collections.defaultdict(int)
listing = con.get_listing(tasks.get_suffix(task))
for filename in listing:
user = storage.get_property(con.get_full_path(filename), DONE_PROPERTY)
if user is not None:
user_progress[user] += 1
total_processed = sum(user_progress.values())
print "%s: %s: %d of %d processed." % (
directory, task, total_processed, len(listing))
print "---"
for user, num_processed in user_progress.iteritems():
print " %s: %d" % (user, num_processed)
def print_progress_stats(task, directory):
con = context.get_context(directory)
user_progress = collections.defaultdict(int)
listing = con.get_listing(tasks.get_suffix(task))
for filename in listing:
user = storage.get_property(con.get_full_path(filename), DONE_PROPERTY)
if user is not None:
user_progress[user] += 1
total_processed = sum(user_progress.values())
total_todo = len(listing)
print datetime.datetime.now()
print "%s: %s: %d of %d processed (%2.1f%%)." % (
directory, task, total_processed, total_todo, (float(total_processed) / float(total_todo)) * 100.)
print "---"
for user, num_processed in user_progress.iteritems():
print " %s: %d" % (user, num_processed)
def print_progress_stats(task, directory):
con = context.get_context(directory)
user_progress = collections.defaultdict(int)
listing = con.get_listing(tasks.get_suffix(task))
for filename in listing:
# if filename.__contains__('p'): # HACK FOR CHECKING P FILES ONLY
user = storage.get_property(con.get_full_path(filename), DONE_PROPERTY)
if user is not None:
user_progress[user] += 1
total_processed = sum(user_progress.values())
total_todo = len([l for l in listing])# if l.__contains__('p')])
print datetime.datetime.now()
print "%s: %s: %d of %d processed (%2.1f%%)." % (
directory, task, total_processed, total_todo, (float(total_processed) / float(total_todo)) * 100.)
print "---"
for user, num_processed in user_progress.iteritems():
print " %s: %d" % (user, num_processed)
def print_progress_stats(task, directory):
con = context.get_context(directory)
user_progress = collections.defaultdict(int)
listing = con.get_listing(tasks.get_suffix(task))
for filename in listing:
# if filename.__contains__('p'): # HACK FOR CHECKING P FILES ONLY
user = storage.get_property(con.get_full_path(filename), DONE_PROPERTY)
if user is not None:
user_progress[user] += 1
total_processed = sum(user_progress.values())
total_todo = len([l for l in listing]) # if l.__contains__('p')])
print(datetime.datetime.now())
print("%s: %s: %d of %d processed (%2.1f%%)." %
(directory, task, total_processed, total_todo,
(float(total_processed) / float(total_todo)) * 100.))
print("---")
for user, num_processed in user_progress.items():
print(" %s: %d" % (user, num_processed))
def get_writer(self):
"""
Get a writer.
This method also makes the output filename be the same as the .track file but with .mpc.
(Currently only works on local filesystem)
"""
if self._writer is None:
suffix = tasks.get_suffix(tasks.TRACK_TASK)
try:
base_name = re.search(
"(?P<base_name>.*?)\.\d*{}".format(suffix),
self.filename).group('base_name')
except:
base_name = os.path.splitext(self.filename)[0]
mpc_filename_pattern = self.output_context.get_full_path(
"{}.?{}".format(base_name, suffix))
mpc_file_count = len(glob(mpc_filename_pattern))
mpc_filename = self.output_context.get_full_path("{}.{}{}".format(
base_name, mpc_file_count, suffix))
self._writer = self._create_writer(mpc_filename)
return self._writer
def setUp(self):
events.unsub_all()
parser = AstromParser()
directory_manager = LocalDirectoryWorkingContext(self._get_working_dir())
self.context = directory_manager
progress_manager = LocalProgressManager(directory_manager)
workunit_provider = WorkUnitProvider(tasks.get_suffix(self._get_task()),
directory_manager, progress_manager,
self._get_workunit_builder(parser, progress_manager))
self.workunit_provider = workunit_provider
self.progress_manager = progress_manager
self.singlet_download_manager = Mock(spec=AsynchronousDownloadManager)
self.triplet_download_manager = Mock(spec=AsynchronousDownloadManager)
self.image_manager = ImageManager(self.singlet_download_manager,
self.triplet_download_manager)
self.model = ValidationModel(self.workunit_provider, self.image_manager, None)
self.custom_setup()
self.model.start_work()
storage.set_property(mv_file, 'done', user_id) # set the .cands.astrom #done tag to the user ID.
uploaded_count += 1
else:
print fn, wasdone
print 'Added unique files:', uploaded_count
return
def fix_tags_on_cands_missing_reals(user_id, vos_dir, property):
"At the moment this just checks for a single user's missing reals. Easy to generalise it to all users: modify the
and"
con = context.get_context(vos_dir)
user_progress = []
listing = con.get_listing(tasks.get_suffix('cands'))
for filename in listing:
user = storage.get_property(con.get_full_path(filename), property)
if (user is not None) and (
user == user_id): # modify here to generalise to all users with work in this directory
user_progress.append(filename)
realsfile = filename.replace('cands', 'reals')
if not storage.exists(con.get_full_path(realsfile)):
print filename, 'no reals file', realsfile
storage.set_property(con.get_full_path(filename), property, None)
print 'Fixed files:', len(user_progress)
return
def get_output_filename(self):
return self.get_filename().replace(tasks.get_suffix(tasks.CANDS_TASK),
tasks.get_suffix(tasks.REALS_TASK))
def _get_done_suffix(self, task):
return tasks.get_suffix(task) + DONE_SUFFIX
def _get_done_suffix(self, task):
return tasks.get_suffix(task) + DONE_SUFFIX
def _create_results_writer(self, full_path, parsed_data):
output_filename = full_path.replace(tasks.get_suffix(tasks.CANDS_TASK),
tasks.get_suffix(tasks.REALS_TASK))
output_filehandle = open(output_filename, "a+b")
return StreamingAstromWriter(output_filehandle, parsed_data.sys_header)
def get_output_filename(self):
return self.get_filename().replace(tasks.get_suffix(tasks.CANDS_TASK),
tasks.get_suffix(tasks.REALS_TASK))
def _create_results_writer(self, full_path, parsed_data):
output_filename = full_path.replace(tasks.get_suffix(tasks.CANDS_TASK),
tasks.get_suffix(tasks.REALS_TASK))
output_filehandle = open(output_filename, "a+b")
return StreamingAstromWriter(output_filehandle, parsed_data.sys_header)
