def test_listdir_for_task(self): directory = self.get_abs_path("data/testdir") listing1 = listdir_for_suffix(directory, tasks.get_suffix(tasks.CANDS_TASK)) assert_that(listing1, contains_inanyorder("xxx1.cands.astrom", "xxx2.cands.astrom")) listing2 = listdir_for_suffix(directory, tasks.get_suffix(tasks.REALS_TASK)) assert_that(listing2, contains_inanyorder("xxx1.reals.astrom", "xxx2.reals.astrom"))
def test_listdir_for_task(self): directory = self.get_abs_path("data/testdir") listing1 = listdir_for_suffix(directory, tasks.get_suffix(tasks.CANDS_TASK)) assert_that( listing1, contains_inanyorder("xxx1.cands.astrom", "xxx2.cands.astrom")) listing2 = listdir_for_suffix(directory, tasks.get_suffix(tasks.REALS_TASK)) assert_that( listing2, contains_inanyorder("xxx1.reals.astrom", "xxx2.reals.astrom"))
def __init__(self, taskname, working_directory): wx_app = wx.App(False) debug_mode = config.read("DEBUG") if debug_mode: wx.lib.inspection.InspectionTool().Show() try: factory = self.task_name_mapping[taskname]() except KeyError: raise ValueError("Unknown task: %s" % taskname) parser = AstromParser() error_handler = DownloadErrorHandler(self) downloader = ImageSliceDownloader() download_manager = AsynchronousImageDownloadManager( downloader, error_handler) directory_context = DirectoryContext(working_directory) progress_manager = ProgressManager(directory_context) builder = factory.create_workunit_builder(parser, progress_manager) workunit_provider = WorkUnitProvider(tasks.get_suffix(taskname), directory_context, progress_manager, builder) model = UIModel(workunit_provider, progress_manager, download_manager) controller = factory.create_controller(model) self.model = model self.view = controller.get_view() wx_app.MainLoop()
def fix_tags_on_cands_missing_reals(user_id, vos_dir, property): "At the moment this just checks for a single user's missing reals. Easy to generalise it to all users." con = context.get_context(vos_dir) user_progress = [] listing = con.get_listing(tasks.get_suffix('reals')) mpc_listing = con.get_listing('mpc') for filename in listing: if not filename.startswith('fk'): user = storage.get_property(con.get_full_path(filename), property) if (user is not None): # and (user == user_id): # modify 'and' to generalise to all users with work in this directory #realsfile = filename.replace('cands', 'reals') #if not con.exists(realsfile): # print filename, 'no reals file', realsfile # go through the listing of .mpc files and see if any match this reals.astrom is_present = False for mpcfile in [ f for f in mpc_listing if not f.startswith('fk') ]: if mpcfile.startswith(filename): print(filename, user, 'exists!', mpcfile) is_present = True if not is_present: user_progress.append(filename) print(filename, user, 'no mpc file') storage.set_property(con.get_full_path(filename), property, None) print('Fixed files:', len(user_progress)) return
def setUp(self): events.unsub_all() parser = AstromParser() directory_manager = LocalDirectoryWorkingContext( self._get_working_dir()) self.context = directory_manager progress_manager = LocalProgressManager(directory_manager) workunit_provider = WorkUnitProvider( tasks.get_suffix(self._get_task()), directory_manager, progress_manager, self._get_workunit_builder(parser, progress_manager)) self.workunit_provider = workunit_provider self.progress_manager = progress_manager self.singlet_download_manager = Mock(spec=AsynchronousDownloadManager) self.triplet_download_manager = Mock(spec=AsynchronousDownloadManager) self.image_manager = ImageManager(self.singlet_download_manager, self.triplet_download_manager) self.model = ValidationModel(self.workunit_provider, self.image_manager, None) self.custom_setup() self.model.start_work()
def fix_tags_on_cands_missing_reals(user_id, vos_dir, property): "At the moment this just checks for a single user's missing reals. Easy to generalise it to all users." con = context.get_context(vos_dir) user_progress = [] listing = con.get_listing(tasks.get_suffix('reals')) mpc_listing = con.get_listing('mpc') for filename in listing: if not filename.startswith('fk'): user = storage.get_property(con.get_full_path(filename), property) if (user is not None): # and (user == user_id): # modify 'and' to generalise to all users with work in this directory #realsfile = filename.replace('cands', 'reals') #if not con.exists(realsfile): # print filename, 'no reals file', realsfile # go through the listing of .mpc files and see if any match this reals.astrom is_present = False for mpcfile in [f for f in mpc_listing if not f.startswith('fk')]: if mpcfile.startswith(filename): print filename, user, 'exists!', mpcfile is_present = True if not is_present: user_progress.append(filename) print filename, user, 'no mpc file' storage.set_property(con.get_full_path(filename), property, None) print 'Fixed files:', len(user_progress) return
def setUp(self): working_directory = self.get_directory_to_clean() context = LocalDirectoryWorkingContext(working_directory) progress_manager = InMemoryProgressManager(context) parser = AstromParser() builder = RealsWorkUnitBuilder(parser, context, context, progress_manager) undertest = WorkUnitProvider(tasks.get_suffix(tasks.REALS_TASK), context, progress_manager, builder) self.progress_manager = progress_manager self.undertest = undertest
def _create_results_writer(self, full_path, parsed_data): # NOTE: this import is only here so that we don't load up secondary # dependencies (like astropy) used in MPCWriter when they are not # needed (i.e. cands task). This is to help reduce the application # startup time. from ossos.mpc import MPCWriter output_filename = full_path.replace(tasks.get_suffix(tasks.REALS_TASK), ".mpc") output_filehandle = open(output_filename, "a+b") return MPCWriter(output_filehandle, auto_flush=False)
def setUp(self): working_directory = self.get_directory_to_clean() directory_manager = DirectoryContext(working_directory) progress_manager = InMemoryProgressManager(directory_manager) parser = AstromParser() builder = RealsWorkUnitBuilder(parser, progress_manager) undertest = WorkUnitProvider(tasks.get_suffix(tasks.REALS_TASK), directory_manager, progress_manager, builder) self.progress_manager = progress_manager self.undertest = undertest
def ensure_cands_have_matching_reals(directory): con = context.get_context(directory) listing = con.get_listing(tasks.get_suffix('cands')) for filename in listing: user = storage.get_property(con.get_full_path(filename), DONE_PROPERTY) if user is not None: reals_file = filename.replace('cands', 'reals') if not con.exists(reals_file): print '.cands.astrom has no matching reals.astrom!', filename, 'done by', user return
def setUp(self): WxWidgetTestCase.setUp(self) parser = AstromParser() context = LocalDirectoryWorkingContext( self.get_abs_path("data/controller_testdir")) progress_manager = LocalProgressManager(context) workunit_provider = WorkUnitProvider(tasks.get_suffix(tasks.REALS_TASK), context, progress_manager, RealsWorkUnitBuilder( parser, context, context, progress_manager)) image_manager = Mock(spec=ImageManager) self.model = ValidationModel(workunit_provider, image_manager, None) self.model.start_work() self.model.get_writer = Mock(return_value=Mock(spec=mpc.MPCWriter)) # We don't actually have any images loaded, so mock this out source_cutout = Mock(spec=SourceCutout) source_cutout.pixel_x = 11 source_cutout.pixel_y = 50 self.model.get_current_cutout = Mock(return_value=source_cutout) x_cen = 10 y_cen = 50 mag = 24.0 self.model.get_current_source_observed_magnitude = Mock(return_value=(x_cen, y_cen, mag)) self.model.is_current_source_adjusted = Mock(return_value=False) self.model.get_current_fits_header = Mock() self.name_generator = Mock(spec=ProvisionalNameGenerator) self.name_generator.generate_name.return_value = TEST_PROVISIONAL_NAME class TestFactory(object): def __init__(self, model, name_generator): self.model = model self.name_generator = name_generator def create_controller(self, view): return ProcessRealsController(self.model, view, self.name_generator) self.view = ApplicationView( TestFactory(self.model, self.name_generator)) self.controller = self.view.controller self.controller.display_current_image = Mock()
def setUp(self): events.unsub_all() parser = AstromParser() directory_manager = DirectoryContext(self._get_working_dir()) progress_manager = ProgressManager(directory_manager) workunit_provider = WorkUnitProvider(tasks.get_suffix(self._get_task()), directory_manager, progress_manager, self._get_workunit_builder(parser, progress_manager)) self.workunit_provider = workunit_provider self.progress_manager = progress_manager self.download_manager = Mock(spec=AsynchronousImageDownloadManager) self.model = self.get_model()
def setUp(self): events.unsub_all() parser = AstromParser() directory_manager = DirectoryContext(self._get_working_dir()) progress_manager = ProgressManager(directory_manager) workunit_provider = WorkUnitProvider( tasks.get_suffix(self._get_task()), directory_manager, progress_manager, self._get_workunit_builder(parser, progress_manager)) self.workunit_provider = workunit_provider self.progress_manager = progress_manager self.download_manager = Mock(spec=AsynchronousImageDownloadManager) self.model = self.get_model()
def what_locks_remain(directory): con = context.get_context(directory) listing = con.get_listing(tasks.get_suffix('cands')) user_progress = collections.defaultdict(int) for filename in listing: user = storage.get_property(con.get_full_path(filename), LOCK_PROPERTY) if user is not None: user_progress[user] += 1 print filename, 'lock_holder=', user storage.set_property(con.get_full_path(filename), LOCK_PROPERTY, None) for user, num_locked in user_progress.iteritems(): print " %s: %d" % (user, num_locked) return
def setUp(self): WxWidgetTestCase.setUp(self) parser = AstromParser() directory_manager = DirectoryContext( self.get_abs_path("data/controller_testdir")) progress_manager = ProgressManager(directory_manager) workunit_provider = WorkUnitProvider(tasks.get_suffix(tasks.REALS_TASK), directory_manager, progress_manager, RealsWorkUnitBuilder(parser, progress_manager)) download_manager = Mock(spec=AsynchronousImageDownloadManager) self.model = UIModel(workunit_provider, progress_manager, download_manager) self.name_generator = Mock() self.controller = ProcessRealsController(self.model, self.name_generator)
def what_locks_remain(directory): con = context.get_context(directory) listing = con.get_listing(tasks.get_suffix('cands')) user_progress = collections.defaultdict(int) for filename in listing: user = storage.get_property(con.get_full_path(filename), LOCK_PROPERTY) if user is not None: user_progress[user] += 1 print(filename, 'lock_holder=', user) storage.set_property(con.get_full_path(filename), LOCK_PROPERTY, None) for user, num_locked in user_progress.items(): print(" %s: %d" % (user, num_locked)) return
def print_progress_stats(task, directory): con = context.get_context(directory) user_progress = collections.defaultdict(int) listing = con.get_listing(tasks.get_suffix(task)) for filename in listing: user = storage.get_property(con.get_full_path(filename), DONE_PROPERTY) if user is not None: user_progress[user] += 1 total_processed = sum(user_progress.values()) print "%s: %s: %d of %d processed." % ( directory, task, total_processed, len(listing)) print "---" for user, num_processed in user_progress.iteritems(): print " %s: %d" % (user, num_processed)
def print_progress_stats(task, directory): con = context.get_context(directory) user_progress = collections.defaultdict(int) listing = con.get_listing(tasks.get_suffix(task)) for filename in listing: user = storage.get_property(con.get_full_path(filename), DONE_PROPERTY) if user is not None: user_progress[user] += 1 total_processed = sum(user_progress.values()) total_todo = len(listing) print datetime.datetime.now() print "%s: %s: %d of %d processed (%2.1f%%)." % ( directory, task, total_processed, total_todo, (float(total_processed) / float(total_todo)) * 100.) print "---" for user, num_processed in user_progress.iteritems(): print " %s: %d" % (user, num_processed)
def print_progress_stats(task, directory): con = context.get_context(directory) user_progress = collections.defaultdict(int) listing = con.get_listing(tasks.get_suffix(task)) for filename in listing: # if filename.__contains__('p'): # HACK FOR CHECKING P FILES ONLY user = storage.get_property(con.get_full_path(filename), DONE_PROPERTY) if user is not None: user_progress[user] += 1 total_processed = sum(user_progress.values()) total_todo = len([l for l in listing])# if l.__contains__('p')]) print datetime.datetime.now() print "%s: %s: %d of %d processed (%2.1f%%)." % ( directory, task, total_processed, total_todo, (float(total_processed) / float(total_todo)) * 100.) print "---" for user, num_processed in user_progress.iteritems(): print " %s: %d" % (user, num_processed)
def print_progress_stats(task, directory): con = context.get_context(directory) user_progress = collections.defaultdict(int) listing = con.get_listing(tasks.get_suffix(task)) for filename in listing: # if filename.__contains__('p'): # HACK FOR CHECKING P FILES ONLY user = storage.get_property(con.get_full_path(filename), DONE_PROPERTY) if user is not None: user_progress[user] += 1 total_processed = sum(user_progress.values()) total_todo = len([l for l in listing]) # if l.__contains__('p')]) print(datetime.datetime.now()) print("%s: %s: %d of %d processed (%2.1f%%)." % (directory, task, total_processed, total_todo, (float(total_processed) / float(total_todo)) * 100.)) print("---") for user, num_processed in user_progress.items(): print(" %s: %d" % (user, num_processed))
def get_writer(self): """ Get a writer. This method also makes the output filename be the same as the .track file but with .mpc. (Currently only works on local filesystem) """ if self._writer is None: suffix = tasks.get_suffix(tasks.TRACK_TASK) try: base_name = re.search( "(?P<base_name>.*?)\.\d*{}".format(suffix), self.filename).group('base_name') except: base_name = os.path.splitext(self.filename)[0] mpc_filename_pattern = self.output_context.get_full_path( "{}.?{}".format(base_name, suffix)) mpc_file_count = len(glob(mpc_filename_pattern)) mpc_filename = self.output_context.get_full_path("{}.{}{}".format( base_name, mpc_file_count, suffix)) self._writer = self._create_writer(mpc_filename) return self._writer
def setUp(self): events.unsub_all() parser = AstromParser() directory_manager = LocalDirectoryWorkingContext(self._get_working_dir()) self.context = directory_manager progress_manager = LocalProgressManager(directory_manager) workunit_provider = WorkUnitProvider(tasks.get_suffix(self._get_task()), directory_manager, progress_manager, self._get_workunit_builder(parser, progress_manager)) self.workunit_provider = workunit_provider self.progress_manager = progress_manager self.singlet_download_manager = Mock(spec=AsynchronousDownloadManager) self.triplet_download_manager = Mock(spec=AsynchronousDownloadManager) self.image_manager = ImageManager(self.singlet_download_manager, self.triplet_download_manager) self.model = ValidationModel(self.workunit_provider, self.image_manager, None) self.custom_setup() self.model.start_work()
storage.set_property(mv_file, 'done', user_id) # set the .cands.astrom #done tag to the user ID. uploaded_count += 1 else: print fn, wasdone print 'Added unique files:', uploaded_count return def fix_tags_on_cands_missing_reals(user_id, vos_dir, property): "At the moment this just checks for a single user's missing reals. Easy to generalise it to all users: modify the and" con = context.get_context(vos_dir) user_progress = [] listing = con.get_listing(tasks.get_suffix('cands')) for filename in listing: user = storage.get_property(con.get_full_path(filename), property) if (user is not None) and ( user == user_id): # modify here to generalise to all users with work in this directory user_progress.append(filename) realsfile = filename.replace('cands', 'reals') if not storage.exists(con.get_full_path(realsfile)): print filename, 'no reals file', realsfile storage.set_property(con.get_full_path(filename), property, None) print 'Fixed files:', len(user_progress) return
def get_output_filename(self): return self.get_filename().replace(tasks.get_suffix(tasks.CANDS_TASK), tasks.get_suffix(tasks.REALS_TASK))
def _get_done_suffix(self, task): return tasks.get_suffix(task) + DONE_SUFFIX
def _create_results_writer(self, full_path, parsed_data): output_filename = full_path.replace(tasks.get_suffix(tasks.CANDS_TASK), tasks.get_suffix(tasks.REALS_TASK)) output_filehandle = open(output_filename, "a+b") return StreamingAstromWriter(output_filehandle, parsed_data.sys_header)