def filter_worker(job_queue, result_queue, **kwargs): signal.signal(signal.SIGINT, signal.SIG_IGN) import time time.sleep(1) while not job_queue.empty(): try: summarizer, sum_args = job_queue.get(block=False) if isinstance(summarizer, APFilteredSummary): event, sim_cutoff = sum_args ap = APSummarizer() ap_tsv_path = ap.get_tsv_path(event) apf_tsv_path = summarizer.get_tsv_path(event, sim_cutoff) if os.path.exists(ap_tsv_path) and \ not os.path.exists(apf_tsv_path): summarizer.make(event, sim_threshold=sim_cutoff) elif isinstance(summarizer, HACFilteredSummary): event, prefix, feature_set, dist_cutoff, sim_cutoff = sum_args hac = HACSummarizer() hac_tsv_path = hac.get_tsv_path(event, dist_cutoff) hacf_tsv_path = summarizer.get_tsv_path( event, dist_cutoff, sim_cutoff) if os.path.exists(hac_tsv_path) and \ not os.path.exists(hacf_tsv_path): try: summarizer.make(event, prefix, feature_set, dist_cutoff=dist_cutoff, sim_threshold=sim_cutoff) except Exception, e: print e print hac_tsv_path elif isinstance(summarizer, APSalienceFilteredSummary) or \ isinstance(summarizer, APSalTRankSalThreshFilteredSummary): event, prefix, feature_set, sal_cutoff, sim_cutoff = sum_args aps = APSalienceSummarizer() aps_tsv_path = aps.get_tsv_path(event, prefix, feature_set) apsf_tsv_path = summarizer.get_tsv_path( event, prefix, feature_set, sal_cutoff, sim_cutoff) if os.path.exists(aps_tsv_path) and \ not os.path.exists(apsf_tsv_path): summarizer.make(event, prefix, feature_set, min_cluster_size=2, center_threshold=sal_cutoff, sim_threshold=sim_cutoff) elif isinstance(summarizer, RankedSalienceFilteredSummary): event, prefix, feature_set, sal_cutoff, sim_cutoff = sum_args rsfs_tsv_path = summarizer.get_tsv_path( event, sal_cutoff, sim_cutoff) if not os.path.exists(rsfs_tsv_path): summarizer.make(event, prefix, feature_set, sal_cutoff, sim_cutoff)
def filter_worker(job_queue, result_queue, **kwargs): signal.signal(signal.SIGINT, signal.SIG_IGN) import time time.sleep(1) while not job_queue.empty(): try: summarizer, sum_args = job_queue.get(block=False) if isinstance(summarizer, APFilteredSummary): event, sim_cutoff = sum_args ap = APSummarizer() ap_tsv_path = ap.get_tsv_path(event) apf_tsv_path = summarizer.get_tsv_path(event, sim_cutoff) if os.path.exists(ap_tsv_path) and \ not os.path.exists(apf_tsv_path): summarizer.make(event, sim_threshold=sim_cutoff) elif isinstance(summarizer, HACFilteredSummary): event, prefix, feature_set, dist_cutoff, sim_cutoff = sum_args hac = HACSummarizer() hac_tsv_path = hac.get_tsv_path(event, dist_cutoff) hacf_tsv_path = summarizer.get_tsv_path( event, dist_cutoff, sim_cutoff) if os.path.exists(hac_tsv_path) and \ not os.path.exists(hacf_tsv_path): try: summarizer.make(event, prefix, feature_set, dist_cutoff=dist_cutoff, sim_threshold=sim_cutoff) except Exception, e: print e print hac_tsv_path elif isinstance(summarizer, APSalienceFilteredSummary) or \ isinstance(summarizer, APSalTRankSalThreshFilteredSummary): event, prefix, feature_set, sal_cutoff, sim_cutoff = sum_args aps = APSalienceSummarizer() aps_tsv_path = aps.get_tsv_path(event, prefix, feature_set) apsf_tsv_path = summarizer.get_tsv_path( event, prefix, feature_set, sal_cutoff, sim_cutoff) if os.path.exists(aps_tsv_path) and \ not os.path.exists(apsf_tsv_path): summarizer.make( event, prefix, feature_set, min_cluster_size=2, center_threshold=sal_cutoff, sim_threshold=sim_cutoff) elif isinstance(summarizer, RankedSalienceFilteredSummary): event, prefix, feature_set, sal_cutoff, sim_cutoff = sum_args rsfs_tsv_path = summarizer.get_tsv_path( event, sal_cutoff, sim_cutoff) if not os.path.exists(rsfs_tsv_path): summarizer.make( event, prefix, feature_set, sal_cutoff, sim_cutoff)