def test_1_hsmetrics(self): """Run hs metrics""" self.app = self.make_app(argv=['production', 'hs-metrics', 'J.Doe_00_01', '-f', '120829_SN0001_0001_AA001AAAXX', '--region_file', 'regionfile', '--force', '-n'], extensions=['scilifelab.pm.ext.ext_hs_metrics']) handler.register(ProductionController) self._run_app() hsmetrics_str = "(DRY_RUN): java -Xmx3g -jar {}/CalculateHsMetrics.jar INPUT={}/120829_SN0001_0001_AA001AAAXX/1_120829_AA001AAAXX_nophix_10-sort-dup.bam TARGET_INTERVALS={}/regionfile BAIT_INTERVALS={}/regionfile OUTPUT={}/120829_SN0001_0001_AA001AAAXX/1_120829_AA001AAAXX_nophix_10-sort-dup.hs_metrics VALIDATION_STRINGENCY=SILENT".format(os.getenv("PICARD_HOME"), self.app.config.get("production", "root"), os.path.abspath(os.curdir), os.path.abspath(os.curdir), self.app.config.get("production", "root")) self.eq(hsmetrics_str, str(self.app._output_data['stderr'].getvalue().split("\n")[0]))
def test_clean(self): before = glob.glob(os.path.join(intermediate, "120829_AA001AAAXX", "*")) self.app = self.make_app(argv=[ 'project', 'clean', 'j_doe_00_02', '--pileup', '--intermediate', '--force' ]) handler.register(ProjectController) self._run_app() after = glob.glob(os.path.join(intermediate, "120829_AA001AAAXX", "*")) diff = [ os.path.basename(x) for x in list(set(before).difference(set(after))) ] self.eq( set(diff), set([ '1_120829_AA001AAAXX_nophix_12-sort-dup-target.pileup.gz', '1_120829_AA001AAAXX_nophix_12-sort-dup-target.pileup', '1_120829_AA001AAAXX_nophix_12-sort-dup-target.crisp_pileup.gz', '1_120829_AA001AAAXX_nophix_12-sort-dup-target.crisp_pileup', '1_120829_AA001AAAXX_nophix_10-sort-dup-target.crisp_pileup.gz', '1_120829_AA001AAAXX_nophix_10-sort-dup-target.crisp_pileup', '1_120829_AA001AAAXX_nophix_10-sort-dup-target.pileup', '1_120829_AA001AAAXX_nophix_10-sort-dup-target.pileup.gz' ]))
def load(): """Called by the framework when the extension is 'loaded'.""" hook.register('post_setup', add_drmaa_option) hook.register('post_setup', add_sbatch_option) hook.register('post_setup', add_shared_distributed_options) hook.register('pre_run', set_distributed_handler) handler.register(DistributedCommandHandler)
def test_runinfo_yaml(self): """Test runinfo yaml list""" self.app = self.make_app(argv=['archive', 'runinfo', '-f', '120829_SN0001_0001_AA001AAAXX', '--as_yaml']) handler.register(ArchiveController) self._run_app() info = Flowcell(runinfo) self.eq(self.app._output_data['stdout'].getvalue(), info.as_yaml())
def test_clean_dry(self): self.app = self.make_app(argv=[ 'project', 'clean', 'j_doe_00_02', '--pileup', '-n', '--intermediate', '--force' ]) handler.register(ProjectController) self._run_app()
def test_project_run_samplefile(self): """Test running project in project folder using a sample file as input.""" self.app = self.make_app(argv = ['project', 'run', 'j_doe_00_04', '--restart', '--amplicon', '--sample', SAMPLEFILE, '--debug', '--force', '--quiet'], extensions=['scilifelab.pm.ext.ext_distributed']) handler.register(ProjectController) handler.register(BcbioRunController) self._run_app() os.chdir(filedir)
def test_clean_fastqbam(self): before = glob.glob( os.path.join(data, "P1_106F_index6/120829_AA001AAAXX/alignments", "*")) self.app = self.make_app(argv=[ 'project', 'clean', 'j_doe_00_02', '--data', '--fastqbam', '--force' ]) handler.register(ProjectController) self._run_app() after = glob.glob( os.path.join(data, "P1_106F_index6/120829_AA001AAAXX/alignments", "*")) diff = len(before) - len(after) self.eq(diff, 2) diff = [ os.path.basename(x) for x in list(set(before).difference(set(after))) ] self.eq( set(diff), set([ '1_120829_AA001AAAXX_nophix_10_1_fastq-fastq.bam', '1_120829_AA001AAAXX_nophix_10_2_fastq-fastq.bam' ]))
def test_decompress(self): """Test decompression of project data""" self.app = self.make_app(argv=[ 'project', 'decompress', 'j_doe_00_01', '--fastq', '--force' ]) handler.register(ProjectController) self._run_app()
def test_hsmetrics_empty(self): """Run hs metrics when no files present""" self.app = self.make_app(argv=['production', 'hs-metrics', 'J.Doe_00_02', '-f', '120829_SN0001_0001_AA001AAAXX','--targets', os.path.join(filedir, 'regionfile'), '--force', '-n'], extensions=[]) handler.register(ProductionController) self._run_app() ## Shouldn't produce any output self.eq([''], self.app._output_data['stdout'].getvalue().split("\n"))
def test_register_duplicate_handler(self): from cement.ext import ext_dummy handler.register(ext_dummy.DummyOutputHandler) try: handler.register(DuplicateHandler) except exc.FrameworkError: raise
def test_hsmetrics_drmaa(self): """Run hs metrics over drmaa""" self.app = self.make_app(argv=['production', 'hs-metrics', 'J.Doe_00_01', '-f', '120829_SN0001_0001_AA001AAAXX', '--targets', os.path.join(filedir, 'regionfile'), '--force', '-A', 'jobaccount', '--jobname', 'jobname', '--partition', 'node', '--time', '10:00:00', '--drmaa', '-n'], extensions=['scilifelab.pm.ext.ext_distributed']) handler.register(ProductionController) self._run_app() hsmetrics_str = "(DRY_RUN): java -Xmx3g -jar {}/CalculateHsMetrics.jar INPUT={}/120829_SN0001_0001_AA001AAAXX/1_120829_AA001AAAXX_nophix_8-sort-dup.bam TARGET_INTERVALS={}/regionfile BAIT_INTERVALS={}/regionfile OUTPUT={}/120829_SN0001_0001_AA001AAAXX/1_120829_AA001AAAXX_nophix_8-sort-dup.hs_metrics VALIDATION_STRINGENCY=SILENT".format(os.getenv("PICARD_HOME"), self.app.config.get("production", "root"), filedir, filedir, self.app.config.get("production", "root")) self.eq(hsmetrics_str, str(sorted(self.app._output_data['stderr'].getvalue().rstrip().split("\n"))[-1]))
def register_controllers(self): self.controllers = [ AuthController, TargetSetController, TargetGetController, CreateTeamController, UpdateTeamController, ShowTeamController, DeleteTeamController, ShowUserController, TeamAddUserController, CreateProjectController, UpdateProjectController, DeleteProjectController, TeamRemoveUserController, ChangePasswordController, ScheduleLoadTestController, ListLoadTestController, InstanceLoadTestController, TrackLoadTestController, ShowResultController, SetDefaultController, GetDefaultController, ] for controller in self.controllers: handler.register(controller)
def test_3_from_pre_to_casava_transfer(self): """Test from pre-casava to casava transfer to project directory""" self.app = self.make_app(argv = ['production', 'transfer', 'J.Doe_00_01', '-f', '120829_SN0001_0001_AA001AAAXX', '--from_pre_casava']) handler.register(ProductionController) self._run_app() res = shell.exec_cmd(["ls", "-1", os.path.join(delivery_dir, "P1_101F_index1", "120829_AA001AAAXX")]) self.eq(['1_120829_AA001AAAXX_barcode', '1_120829_AA001AAAXX_nophix_1-sort-dup.align_metrics', '1_120829_AA001AAAXX_nophix_1-sort-dup.bam', '1_120829_AA001AAAXX_nophix_1-sort-dup.dup_metrics', '1_120829_AA001AAAXX_nophix_1-sort-dup.hs_metrics', '1_120829_AA001AAAXX_nophix_1-sort-dup.insert_metrics', '1_120829_AA001AAAXX_nophix_1-sort.bam', 'P1_101F_index1-bcbb-config.yaml', 'alignments'], res[0].split())
def test_4_from_pre_to_pre_casava_transfer(self): """Test pre_casava transfer to project directory""" self.app = self.make_app(argv = ['production', 'transfer', 'J.Doe_00_01', '-f', '120829_SN0001_0001_AA001AAAXX', '--from_pre_casava', '--to_pre_casava', '--quiet']) handler.register(ProductionController) self._run_app() ## Assert data output res = shell.exec_cmd(["ls", "-1", os.path.join(delivery_dir, "120829_AA001AAAXX", "1_120829_AA001AAAXX_barcode")]) res_files = ['1_120829_AA001AAAXX_nophix_10_1_fastq.txt', '1_120829_AA001AAAXX_nophix_10_2_fastq.txt', '1_120829_AA001AAAXX_nophix_12_1_fastq.txt','1_120829_AA001AAAXX_nophix_12_2_fastq.txt', '1_120829_AA001AAAXX_nophix_1_1_fastq.txt','1_120829_AA001AAAXX_nophix_1_2_fastq.txt', '1_120829_AA001AAAXX_nophix_2_1_fastq.txt','1_120829_AA001AAAXX_nophix_2_2_fastq.txt', '1_120829_AA001AAAXX_nophix_3_1_fastq.txt','1_120829_AA001AAAXX_nophix_3_2_fastq.txt', '1_120829_AA001AAAXX_nophix_4_1_fastq.txt','1_120829_AA001AAAXX_nophix_4_2_fastq.txt', '1_120829_AA001AAAXX_nophix_8_1_fastq.txt','1_120829_AA001AAAXX_nophix_8_2_fastq.txt'] self.eq(set(res_files), set(res[0].split())) ## Assert intermediate delivery output res = shell.exec_cmd(["ls", "-1", os.path.join(intermediate_delivery_dir, "120829_AA001AAAXX")]) self.eq(['1_120829_AA001AAAXX_nophix_1-sort-dup.align_metrics','1_120829_AA001AAAXX_nophix_1-sort-dup.bam'], res[0].split()[0:2]) self.eq(['1_120829_AA001AAAXX_nophix_8-sort-dup.insert_metrics','1_120829_AA001AAAXX_nophix_8-sort.bam', 'alignments'], res[0].split()[-3:]) ## Assert pruned yaml file contents with open(os.path.join(delivery_dir, "120829_AA001AAAXX", "project_run_info.yaml")) as fh: runinfo_yaml = yaml.load(fh) self.eq(runinfo_yaml['details'][0]['multiplex'][0]['name'], 'P1_101F_index1') self.eq(runinfo_yaml['details'][0]['multiplex'][0]['description'], 'J.Doe_00_01_P1_101F_index1') self.eq(set(runinfo_yaml['details'][0]['multiplex'][0]['files']), set([os.path.join(delivery_dir,"120829_AA001AAAXX", "1_120829_AA001AAAXX_barcode", os.path.basename(x)) for x in ['1_120829_AA001AAAXX_nophix_1_1_fastq.txt','1_120829_AA001AAAXX_nophix_1_2_fastq.txt']]))
def setup(self): # Add hooks hook.register('post_argument_parsing', hooks.pre_run_hook) # Add controllers controllers = [ InitController, CreateController, EventsController, LogsController, PrintEnvController, DeployController, StatusController, TerminateController, ConfigController, SwapController, OpenController, ConsoleController, ScaleController, SSHController, UseController, SetEnvController, ListController, PlatformController, CloneController, UpgradeController, AbortController, LabsController, LocalController, HealthController, CodeSourceController, ] # register all controllers for c in controllers: c._add_to_handler(handler) # Add special controllers handler.register(CompleterController) super(EB, self).setup() #Register global arguments self.add_arg('-v', '--verbose', action='store_true', help=flag_text['base.verbose']) self.add_arg('--profile', help=flag_text['base.profile']) self.add_arg('-r', '--region', help=flag_text['base.region']) self.add_arg('--endpoint-url', help=SUPPRESS) self.add_arg('--no-verify-ssl', action='store_true', help=flag_text['base.noverify']) self.add_arg( '--debugboto', # show debug info for botocore action='store_true', help=SUPPRESS) ebglobals.app = self
def test_1_clean(self): before = glob.glob(os.path.join(intermediate, "120829_AA001AAAXX", "*")) self.app = self.make_app(argv = ['project', 'clean', 'j_doe_00_02', '--pileup', '--intermediate', '--force']) handler.register(ProjectController) self._run_app() after = glob.glob(os.path.join(intermediate, "120829_AA001AAAXX", "*")) diff = [os.path.basename(x) for x in list(set(before).difference(set(after)))] self.eq(set(diff), set(['1_120829_AA001AAAXX_nophix_12-sort-dup-target.pileup.gz', '1_120829_AA001AAAXX_nophix_12-sort-dup-target.pileup', '1_120829_AA001AAAXX_nophix_12-sort-dup-target.crisp_pileup.gz', '1_120829_AA001AAAXX_nophix_12-sort-dup-target.crisp_pileup', '1_120829_AA001AAAXX_nophix_10-sort-dup-target.crisp_pileup.gz', '1_120829_AA001AAAXX_nophix_10-sort-dup-target.crisp_pileup', '1_120829_AA001AAAXX_nophix_10-sort-dup-target.pileup', '1_120829_AA001AAAXX_nophix_10-sort-dup-target.pileup.gz']))
def test_production(self): self.app = self.make_app(argv=[ 'production', 'run', 'J.Doe_00_04', '--debug', '--force', '--amplicon', '--restart' ]) handler.register(ProductionController) self._run_app() os.chdir(filedir)
def test_3_project_status_ps_map(self): self.app = self.make_app(argv=[ 'report', 'project_status', '--user', self.user, '--password', self.pw, '--url', self.url, self.examples["project"], '--debug' ], extensions=['scilifelab.pm.ext.ext_couchdb']) handler.register(DeliveryReportController) self._run_app()
def test_1_ls(self): self.app = self.make_app(argv=['production', 'ls']) handler.register(ProductionController) self._run_app() self.eq( self.app._output_data['stdout'].getvalue(), '120829_SN0001_0001_AA001AAAXX\n120829_SN0001_0002_BB001BBBXX\n120924_SN0002_0003_CC003CCCXX\nJ.Doe_00_04\nJ.Doe_00_05\nJ.Doe_00_06' )
def test_change_platform_args(self): """Test that passing --time actually changes platform arguments. These arguments should have precedence over whatever is written in the config file.""" self.app = self.make_app(argv = ['production', 'run', 'J.Doe_00_04', '--debug', '--force', '--amplicon', '--restart', '--sample', SAMPLES[1], '--drmaa', '--time', '00:01:00', '-n'], extensions=['scilifelab.pm.ext.ext_distributed']) handler.register(ProductionController) self._run_app() os.chdir(filedir)
def test_rm_analysis_1_dry(self): """Test dry removal of one intermediate analysis""" self.app = self.make_app(argv=[ 'project', 'rm', 'j_doe_00_04', 'analysis_1', '-n', '--force' ]) handler.register(ProjectController) handler.register(ProjectRmController) self._run_app()
def test_project_status(self): self.app = self.make_app( argv=['report', 'project_status', self.examples["project"]], extensions=['scilifelab.pm.ext.ext_couchdb']) handler.register(DeliveryReportController) self._run_app() data = ast.literal_eval(self.app._output_data['debug'].getvalue()) self.assertEqual(data['param']['project_name'], 'J.Doe_00_01') self.assertEqual(data['param']['ordered_amount'], 0.1)
def load(app): # register the plugin class.. this only happens if the plugin is enabled handler.register(EELogController) handler.register(EELogShowController) handler.register(EELogResetController) handler.register(EELogGzipController) handler.register(EELogMailController) # register a hook (function) to run after arguments are parsed. hook.register('post_argument_parsing', ee_log_hook)
def test_register_invalid_handler_type(self): self.app.setup() class BadInterface: class IMeta: label = 'bad_interface' class BadHandler(TestHandler): class Meta: interface = BadInterface handler.register(BadHandler)
def load(app): # register the plugin class.. this only happens if the plugin is enabled handler.register(SSSSiteController) handler.register(SSSSiteCreateController) handler.register(SSSSiteEditController) handler.register(SSSSiteDeleteController) handler.register(SSSSiteListController) # register a hook (function) to run after arguments are parsed. hook.register('post_argument_parsing', sss_site_hook)
def main(): app = MyApp() handler.register(MySecondController) try: app.setup() app.run() finally: app.close()
def test_5_quiet(self): """Test pre_casava delivery to project directory with quiet flag""" self.app = self.make_app(argv=[ 'production', 'transfer', 'J.Doe_00_01', '-f', '120829_SN0001_0001_AA001AAAXX', '--from_pre_casava', '--to_pre_casava', '--quiet' ]) handler.register(ProductionController) self._run_app()
def main(): app = ArtnetApp() handler.register(ArtnetScriptController) try: app.setup() app.run() finally: app.close()
def setUp(self): super(BaseTest, self).setUp() self.reset_backend() self.app = DroopeScan(argv=[], plugin_config_dir="./plugins.d", plugin_dir="./plugins") handler.register(AbstractArgumentController) self.app.testing = True self.app.setup()
def test_rm_analysis_1(self): """Test removal of non-existing intermediate analysis""" self.app = self.make_app(argv = ['project', 'rm', 'j_doe_00_04', 'analysisoe', '--force']) handler.register(ProjectController) handler.register(ProjectRmController) try: self._run_app() except: raise Exception
def test_remove_finished(self): self.app = self.make_app(argv = ['production', 'touch-finished', 'J.Doe_00_04', '--debug', '--force', '--sample', SAMPLES[1]], extensions=[]) handler.register(ProductionController) self._run_app() self.assertTrue(os.path.exists(FINISHED['J.Doe_00_04'][SAMPLES[1]])) ## Remove file, dry self.app = self.make_app(argv = ['production', 'remove-finished', 'J.Doe_00_04', '--debug', '--force', '-n'], extensions=[]) handler.register(ProductionController) self._run_app()
def load(): # register hook hook.register('pre_setup', init_configuration_file) # register controllers # handler.define(IRESTApi) handler.register(P4CMInfoController) handler.register(P4CMConfigController)
def test_rm_analysis_1_intermediate(self): """Test removal of one intermediate analysis""" self.app = self.make_app(argv = ['project', 'rm', 'j_doe_00_04', 'analysis_1', '--force']) handler.register(ProjectController) handler.register(ProjectRmController) self._run_app() try: os.listdir(os.path.join(filedir, "data", "projects", "j_doe_00_04", "intermediate", "analysis_1")) except: raise Exception
def load(): """Called by the framework when the extension is 'loaded'.""" if not os.getenv("DRMAA_LIBRARY_PATH"): LOG.debug("No environment variable $DRMAA_LIBRARY_PATH: loading {} failed".format(__name__)) return hook.register('post_setup', add_drmaa_option) hook.register('post_setup', add_shared_distributed_options) hook.register('pre_run', set_distributed_handler) hook.register('post_run', run_batch_command) handler.register(DistributedCommandHandler)
def test_from_casava_to_casava_custom_transfer(self): """Test from casava to casava transfer to custom project directory""" self.app = self.make_app(argv = ['production', 'transfer', 'J.Doe_00_04', '--transfer_dir', 'j_doe_00_04_custom']) handler.register(ProductionController) self._run_app() delivery_dir = os.path.abspath(os.path.join(filedir, "data", "projects", "j_doe_00_04_custom", "data")) with open(os.path.join(delivery_dir, "P001_101_index3", "120924_CC003CCCXX", "P001_101_index3-bcbb-config.yaml")) as fh: runinfo_yaml = yaml.load(fh) res = shell.exec_cmd(["ls", "-1", os.path.join(delivery_dir, "P001_101_index3", "120924_CC003CCCXX")]) self.eq(len(set(res[0].split())), 22)
def test_batch_submission(self): """Test that adding --batch groups commands into a batch submission""" pp = os.path.join(j_doe_00_04, SAMPLES[1], FLOWCELL, "{}-post_process.yaml".format(SAMPLES[1])) with open(pp) as fh: config = yaml.load(fh) platform_args = config["distributed"]["platform_args"].split() account = platform_args[platform_args.index("-A")+1] self.app = self.make_app(argv = ['production', 'compress', 'J.Doe_00_04', '--debug', '--force', '--jobname', 'batchsubmission', '--drmaa', '--batch', '--partition', 'devel', '--time', '01:00:00', '-A', account], extensions=['scilifelab.pm.ext.ext_distributed']) handler.register(ProductionController) self._run_app()
def test_2_clean_fastqbam(self): before = glob.glob(os.path.join(data, "P1_106F_index6/120829_AA001AAAXX/alignments", "*")) self.app = self.make_app(argv = ['project', 'clean', 'j_doe_00_02', '--data', '--fastqbam', '--force']) handler.register(ProjectController) self._run_app() after = glob.glob(os.path.join(data, "P1_106F_index6/120829_AA001AAAXX/alignments", "*")) diff = len(before) - len(after) self.eq(diff, 2) diff = [os.path.basename(x) for x in list(set(before).difference(set(after)))] self.eq(set(diff), set(['1_120829_AA001AAAXX_nophix_10_1_fastq-fastq.bam', '1_120829_AA001AAAXX_nophix_10_2_fastq-fastq.bam']))
def main(): app = DigitalOceanApp() handler.register(ApiController) hook.register('post_argument_parsing', token_check_hook, weight=0) hook.register('post_argument_parsing', setup_manager_hook, weight=1) try: app.setup() app.run() finally: app.close()
def load(): # Subcontrollers for each functional component handler.register(cli.cloudlet.NephoCloudletController) handler.register(cli.blueprint.NephoBlueprintController) handler.register(cli.stack.NephoStackController) handler.register(cli.parameter.NephoParameterController) handler.register(cli.scope.NephoScopeController) hook.register('post_argument_parsing', cli.hooks.set_scope) hook.register('post_setup', cli.hooks.process_config)
def test_project_run_only_failed(self): """Test running project in project folder on only failed samples. """ psummary = os.path.join(j_doe_00_04, "P001_101_index3", "121015_BB002BBBXX", "project_summary.csv") if os.path.exists(psummary): os.unlink(psummary) self.app = self.make_app(argv = ['project', 'run', 'j_doe_00_04', '-n', '--only_failed', '--debug', '--force', '--quiet'], extensions=['scilifelab.pm.ext.ext_distributed']) handler.register(ProjectController) handler.register(BcbioRunController) self._run_app() os.chdir(filedir)
def test_production_setup(self): self.app = self.make_app( argv=[ 'production', 'run', 'J.Doe_00_04', '--debug', '--force', '--only_setup', '--restart', '--drmaa' ], extensions=['scilifelab.pm.ext.ext_distributed']) handler.register(ProductionController) self._run_app() os.chdir(filedir)
def test_rm_analysis_1(self): """Test removal of non-existing intermediate analysis""" self.app = self.make_app( argv=['project', 'rm', 'j_doe_00_04', 'analysisoe', '--force']) handler.register(ProjectController) handler.register(ProjectRmController) try: self._run_app() except: raise Exception
def main(): 'Initialise and execute the cement app.' histogram_app = foundation.CementApp('histogram') handler.register(HistogramBaseController) try: histogram_app.setup() histogram_app.run() finally: histogram_app.close()
def test_2_project_status_custom(self): """Test project status note generation with command line customizations""" self.app = self.make_app(argv=[ 'report', 'project_status', '--user', self.user, '--password', self.pw, '--url', self.url, self.examples["project"], '--debug', '--customer_reference', 'MyCustomerReference', '--uppnex_id', 'MyUppnexID', '--use_bc_map', '--ordered_million_reads', '1' ], extensions=['scilifelab.pm.ext.ext_couchdb']) handler.register(DeliveryReportController) self._run_app()
def test_nested_controller(self): app = self.make_app(argv=['nested-controller']) handler.register(TestController) handler.register(Nested) app.setup() app.run() check = 'nested-controller' in app.controller._visible_commands self.ok(check) self.eq(app.controller._dispatch_command['func_name'], '_dispatch')