def main(): argparser = argparse.ArgumentParser( description="Run WDL compiler tests on the platform") argparser.add_argument("--compile-only", help="Only compile the workflows, don't run them", action="store_true", default=False) args = argparser.parse_args() version_id = util.get_version_id(top_dir) wdl_source_file = os.path.join(test_dir, "multi_region/trivial.wdl") dx_objects = [] test_exec_objs = [] # build version of the applet on all regions for proj_name in projects: dx_proj = util.get_project(proj_name) if dx_proj is None: raise RuntimeError("Could not find project {}".format(proj_name)) oid = build_test(wdl_source_file, dx_proj, target_folder, version_id) if args.compile_only: continue anl = run_workflow(dx_proj, target_folder, oid) print("Running {}".format(oid)) test_exec_objs.append(anl) # Wait for completion print("executables: " + ", ".join([a.get_id() for a in test_exec_objs])) wait_for_completion(test_exec_objs)
def main(): global test_locked argparser = argparse.ArgumentParser(description="Run WDL proxy tests") argparser.add_argument("--do-not-build", help="Do not assemble the dxWDL jar file", action="store_true", default=False) argparser.add_argument("--project", help="DNAnexus project ID", default="dxWDL_playground") argparser.add_argument("--verbose", help="Verbose compilation", action="store_true", default=False) args = argparser.parse_args() print("top_dir={} test_dir={}".format(top_dir, test_dir)) version_id = util.get_version_id(top_dir) project = util.get_project(args.project) if project is None: raise RuntimeError("Could not find project {}".format(args.project)) folder = build_dirs(project, version_id) print("project: {} ({})".format(project.name, project.get_id())) print("folder: {}".format(folder)) test_dict = {"aws:us-east-1": project.name + ":" + folder} # build the dxWDL jar file, only on us-east-1 if not args.do_not_build: util.build(project, folder, version_id, top_dir, test_dict) try: # make sure that squid is installed and running setup() # test_deny(project, folder, version_id) test_allow(project, folder, version_id) # test_allow_auth(project, folder, version_id) test_ntlm_auth(project, folder, version_id) # test_network_blocking(project, folder, version_id) # test_squid_allows_bypassing_firewall(project, folder, version_id) finally: print("Test complete") shutdown()
def copy_wf_test(tname, src_proj, src_folder, debug_flag): alt_proj_name = "dxWDL_playground_2" trg_proj = util.get_project(alt_proj_name) if trg_proj is None: raise RuntimeError("Could not find project {}".format(alt_proj_name)) test_folder = "/test" # clean up target print("cleaning up target") trg_proj.remove_folder(test_folder, recurse=True) trg_proj.new_folder(test_folder, parents=True) # copy to alternate project src_wf_id = lookup_dataobj(tname, src_proj, src_folder) print("copy {} to alternate project".format(src_wf_id)) wf = dxpy.DXWorkflow(dxid=src_wf_id) wf2 = wf.clone(trg_proj.get_id(), folder=test_folder) # Run the workflow, and wait for completion runnable = {tname: wf2.get_id()} run_test_subset(trg_proj, runnable, test_folder, debug_flag)
def main(): global test_locked argparser = argparse.ArgumentParser(description="Run WDL proxy tests") argparser.add_argument("--do-not-build", help="Do not assemble the dxWDL jar file", action="store_true", default=False) argparser.add_argument("--project", help="DNAnexus project ID", default="dxWDL_playground") argparser.add_argument("--verbose", help="Verbose compilation", action="store_true", default=False) args = argparser.parse_args() print("top_dir={} test_dir={}".format(top_dir, test_dir)) version_id = util.get_version_id(top_dir) project = util.get_project(args.project) if project is None: raise RuntimeError("Could not find project {}".format(args.project)) folder = util.build_dirs(project, version_id) print("project: {} ({})".format(project.name, project.get_id())) print("folder: {}".format(folder)) test_dict = {"aws:us-east-1": project.name + ":" + folder} # build the dxWDL jar file, only on us-east-1 #if not args.do_not_build: # util.build(project, folder, version_id, top_dir, test_dict) # For some reason, the ip-tables state is not completely # cleared. Therefore, we need to run those tests first. test_networking(project, folder, version_id) test_authorization(project, folder, version_id)
def main(): global test_locked argparser = argparse.ArgumentParser( description="Run WDL compiler tests on the platform") argparser.add_argument("--archive", help="Archive old applets", action="store_true", default=False) argparser.add_argument("--compile-only", help="Only compile the workflows, don't run them", action="store_true", default=False) argparser.add_argument("--compile-mode", help="Compilation mode") argparser.add_argument("--delay-workspace-destruction", help="Flag passed to workflow run", action="store_true", default=False) argparser.add_argument("--do-not-build", help="Do not assemble the dxWDL jar file", action="store_true", default=False) argparser.add_argument("--force", help="Remove old versions of applets and workflows", action="store_true", default=False) argparser.add_argument( "--folder", help="Use an existing folder, instead of building dxWDL") argparser.add_argument("--lazy", help="Only compile workflows that are unbuilt", action="store_true", default=False) argparser.add_argument("--locked", help="Generate locked-down workflows", action="store_true", default=False) argparser.add_argument("--regular", help="Generate only regular workflows", action="store_true", default=False) argparser.add_argument("--no-wait", help="Exit immediately after launching tests", action="store_true", default=False) argparser.add_argument("--project", help="DNAnexus project ID", default="dxWDL_playground") argparser.add_argument( "--runtime-debug-level", help="printing verbosity of task/workflow runner, {0,1,2}") argparser.add_argument("--test", help="Run a test, or a subgroup of tests", action="append", default=[]) argparser.add_argument("--test-list", help="Print a list of available tests", action="store_true", default=False) argparser.add_argument("--verbose", help="Verbose compilation", action="store_true", default=False) args = argparser.parse_args() register_all_tests() if args.test_list: print_test_list() exit(0) test_names = [] if len(args.test) == 0: args.test = 'M' for t in args.test: test_names += choose_tests(t) print("Running tests {}".format(test_names)) version_id = util.get_version_id(top_dir) project = util.get_project(args.project) if project is None: raise RuntimeError("Could not find project {}".format(args.project)) if args.folder is None: base_folder = build_dirs(project, version_id) else: # Use existing prebuilt folder base_folder = args.folder applet_folder = base_folder + "/applets" test_folder = base_folder + "/test" print("project: {} ({})".format(project.name, project.get_id())) print("folder: {}".format(base_folder)) test_dict = {"aws:us-east-1": project.name + ":" + base_folder} # build the dxWDL jar file, only on us-east-1 if not args.do_not_build: util.build(project, base_folder, version_id, top_dir) util.build_compiler_jar(version_id, top_dir, test_dict) if args.regular: # Disable all locked workflows args.locked = False test_locked = [] compiler_flags = [] if args.locked: compiler_flags.append("-locked") test_locked += test_names if args.archive: compiler_flags.append("-archive") if args.compile_mode: compiler_flags += ["-compileMode", args.compile_mode] if args.force: compiler_flags.append("-force") if args.verbose: compiler_flags.append("-verbose") if args.runtime_debug_level: compiler_flags += ["-runtimeDebugLevel", args.runtime_debug_level] compiler_flags += ["--extras", os.path.join(top_dir, "test/extras.json")] if "call_native" in test_names: native_call_setup(project, applet_folder, version_id) if "call_native_app" in test_names: native_call_app_setup(version_id) try: # Compile the WDL files to dx:workflows and dx:applets runnable = {} for tname in test_names: oid = None if args.lazy: oid = lookup_dataobj(tname, project, applet_folder) if oid is None: c_flags = compiler_flags[:] + compiler_per_test_flags(tname) oid = build_test(tname, project, applet_folder, version_id, c_flags) runnable[tname] = oid print("runnable({}) = {}".format(tname, oid)) if not args.compile_only: run_test_subset(project, runnable, test_folder, args.delay_workspace_destruction, args.no_wait) finally: print("Test complete")
def main(): argparser = argparse.ArgumentParser( description="Build a dxCompiler release") argparser.add_argument("--force", help="Build even if there is an existing version", action='store_true', default=False) argparser.add_argument("--multi-region", help="Copy to all supported regions", action='store_true', default=False) argparser.add_argument("--dry-run", help="Don't build any artifacts", action='store_true', default=False) args = argparser.parse_args() # build multi-region jar for releases, or # if explicitly specified multi_region = args.multi_region # Choose which dictionary to use if multi_region: project_dict = RELEASE_DICT else: project_dict = TEST_DICT project = util.get_project(project_dict[HOME_REGION]) print("project: {} ({})".format(project.name, project.get_id())) # Figure out what the current version is version_id = util.get_version_id(top_dir) print("version: {}".format(version_id)) # Set the folder folder = "/releases/{}".format(version_id) print("folder: {}".format(folder)) if args.dry_run: args.force = False # remove the existing directory paths if args.force: for proj_name in project_dict.values(): print("removing path {}:{}".format(proj_name, folder)) dx_proj = util.get_project(proj_name) try: dx_proj.remove_folder(folder, recurse=True) except dxpy.DXError: pass # Make sure the target directory exists project.new_folder(folder, parents=True) # Build the asset, and the compiler jar file. path_dict = dict( map(lambda kv: (kv[0], kv[1] + ":" + folder), project_dict.items())) if args.dry_run: return home_ad = util.build(project, folder, version_id, top_dir, path_dict) if multi_region: for lang, asset_desc in home_ad.asset_ids.items(): home_rec = dxpy.DXRecord(asset_desc) all_regions = project_dict.keys() # Leave only regions where the asset is missing target_regions = [] for dest_region in all_regions: dest_proj = util.get_project(project_dict[dest_region]) dest_asset = util.find_asset(dest_proj, folder, lang) if dest_asset == None: target_regions.append(dest_region) _clone_asset(home_rec, folder, target_regions, project_dict)
def _clone_asset(record, folder, regions, project_dict): """ This function will attempt to clone the given record into all of the given regions. It will return a dictionary with the regions as keys and the record-ids of the corresponding asset as the values. If an asset is not able to be created in a given region, the value will be set to None. """ # Get the asset record fid = record.get_details()['archiveFileId']['$dnanexus_link'] curr_region = dxpy.describe(record.project)['region'] # Only run once per region regions = set(regions) - set([curr_region]) if len(regions) == 0: # there is nothing to do return app_supported_regions = set( COPY_FILE_APP.describe()['regionalOptions'].keys()) if len(regions - app_supported_regions) > 0: print('Currently no support for the following region(s): [{regions}]'. format(regions=', '.join(regions - app_supported_regions)), file=sys.stderr) sys.exit(1) # Get information about the asset asset_properties = record.get_properties() asset_properties['cloned_from'] = record.get_id() asset_file_name = dxpy.describe(fid)['name'] url = dxpy.DXFile(fid).get_download_url( preauthenticated=True, project=dxpy.DXFile.NO_PROJECT_HINT, duration=URL_DURATION)[0] # setup target folders region2projid = {} for region in regions: dest_proj = util.get_project(project_dict[region]) dest_proj.new_folder(folder, parents=True) region2projid[region] = dest_proj.get_id() print(region2projid) # Fire off a clone process for each region # Wait for the cloning to complete for i in [1, 2, 3]: jobs = _clone_to_all_regions(region2projid, regions, asset_file_name, folder, url) retval = _wait_for_completion(jobs) if retval: break # make records for each file for region in regions: dest_proj_id = region2projid[region] results = list( dxpy.find_data_objects(classname="file", visibility="hidden", name=asset_file_name, project=dest_proj_id, folder=folder)) file_ids = [p["id"] for p in results] if len(file_ids) == 0: raise RuntimeError("Found no files {}:{}/{}".format( dest_proj_id, folder, asset_file_name)) if len(file_ids) > 1: raise RuntimeError( "Found {} files {}:{}/{}, instead of just one".format( len(dxfiles), dest_proj_id, folder, asset_file_name)) dest_asset = dxpy.new_dxrecord( name=record.name, types=['AssetBundle'], details={'archiveFileId': dxpy.dxlink(file_ids[0])}, properties=record.get_properties(), project=dest_proj_id, folder=folder, close=True)
def main(): global test_unlocked argparser = argparse.ArgumentParser( description="Run WDL compiler tests on the platform") argparser.add_argument("--archive", help="Archive old applets", action="store_true", default=False) argparser.add_argument("--compile-only", help="Only compile the workflows, don't run them", action="store_true", default=False) argparser.add_argument("--compile-mode", help="Compilation mode") argparser.add_argument("--debug", help="Run applets with debug-hold, and allow ssh", action="store_true", default=False) argparser.add_argument("--do-not-build", help="Do not assemble the dxWDL jar file", action="store_true", default=False) argparser.add_argument("--extras", help="run extra tests", action="store_true", default=False) argparser.add_argument("--force", help="Remove old versions of applets and workflows", action="store_true", default=False) argparser.add_argument( "--folder", help="Use an existing folder, instead of building dxWDL") argparser.add_argument("--lazy", help="Only compile workflows that are unbuilt", action="store_true", default=False) argparser.add_argument("--list", "--test-list", help="Print a list of available tests", action="store_true", dest="test_list", default=False) argparser.add_argument("--locked", help="Generate locked-down workflows", action="store_true", default=False) argparser.add_argument("--project", help="DNAnexus project ID", default="dxWDL_playground") argparser.add_argument( "--runtime-debug-level", help="printing verbosity of task/workflow runner, {0,1,2}") argparser.add_argument("--test", help="Run a test, or a subgroup of tests", action="append", default=[]) argparser.add_argument("--unlocked", help="Generate only unlocked workflows", action="store_true", default=False) argparser.add_argument("--verbose", help="Verbose compilation", action="store_true", default=False) argparser.add_argument("--verbose-key", help="Verbose compilation", action="append", default=[]) args = argparser.parse_args() print("top_dir={} test_dir={}".format(top_dir, test_dir)) register_all_tests() if args.test_list: print_test_list() exit(0) test_names = [] if len(args.test) == 0: args.test = 'M' for t in args.test: test_names += choose_tests(t) print("Running tests {}".format(test_names)) version_id = util.get_version_id(top_dir) project = util.get_project(args.project) if project is None: raise RuntimeError("Could not find project {}".format(args.project)) if args.folder is None: base_folder = build_dirs(project, version_id) else: # Use existing prebuilt folder base_folder = args.folder applet_folder = base_folder + "/applets" test_folder = base_folder + "/test" print("project: {} ({})".format(project.name, project.get_id())) print("folder: {}".format(base_folder)) test_dict = {"aws:us-east-1": project.name + ":" + base_folder} # build the dxWDL jar file, only on us-east-1 if not args.do_not_build: util.build(project, base_folder, version_id, top_dir, test_dict) if args.unlocked: # Disable all locked workflows args.locked = False test_unlocked = test_names compiler_flags = [] if args.locked: compiler_flags.append("-locked") test_unlocked = [] if args.archive: compiler_flags.append("-archive") if args.compile_mode: compiler_flags += ["-compileMode", args.compile_mode] if args.force: compiler_flags.append("-force") if args.verbose: compiler_flags.append("-verbose") if args.verbose_key: for key in args.verbose_key: compiler_flags += ["-verboseKey", key] if args.runtime_debug_level: compiler_flags += ["-runtimeDebugLevel", args.runtime_debug_level] # is "native" included in one of the test names? if ("call_native" in test_names or "call_native_v1" in test_names): native_call_setup(project, applet_folder, version_id) if "call_native_app" in test_names: native_call_app_setup(version_id) try: # Compile the WDL files to dx:workflows and dx:applets runnable = compile_tests_to_project(project, test_names, applet_folder, compiler_flags, version_id, args.lazy) if not args.compile_only: run_test_subset(project, runnable, test_folder, args.debug) finally: print("Completed running tasks in {}".format(args.project)) if args.extras: print("Copy workflow and run in alternate project ") copy_wf_test("linear", project, applet_folder, args.debug)
def main(): argparser = argparse.ArgumentParser(description="Build the dxWDL jar file") argparser.add_argument("--folder", help="Destination folder") argparser.add_argument("--multi-region", help="Copy to all supported regions", action='store_true', default=False) argparser.add_argument("--release", help="Create a dxWDL release, implies multi-region", action='store_true', default=False) args = argparser.parse_args() # resolve project project_dict = None if args.release: project_dict = RELEASE_DICT else: project_dict = TEST_DICT project = util.get_project(project_dict[HOME_REGION]) print("project: {} ({})".format(project.name, project.get_id())) # Set the folder, build one if necessary if args.folder is not None: folder = args.folder elif args.release: folder = time.strftime("/releases/%Y-%m-%d/%H%M%S") project.new_folder(folder, parents=True) else: folder = time.strftime("/builds/%Y-%m-%d/%H%M%S") project.new_folder(folder, parents=True) print("folder: {}".format(folder)) # build multi-region jar for releases, or # if explicitly specified multi_region = args.multi_region if args.release: multi_region = True # Figure out what the current version is version_id = util.get_version_id(top_dir) print("version: {}".format(version_id)) # build the asset home_ad = util.build(project, folder, version_id, top_dir) ad_all = [home_ad] if multi_region: # download dxWDL runtime library home_rec = dxpy.DXRecord(home_ad.asset_id) fid = home_rec.get_details()['archiveFileId']['$dnanexus_link'] fn = dxpy.describe(fid)['name'] rtlib_path = "/tmp/{}".format(fn) print("Download asset file {}".format(fn)) dxpy.download_dxfile(fid, rtlib_path, show_progress=True) # copy to all other regions for region in project_dict.keys(): if region != home_ad.region: proj = project_dict[region] if proj is None: raise Exception( "No project configured for region {}".format(region)) dest_proj = util.get_project(proj) dest_ad = util.copy_across_regions(rtlib_path, home_rec, region, dest_proj, folder) ad_all.append(dest_ad) # build the final jar file, containing a list of the per-region # assets jar_path = util.build_final_jar(version_id, top_dir, ad_all) # Upload compiler jar file if args.release: util.upload_local_file(jar_path, project, folder)
def main(): global test_unlocked argparser = argparse.ArgumentParser(description="Run WDL compiler tests on the platform") argparser.add_argument("--archive", help="Archive old applets", action="store_true", default=False) argparser.add_argument("--compile-only", help="Only compile the workflows, don't run them", action="store_true", default=False) argparser.add_argument("--compile-mode", help="Compilation mode") argparser.add_argument("--debug", help="Run applets with debug-hold, and allow ssh", action="store_true", default=False) argparser.add_argument("--delay-workspace-destruction", help="Run applets with delayWorkspaceDestruction", action="store_true", default=False) argparser.add_argument("--force", help="Remove old versions of applets and workflows", action="store_true", default=False) argparser.add_argument("--folder", help="Use an existing folder, instead of building dxCompiler") argparser.add_argument("--lazy", help="Only compile workflows that are unbuilt", action="store_true", default=False) argparser.add_argument("--list", "--test-list", help="Print a list of available tests", action="store_true", dest="test_list", default=False) argparser.add_argument("--clean", help="Remove build directory in the project after running tests", action="store_true", default=False) argparser.add_argument("--locked", help="Generate locked-down workflows", action="store_true", default=False) argparser.add_argument("--project", help="DNAnexus project ID", default="dxCompiler_playground") argparser.add_argument("--project-wide-reuse", help="look for existing applets in the entire project", action="store_true", default=False) argparser.add_argument("--stream-all-files", help="Stream all input files with dxfs2", action="store_true", default=False) argparser.add_argument("--runtime-debug-level", help="printing verbosity of task/workflow runner, {0,1,2}") argparser.add_argument("--test", help="Run a test, or a subgroup of tests", action="append", default=[]) argparser.add_argument("--unlocked", help="Generate only unlocked workflows", action="store_true", default=False) argparser.add_argument("--verbose", help="Verbose compilation", action="store_true", default=False) argparser.add_argument("--verbose-key", help="Verbose compilation", action="append", default=[]) args = argparser.parse_args() print("top_dir={} test_dir={}".format(top_dir, test_dir)) register_all_tests(args.verbose) if args.test_list: print_test_list() exit(0) test_names = [] if len(args.test) == 0: args.test = 'M' for t in args.test: test_names += choose_tests(t) print("Running tests {}".format(test_names)) version_id = util.get_version_id(top_dir) project = util.get_project(args.project) if project is None: raise RuntimeError("Could not find project {}".format(args.project)) if args.folder is None: base_folder = util.create_build_dirs(project, version_id) else: # Use existing prebuilt base folder base_folder = args.folder util.create_build_subdirs(project, base_folder) applet_folder = base_folder + "/applets" test_folder = base_folder + "/test" print("project: {} ({})".format(project.name, project.get_id())) print("folder: {}".format(base_folder)) test_dict = { "aws:us-east-1" : project.name + ":" + base_folder } # build the dxCompiler jar file, only on us-east-1 assets = util.build(project, base_folder, version_id, top_dir, test_dict) print("assets: {}".format(assets)) if args.unlocked: # Disable all locked workflows args.locked = False test_unlocked = test_names compiler_flags = [] if args.locked: compiler_flags.append("-locked") test_unlocked = [] if args.archive: compiler_flags.append("-archive") if args.compile_mode: compiler_flags += ["-compileMode", args.compile_mode] if args.force: compiler_flags.append("-force") if args.verbose: compiler_flags.append("-verbose") if args.stream_all_files: compiler_flags.append("-streamAllFiles") if args.verbose_key: for key in args.verbose_key: compiler_flags += ["-verboseKey", key] if args.runtime_debug_level: compiler_flags += ["-runtimeDebugLevel", args.runtime_debug_level] if args.project_wide_reuse: compiler_flags.append("-projectWideReuse") # is "native" included in one of the test names? if ("call_native" in test_names or "call_native_v1" in test_names): native_call_setup(project, applet_folder, version_id, args.verbose) if "call_native_app" in test_names: native_call_app_setup(project, version_id, args.verbose) try: # Compile the WDL files to dx:workflows and dx:applets runnable = compile_tests_to_project(project, test_names, applet_folder, compiler_flags, version_id, args.lazy) if not args.compile_only: run_test_subset(project, runnable, test_folder, args.debug, args.delay_workspace_destruction) finally: if args.clean: project.remove_folder(base_folder, recurse=True, force=True) print("Completed running tasks in {}".format(args.project))
def main(): argparser = argparse.ArgumentParser(description="Build a dxWDL release") argparser.add_argument( "--force", help="Build even if the there is an existing version", action='store_true', default=False) argparser.add_argument("--multi-region", help="Copy to all supported regions", action='store_true', default=False) args = argparser.parse_args() # build multi-region jar for releases, or # if explicitly specified multi_region = args.multi_region # Choose which dictionary to use if multi_region: project_dict = RELEASE_DICT else: project_dict = TEST_DICT project = util.get_project(project_dict[HOME_REGION]) print("project: {} ({})".format(project.name, project.get_id())) # Figure out what the current version is version_id = util.get_version_id(top_dir) print("version: {}".format(version_id)) # Set the folder folder = "/releases/{}".format(version_id) print("folder: {}".format(folder)) # remove the existing directory paths if args.force: for proj_name in project_dict.values(): print("removing path {}:{}".format(proj_name, folder)) dx_proj = util.get_project(proj_name) try: dx_proj.remove_folder(folder, recurse=True) except dxpy.DXError: pass # Make sure the target directory exists project.new_folder(folder, parents=True) # Build the asset, and the compiler jar file. path_dict = dict( map(lambda kv: (kv[0], kv[1] + ":" + folder), project_dict.items())) (jar_path, home_ad) = util.build(project, folder, version_id, top_dir, path_dict) if multi_region: # download dxWDL runtime library home_rec = dxpy.DXRecord(home_ad.asset_id) fid = home_rec.get_details()['archiveFileId']['$dnanexus_link'] fn = dxpy.describe(fid)['name'] rtlib_path = "/tmp/{}".format(fn) print("Download asset file {}".format(fn)) dxpy.download_dxfile(fid, rtlib_path, show_progress=True) # copy to all other regions for region in project_dict.keys(): if region != home_ad.region: proj = project_dict[region] if proj is None: raise Exception( "No project configured for region {}".format(region)) dest_proj = util.get_project(proj) if dest_proj is not None: dest_ad = util.copy_across_regions(rtlib_path, home_rec, region, dest_proj, folder) else: print("No project named {}".format(proj)) # Upload compiler jar file util.upload_local_file(jar_path, project, folder)
def get(self, eId): projectList = findTopKProjectsPerUser(eId) result = get_project(projectList) #result = {'projectsRecommended': projectList} return jsonify(result)