def partition_files(src_directory, dst_directory, max_size, exclude_directories=[], exclude_files=native_binaries_to_ignore): """ Copy bucketized files based on size to destination folder. Args: src_directory (string): Source folder containing files to be copied. dst_directory (string): Destination folder where files should be copied. max_size (int): Maximum partition size in bytes exclude_directories ([string]): List of folder names to be excluded. exclude_files ([string]): List of files names to be excluded. """ print('Partitioning files from {0} to {1}'.format(src_directory, dst_directory)) sorted_by_size = get_files_sorted_by_size(src_directory, exclude_directories, exclude_files) partitions = first_fit(sorted_by_size, max_size) index = 0 for p_index in partitions: file_names = [curr_file[0] for curr_file in partitions[p_index]] curr_dst_path = os.path.join(dst_directory, str(index), "binaries") copy_files(src_directory, curr_dst_path, file_names) index += 1
def main(main_args): """ Main entrypoint Args: main_args ([type]): Arguments to the script """ coreclr_args = setup_args(main_args) source_directory = coreclr_args.source_directory # CorrelationPayload directories correlation_payload_directory = os.path.join(coreclr_args.source_directory, "payload") superpmi_src_directory = os.path.join(source_directory, 'src', 'coreclr', 'scripts') superpmi_dst_directory = os.path.join(correlation_payload_directory, "superpmi") arch = coreclr_args.arch helix_source_prefix = "official" creator = "" ci = True if is_windows: helix_queue = "Windows.10.Arm64" if arch == "arm64" else "Windows.10.Amd64.X86.Rt" else: if arch == "arm": helix_queue = "(Ubuntu.1804.Arm32)[email protected]/dotnet-buildtools/prereqs:ubuntu-18.04-helix-arm32v7-bfcd90a-20200121150440" elif arch == "arm64": helix_queue = "(Ubuntu.1804.Arm64)[email protected]/dotnet-buildtools/prereqs:ubuntu-18.04-helix-arm64v8-20210531091519-97d8652" else: helix_queue = "Ubuntu.1804.Amd64" # create superpmi directory print('Copying {} -> {}'.format(superpmi_src_directory, superpmi_dst_directory)) copy_directory(superpmi_src_directory, superpmi_dst_directory, verbose_output=True, match_func=lambda path: any( path.endswith(extension) for extension in [".py"])) if is_windows: acceptable_copy = lambda path: any( path.endswith(extension) for extension in [".py", ".dll", ".exe", ".json"]) else: # Need to accept files without any extension, which is how executable file's names look. acceptable_copy = lambda path: (os.path.basename(path).find( ".") == -1) or any( path.endswith(extension) for extension in [".py", ".dll", ".so", ".json"]) print('Copying {} -> {}'.format(coreclr_args.core_root_directory, superpmi_dst_directory)) copy_directory(coreclr_args.core_root_directory, superpmi_dst_directory, verbose_output=True, match_func=acceptable_copy) # Copy all the test files to CORE_ROOT # The reason is there are lot of dependencies with *.Tests.dll and to ensure we do not get # Reflection errors, just copy everything to CORE_ROOT so for all individual partitions, the # references will be present in CORE_ROOT. if coreclr_args.collection_name == "libraries_tests": print('Copying {} -> {}'.format(coreclr_args.input_directory, superpmi_dst_directory)) def make_readable(folder_name): """Make file executable by changing the permission Args: folder_name (string): folder to mark with 744 """ if is_windows: return print("Inside make_readable") run_command(["ls", "-l", folder_name]) for file_path, dirs, files in os.walk(folder_name, topdown=True): for d in dirs: os.chmod( os.path.join(file_path, d), # read+write+execute for owner (stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR) | # read for group (stat.S_IRGRP) | # read for other (stat.S_IROTH)) for f in files: os.chmod( os.path.join(file_path, f), # read+write+execute for owner (stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR) | # read for group (stat.S_IRGRP) | # read for other (stat.S_IROTH)) run_command(["ls", "-l", folder_name]) make_readable(coreclr_args.input_directory) copy_directory(coreclr_args.input_directory, superpmi_dst_directory, verbose_output=True, match_func=acceptable_copy) # Workitem directories workitem_directory = os.path.join(source_directory, "workitem") input_artifacts = "" if coreclr_args.collection_name == "benchmarks": # Setup microbenchmarks setup_microbenchmark(workitem_directory, arch) else: # Setup for pmi/crossgen runs # Clone and build jitutils try: with TempDir() as jitutils_directory: run_command([ "git", "clone", "--quiet", "--depth", "1", "https://github.com/dotnet/jitutils", jitutils_directory ]) # Make sure ".dotnet" directory exists, by running the script at least once dotnet_script_name = "dotnet.cmd" if is_windows else "dotnet.sh" dotnet_script_path = os.path.join(source_directory, dotnet_script_name) run_command([dotnet_script_path, "--info"], jitutils_directory) # Set dotnet path to run build os.environ["PATH"] = os.path.join( source_directory, ".dotnet") + os.pathsep + os.environ["PATH"] build_file = "build.cmd" if is_windows else "build.sh" run_command( [os.path.join(jitutils_directory, build_file), "-p"], jitutils_directory) copy_files( os.path.join(jitutils_directory, "bin"), superpmi_dst_directory, [os.path.join(jitutils_directory, "bin", "pmi.dll")]) except PermissionError as pe_error: # Details: https://bugs.python.org/issue26660 print('Ignoring PermissionError: {0}'.format(pe_error)) # NOTE: we can't use the build machine ".dotnet" to run on all platforms. E.g., the Windows x86 build uses a # Windows x64 .dotnet\dotnet.exe that can't load a 32-bit shim. Thus, we always use corerun from Core_Root to invoke crossgen2. # The following will copy .dotnet to the correlation payload in case we change our mind, and need or want to use it for some scenarios. # # Copy ".dotnet" to correlation_payload_directory for crossgen2 job; it is needed to invoke crossgen2.dll # if coreclr_args.collection_type == "crossgen2": # dotnet_src_directory = os.path.join(source_directory, ".dotnet") # dotnet_dst_directory = os.path.join(correlation_payload_directory, ".dotnet") # print('Copying {} -> {}'.format(dotnet_src_directory, dotnet_dst_directory)) # copy_directory(dotnet_src_directory, dotnet_dst_directory, verbose_output=False) # payload pmiassemblies_directory = os.path.join(workitem_directory, "pmiAssembliesDirectory") input_artifacts = os.path.join(pmiassemblies_directory, coreclr_args.collection_name) exclude_directory = [ 'Core_Root' ] if coreclr_args.collection_name == "coreclr_tests" else [] exclude_files = native_binaries_to_ignore if coreclr_args.collection_type == "crossgen2": print('Adding exclusions for crossgen2') # Currently, trying to crossgen2 R2RTest\Microsoft.Build.dll causes a pop-up failure, so exclude it. exclude_files += ["Microsoft.Build.dll"] if coreclr_args.collection_name == "libraries_tests": # libraries_tests artifacts contains files from core_root folder. Exclude them. core_root_dir = coreclr_args.core_root_directory exclude_files += [ item for item in os.listdir(core_root_dir) if os.path.isfile(os.path.join(core_root_dir, item)) and ( item.endswith(".dll") or item.endswith(".exe")) ] partition_files(coreclr_args.input_directory, input_artifacts, coreclr_args.max_size, exclude_directory, exclude_files) # Set variables print('Setting pipeline variables:') set_pipeline_variable("CorrelationPayloadDirectory", correlation_payload_directory) set_pipeline_variable("WorkItemDirectory", workitem_directory) set_pipeline_variable("InputArtifacts", input_artifacts) set_pipeline_variable("Python", ' '.join(get_python_name())) set_pipeline_variable("Architecture", arch) set_pipeline_variable("Creator", creator) set_pipeline_variable("Queue", helix_queue) set_pipeline_variable("HelixSourcePrefix", helix_source_prefix) set_pipeline_variable("MchFileTag", coreclr_args.mch_file_tag)