Esempio n. 1
0
  def _set_up(self, sys_argv):
    options, args = self._parse_args(sys_argv, 1)
    dump_path = args[1]
    shared_first_dump_paths = args[2:]
    alternative_dirs_dict = {}
    if options.alternative_dirs:
      for alternative_dir_pair in options.alternative_dirs.split(':'):
        target_path, host_path = alternative_dir_pair.split('@', 1)
        alternative_dirs_dict[target_path] = host_path
    (bucket_set, dumps) = SubCommand.load_basic_files(
        dump_path, True, alternative_dirs=alternative_dirs_dict)

    self._timestamp = options.timestamp

    pfn_counts_dict = {}
    for shared_first_dump_path in shared_first_dump_paths:
      shared_dumps = SubCommand._find_all_dumps(shared_first_dump_path)
      for shared_dump in shared_dumps:
        pfn_counts = PFNCounts.load(shared_dump)
        if pfn_counts.pid not in pfn_counts_dict:
          pfn_counts_dict[pfn_counts.pid] = []
        pfn_counts_dict[pfn_counts.pid].append(pfn_counts)

    policy_set = PolicySet.load(SubCommand._parse_policy_list(options.policy))
    return policy_set, dumps, pfn_counts_dict, bucket_set
Esempio n. 2
0
    def _set_up(self, sys_argv):
        options, args = self._parse_args(sys_argv, 1)
        dump_path = args[1]
        shared_first_dump_paths = args[2:]
        alternative_dirs_dict = {}
        if options.alternative_dirs:
            for alternative_dir_pair in options.alternative_dirs.split(':'):
                target_path, host_path = alternative_dir_pair.split('@', 1)
                alternative_dirs_dict[target_path] = host_path
        (bucket_set, dumps) = SubCommand.load_basic_files(
            dump_path, True, alternative_dirs=alternative_dirs_dict)

        self._timestamp = options.timestamp

        pfn_counts_dict = {}
        for shared_first_dump_path in shared_first_dump_paths:
            shared_dumps = SubCommand._find_all_dumps(shared_first_dump_path)
            for shared_dump in shared_dumps:
                pfn_counts = PFNCounts.load(shared_dump)
                if pfn_counts.pid not in pfn_counts_dict:
                    pfn_counts_dict[pfn_counts.pid] = []
                pfn_counts_dict[pfn_counts.pid].append(pfn_counts)

        policy_set = PolicySet.load(
            SubCommand._parse_policy_list(options.policy))
        return policy_set, dumps, pfn_counts_dict, bucket_set
Esempio n. 3
0
    def do(self, sys_argv):
        options, args = self._parse_args(sys_argv, 2)
        dump_path = args[1]
        gs_path = args[2]

        dump_files = SubCommand._find_all_dumps(dump_path)
        bucket_files = SubCommand._find_all_buckets(dump_path)
        prefix = SubCommand._find_prefix(dump_path)
        symbol_data_sources = SymbolDataSources(prefix)
        symbol_data_sources.prepare()
        symbol_path = symbol_data_sources.path()

        handle_zip, filename_zip = tempfile.mkstemp('.zip', 'dmprof')
        os.close(handle_zip)

        try:
            file_zip = zipfile.ZipFile(filename_zip, 'w', zipfile.ZIP_DEFLATED)
            for filename in dump_files:
                file_zip.write(filename,
                               os.path.basename(os.path.abspath(filename)))
            for filename in bucket_files:
                file_zip.write(filename,
                               os.path.basename(os.path.abspath(filename)))

            symbol_basename = os.path.basename(os.path.abspath(symbol_path))
            for filename in os.listdir(symbol_path):
                if not filename.startswith('.'):
                    file_zip.write(
                        os.path.join(symbol_path, filename),
                        os.path.join(
                            symbol_basename,
                            os.path.basename(os.path.abspath(filename))))
            file_zip.close()

            returncode = UploadCommand._run_gsutil(options.gsutil, 'cp', '-a',
                                                   'public-read', filename_zip,
                                                   gs_path)
        finally:
            os.remove(filename_zip)

        return returncode
Esempio n. 4
0
  def do(self, sys_argv):
    options, args = self._parse_args(sys_argv, 2)
    dump_path = args[1]
    gs_path = args[2]

    dump_files = SubCommand._find_all_dumps(dump_path)
    bucket_files = SubCommand._find_all_buckets(dump_path)
    prefix = SubCommand._find_prefix(dump_path)
    symbol_data_sources = SymbolDataSources(prefix)
    symbol_data_sources.prepare()
    symbol_path = symbol_data_sources.path()

    handle_zip, filename_zip = tempfile.mkstemp('.zip', 'dmprof')
    os.close(handle_zip)

    try:
      file_zip = zipfile.ZipFile(filename_zip, 'w', zipfile.ZIP_DEFLATED)
      for filename in dump_files:
        file_zip.write(filename, os.path.basename(os.path.abspath(filename)))
      for filename in bucket_files:
        file_zip.write(filename, os.path.basename(os.path.abspath(filename)))

      symbol_basename = os.path.basename(os.path.abspath(symbol_path))
      for filename in os.listdir(symbol_path):
        if not filename.startswith('.'):
          file_zip.write(os.path.join(symbol_path, filename),
                         os.path.join(symbol_basename, os.path.basename(
                             os.path.abspath(filename))))
      file_zip.close()

      returncode = UploadCommand._run_gsutil(
          options.gsutil, 'cp', '-a', 'public-read', filename_zip, gs_path)
    finally:
      os.remove(filename_zip)

    return returncode