def _set_up(self, sys_argv):
options, args = self._parse_args(sys_argv, 1)
dump_path = args[1]
shared_first_dump_paths = args[2:]
alternative_dirs_dict = {}
if options.alternative_dirs:
for alternative_dir_pair in options.alternative_dirs.split(':'):
target_path, host_path = alternative_dir_pair.split('@', 1)
alternative_dirs_dict[target_path] = host_path
(bucket_set, dumps) = SubCommand.load_basic_files(
dump_path, True, alternative_dirs=alternative_dirs_dict)
self._timestamp = options.timestamp
pfn_counts_dict = {}
for shared_first_dump_path in shared_first_dump_paths:
shared_dumps = SubCommand._find_all_dumps(shared_first_dump_path)
for shared_dump in shared_dumps:
pfn_counts = PFNCounts.load(shared_dump)
if pfn_counts.pid not in pfn_counts_dict:
pfn_counts_dict[pfn_counts.pid] = []
pfn_counts_dict[pfn_counts.pid].append(pfn_counts)
policy_set = PolicySet.load(SubCommand._parse_policy_list(options.policy))
return policy_set, dumps, pfn_counts_dict, bucket_set
def _set_up(self, sys_argv):
options, args = self._parse_args(sys_argv, 1)
dump_path = args[1]
shared_first_dump_paths = args[2:]
alternative_dirs_dict = {}
if options.alternative_dirs:
for alternative_dir_pair in options.alternative_dirs.split(':'):
target_path, host_path = alternative_dir_pair.split('@', 1)
alternative_dirs_dict[target_path] = host_path
(bucket_set, dumps) = SubCommand.load_basic_files(
dump_path, True, alternative_dirs=alternative_dirs_dict)
self._timestamp = options.timestamp
pfn_counts_dict = {}
for shared_first_dump_path in shared_first_dump_paths:
shared_dumps = SubCommand._find_all_dumps(shared_first_dump_path)
for shared_dump in shared_dumps:
pfn_counts = PFNCounts.load(shared_dump)
if pfn_counts.pid not in pfn_counts_dict:
pfn_counts_dict[pfn_counts.pid] = []
pfn_counts_dict[pfn_counts.pid].append(pfn_counts)
policy_set = PolicySet.load(
SubCommand._parse_policy_list(options.policy))
return policy_set, dumps, pfn_counts_dict, bucket_set
def do(self, sys_argv):
options, args = self._parse_args(sys_argv, 2)
dump_path = args[1]
gs_path = args[2]
dump_files = SubCommand._find_all_dumps(dump_path)
bucket_files = SubCommand._find_all_buckets(dump_path)
prefix = SubCommand._find_prefix(dump_path)
symbol_data_sources = SymbolDataSources(prefix)
symbol_data_sources.prepare()
symbol_path = symbol_data_sources.path()
handle_zip, filename_zip = tempfile.mkstemp('.zip', 'dmprof')
os.close(handle_zip)
try:
file_zip = zipfile.ZipFile(filename_zip, 'w', zipfile.ZIP_DEFLATED)
for filename in dump_files:
file_zip.write(filename,
os.path.basename(os.path.abspath(filename)))
for filename in bucket_files:
file_zip.write(filename,
os.path.basename(os.path.abspath(filename)))
symbol_basename = os.path.basename(os.path.abspath(symbol_path))
for filename in os.listdir(symbol_path):
if not filename.startswith('.'):
file_zip.write(
os.path.join(symbol_path, filename),
os.path.join(
symbol_basename,
os.path.basename(os.path.abspath(filename))))
file_zip.close()
returncode = UploadCommand._run_gsutil(options.gsutil, 'cp', '-a',
'public-read', filename_zip,
gs_path)
finally:
os.remove(filename_zip)
return returncode
def do(self, sys_argv):
options, args = self._parse_args(sys_argv, 2)
dump_path = args[1]
gs_path = args[2]
dump_files = SubCommand._find_all_dumps(dump_path)
bucket_files = SubCommand._find_all_buckets(dump_path)
prefix = SubCommand._find_prefix(dump_path)
symbol_data_sources = SymbolDataSources(prefix)
symbol_data_sources.prepare()
symbol_path = symbol_data_sources.path()
handle_zip, filename_zip = tempfile.mkstemp('.zip', 'dmprof')
os.close(handle_zip)
try:
file_zip = zipfile.ZipFile(filename_zip, 'w', zipfile.ZIP_DEFLATED)
for filename in dump_files:
file_zip.write(filename, os.path.basename(os.path.abspath(filename)))
for filename in bucket_files:
file_zip.write(filename, os.path.basename(os.path.abspath(filename)))
symbol_basename = os.path.basename(os.path.abspath(symbol_path))
for filename in os.listdir(symbol_path):
if not filename.startswith('.'):
file_zip.write(os.path.join(symbol_path, filename),
os.path.join(symbol_basename, os.path.basename(
os.path.abspath(filename))))
file_zip.close()
returncode = UploadCommand._run_gsutil(
options.gsutil, 'cp', '-a', 'public-read', filename_zip, gs_path)
finally:
os.remove(filename_zip)
return returncode