def testMkdirsFail(self):
# There are no writing permissions to create such top-level directories
self.assertEqual(hdfs.mkdirs(self.nonExistingHDFSdir), -1)
self.assertEqual(hdfs.mkdirs(self.nonExistingHomeDir), -1)
# The function should raise an exception since the target path for making a directory is a file
self.assertRaises(hdfsException,
lambda: hdfs.mkdirs(self.userHDFSfile))
self.assertRaises(hdfsException,
lambda: hdfs.mkdirs(self.userHomeFile))
def create_output_dir(file_name):
output_dir = os.path.dirname(file_name)
if not os.path.isdir(output_dir):
if not hdfs.mkdirs(output_dir):
raise RuntimeError(
"Unable to create directory for the output file: %s" %
file_name)
def create_if_not_exists(dir_fullpath):
"""Creates a given directory if it doesn't exist yet
Args:
dir_fullpath: full path to the directory
"""
#print("create_if_not_exists: dir_fullpath = '%s'" % dir_fullpath)
if hdfs.mkdirs(dir_fullpath) != 0:
raise RuntimeError("Unable to create directory %s" % dir_fullpath)
def create_if_not_exists(dir_fullpath):
"""Creates a given directory if it doesn't exist yet
Args:
dir_fullpath: full path to the directory
"""
from tthAnalysis.HiggsToTauTau.hdfs import hdfs
if hdfs.mkdirs(dir_fullpath) != 0:
raise RuntimeError("Unable to create directory %s" % dir_fullpath)
def testMkdirs(self):
self.assertEqual(hdfs.mkdirs(self.userHDFSsubDir), 0)
self.assertEqual(hdfs.mkdirs(self.userHomeSubDir), 0)
def testNegative(self):
testArgs = self.testArguments['negative']
self.manager.submitJob(
inputFiles=[],
executable=testArgs['cmd'],
command_line_parameter="",
outputFilePath="",
outputFiles=[],
scriptFile=os.path.join(testDir, '{}.sh'.format(testArgs['name'])),
)
# if passes, true negative; otherwise true positive
self.assertRaises(sbatchManagerRuntimeError, self.manager.waitForJobs)
def suite():
testSuite = unittest.TestSuite()
testSuite.addTest(unittest.makeSuite(SbatchTestCase))
return testSuite
if not hdfs.isdir(testDir):
hdfs.mkdirs(testDir)
suite_instance = suite()
runner = unittest.TextTestRunner()
runner.run(suite_instance)
if hdfs.isdir(testDir):
shutil.rmtree(testDir)
offset = get_file_idx(file_list_1[-1])
new_idx = offset
for root_file in file_list_2:
new_idx += 1
root_file_basename = 'tree_%d.root' % new_idx
root_file_idx = new_idx // 1000
dst_subdir = os.path.join(destination, '%04d' % root_file_idx)
if dst_subdir not in missing_subdirs:
missing_subdirs.append(dst_subdir)
copy_relations[root_file] = os.path.join(dst_subdir, root_file_basename)
if args.copy:
for missing_subdir in missing_subdirs:
if not hdfs.isdir(missing_subdir):
logging.info('Created subdirectory {}'.format(missing_subdir))
if hdfs.mkdirs(missing_subdir) != 0:
raise RuntimeError("Unable to create directory: %s" % missing_subdir)
for src_file, dst_file in copy_relations.items():
logging.debug('Copying file {} to {}'.format(src_file, dst_file))
if args.copy:
if hdfs.copy(src_file, dst_file, overwrite = False) != 0:
raise RuntimeError("Unable to copy file from %s to %s" % (src_file, dst_file))
logging.info('Copying done')
new_lines[os.path.dirname(destination)] = (
chunks[chunk_1] * len(file_list_1) / 100. + chunks[chunk_2] * len(file_list_2) / 100.
) / (len(file_list_1) + len(file_list_2)) * 100.
if args.modify:
with open(args.input, 'w') as input_list_file:
sys.exit(1)
# check if the directory into which we have to write the output ROOT file already exists
out_parent_dir = os.path.dirname(out_filename)
if not hdfs.isdir(out_parent_dir):
if not force:
logging.error("Parent directory of the output file {out_filename} does not exist".format(
out_filename = out_filename),
)
sys.exit(1)
else:
logging.debug("Output directory {out_parent_dir} does not exist, attempting to create one".format(
out_parent_dir = out_parent_dir,
))
try:
hdfs.mkdirs(out_parent_dir)
except IOError as err:
logging.error("Could not create directory {out_parent_dir}".format(out_parent_dir = out_parent_dir))
sys.exit(1)
# check if output ROOT file exists (if it does then the user is obliged to use '-f' flag!)
if hdfs.exists(out_filename) and not force:
logging.error("File {out_filename} already exists!".format(out_filename = out_filename))
sys.exit(1)
if grep_dir and not hdfs.isdir(grep_dir):
logging.error("Directory {grep_dir} does not exist!".format(grep_dir = grep_dir))
sys.exit(1)
# check if the provided sample name is in our 2016 py dictionary
samples = load_samples(args.era, is_postproc = args.post_processed)