def test_safe_save_threadcalls():
"""Test that XML files don't disappear - See Github Issue #185"""
from Ganga.Core.GangaRepository.GangaRepositoryXML import safe_save
def my_to_file(obj, fhandle, ignore_subs):
fhandle.write("!" * 1000)
# Create lots of threads that will keep hitting safe_save
testfn = '/tmp/xmltest.tmp' + str(uuid.uuid4())
ths = []
o = LocalFile()
for i in range(0, 500):
ths.append(
threading.Thread(target=safe_save, args=(testfn, o, my_to_file)))
for th in ths:
th.start()
for th in ths:
th.join()
assert os.path.isfile(testfn)
os.remove(testfn)
assert os.path.isfile(testfn + '~')
os.remove(testfn + '~')
assert not os.path.isfile(testfn + '.new')
def constructExtraFiles(self, job):
"""
This constructs or appends to an uncompressed archive containing all of the opts files which are required to run on the grid
Args:
job (Job): The parent job of this application, we don't care if it's unique or not
"""
master_job = job.master or job
df = master_job.application.jobScriptArchive
folder_dir = master_job.getInputWorkspace(create=True).getPath()
if not df or df.namePattern == '':
unique_name = GaudiExec.sharedOptsFile_baseName % uuid.uuid4()
master_job.application.jobScriptArchive = LocalFile(
namePattern=unique_name, localDir=folder_dir)
tar_filename = path.join(folder_dir, unique_name)
if not path.isfile(tar_filename):
with tarfile.open(tar_filename, "w"):
pass
with tarfile.open(tar_filename, "a") as tar_file:
tinfo = tarfile.TarInfo('__timestamp__')
tinfo.mtime = time.time()
fileobj = StringIO(getTimestampContent())
tinfo.size = fileobj.len
tar_file.addfile(tinfo, fileobj)
else:
unique_name = master_job.application.jobScriptArchive.namePattern
extra_opts_file = self.getExtraOptsFileName()
# First construct if needed
if not path.isfile(path.join(folder_dir, unique_name)):
with tarfile.open(path.join(folder_dir, unique_name), "w"):
pass
# Now append the extra_opts file here when needed
with tarfile.open(path.join(folder_dir, unique_name), "a") as tar_file:
# Add the extra opts file to the job
tinfo = tarfile.TarInfo(extra_opts_file)
tinfo.mtime = time.time()
fileobj = StringIO(self.extraOpts)
tinfo.size = fileobj.len
tar_file.addfile(tinfo, fileobj)
if not self.useGaudiRun:
# Add the WN script for wrapping the job
logger.info("Constructing: %s" % self.getWrapperScriptName())
tinfo2 = tarfile.TarInfo(self.getWrapperScriptName())
tinfo2.mtime = time.time()
fileobj2 = StringIO(self.getWNPythonContents())
tinfo2.size = fileobj2.len
tar_file.addfile(tinfo2, fileobj2)
def string_file_shortcut(v, item):
if isinstance(v, str):
# use proxy class to enable all user conversions on the value itself
# but return the implementation object (not proxy)
key = findOutputFileTypeByFileName(v)
if key is not None:
if key == 'MassStorageFile':
from .MassStorageFile import MassStorageFile
return stripProxy(MassStorageFile._proxyClass(v))
elif key == 'LCGSEFile':
from .LCGSEFile import LCGSEFile
return stripProxy(LCGSEFile._proxyClass(v))
elif key == 'DiracFile':
try:
from GangaDirac.Lib.Files.DiracFile import DiracFile
return stripProxy(DiracFile._proxyClass(v))
except:
Ganga.Utility.logging.log_unknown_exception()
pass
return stripProxy(LocalFile._proxyClass(v))
return None
def string_file_shortcut(v, item):
if isinstance(v, str):
# use proxy class to enable all user conversions on the value itself
# but return the implementation object (not proxy)
key = findOutputFileTypeByFileName(v)
if key is not None:
if key == 'MassStorageFile':
from .MassStorageFile import MassStorageFile
return stripProxy(MassStorageFile._proxyClass(v))
elif key == 'LCGSEFile':
from .LCGSEFile import LCGSEFile
return stripProxy(LCGSEFile._proxyClass(v))
elif key == 'DiracFile':
try:
from GangaDirac.Lib.Files.DiracFile import DiracFile
return stripProxy(DiracFile._proxyClass(v))
except:
Ganga.Utility.logging.log_unknown_exception()
pass
return stripProxy(LocalFile._proxyClass(v))
return None