def get_scratch_dir():
scratch = ''
# Get scratch directory path.
if 'TMPDIR' in os.environ:
scratch = os.environ['TMPDIR']
elif 'SCRATCH' in os.environ:
scratch = os.environ['SCRATCH']
else:
scratch = '/scratch/%s/%s' % (get_experiment(), get_user())
if not larbatch_posix.isdir(scratch) or not larbatch_posix.access(
scratch, os.W_OK):
scratch = '/%s/data/users/%s' % (get_experiment(), get_user())
# Checkout.
if scratch == '':
raise RuntimeError('No scratch directory specified.')
if not larbatch_posix.isdir(scratch) or not larbatch_posix.access(
scratch, os.W_OK):
raise RuntimeError(
'Scratch directory %s does not exist or is not writeable.' %
scratch)
return scratch
def makeDummyDef(defname):
if not defExists(defname):
# Make dummy definition.
print('Making dummy dataset definition %s' % defname)
test_kca()
samweb().createDefinition(defname, 'file_id 0', user=get_user(), group=get_experiment())
def makeFileListDefinition(list_or_dim):
# Make sure we have a kca certificate.
test_kca()
# Make file list dimension.
flist = []
if type(list_or_dim) == type([]) or type(list_or_dim) == type(set()):
flist = list_or_dim
print('Making file list definition from %s with %d elements.' %
(type(list_or_dim), len(list_or_dim)))
else:
flist = listFiles(list_or_dim)
print('Making file list definition using dimension "%s"' % list_or_dim)
listdim = ''
for filename in flist:
if listdim == '':
listdim = 'file_name %s' % filename
else:
listdim += ', %s' % filename
if listdim == '':
listdim = 'file_id 0'
# Maybe construct a new unique definition name.
defname = get_user() + '_filelist_' + str(uuid.uuid4())
# Create definition.
samweb().createDefinition(defname,
listdim,
user=get_user(),
group=get_experiment())
# Done.
return defname
def create_limited_dataset(defname, run, subruns):
if len(subruns) == 0:
return ''
# Construct comma-separated list of run-subrun pairs in a form that is
# acceptable as sam dimension constraint.
run_subrun_dim = ''
for subrun in subruns:
if run_subrun_dim != '':
run_subrun_dim += ','
run_subrun_dim += "%d.%d" % (run, subrun)
# Construct dimension including run and subrun constraints.
dim = "defname: %s and run_number %s" % (defname, run_subrun_dim)
# Test the new dimension.
nfiles = samweb().countFiles(dimensions=dim)
if nfiles == 0:
return ''
# Make sure we have a kca certificate.
test_kca()
# Construct a new unique definition name.
newdefname = defname + '_' + str(uuid.uuid4())
# Create definition.
samweb().createDefinition(newdefname,
dim,
user=get_user(),
group=get_experiment())
# Done (return definition name).
return newdefname
def make_active_project_dataset(defname, dropboxwait, active_defname,
wait_defname):
s = samweb()
test_kca()
# Get list of active projects.
prjs = active_projects(defname) | active_projects2(defname)
# Make sam dimension.
dim = ''
for prj in prjs:
if dim == '':
dim = 'snapshot_for_project_name %s' % prj
else:
dim += ',%s' % prj
# If there were no matching projects, make up some legal dimension that won't
# match any files.
if dim == '':
dim = 'file_id 0'
# Create or update active_defname.
if defExists(active_defname):
print('Updating dataset definition %s' % active_defname)
s.deleteDefinition(active_defname)
else:
print('Creating dataset definition %s' % active_defname)
s.createDefinition(active_defname,
dim,
user=get_user(),
group=get_experiment())
# If the dropbox waiting interval is nonzero, create a dataset for
# dropbox waiting files.
dim = ''
if dropboxwait > 0.:
# Convert the waiting interval to a datetime.timedelta object.
dt = datetime.timedelta(int(dropboxwait), int(dropboxwait % 1 * 86400))
# Get the earliest allowed time.
tmin = datetime.datetime.utcnow() - dt
# Format time in a form acceptable to sam.
tminstr = tmin.strftime('%Y-%m-%dT%H:%M:%S')
# Append sam dimension.
dim = "isparentof: (create_date > '%s' and availability: virtual)" % tminstr
else:
# Otherwise make dummy dataset.
dim = 'file_id 0'
# Create or update active_defname.
if defExists(wait_defname):
print('Updating dataset definition %s' % wait_defname)
s.deleteDefinition(wait_defname)
else:
print('Creating dataset definition %s' % wait_defname)
s.createDefinition(wait_defname,
dim,
user=get_user(),
group=get_experiment())
def start_project(defname, default_prjname, max_files, force_snapshot,
filelistdef):
# Check project name.
s = samweb()
prjname = default_prjname
if prjname == '':
prjname = s.makeProjectName(defname)
print('Starting project %s' % prjname)
# Make sure we have a certificate.
test_kca()
# Figure out how many files are in the input dataset.
nf = 0
if filelistdef:
files = listFiles('defname: %s' % defname)
nf = len(files)
else:
nf = s.countFiles('defname: %s' % defname)
print('Input dataset has %d files.' % nf)
if nf == 0:
return 1
# Make limited dataset?
if max_files > 0 and nf > max_files:
limitdef = '%s_limit_%d' % (prjname, max_files)
# Figure out whether limitdef already exists.
if defExists(limitdef) and not filelistdef:
print('Using already created limited dataset definition %s.' %
limitdef)
else:
dim = 'defname: %s with limit %d' % (defname, max_files)
if filelistdef:
limitdef = makeFileListDefinition(dim)
else:
print('Creating limited dataset definition %s.' % limitdef)
s.createDefinition(limitdef,
dim,
user=get_user(),
group=get_experiment())
defname = limitdef
nf = max_files
elif filelistdef:
dim = 'defname: %s' % defname
defname = makeFileListDefinition(dim)
# Force snapshot?
if force_snapshot:
print('Forcing snapthot.')
defname = '%s:force' % defname
# Start the project.
print('Starting project %s.' % prjname)
s.startProject(prjname,
defname=defname,
station=get_experiment(),
group=get_experiment(),
user=get_user())
# Done.
return 0
