def get_scratch_dir(): scratch = '' # Get scratch directory path. if 'TMPDIR' in os.environ: scratch = os.environ['TMPDIR'] elif 'SCRATCH' in os.environ: scratch = os.environ['SCRATCH'] else: scratch = '/scratch/%s/%s' % (get_experiment(), get_user()) if not larbatch_posix.isdir(scratch) or not larbatch_posix.access( scratch, os.W_OK): scratch = '/%s/data/users/%s' % (get_experiment(), get_user()) # Checkout. if scratch == '': raise RuntimeError('No scratch directory specified.') if not larbatch_posix.isdir(scratch) or not larbatch_posix.access( scratch, os.W_OK): raise RuntimeError( 'Scratch directory %s does not exist or is not writeable.' % scratch) return scratch
def makeDummyDef(defname): if not defExists(defname): # Make dummy definition. print('Making dummy dataset definition %s' % defname) test_kca() samweb().createDefinition(defname, 'file_id 0', user=get_user(), group=get_experiment())
def makeFileListDefinition(list_or_dim): # Make sure we have a kca certificate. test_kca() # Make file list dimension. flist = [] if type(list_or_dim) == type([]) or type(list_or_dim) == type(set()): flist = list_or_dim print('Making file list definition from %s with %d elements.' % (type(list_or_dim), len(list_or_dim))) else: flist = listFiles(list_or_dim) print('Making file list definition using dimension "%s"' % list_or_dim) listdim = '' for filename in flist: if listdim == '': listdim = 'file_name %s' % filename else: listdim += ', %s' % filename if listdim == '': listdim = 'file_id 0' # Maybe construct a new unique definition name. defname = get_user() + '_filelist_' + str(uuid.uuid4()) # Create definition. samweb().createDefinition(defname, listdim, user=get_user(), group=get_experiment()) # Done. return defname
def create_limited_dataset(defname, run, subruns): if len(subruns) == 0: return '' # Construct comma-separated list of run-subrun pairs in a form that is # acceptable as sam dimension constraint. run_subrun_dim = '' for subrun in subruns: if run_subrun_dim != '': run_subrun_dim += ',' run_subrun_dim += "%d.%d" % (run, subrun) # Construct dimension including run and subrun constraints. dim = "defname: %s and run_number %s" % (defname, run_subrun_dim) # Test the new dimension. nfiles = samweb().countFiles(dimensions=dim) if nfiles == 0: return '' # Make sure we have a kca certificate. test_kca() # Construct a new unique definition name. newdefname = defname + '_' + str(uuid.uuid4()) # Create definition. samweb().createDefinition(newdefname, dim, user=get_user(), group=get_experiment()) # Done (return definition name). return newdefname
def make_active_project_dataset(defname, dropboxwait, active_defname, wait_defname): s = samweb() test_kca() # Get list of active projects. prjs = active_projects(defname) | active_projects2(defname) # Make sam dimension. dim = '' for prj in prjs: if dim == '': dim = 'snapshot_for_project_name %s' % prj else: dim += ',%s' % prj # If there were no matching projects, make up some legal dimension that won't # match any files. if dim == '': dim = 'file_id 0' # Create or update active_defname. if defExists(active_defname): print('Updating dataset definition %s' % active_defname) s.deleteDefinition(active_defname) else: print('Creating dataset definition %s' % active_defname) s.createDefinition(active_defname, dim, user=get_user(), group=get_experiment()) # If the dropbox waiting interval is nonzero, create a dataset for # dropbox waiting files. dim = '' if dropboxwait > 0.: # Convert the waiting interval to a datetime.timedelta object. dt = datetime.timedelta(int(dropboxwait), int(dropboxwait % 1 * 86400)) # Get the earliest allowed time. tmin = datetime.datetime.utcnow() - dt # Format time in a form acceptable to sam. tminstr = tmin.strftime('%Y-%m-%dT%H:%M:%S') # Append sam dimension. dim = "isparentof: (create_date > '%s' and availability: virtual)" % tminstr else: # Otherwise make dummy dataset. dim = 'file_id 0' # Create or update active_defname. if defExists(wait_defname): print('Updating dataset definition %s' % wait_defname) s.deleteDefinition(wait_defname) else: print('Creating dataset definition %s' % wait_defname) s.createDefinition(wait_defname, dim, user=get_user(), group=get_experiment())
def start_project(defname, default_prjname, max_files, force_snapshot, filelistdef): # Check project name. s = samweb() prjname = default_prjname if prjname == '': prjname = s.makeProjectName(defname) print('Starting project %s' % prjname) # Make sure we have a certificate. test_kca() # Figure out how many files are in the input dataset. nf = 0 if filelistdef: files = listFiles('defname: %s' % defname) nf = len(files) else: nf = s.countFiles('defname: %s' % defname) print('Input dataset has %d files.' % nf) if nf == 0: return 1 # Make limited dataset? if max_files > 0 and nf > max_files: limitdef = '%s_limit_%d' % (prjname, max_files) # Figure out whether limitdef already exists. if defExists(limitdef) and not filelistdef: print('Using already created limited dataset definition %s.' % limitdef) else: dim = 'defname: %s with limit %d' % (defname, max_files) if filelistdef: limitdef = makeFileListDefinition(dim) else: print('Creating limited dataset definition %s.' % limitdef) s.createDefinition(limitdef, dim, user=get_user(), group=get_experiment()) defname = limitdef nf = max_files elif filelistdef: dim = 'defname: %s' % defname defname = makeFileListDefinition(dim) # Force snapshot? if force_snapshot: print('Forcing snapthot.') defname = '%s:force' % defname # Start the project. print('Starting project %s.' % prjname) s.startProject(prjname, defname=defname, station=get_experiment(), group=get_experiment(), user=get_user()) # Done. return 0