def get_scratch_dir(): scratch = '' # Get scratch directory path. if 'TMPDIR' in os.environ: scratch = os.environ['TMPDIR'] elif 'SCRATCH' in os.environ: scratch = os.environ['SCRATCH'] else: scratch = '/scratch/%s/%s' % (get_experiment(), get_user()) if not larbatch_posix.isdir(scratch) or not larbatch_posix.access( scratch, os.W_OK): scratch = '/%s/data/users/%s' % (get_experiment(), get_user()) # Checkout. if scratch == '': raise RuntimeError('No scratch directory specified.') if not larbatch_posix.isdir(scratch) or not larbatch_posix.access( scratch, os.W_OK): raise RuntimeError( 'Scratch directory %s does not exist or is not writeable.' % scratch) return scratch
def active_projects(defname=''): result = set() # Get project name stem. s = samweb() prjstem = '' if defname != '': prjstem = '%s_' % s.makeProjectName(defname).rsplit('_', 1)[0] # Dump station url = '%s/dumpStation?station=%s' % (s.get_baseurl(), get_experiment()) furl = urlrequest.urlopen(url) # Parse response. for line in furl.readlines(): words = line.split() if len(words) > 5: prjname = convert_str(words[0]) if prjstem == '' or prjname.startswith(prjstem): result.add(prjname) # Done. return result
def makeDummyDef(defname): if not defExists(defname): # Make dummy definition. print('Making dummy dataset definition %s' % defname) test_kca() samweb().createDefinition(defname, 'file_id 0', user=get_user(), group=get_experiment())
def samweb(): global samweb_obj if samweb_obj == None: samweb_obj = samweb_cli.SAMWebClient(experiment=get_experiment()) os.environ['SSL_CERT_DIR'] = '/etc/grid-security/certificates' return samweb_obj
def active_projects2(defname=''): result = set() # Get project name stem. s = samweb() prjstem = '' if defname != '': prjstem = '%s_' % s.makeProjectName(defname).rsplit('_', 1)[0] # Query a list of projects started within the last 72 hours. dt = datetime.timedelta(3, 0) tmin = datetime.datetime.utcnow() - dt tminstr = tmin.strftime('%Y-%m-%dT%H:%M:%S') prjnames = s.listProjects(started_after=tminstr) # Loop over projects to check end times. for prjname in prjnames: if prjstem == '' or prjname.startswith(prjstem): # This project is a candidate for inclusion in result. # Check end time. age = 0 prjurl = s.findProject(project=prjname, station=get_experiment()) if prjurl != '': prjsum = s.projectSummary(prjurl) if 'project_end_time' in prjsum: tendstr = prjsum['project_end_time'] if len(tendstr) >= 19: try: tend = datetime.datetime.strptime( tendstr[:19], '%Y-%m-%dT%H:%M:%S') tage = datetime.datetime.utcnow() - tend age = tage.total_seconds() except: pass # Keep this project if there is no end time. if age == 0: result.add(prjname) # Done. return result
def create_limited_dataset(defname, run, subruns): if len(subruns) == 0: return '' # Construct comma-separated list of run-subrun pairs in a form that is # acceptable as sam dimension constraint. run_subrun_dim = '' for subrun in subruns: if run_subrun_dim != '': run_subrun_dim += ',' run_subrun_dim += "%d.%d" % (run, subrun) # Construct dimension including run and subrun constraints. dim = "defname: %s and run_number %s" % (defname, run_subrun_dim) # Test the new dimension. nfiles = samweb().countFiles(dimensions=dim) if nfiles == 0: return '' # Make sure we have a kca certificate. test_kca() # Construct a new unique definition name. newdefname = defname + '_' + str(uuid.uuid4()) # Create definition. samweb().createDefinition(newdefname, dim, user=get_user(), group=get_experiment()) # Done (return definition name). return newdefname
def makeFileListDefinition(list_or_dim): # Make sure we have a kca certificate. test_kca() # Make file list dimension. flist = [] if type(list_or_dim) == type([]) or type(list_or_dim) == type(set()): flist = list_or_dim print('Making file list definition from %s with %d elements.' % (type(list_or_dim), len(list_or_dim))) else: flist = listFiles(list_or_dim) print('Making file list definition using dimension "%s"' % list_or_dim) listdim = '' for filename in flist: if listdim == '': listdim = 'file_name %s' % filename else: listdim += ', %s' % filename if listdim == '': listdim = 'file_id 0' # Maybe construct a new unique definition name. defname = get_user() + '_filelist_' + str(uuid.uuid4()) # Create definition. samweb().createDefinition(defname, listdim, user=get_user(), group=get_experiment()) # Done. return defname
def active_projects(defname=''): result = set() # Get project name stem. s = samweb() prjstem = '' if defname != '': prjstem = '%s_' % s.makeProjectName(defname).rsplit('_', 1)[0] # Dump station url = '%s/dumpStation?station=%s' % (s.get_baseurl(), get_experiment()) buffer = StringIO.StringIO() c = pycurl.Curl() c.setopt(c.URL, url) c.setopt(c.USERPWD, 'uboone:argon!') c.setopt(c.FOLLOWLOCATION, True) c.setopt(c.WRITEFUNCTION, buffer.write) c.perform() c.close() # Parse response. buffer.seek(0) for line in buffer.readlines(): words = line.split() if len(words) > 0 and words[0] == 'project': prjname = words[1].split('(')[0] if prjstem == '' or prjname.startswith(prjstem): result.add(prjname) # Done. return result
def default_jobsub_submit_options(): opt = '--append_condor_requirements=\'(TARGET.HAS_CVMFS_%s_opensciencegrid_org==true)\'' % get_experiment( ) return opt
def make_active_project_dataset(defname, dropboxwait, active_defname, wait_defname): s = samweb() test_kca() # Get list of active projects. prjs = active_projects(defname) | active_projects2(defname) # Make sam dimension. dim = '' for prj in prjs: if dim == '': dim = 'snapshot_for_project_name %s' % prj else: dim += ',%s' % prj # If there were no matching projects, make up some legal dimension that won't # match any files. if dim == '': dim = 'file_id 0' # Create or update active_defname. if defExists(active_defname): print('Updating dataset definition %s' % active_defname) s.deleteDefinition(active_defname) else: print('Creating dataset definition %s' % active_defname) s.createDefinition(active_defname, dim, user=get_user(), group=get_experiment()) # If the dropbox waiting interval is nonzero, create a dataset for # dropbox waiting files. dim = '' if dropboxwait > 0.: # Convert the waiting interval to a datetime.timedelta object. dt = datetime.timedelta(int(dropboxwait), int(dropboxwait % 1 * 86400)) # Get the earliest allowed time. tmin = datetime.datetime.utcnow() - dt # Format time in a form acceptable to sam. tminstr = tmin.strftime('%Y-%m-%dT%H:%M:%S') # Append sam dimension. dim = "isparentof: (create_date > '%s' and availability: virtual)" % tminstr else: # Otherwise make dummy dataset. dim = 'file_id 0' # Create or update active_defname. if defExists(wait_defname): print('Updating dataset definition %s' % wait_defname) s.deleteDefinition(wait_defname) else: print('Creating dataset definition %s' % wait_defname) s.createDefinition(wait_defname, dim, user=get_user(), group=get_experiment())
def start_project(defname, default_prjname, max_files, force_snapshot, filelistdef): # Check project name. s = samweb() prjname = default_prjname if prjname == '': prjname = s.makeProjectName(defname) print('Starting project %s' % prjname) # Make sure we have a certificate. test_kca() # Figure out how many files are in the input dataset. nf = 0 if filelistdef: files = listFiles('defname: %s' % defname) nf = len(files) else: nf = s.countFiles('defname: %s' % defname) print('Input dataset has %d files.' % nf) if nf == 0: return 1 # Make limited dataset? if max_files > 0 and nf > max_files: limitdef = '%s_limit_%d' % (prjname, max_files) # Figure out whether limitdef already exists. if defExists(limitdef) and not filelistdef: print('Using already created limited dataset definition %s.' % limitdef) else: dim = 'defname: %s with limit %d' % (defname, max_files) if filelistdef: limitdef = makeFileListDefinition(dim) else: print('Creating limited dataset definition %s.' % limitdef) s.createDefinition(limitdef, dim, user=get_user(), group=get_experiment()) defname = limitdef nf = max_files elif filelistdef: dim = 'defname: %s' % defname defname = makeFileListDefinition(dim) # Force snapshot? if force_snapshot: print('Forcing snapthot.') defname = '%s:force' % defname # Start the project. print('Starting project %s.' % prjname) s.startProject(prjname, defname=defname, station=get_experiment(), group=get_experiment(), user=get_user()) # Done. return 0