def prestageDataset(
samweb,
projectname=None,
defname=None,
snapshot_id=None,
maxFiles=0,
station=None,
deliveryLocation=None,
node=None,
nparallel=1,
):
""" Prestage the given dataset. This is really the same as run-project with names set appropriately """
if nparallel is None or nparallel < 2:
nparallel = 1
def prestage(fileurl):
if nparallel > 1:
import threading
prefix = "%s: " % threading.currentThread().getName()
else:
prefix = ""
print "%sFile %s is staged" % (prefix, os.path.basename(fileurl))
return True
if not projectname:
projectname = "prestage"
if defname:
projectname = samweb.makeProjectName("%s_%s" % (defname, projectname))
elif snapshot_id:
projectname = samweb.makeProjectName("snapshot_id_%d_%s" % (snapshot_id, projectname))
samweb.runProject(
projectname=projectname,
defname=defname,
snapshot_id=snapshot_id,
application=("prestage", "prestage", get_version()),
callback=prestage,
maxFiles=maxFiles,
station=station,
deliveryLocation=deliveryLocation,
node=node,
nparallel=nparallel,
)
def prestageDataset(samweb,
projectname=None,
defname=None,
snapshot_id=None,
maxFiles=0,
station=None,
deliveryLocation=None,
node=None,
nparallel=1):
""" Prestage the given dataset. This is really the same as run-project with names set appropriately """
if nparallel is None or nparallel < 2: nparallel = 1
def prestage(fileurl):
if nparallel > 1:
import threading
prefix = '%s: ' % threading.currentThread().getName()
else:
prefix = ''
print "%sFile %s is staged" % (prefix, os.path.basename(fileurl))
return True
if not projectname:
projectname = 'prestage'
if defname:
projectname = samweb.makeProjectName('%s_%s' %
(defname, projectname))
elif snapshot_id:
projectname = samweb.makeProjectName('snapshot_id_%d_%s' %
(snapshot_id, projectname))
samweb.runProject(projectname=projectname,
defname=defname,
snapshot_id=snapshot_id,
application=('prestage', 'prestage', get_version()),
callback=prestage,
maxFiles=maxFiles,
station=station,
deliveryLocation=deliveryLocation,
node=node,
nparallel=nparallel)
def runProject(
samweb,
projectname=None,
defname=None,
snapshot_id=None,
callback=None,
deliveryLocation=None,
node=None,
station=None,
maxFiles=0,
schemas=None,
application=("runproject", "runproject", get_version()),
nparallel=1,
quiet=False,
):
""" Run a project
arguments (use keyword arguments, all default to None):
projectname: the name for the project
defname: the defname to use
snapshot_id: snapshot_id to use
callback: a single argument function invoked on each file
deliveryLocation
node
station
maxFiles
schemas
application: a three element sequence of (family, name, version)
nparallel: number of processes to run in parallel
quiet: If true, suppress normal output
"""
if callback is None:
def _print(fileurl):
print fileurl
return True
callback = _print
if not projectname:
if defname:
projectname = samweb.makeProjectName(defname)
elif snapshot_id:
projectname = samweb.makeProjectName("snapshot_id_%d" % snapshot_id)
if quiet:
def write(s):
pass
else:
import sys
write = sys.stdout.write
project = samweb.startProject(projectname, defname=defname, snapshot_id=snapshot_id, station=station)
write("Started project %s\n" % projectname)
projecturl = project["projectURL"]
process_description = ""
appFamily, appName, appVersion = application
if nparallel is None or nparallel < 2:
nparallel = 1
if nparallel > 1:
import threading
maxFiles = (maxFiles + nparallel - 1) // nparallel
def runProcess():
cpid = samweb.startProcess(
projecturl,
appFamily,
appName,
appVersion,
deliveryLocation,
node=node,
description=process_description,
maxFiles=maxFiles,
schemas=schemas,
)
write("Started consumer processs ID %s\n" % (cpid,))
if nparallel > 1:
threading.currentThread().setName("CPID-%s" % cpid)
log_prefix = "%s: " % threading.currentThread().getName()
else:
log_prefix = ""
processurl = samweb.makeProcessUrl(projecturl, cpid)
while True:
try:
newfile = samweb.getNextFile(processurl)["url"]
try:
rval = callback(newfile)
except Exception, ex:
write("%s%s\n" % (log_prefix, ex))
rval = 1
except NoMoreFiles:
break
if rval:
status = "ok"
else:
status = "bad"
samweb.releaseFile(processurl, newfile, status)
samweb.setProcessStatus("completed", processurl)
if nparallel < 2:
runProcess()
else:
threads = []
for i in range(nparallel):
t = threading.Thread(target=runProcess, name="Thread-%02d" % (i + 1,))
t.start()
threads.append(t)
for t in threads:
t.join()
samweb.stopProject(projecturl)
write("Stopped project %s\n" % projectname)
return projectname
def runProject(samweb,
projectname=None,
defname=None,
snapshot_id=None,
callback=None,
deliveryLocation=None,
node=None,
station=None,
maxFiles=0,
schemas=None,
application=('runproject', 'runproject', get_version()),
nparallel=1,
quiet=False):
""" Run a project
arguments (use keyword arguments, all default to None):
projectname: the name for the project
defname: the defname to use
snapshot_id: snapshot_id to use
callback: a single argument function invoked on each file
deliveryLocation
node
station
maxFiles
schemas
application: a three element sequence of (family, name, version)
nparallel: number of processes to run in parallel
quiet: If true, suppress normal output
"""
if callback is None:
def _print(fileurl):
print fileurl
return True
callback = _print
if not projectname:
if defname:
projectname = samweb.makeProjectName(defname)
elif snapshot_id:
projectname = samweb.makeProjectName('snapshot_id_%d' %
snapshot_id)
if quiet:
def write(s):
pass
else:
import sys
write = sys.stdout.write
project = samweb.startProject(projectname,
defname=defname,
snapshot_id=snapshot_id,
station=station)
write("Started project %s\n" % projectname)
projecturl = project['projectURL']
process_description = ""
appFamily, appName, appVersion = application
if nparallel is None or nparallel < 2:
nparallel = 1
if nparallel > 1:
import threading
maxFiles = (maxFiles + nparallel - 1) // nparallel
def runProcess():
cpid = samweb.startProcess(projecturl,
appFamily,
appName,
appVersion,
deliveryLocation,
node=node,
description=process_description,
maxFiles=maxFiles,
schemas=schemas)
write("Started consumer processs ID %s\n" % (cpid, ))
if nparallel > 1:
threading.currentThread().setName('CPID-%s' % cpid)
log_prefix = '%s: ' % threading.currentThread().getName()
else:
log_prefix = ''
processurl = samweb.makeProcessUrl(projecturl, cpid)
while True:
try:
newfile = samweb.getNextFile(processurl)['url']
try:
rval = callback(newfile)
except Exception, ex:
write('%s%s\n' % (log_prefix, ex))
rval = 1
except NoMoreFiles:
break
if rval: status = 'ok'
else: status = 'bad'
samweb.releaseFile(processurl, newfile, status)
samweb.setProcessStatus('completed', processurl)
if nparallel < 2:
runProcess()
else:
threads = []
for i in range(nparallel):
t = threading.Thread(target=runProcess,
name='Thread-%02d' % (i + 1, ))
t.start()
threads.append(t)
for t in threads:
t.join()
samweb.stopProject(projecturl)
write("Stopped project %s\n" % projectname)
return projectname