def conts_list():
"""
Get list of containers method for ajax
:return: List of Container obj
"""
user = g.user
hours_limit = session.get('hours_limit', current_app.config['HOURS_LIMIT'])
display_limit = session.get('display_limit', current_app.config['DISPLAY_LIMIT'])
scope = getScope(user.username)
# show users jobs
conts = conts_.find(status='user').order_by(Container.id.desc()).limit(30)
# prepare json
conts_o = []
for cont in conts:
cont_o = {}
cont_o['id'] = cont.id
cont_o['guid'] = cont.guid
cont_o['status'] = cont.status
cont_o['n'] = len(cont.files)
conts_o.append(cont_o)
data = {}
data['data'] = conts_o
return make_response(jsonify(data), 200)
def container():
"""
New container form view
:return: Response obj
"""
form = NewContainerForm()
if request.method == 'POST':
user = g.user
scope = getScope(user.username)
ftpdir = form.ftpdir.data
#Create a unique container quid for this particular batch of uploads.
cguid = 'job.' + commands.getoutput('uuidgen')
# Create new container
container = Container()
container.guid = cguid
container.status = 'open'
conts_.save(container)
# Check if ftpdir empty
if ftpdir and len(ftpdir) > 0:
async_uploadContainer.delay(ftpdir, scope, container.guid)
return redirect(url_for('conts.cont_info', guid=container.guid))
return render_template("dashboard/cont/cont_new.html", form=form)
def jobLog(id):
"""
Returns job stdout & stderr
:param id: Job id
:return: json data
"""
job = jobs_.get(id=id)
extractLog(id)
locdir = '/%s/.sys/%s' % (getScope(job.owner.username), job.container.guid)
absdir = ddm_getlocalabspath(locdir)
fout = find('payload_stdout.txt', absdir)
ferr = find('payload_stderr.txt', absdir)
out = ''
err = ''
if len(fout) > 0:
with open(fout[0]) as f:
out = f.read()
if len(ferr) > 0:
with open(ferr[0]) as f:
err = f.read()
data = {}
data['id'] = id
data['out'] = out
data['err'] = err
return make_response(jsonify({'data': data}), 200)
def files_list():
"""
Get list of files method for ajax
:return: List of File obj
"""
user = g.user
hours_limit = session.get('hours_limit', current_app.config['HOURS_LIMIT'])
display_limit = session.get('display_limit',
current_app.config['DISPLAY_LIMIT'])
scope = getScope(user.username)
# show users jobs
files = files_.find(scope=scope).order_by(File.id).limit(display_limit)
# prepare json
files_o = []
for file in files:
file_o = {}
file_o['id'] = file.id
file_o['scope'] = file.scope
file_o['guid'] = file.guid
file_o['type'] = file.type
file_o['lfn'] = file.lfn
file_o['status'] = file.status
files_o.append(file_o)
data = {}
data['data'] = files_o
return make_response(jsonify(data), 200)
def file_download(guid):
"""
Get file as stream
:param guid: guid of file
:return: Response obj
"""
try:
file = files_.first(guid=guid)
except (Exception):
_logger.error(Exception.message)
return make_response(jsonify({'error': 'File not found'}), 404)
if file.scope != getScope(g.user.username):
return make_response(jsonify({'error': 'File is not in your scope'}),
403)
replicas = file.replicas
for replica in replicas:
if replica.se == current_app.config[
'DEFAULT_SE'] and replica.status == 'ready':
fullpath = current_app.config['DATA_PATH'] + replica.lfn
f = open(fullpath, 'r')
rr = Response(f.read(),
status=200,
content_type='application/octet-stream')
rr.headers[
'Content-Disposition'] = 'inline; filename="%s"' % file.lfn
rr.headers['Content-MD5'] = file.md5sum
file.downloaded += 1
files_.save(file)
return rr
#TODO: add support message
return make_response(jsonify({'error': 'No ready replica'}), 404)
def container():
"""
New container form view
:return: Response obj
"""
form = NewContainerForm()
if request.method == 'POST':
user = g.user
scope = getScope(user.username)
ftpdir = form.ftpdir.data
#Create a unique container quid for this particular batch of uploads.
cguid = 'job.' + commands.getoutput('uuidgen')
# Create new container
container = Container()
container.guid = cguid
container.status = 'open'
conts_.save(container)
# Check if ftpdir empty
if ftpdir and len(ftpdir) > 0:
async_uploadContainer.delay(ftpdir, scope, container.guid)
return redirect(url_for('conts.cont_info', guid=container.guid))
return render_template("dashboard/cont/cont_new.html", form=form)
def conts_list():
"""
Get list of containers method for ajax
:return: List of Container obj
"""
user = g.user
hours_limit = session.get('hours_limit', current_app.config['HOURS_LIMIT'])
display_limit = session.get('display_limit',
current_app.config['DISPLAY_LIMIT'])
scope = getScope(user.username)
# show users jobs
conts = conts_.find(status='user').order_by(Container.id.desc()).limit(30)
# prepare json
conts_o = []
for cont in conts:
cont_o = {}
cont_o['id'] = cont.id
cont_o['guid'] = cont.guid
cont_o['status'] = cont.status
cont_o['n'] = len(cont.files)
conts_o.append(cont_o)
data = {}
data['data'] = conts_o
return make_response(jsonify(data), 200)
def reg_file_in_cont_byname(user, lfn, c, t):
"""
Registers file in catalog by filename
:param user: File owner
:param lfn: Local FileName
:param c: Container to register in
:param t: Type of file (input, output, log)
:type user: User
:type lfn: str
:type c: Container
:type t: str
:return: True/False
"""
if not (isinstance(lfn, str) or isinstance(lfn, unicode)):
raise Exception("Illegal lfn class: not str")
if len(lfn) == 0:
raise Exception("Illegal lfn length: zero")
# Prepare File obj
f = File()
f.scope = getScope(user.username)
f.attemptn = 0
f.guid = getGUID(f.scope, None)
f.lfn = lfn
f.status = "defined"
f.transfertask = None
# f.fsize =
# md5sum =
# checksum =
f.modification_time = datetime.utcnow()
f.downloaded = 0
# Save to fc
files_.save(f)
if not isinstance(f, File):
raise Exception("Illegal file class: not File")
if not isinstance(c, Container):
raise Exception("Illegal catalog class: not Container")
if not isinstance(t, str):
raise Exception("Illegal type class: not str")
if t not in ['input', 'output', 'log', 'intermediate']:
raise Exception("Illegal type value: " + t)
catalog_item = Catalog()
catalog_item.file = f
catalog_item.cont = c
catalog_item.type = t
# TODO: Add registration time
catalog_.save(catalog_item)
return True
def new_cont():
"""
POST: /pilot/container
Saves new container
:return: ftp/guid
:rtype: json
"""
cont = Container()
guid = 'job.' + commands.getoutput('uuidgen')
cont.guid = guid
cont.status = 'open'
conts_.save(cont)
url = '%s/%s' % (current_app.config['FTP'], guid)
os.makedirs(os.path.join(current_app.config['UPLOAD_FOLDER'], getScope(g.user.username), cont.guid))
return {'ftp': url, 'guid': cont.guid}
def cont_close(guid):
"""
POST: /pilot/container/<guid>/close
Changes container status to 'close'
:param guid: Container guid
:type guid: str
"""
cont = conts_.first(guid=guid)
if cont is None:
raise WebpandaError("Container not found")
path = os.path.join(current_app.config['UPLOAD_FOLDER'], getScope(g.user.username), cont.guid)
os.path.walk(path, registerLocalFile, cont.guid)
cont.status = 'close'
conts_.save(cont)
return {'response': 'Container status: close'}
def cont_close(guid):
"""
POST: /pilot/container/<guid>/close
Changes container status to 'close'
:param guid: Container guid
:type guid: str
"""
cont = conts_.first(guid=guid)
if cont is None:
raise WebpandaError("Container not found")
path = os.path.join(current_app.config['UPLOAD_FOLDER'],
getScope(g.user.username), cont.guid)
os.path.walk(path, registerLocalFile, cont.guid)
cont.status = 'close'
conts_.save(cont)
return {'response': 'Container status: close'}
def new_cont():
"""
POST: /pilot/container
Saves new container
:return: ftp/guid
:rtype: json
"""
cont = Container()
guid = 'job.' + commands.getoutput('uuidgen')
cont.guid = guid
cont.status = 'open'
conts_.save(cont)
url = '%s/%s' % (current_app.config['FTP'], guid)
os.makedirs(
os.path.join(current_app.config['UPLOAD_FOLDER'],
getScope(g.user.username), cont.guid))
return {'ftp': url, 'guid': cont.guid}
def jobLogAPI(id):
"""Returns job stdout & stderr"""
g.user = request.oauth.user
job = jobs_.get(id)
extractLog(id)
locdir = '/%s/.sys/%s' % (getScope(job.owner.username), job.container.guid)
absdir = ddm_getlocalabspath(locdir)
fout = find('payload.stdout', absdir)
ferr = find('payload.stderr', absdir)
out = ''
err = ''
if len(fout) > 0:
with open(fout[0]) as f:
out = f.read()
if len(ferr) > 0:
with open(ferr[0]) as f:
err = f.read()
data = dict()
data['id'] = id
data['out'] = out
data['err'] = err
return data
def jobLogAPI(id):
"""Returns job stdout & stderr"""
g.user = request.oauth.user
job = jobs_.get(id)
extractLog(id)
locdir = '/%s/.sys/%s' % (getScope(job.owner.username), job.container.guid)
absdir = ddm_getlocalabspath(locdir)
fout = find('payload.stdout', absdir)
ferr = find('payload.stderr', absdir)
out = ''
err = ''
if len(fout) > 0:
with open(fout[0]) as f:
out = f.read()
if len(ferr) > 0:
with open(ferr[0]) as f:
err = f.read()
data = dict()
data['id'] = id
data['out'] = out
data['err'] = err
return data
def new_file(user, lfn):
"""
Creates new file object
:param user: File owner
:param lfn: Local FileName
:type user: User
:type lfn: str
:return: Created file
:rtype: File
"""
if not isinstance(user, User):
raise Exception("Illegal user class: not User")
if not isinstance(lfn, str):
raise Exception("Illegal lfn class: not str")
if len(lfn) == 0:
raise Exception("Illegal lfn length: zero")
# Prepare File obj
f = File()
f.scope = getScope(user.username)
f.attemptn = 0
f.guid = getGUID(f.scope, None)
f.lfn = lfn
f.status = "defined"
f.transfertask = None
# f.fsize =
# md5sum =
# checksum =
f.modification_time = datetime.utcnow()
f.downloaded = 0
# Save to fc
files_.save(f)
return f
def send_job(jobid, siteid):
_logger.debug('Jobid: ' + str(jobid))
site = sites_.get(siteid)
job = jobs_.get(int(jobid))
cont = job.container
files_catalog = cont.files
fscope = getScope(job.owner.username)
datasetName = '{}:{}'.format(fscope, cont.guid)
distributive = job.distr.name
release = job.distr.release
# Prepare runScript
parameters = job.distr.command
parameters = parameters.replace("$COMMAND$", job.params)
parameters = parameters.replace("$USERNAME$", job.owner.username)
parameters = parameters.replace("$WORKINGGROUP$", job.owner.working_group)
# Prepare metadata
metadata = dict(user=job.owner.username)
# Prepare PanDA Object
pandajob = JobSpec()
pandajob.jobDefinitionID = int(time.time()) % 10000
pandajob.jobName = cont.guid
pandajob.transformation = client_config.DEFAULT_TRF
pandajob.destinationDBlock = datasetName
pandajob.destinationSE = site.se
pandajob.currentPriority = 1000
pandajob.prodSourceLabel = 'user'
pandajob.computingSite = site.ce
pandajob.cloud = 'RU'
pandajob.VO = 'atlas'
pandajob.prodDBlock = "%s:%s" % (fscope, pandajob.jobName)
pandajob.coreCount = job.corecount
pandajob.metadata = json.dumps(metadata)
#pandajob.workingGroup = job.owner.working_group
if site.encode_commands:
# It requires script wrapper on cluster side
pandajob.jobParameters = '%s %s %s "%s"' % (cont.guid, release,
distributive, parameters)
else:
pandajob.jobParameters = parameters
has_input = False
for fcc in files_catalog:
if fcc.type == 'input':
f = fcc.file
guid = f.guid
fileIT = FileSpec()
fileIT.lfn = f.lfn
fileIT.dataset = pandajob.prodDBlock
fileIT.prodDBlock = pandajob.prodDBlock
fileIT.type = 'input'
fileIT.scope = fscope
fileIT.status = 'ready'
fileIT.GUID = guid
pandajob.addFile(fileIT)
has_input = True
if fcc.type == 'output':
f = fcc.file
fileOT = FileSpec()
fileOT.lfn = f.lfn
fileOT.destinationDBlock = pandajob.prodDBlock
fileOT.destinationSE = pandajob.destinationSE
fileOT.dataset = pandajob.prodDBlock
fileOT.type = 'output'
fileOT.scope = fscope
fileOT.GUID = f.guid
pandajob.addFile(fileOT)
# Save replica meta
fc.new_replica(f, site)
if not has_input:
# Add fake input
fileIT = FileSpec()
fileIT.lfn = "fake.input"
fileIT.dataset = pandajob.prodDBlock
fileIT.prodDBlock = pandajob.prodDBlock
fileIT.type = 'input'
fileIT.scope = fscope
fileIT.status = 'ready'
fileIT.GUID = "fake.guid"
pandajob.addFile(fileIT)
# Prepare lof file
fileOL = FileSpec()
fileOL.lfn = "%s.log.tgz" % pandajob.jobName
fileOL.destinationDBlock = pandajob.destinationDBlock
fileOL.destinationSE = pandajob.destinationSE
fileOL.dataset = '{}:logs'.format(fscope)
fileOL.type = 'log'
fileOL.scope = 'panda'
pandajob.addFile(fileOL)
# Save log meta
log = File()
log.scope = fscope
log.lfn = fileOL.lfn
log.guid = getGUID(log.scope, log.lfn)
log.type = 'log'
log.status = 'defined'
files_.save(log)
# Save replica meta
fc.new_replica(log, site)
# Register file in container
fc.reg_file_in_cont(log, cont, 'log')
# Submit job
o = submitJobs([pandajob])
x = o[0]
try:
#update PandaID
PandaID = int(x[0])
job.pandaid = PandaID
job.ce = site.ce
except:
job.status = 'submit_error'
jobs_.save(job)
return 0
def new_job():
"""Creates new job
"""
g.user = request.oauth.user
scope = getScope(request.oauth.user.username)
js = request.json
data = js['data']
distr_id = data['sw_id']
params = data['script']
corecount = data['cores']
site = sites_.first(ce=current_app.config['DEFAULT_CE'])
distr = distrs_.get(id)
container = Container()
guid = 'job.' + commands.getoutput('uuidgen')
container.guid = guid
container.status = 'open'
conts_.save(container)
# Process ftp files
if 'ftp_dir' in data.keys():
ftp_dir = data['ftp_dir']
register_ftp_files(ftp_dir, scope, container.guid)
# Process guid list
if 'guids' in data.keys():
guids = data['guids']
for f in guids:
if f != '':
file_ = files_.first(guid=f)
if file_ is not None:
# Register file in catalog
fc.reg_file_in_cont(file_, container, 'input')
else:
raise WebpandaError('File with guid %s not found' % f)
ofiles = ['results.tgz']
# Starts cloneReplica tasks
ftasks = prepareInputFiles(container.id, site.se)
# Saves output files meta
for lfn in ofiles:
file = File()
file.scope = scope
file.guid = getGUID(scope, lfn)
file.lfn = lfn
file.status = 'defined'
files_.save(file)
# Register file in catalog
fc.reg_file_in_cont(file, container, 'output')
# Counts files
allfiles = container.files
nifiles = 0
nofiles = 0
for f in allfiles:
if f.type == 'input':
nifiles += 1
if f.type == 'output':
nofiles += 1
# Defines job meta
job = Job()
job.pandaid = None
job.status = 'pending'
job.owner = request.oauth.user
job.params = params
job.distr = distr
job.container = container
job.creation_time = datetime.utcnow()
job.modification_time = datetime.utcnow()
job.ninputfiles = nifiles
job.noutputfiles = nofiles
job.corecount = corecount
job.tags = data['tags'] if 'tags' in data.keys() else ""
jobs_.save(job)
# Async sendjob
res = chord(ftasks)(async_send_job.s(jobid=job.id, siteid=site.id))
return {'id': job.id, 'container_id': guid}
def upload():
form = request.form
# Create a unique container quid for this particular batch of uploads.
cguid = 'job.' + commands.getoutput('uuidgen')
# Is the upload using Ajax, or a direct POST by the form?
is_ajax = False
if form.get("__ajax", None) == "true":
is_ajax = True
# Create new container
container = Container()
container.guid = cguid
container.status = 'open'
conts_.save(container)
# Process files in request
for upload in request.files.getlist("file"):
# Define file params
lfn = upload.filename.rsplit("/")[0]
scope = getScope(g.user.username)
guid = getGUID(scope, lfn)
site = sites_.first(se=current_app.config['DEFAULT_SE'])
# Target folder for these uploads.
dir = '/' + os.path.join('system', scope, guid)
target = site.datadir + dir
try:
os.makedirs(target)
except:
if is_ajax:
return ajax_response(False, "Couldn't create upload directory: %s" % target)
else:
return "Couldn't create upload directory: %s" % target
replfn = os.path.join(dir, lfn)
destination = os.path.join(target, lfn)
upload.save(destination)
if os.path.isfile(destination):
# Check file existence in catalog
adler = adler32(destination)
md5 = md5sum(destination)
size = fsize(destination)
file_id = ddm_checkifexists(lfn, size, adler, md5)
if file_id:
# If file exists
file = files_.get(file_id)
else:
# Otherwise create new
file = File()
file.scope = scope
file.guid = guid
file.type = 'input'
file.lfn = lfn
file.token = ''
file.status = 'defined'
files_.save(file)
setFileMeta(file.id, destination)
replica = Replica()
replica.se = site.se
replica.status = 'ready'
replica.lfn = replfn
replica.original = file
replicas_.save(replica)
# Register file in container
fc.reg_file_in_cont(file, container, 'input')
else:
return ajax_response(False, "Couldn't save file: %s" % target)
if is_ajax:
return ajax_response(True, cguid)
else:
return redirect(url_for("jobs.jobs"))
def send_job(jobid, siteid):
_logger.debug('Jobid: ' + str(jobid))
site = sites_.get(siteid)
job = jobs_.get(int(jobid))
cont = job.container
files_catalog = cont.files
fscope = getScope(job.owner.username)
datasetName = '{}:{}'.format(fscope, cont.guid)
distributive = job.distr.name
release = job.distr.release
# Prepare runScript
parameters = job.distr.command
parameters = parameters.replace("$COMMAND$", job.params)
parameters = parameters.replace("$USERNAME$", job.owner.username)
parameters = parameters.replace("$WORKINGGROUP$", job.owner.working_group)
# Prepare metadata
metadata = dict(user=job.owner.username)
# Prepare PanDA Object
pandajob = JobSpec()
pandajob.jobDefinitionID = int(time.time()) % 10000
pandajob.jobName = cont.guid
pandajob.transformation = client_config.DEFAULT_TRF
pandajob.destinationDBlock = datasetName
pandajob.destinationSE = site.se
pandajob.currentPriority = 1000
pandajob.prodSourceLabel = 'user'
pandajob.computingSite = site.ce
pandajob.cloud = 'RU'
pandajob.VO = 'atlas'
pandajob.prodDBlock = "%s:%s" % (fscope, pandajob.jobName)
pandajob.coreCount = job.corecount
pandajob.metadata = json.dumps(metadata)
#pandajob.workingGroup = job.owner.working_group
if site.encode_commands:
# It requires script wrapper on cluster side
pandajob.jobParameters = '%s %s %s "%s"' % (cont.guid, release, distributive, parameters)
else:
pandajob.jobParameters = parameters
has_input = False
for fcc in files_catalog:
if fcc.type == 'input':
f = fcc.file
guid = f.guid
fileIT = FileSpec()
fileIT.lfn = f.lfn
fileIT.dataset = pandajob.prodDBlock
fileIT.prodDBlock = pandajob.prodDBlock
fileIT.type = 'input'
fileIT.scope = fscope
fileIT.status = 'ready'
fileIT.GUID = guid
pandajob.addFile(fileIT)
has_input = True
if fcc.type == 'output':
f = fcc.file
fileOT = FileSpec()
fileOT.lfn = f.lfn
fileOT.destinationDBlock = pandajob.prodDBlock
fileOT.destinationSE = pandajob.destinationSE
fileOT.dataset = pandajob.prodDBlock
fileOT.type = 'output'
fileOT.scope = fscope
fileOT.GUID = f.guid
pandajob.addFile(fileOT)
# Save replica meta
fc.new_replica(f, site)
if not has_input:
# Add fake input
fileIT = FileSpec()
fileIT.lfn = "fake.input"
fileIT.dataset = pandajob.prodDBlock
fileIT.prodDBlock = pandajob.prodDBlock
fileIT.type = 'input'
fileIT.scope = fscope
fileIT.status = 'ready'
fileIT.GUID = "fake.guid"
pandajob.addFile(fileIT)
# Prepare lof file
fileOL = FileSpec()
fileOL.lfn = "%s.log.tgz" % pandajob.jobName
fileOL.destinationDBlock = pandajob.destinationDBlock
fileOL.destinationSE = pandajob.destinationSE
fileOL.dataset = '{}:logs'.format(fscope)
fileOL.type = 'log'
fileOL.scope = 'panda'
pandajob.addFile(fileOL)
# Save log meta
log = File()
log.scope = fscope
log.lfn = fileOL.lfn
log.guid = getGUID(log.scope, log.lfn)
log.type = 'log'
log.status = 'defined'
files_.save(log)
# Save replica meta
fc.new_replica(log, site)
# Register file in container
fc.reg_file_in_cont(log, cont, 'log')
# Submit job
o = submitJobs([pandajob])
x = o[0]
try:
#update PandaID
PandaID = int(x[0])
job.pandaid = PandaID
job.ce = site.ce
except:
job.status = 'submit_error'
jobs_.save(job)
return 0
def job():
"""
New job form view
:return: Response obj
"""
form = NewJobForm(request.form)
if request.method == 'POST':
site = sites_.get(int(form.site.data))
distr_name, distr_release = form.distr.data.split(':')
distr = distrs_.first(name=distr_name, release=int(distr_release))
container_guid = form.container.data
try:
container = conts_.first(guid=container_guid)
except(Exception):
_logger.error(Exception.message)
return make_response(jsonify({'error': 'Container not found'}), 404)
if site.encode_commands:
# By default frontend encodes with base64 job script parts separated by ";"
# It requires script wrapper on cluster side
jparams = form.params.data
else:
# Set site.encode_commands as False if you want to send command string without base64 encoding
jparams = ';'.join([b64decode(command) for command in form.params.data.split(';')])
ifiles = request.form.getlist('ifiles[]')
iguids = request.form.getlist('iguids[]')
iconts = request.form.getlist('iconts[]')
ofiles = ['{guid}.out.tgz'.format(guid=container.guid)]
scope = getScope(g.user.username)
# Process ftp files
ftp_dir = form.ftpdir.data
register_ftp_files(ftp_dir, scope, container.guid)
# Process guid list
for f in iguids:
if f != '':
file = files_.first(guid=f)
if file is not None:
# Register files in container
fc.reg_file_in_cont(file, container, 'input')
else:
return make_response(jsonify({'error': "GUID {} not found".format(f)}))
# Process containers
for c in iconts:
if c != '':
try:
form_cont = conts_.first(guid=c)
except(Exception):
_logger.error(Exception.message)
return make_response(jsonify({'error': 'Container in form not found'}), 404)
for f in form_cont.files:
# Register file in catalog
fc.reg_file_in_cont(f.file, container, 'input')
# Processes urls
for f in ifiles:
if f != '':
from_se, path, token = getUrlInfo(f)
replfn = ':/'.join([from_se, path])
# Check if used before
file_id = ddm_checkexternalifexists('', replfn)
if file_id:
file = files_.get(file_id)
else:
lfn = path.split('/')[-1]
guid = getGUID(scope, lfn)
file = File()
file.scope = scope
file.guid = guid
file.type = 'input'
file.lfn = lfn
file.status = 'defined'
files_.save(file)
replica = Replica()
replica.se = from_se
replica.status = 'link'
# Separate url & token
replica.lfn = replfn
replica.token = token
replica.original = file
replicas_.save(replica)
# Register file in container
fc.reg_file_in_cont(file, container, 'input')
# Starts cloneReplica tasks
ftasks = prepareInputFiles(container.id, site.se)
# Saves output files meta
for lfn in ofiles:
file = File()
file.scope = scope
file.guid = getGUID(scope, lfn)
file.type = 'output'
file.lfn = lfn
file.status = 'defined'
files_.save(file)
# Register file in container
fc.reg_file_in_cont(file, container, 'output')
# Counts files
allfiles = container.files
nifiles = 0
nofiles = 0
for f in allfiles:
if f.type == 'input':
nifiles += 1
if f.type == 'output':
nofiles += 1
# Defines job meta
job = Job()
job.pandaid = None
job.status = 'pending'
job.owner = g.user
job.params = jparams
job.distr = distr
job.container = container
job.creation_time = datetime.utcnow()
job.modification_time = datetime.utcnow()
job.ninputfiles = nifiles
job.noutputfiles = nofiles
job.corecount = form.corecount.data
job.tags = form.tags.data if form.tags.data != "" else None
jobs_.save(job)
# Async sendjob
res = chord(ftasks)(async_send_job.s(jobid=job.id, siteid=site.id))
return redirect(url_for('jobs.jobs'))
form.distr.choices = [("%s:%s" % (distr.name, distr.release), "%s: %s" % (distr.name, distr.version)) for distr in distrs_.find().order_by('name').order_by('version')]
form.site.choices = [(site.id, "{ce}".format(ce=site.ce)) for site in sites_.find(active=1).order_by('ce')]
return render_template("dashboard/jobs/new.html", form=form)
def upload():
form = request.form
# Create a unique container quid for this particular batch of uploads.
cguid = 'job.' + commands.getoutput('uuidgen')
# Is the upload using Ajax, or a direct POST by the form?
is_ajax = False
if form.get("__ajax", None) == "true":
is_ajax = True
# Create new container
container = Container()
container.guid = cguid
container.status = 'open'
conts_.save(container)
# Process files in request
for upload in request.files.getlist("file"):
# Define file params
lfn = upload.filename.rsplit("/")[0]
scope = getScope(g.user.username)
guid = getGUID(scope, lfn)
site = sites_.first(se=current_app.config['DEFAULT_SE'])
# Target folder for these uploads.
dir = '/' + os.path.join('system', scope, guid)
target = site.datadir + dir
try:
os.makedirs(target)
except:
if is_ajax:
return ajax_response(
False, "Couldn't create upload directory: %s" % target)
else:
return "Couldn't create upload directory: %s" % target
replfn = os.path.join(dir, lfn)
destination = os.path.join(target, lfn)
upload.save(destination)
if os.path.isfile(destination):
# Check file existence in catalog
adler = adler32(destination)
md5 = md5sum(destination)
size = fsize(destination)
file_id = ddm_checkifexists(lfn, size, adler, md5)
if file_id:
# If file exists
file = files_.get(file_id)
else:
# Otherwise create new
file = File()
file.scope = scope
file.guid = guid
file.type = 'input'
file.lfn = lfn
file.token = ''
file.status = 'defined'
files_.save(file)
setFileMeta(file.id, destination)
replica = Replica()
replica.se = site.se
replica.status = 'ready'
replica.lfn = replfn
replica.original = file
replicas_.save(replica)
# Register file in container
fc.reg_file_in_cont(file, container, 'input')
else:
return ajax_response(False, "Couldn't save file: %s" % target)
if is_ajax:
return ajax_response(True, cguid)
else:
return redirect(url_for("jobs.jobs"))
def file_save(container_guid, lfn):
"""
POST: /pilot/file/<container_guid>/<lfn>/save
Saves file from request, returns file guid
:param container_guid: Guid of container
:type container_guid: str
:param lfn: Local FileName
:type lfn: str
:return: guid
:rtype: json
"""
site = sites_.first(se=current_app.config['DEFAULT_SE'])
if ':' in container_guid:
container_guid = container_guid.split(':')[-1]
container = conts_.first(guid=container_guid)
if container.status != 'open':
raise WebpandaError('Unable to upload: Container is not open')
cc = container.files
ff = None
for c in cc:
f = c.file
if f.lfn == lfn:
ff = f
if not ff:
ff = File()
ff.scope = getScope(g.user.username)
ff.lfn = lfn
ff.guid = getGUID(ff.scope, ff.lfn)
ff.status = 'defined'
files_.save(ff)
# Register file in container
fc.reg_file_in_cont(ff, container, 'input')
path = os.path.join(site.datadir, getScope(g.user.username), container.guid)
replfn = '/' + os.path.join(getScope(g.user.username), container.guid, ff.lfn)
destination = os.path.join(path, ff.lfn)
for r in ff.replicas:
if r.se == site.se:
destination = site.datadir + r.lfn
file_dir = '/'.join(destination.split('/')[:-1])
if r.status == 'ready':
if os.path.isfile(destination): # Check fsize, md5 or adler
raise WebpandaError('Replica exists')
else:
r.status = 'broken'
replicas_.save(r)
raise WebpandaError('Broken replica')
elif r.status == 'defined':
try:
os.makedirs(file_dir)
except(Exception):
pass
f = open(destination, 'wb')
f.write(request.data)
f.close()
# Update file info
setFileMeta(ff.id, destination)
r.status = 'ready'
replicas_.save(r)
return {'guid': ff.guid}
else:
raise WebpandaError('Replica status: %s' % r.status)
replica = Replica()
if os.path.isfile(destination):
raise WebpandaError('Unable to upload: File exists')
try:
os.makedirs(path)
except(Exception):
_logger.debug('Path exists: %s' % path)
f = open(destination, 'wb')
f.write(request.data)
f.close()
# Update file info
setFileMeta(ff.id, destination)
# Create/change replica
replica.se = site.se
replica.status = 'ready'
replica.lfn = replfn
replica.token = ''
replica.original = ff
replicas_.save(replica)
return {'guid': ff.guid}
def file_save(container_guid, lfn):
"""
POST: /pilot/file/<container_guid>/<lfn>/save
Saves file from request, returns file guid
:param container_guid: Guid of container
:type container_guid: str
:param lfn: Local FileName
:type lfn: str
:return: guid
:rtype: json
"""
site = sites_.first(se=current_app.config['DEFAULT_SE'])
if ':' in container_guid:
container_guid = container_guid.split(':')[-1]
container = conts_.first(guid=container_guid)
if container.status != 'open':
raise WebpandaError('Unable to upload: Container is not open')
cc = container.files
ff = None
for c in cc:
f = c.file
if f.lfn == lfn:
ff = f
if not ff:
ff = File()
ff.scope = getScope(g.user.username)
ff.lfn = lfn
ff.guid = getGUID(ff.scope, ff.lfn)
ff.status = 'defined'
files_.save(ff)
# Register file in container
fc.reg_file_in_cont(ff, container, 'input')
path = os.path.join(site.datadir, getScope(g.user.username),
container.guid)
replfn = '/' + os.path.join(getScope(g.user.username), container.guid,
ff.lfn)
destination = os.path.join(path, ff.lfn)
for r in ff.replicas:
if r.se == site.se:
destination = site.datadir + r.lfn
file_dir = '/'.join(destination.split('/')[:-1])
if r.status == 'ready':
if os.path.isfile(destination): # Check fsize, md5 or adler
raise WebpandaError('Replica exists')
else:
r.status = 'broken'
replicas_.save(r)
raise WebpandaError('Broken replica')
elif r.status == 'defined':
try:
os.makedirs(file_dir)
except (Exception):
pass
f = open(destination, 'wb')
f.write(request.data)
f.close()
# Update file info
setFileMeta(ff.id, destination)
r.status = 'ready'
replicas_.save(r)
return {'guid': ff.guid}
else:
raise WebpandaError('Replica status: %s' % r.status)
replica = Replica()
if os.path.isfile(destination):
raise WebpandaError('Unable to upload: File exists')
try:
os.makedirs(path)
except (Exception):
_logger.debug('Path exists: %s' % path)
f = open(destination, 'wb')
f.write(request.data)
f.close()
# Update file info
setFileMeta(ff.id, destination)
# Create/change replica
replica.se = site.se
replica.status = 'ready'
replica.lfn = replfn
replica.token = ''
replica.original = ff
replicas_.save(replica)
return {'guid': ff.guid}
def new_job():
"""Creates new job
"""
g.user = request.oauth.user
scope = getScope(request.oauth.user.username)
js = request.json
data = js['data']
distr_id = data['sw_id']
params = data['script']
corecount = data['cores']
site = sites_.first(ce=current_app.config['DEFAULT_CE'])
distr = distrs_.get(id)
container = Container()
guid = 'job.' + commands.getoutput('uuidgen')
container.guid = guid
container.status = 'open'
conts_.save(container)
# Process ftp files
if 'ftp_dir' in data.keys():
ftp_dir = data['ftp_dir']
register_ftp_files(ftp_dir, scope, container.guid)
# Process guid list
if 'guids' in data.keys():
guids = data['guids']
for f in guids:
if f != '':
file_ = files_.first(guid=f)
if file_ is not None:
# Register file in catalog
fc.reg_file_in_cont(file_, container, 'input')
else:
raise WebpandaError('File with guid %s not found' % f)
ofiles = ['results.tgz']
# Starts cloneReplica tasks
ftasks = prepareInputFiles(container.id, site.se)
# Saves output files meta
for lfn in ofiles:
file = File()
file.scope = scope
file.guid = getGUID(scope, lfn)
file.lfn = lfn
file.status = 'defined'
files_.save(file)
# Register file in catalog
fc.reg_file_in_cont(file, container, 'output')
# Counts files
allfiles = container.files
nifiles = 0
nofiles = 0
for f in allfiles:
if f.type == 'input':
nifiles += 1
if f.type == 'output':
nofiles += 1
# Defines job meta
job = Job()
job.pandaid = None
job.status = 'pending'
job.owner = request.oauth.user
job.params = params
job.distr = distr
job.container = container
job.creation_time = datetime.utcnow()
job.modification_time = datetime.utcnow()
job.ninputfiles = nifiles
job.noutputfiles = nofiles
job.corecount = corecount
job.tags = data['tags'] if 'tags' in data.keys() else ""
jobs_.save(job)
# Async sendjob
res = chord(ftasks)(async_send_job.s(jobid=job.id, siteid=site.id))
return {'id': job.id, 'container_id': guid}