def transfer_outputs(ids=[]):
if len(ids) == 0:
return 0
to_site = sites_.first(se=current_app.config['DEFAULT_SE'])
jobs = Job.query.filter(Job.id.in_(ids)).all()
for job in jobs:
from_site = sites_.first(ce=job.ce)
cont = job.container
files = cont.files
for file in files:
if file.type in ['log', 'output']:
replicas = file.replicas
needReplica = False
fromReplica = 0
hasReplica = False
for replica in replicas:
if replica.se == from_site.se and replica.status == 'ready':
needReplica = True
fromReplica = replica.id
if replica.se == to_site.se:
if replica.status == 'ready':
hasReplica = True
setFileMeta(
file.id,
current_app.config['DATA_PATH'] + replica.lfn)
if replica.status != 'ready':
raise Exception('Broken replica. File: %s' %
file.guid)
if needReplica and not hasReplica:
task = async_cloneReplica.delay(fromReplica, to_site.se)
return 0
def transfer_outputs(ids=[]):
if len(ids) == 0:
return 0
to_site = sites_.first(se=current_app.config['DEFAULT_SE'])
jobs = Job.query.filter(Job.id.in_(ids)).all()
for job in jobs:
from_site = sites_.first(ce=job.ce)
cont = job.container
files = cont.files
for file in files:
if file.type in ['log', 'output']:
replicas = file.replicas
needReplica = False
fromReplica = 0
hasReplica = False
for replica in replicas:
if replica.se == from_site.se and replica.status == 'ready':
needReplica = True
fromReplica = replica.id
if replica.se == to_site.se:
if replica.status == 'ready':
hasReplica = True
setFileMeta(file.id, current_app.config['DATA_PATH'] + replica.lfn)
if replica.status != 'ready':
raise Exception('Broken replica. File: %s' % file.guid)
if needReplica and not hasReplica:
task = async_cloneReplica.delay(fromReplica, to_site.se)
return 0
def index():
"""
Register files from HPC
:return: Response obj
"""
form = NewFileForm()
if request.method == 'POST':
path = form.path.data
user = g.user
# Define SE connector
se_name = 'RRC-KI-HPC'
se = sites_.first(se=se_name)
# Get container
cont = fc.new_cont()
cont.status = 'user'
fc.save(cont)
# Upload files to container
async_upload_dir.delay(user.id, cont.id, se.id, path)
# Return container page
return redirect(url_for('conts.cont_info', guid=cont.guid))
return render_template("dashboard/files/file_new.html", form=form)
def link_file(container_guid, lfn):
"""
GET: /pilot/file/<container_guid>/<lfn>/link
Returns ftp link to file
:param container_guid: Guid of container
:type container_guid: str
:param lfn: Local FileName
:type lfn: str
:return: id/guid/ftp
:rtype: json
"""
if ':' in container_guid:
container_guid = container_guid.split(':')[-1]
container = conts_.first(guid=container_guid)
site = sites_.first(se=current_app.config['DEFAULT_SE'])
cc = container.files
for c in cc:
f = c.file
if f.lfn == lfn:
replicas = f.replicas
for r in replicas:
if r.se == site.se and r.status == 'ready':
data = {}
data['lfn'] = f.lfn
data['guid'] = f.guid
data['ftp'] = getFtpLink(r.lfn)
return make_response(jsonify(data), 200)
raise WebpandaError('File not found')
def send_job_(task, container, script):
"""
Prepares a job for task with container and trf script
:param task: Task obj to append a job
:param container: Container obj for job
:param script: str Trf script
:return: True
"""
# Get default ComputingElement
site = sites_.first(ce=current_app.config['DEFAULT_CE'])
if site is None:
raise WebpandaError("ComputingElement not found")
# Get distributive
distr = task.task_type.distr
# Define jobs
job = Job()
job.pandaid = None
job.status = 'pending'
job.owner = users_.get(task.owner_id)
job.params = b64encode(script)
job.distr = distr
job.container = container
job.creation_time = datetime.utcnow()
job.modification_time = datetime.utcnow()
#job.ninputfiles = 0
#job.noutputfiles = 0
job.corecount = 1
job.tags = task.tag
jobs_.save(job)
# Async sendjob
async_send_job.delay(jobid=job.id, siteid=site.id)
return True
def link_file(container_guid, lfn):
"""
GET: /pilot/file/<container_guid>/<lfn>/link
Returns ftp link to file
:param container_guid: Guid of container
:type container_guid: str
:param lfn: Local FileName
:type lfn: str
:return: id/guid/ftp
:rtype: json
"""
if ':' in container_guid:
container_guid = container_guid.split(':')[-1]
container = conts_.first(guid=container_guid)
site = sites_.first(se=current_app.config['DEFAULT_SE'])
cc = container.files
for c in cc:
f = c.file
if f.lfn == lfn:
replicas = f.replicas
for r in replicas:
if r.se == site.se and r.status == 'ready':
data = {}
data['lfn'] = f.lfn
data['guid'] = f.guid
data['ftp'] = getFtpLink(r.lfn)
return make_response(jsonify(data), 200)
raise WebpandaError('File not found')
def cloneReplica(replicaid, se):
replica = replicas_.get(replicaid)
file = replica.original
replicas = file.replicas
for r in replicas:
if se == r.se:
_logger.debug('Replica exists: id=%s' % r.id)
# Update expired time
return r.id
# Define base replica
from_se = sites_.first(se=replica.se)
fromParams = {}
if replica.status == 'link':
lfn = getLinkLFN(file.scope, replica.lfn)
else:
lfn = replica.lfn
# Define result replica params
to_se = sites_.first(se=se)
dest = '/'.join(lfn.split('/')[:-1])
toParams = {'dest': dest}
ec, filesinfo = movedata({}, [replica.lfn], from_se.plugin, fromParams,
to_se.plugin, toParams)
if ec == 0:
r = Replica()
if file.fsize is None:
file.fsize = filesinfo[replica.lfn]['fsize']
if file.md5sum is None:
file.md5sum = filesinfo[replica.lfn]['md5sum']
if file.checksum is None:
file.checksum = filesinfo[replica.lfn]['checksum']
r.se = se
r.status = 'ready'
r.lfn = lfn
replicas_.save(r)
file.modification_time = datetime.utcnow()
file.replicas.append(r)
files_.save(file)
for cont in file.containers:
linkReplica(
r.id, '/%s/%s' % (client_config.DEFAULT_SCOPE, cont.cont.guid))
return r.id
raise Exception('movedata return code: %s' % ec)
def register_outputs():
jobs = jobs_.find().filter(Job.status.in_(['finished', 'failed', 'cancelled']))\
.filter(Job.registered != 1)\
.all()
ids = []
conn_factory = SEFactory()
for job in jobs:
ids.append(job.id)
user = users_.get(job.owner_id)
site = sites_.first(ce=job.ce)
cont = job.container
cont_path = fc.get_cont_dir(cont, fc.get_scope(user))
cc = cont.files
out_ready = False
if job.status == 'finished':
out_ready = True
for c in cc:
if c.type not in ["log", "output"]:
continue
f = c.file
for replica in f.replicas:
if replica.se == site.se:
if c.type == 'output' and not out_ready:
# Update replica status
replica.status = "failed"
replicas_.save(replica)
else:
connector = conn_factory.getSE(site.plugin, None)
# link real file to saved replica
fpath = os.path.join(cont_path, f.lfn)
# append attemptnr
if job.attemptnr > 0:
fpath += "." + str(job.attemptnr)
#connector.link(os.path.join(cont_path, f.lfn), replica_dir, rel=True)
connector.mv(fpath, replica.lfn, rel=True)
# get replica info
f.fsize = connector.fsize(replica.lfn)
f.md5sum = connector.md5sum(replica.lfn)
f.checksum = connector.adler32(replica.lfn)
f.modification_time = datetime.utcnow()
fc.save(f)
# Update replica status
replica.status = "ready"
replicas_.save(replica)
job.registered = 1
job.registation_time = datetime.utcnow()
jobs_.save(job)
return ids
def register_outputs():
jobs = jobs_.find().filter(Job.status.in_(['finished', 'failed', 'cancelled']))\
.filter(Job.registered != 1)\
.all()
ids = []
conn_factory = SEFactory()
for job in jobs:
ids.append(job.id)
user = users_.get(job.owner_id)
site = sites_.first(ce=job.ce)
cont = job.container
cont_path = fc.get_cont_dir(cont, fc.get_scope(user))
cc = cont.files
out_ready = False
if job.status == 'finished':
out_ready = True
for c in cc:
if c.type not in ["log", "output"]:
continue
f = c.file
for replica in f.replicas:
if replica.se == site.se:
if c.type == 'output' and not out_ready:
# Update replica status
replica.status = "failed"
replicas_.save(replica)
else:
connector = conn_factory.getSE(site.plugin, None)
# link real file to saved replica
fpath = os.path.join(cont_path, f.lfn)
# append attemptnr
if job.attemptnr > 0:
fpath += "." + str(job.attemptnr)
#connector.link(os.path.join(cont_path, f.lfn), replica_dir, rel=True)
connector.mv(fpath, replica.lfn, rel=True)
# get replica info
f.fsize = connector.fsize(replica.lfn)
f.md5sum = connector.md5sum(replica.lfn)
f.checksum = connector.adler32(replica.lfn)
f.modification_time = datetime.utcnow()
fc.save(f)
# Update replica status
replica.status = "ready"
replicas_.save(replica)
job.registered = 1
job.registation_time = datetime.utcnow()
jobs_.save(job)
return ids
def cloneReplica(replicaid, se):
replica = replicas_.get(replicaid)
file = replica.original
replicas = file.replicas
for r in replicas:
if se == r.se:
_logger.debug('Replica exists: id=%s' % r.id)
# Update expired time
return r.id
# Define base replica
from_se = sites_.first(se=replica.se)
fromParams = {}
if replica.status == 'link':
lfn = getLinkLFN(file.scope, replica.lfn)
else:
lfn = replica.lfn
# Define result replica params
to_se = sites_.first(se=se)
dest = '/'.join(lfn.split('/')[:-1])
toParams = {'dest': dest}
ec, filesinfo = movedata({}, [replica.lfn], from_se.plugin, fromParams, to_se.plugin, toParams)
if ec == 0:
r = Replica()
if file.fsize is None:
file.fsize = filesinfo[replica.lfn]['fsize']
if file.md5sum is None:
file.md5sum = filesinfo[replica.lfn]['md5sum']
if file.checksum is None:
file.checksum = filesinfo[replica.lfn]['checksum']
r.se = se
r.status = 'ready'
r.lfn = lfn
replicas_.save(r)
file.modification_time = datetime.utcnow()
file.replicas.append(r)
files_.save(file)
for cont in file.containers:
linkReplica(r.id, '/%s/%s' % (client_config.DEFAULT_SCOPE, cont.cont.guid))
return r.id
raise Exception('movedata return code: %s' % ec)
def registerLocalFile(arg, dirname, names, scope):
"""Register files from local dir to container
:param arg: Container guid
:param dirname: Abs dir
:param names: File name
:param scope: Scope to upload files in
:return:
"""
site = sites_.first(se=client_config.DEFAULT_SE)
_logger.debug(str(arg))
cont = conts_.first(guid=arg)
files = cont.files
for name in names:
fpath = os.path.join(dirname, name)
fobj = None
# Check in container
for file in files:
if file.lfn == name:
fobj = file
# Check in catalog
if not fobj:
destination = os.path.join(dirname, name)
adler = adler32(destination)
md5 = md5sum(destination)
size = fsize(destination)
file_id = ddm_checkifexists(name, size, adler, md5)
if file_id:
# If file exists
fobj = files_.get(file_id)
if not fobj:
fobj = File()
fobj.scope = scope
fobj.lfn = name
fobj.guid = getGUID(fobj.scope, fobj.lfn)
fobj.type = 'input'
fobj.status = 'defined'
files_.save(fobj)
setFileMeta(fobj.id, fpath)
# Register file in catalog
fc.reg_file_in_cont(fobj, cont, "input")
replicas = fobj.replicas
replica = None
for r in replicas:
if r.se == site.se and r.status == 'ready':
replica = r
if not replica:
ldir = '/' + os.path.join('system', fobj.scope, fobj.guid)
ddm_localmakedirs(ldir)
ddm_localcp(fpath[len(site.datadir):], ldir)
replica = Replica()
replica.se = site.se
replica.status = 'ready'
replica.token = ''
replica.lfn = os.path.join(ldir, fobj.lfn)
replica.original = fobj
replicas_.save(replica)
def new_job():
"""Creates new job
"""
g.user = request.oauth.user
scope = getScope(request.oauth.user.username)
js = request.json
data = js['data']
distr_id = data['sw_id']
params = data['script']
corecount = data['cores']
site = sites_.first(ce=current_app.config['DEFAULT_CE'])
distr = distrs_.get(id)
container = Container()
guid = 'job.' + commands.getoutput('uuidgen')
container.guid = guid
container.status = 'open'
conts_.save(container)
# Process ftp files
if 'ftp_dir' in data.keys():
ftp_dir = data['ftp_dir']
register_ftp_files(ftp_dir, scope, container.guid)
# Process guid list
if 'guids' in data.keys():
guids = data['guids']
for f in guids:
if f != '':
file_ = files_.first(guid=f)
if file_ is not None:
# Register file in catalog
fc.reg_file_in_cont(file_, container, 'input')
else:
raise WebpandaError('File with guid %s not found' % f)
ofiles = ['results.tgz']
# Starts cloneReplica tasks
ftasks = prepareInputFiles(container.id, site.se)
# Saves output files meta
for lfn in ofiles:
file = File()
file.scope = scope
file.guid = getGUID(scope, lfn)
file.lfn = lfn
file.status = 'defined'
files_.save(file)
# Register file in catalog
fc.reg_file_in_cont(file, container, 'output')
# Counts files
allfiles = container.files
nifiles = 0
nofiles = 0
for f in allfiles:
if f.type == 'input':
nifiles += 1
if f.type == 'output':
nofiles += 1
# Defines job meta
job = Job()
job.pandaid = None
job.status = 'pending'
job.owner = request.oauth.user
job.params = params
job.distr = distr
job.container = container
job.creation_time = datetime.utcnow()
job.modification_time = datetime.utcnow()
job.ninputfiles = nifiles
job.noutputfiles = nofiles
job.corecount = corecount
job.tags = data['tags'] if 'tags' in data.keys() else ""
jobs_.save(job)
# Async sendjob
res = chord(ftasks)(async_send_job.s(jobid=job.id, siteid=site.id))
return {'id': job.id, 'container_id': guid}
def upload():
form = request.form
# Create a unique container quid for this particular batch of uploads.
cguid = 'job.' + commands.getoutput('uuidgen')
# Is the upload using Ajax, or a direct POST by the form?
is_ajax = False
if form.get("__ajax", None) == "true":
is_ajax = True
# Create new container
container = Container()
container.guid = cguid
container.status = 'open'
conts_.save(container)
# Process files in request
for upload in request.files.getlist("file"):
# Define file params
lfn = upload.filename.rsplit("/")[0]
scope = getScope(g.user.username)
guid = getGUID(scope, lfn)
site = sites_.first(se=current_app.config['DEFAULT_SE'])
# Target folder for these uploads.
dir = '/' + os.path.join('system', scope, guid)
target = site.datadir + dir
try:
os.makedirs(target)
except:
if is_ajax:
return ajax_response(False, "Couldn't create upload directory: %s" % target)
else:
return "Couldn't create upload directory: %s" % target
replfn = os.path.join(dir, lfn)
destination = os.path.join(target, lfn)
upload.save(destination)
if os.path.isfile(destination):
# Check file existence in catalog
adler = adler32(destination)
md5 = md5sum(destination)
size = fsize(destination)
file_id = ddm_checkifexists(lfn, size, adler, md5)
if file_id:
# If file exists
file = files_.get(file_id)
else:
# Otherwise create new
file = File()
file.scope = scope
file.guid = guid
file.type = 'input'
file.lfn = lfn
file.token = ''
file.status = 'defined'
files_.save(file)
setFileMeta(file.id, destination)
replica = Replica()
replica.se = site.se
replica.status = 'ready'
replica.lfn = replfn
replica.original = file
replicas_.save(replica)
# Register file in container
fc.reg_file_in_cont(file, container, 'input')
else:
return ajax_response(False, "Couldn't save file: %s" % target)
if is_ajax:
return ajax_response(True, cguid)
else:
return redirect(url_for("jobs.jobs"))
def linkReplica(replicaid, dir):
replica = replicas_.get(replicaid)
site = sites_.first(se=replica.se)
lfn = replica.lfn
linkdata(site.plugin, {}, lfn, dir)
return 0
def new_job():
"""Creates new job
"""
g.user = request.oauth.user
scope = getScope(request.oauth.user.username)
js = request.json
data = js['data']
distr_id = data['sw_id']
params = data['script']
corecount = data['cores']
site = sites_.first(ce=current_app.config['DEFAULT_CE'])
distr = distrs_.get(id)
container = Container()
guid = 'job.' + commands.getoutput('uuidgen')
container.guid = guid
container.status = 'open'
conts_.save(container)
# Process ftp files
if 'ftp_dir' in data.keys():
ftp_dir = data['ftp_dir']
register_ftp_files(ftp_dir, scope, container.guid)
# Process guid list
if 'guids' in data.keys():
guids = data['guids']
for f in guids:
if f != '':
file_ = files_.first(guid=f)
if file_ is not None:
# Register file in catalog
fc.reg_file_in_cont(file_, container, 'input')
else:
raise WebpandaError('File with guid %s not found' % f)
ofiles = ['results.tgz']
# Starts cloneReplica tasks
ftasks = prepareInputFiles(container.id, site.se)
# Saves output files meta
for lfn in ofiles:
file = File()
file.scope = scope
file.guid = getGUID(scope, lfn)
file.lfn = lfn
file.status = 'defined'
files_.save(file)
# Register file in catalog
fc.reg_file_in_cont(file, container, 'output')
# Counts files
allfiles = container.files
nifiles = 0
nofiles = 0
for f in allfiles:
if f.type == 'input':
nifiles += 1
if f.type == 'output':
nofiles += 1
# Defines job meta
job = Job()
job.pandaid = None
job.status = 'pending'
job.owner = request.oauth.user
job.params = params
job.distr = distr
job.container = container
job.creation_time = datetime.utcnow()
job.modification_time = datetime.utcnow()
job.ninputfiles = nifiles
job.noutputfiles = nofiles
job.corecount = corecount
job.tags = data['tags'] if 'tags' in data.keys() else ""
jobs_.save(job)
# Async sendjob
res = chord(ftasks)(async_send_job.s(jobid=job.id, siteid=site.id))
return {'id': job.id, 'container_id': guid}
def file_save(container_guid, lfn):
"""
POST: /pilot/file/<container_guid>/<lfn>/save
Saves file from request, returns file guid
:param container_guid: Guid of container
:type container_guid: str
:param lfn: Local FileName
:type lfn: str
:return: guid
:rtype: json
"""
site = sites_.first(se=current_app.config['DEFAULT_SE'])
if ':' in container_guid:
container_guid = container_guid.split(':')[-1]
container = conts_.first(guid=container_guid)
if container.status != 'open':
raise WebpandaError('Unable to upload: Container is not open')
cc = container.files
ff = None
for c in cc:
f = c.file
if f.lfn == lfn:
ff = f
if not ff:
ff = File()
ff.scope = getScope(g.user.username)
ff.lfn = lfn
ff.guid = getGUID(ff.scope, ff.lfn)
ff.status = 'defined'
files_.save(ff)
# Register file in container
fc.reg_file_in_cont(ff, container, 'input')
path = os.path.join(site.datadir, getScope(g.user.username), container.guid)
replfn = '/' + os.path.join(getScope(g.user.username), container.guid, ff.lfn)
destination = os.path.join(path, ff.lfn)
for r in ff.replicas:
if r.se == site.se:
destination = site.datadir + r.lfn
file_dir = '/'.join(destination.split('/')[:-1])
if r.status == 'ready':
if os.path.isfile(destination): # Check fsize, md5 or adler
raise WebpandaError('Replica exists')
else:
r.status = 'broken'
replicas_.save(r)
raise WebpandaError('Broken replica')
elif r.status == 'defined':
try:
os.makedirs(file_dir)
except(Exception):
pass
f = open(destination, 'wb')
f.write(request.data)
f.close()
# Update file info
setFileMeta(ff.id, destination)
r.status = 'ready'
replicas_.save(r)
return {'guid': ff.guid}
else:
raise WebpandaError('Replica status: %s' % r.status)
replica = Replica()
if os.path.isfile(destination):
raise WebpandaError('Unable to upload: File exists')
try:
os.makedirs(path)
except(Exception):
_logger.debug('Path exists: %s' % path)
f = open(destination, 'wb')
f.write(request.data)
f.close()
# Update file info
setFileMeta(ff.id, destination)
# Create/change replica
replica.se = site.se
replica.status = 'ready'
replica.lfn = replfn
replica.token = ''
replica.original = ff
replicas_.save(replica)
return {'guid': ff.guid}
def file_save(container_guid, lfn):
"""
POST: /pilot/file/<container_guid>/<lfn>/save
Saves file from request, returns file guid
:param container_guid: Guid of container
:type container_guid: str
:param lfn: Local FileName
:type lfn: str
:return: guid
:rtype: json
"""
site = sites_.first(se=current_app.config['DEFAULT_SE'])
if ':' in container_guid:
container_guid = container_guid.split(':')[-1]
container = conts_.first(guid=container_guid)
if container.status != 'open':
raise WebpandaError('Unable to upload: Container is not open')
cc = container.files
ff = None
for c in cc:
f = c.file
if f.lfn == lfn:
ff = f
if not ff:
ff = File()
ff.scope = getScope(g.user.username)
ff.lfn = lfn
ff.guid = getGUID(ff.scope, ff.lfn)
ff.status = 'defined'
files_.save(ff)
# Register file in container
fc.reg_file_in_cont(ff, container, 'input')
path = os.path.join(site.datadir, getScope(g.user.username),
container.guid)
replfn = '/' + os.path.join(getScope(g.user.username), container.guid,
ff.lfn)
destination = os.path.join(path, ff.lfn)
for r in ff.replicas:
if r.se == site.se:
destination = site.datadir + r.lfn
file_dir = '/'.join(destination.split('/')[:-1])
if r.status == 'ready':
if os.path.isfile(destination): # Check fsize, md5 or adler
raise WebpandaError('Replica exists')
else:
r.status = 'broken'
replicas_.save(r)
raise WebpandaError('Broken replica')
elif r.status == 'defined':
try:
os.makedirs(file_dir)
except (Exception):
pass
f = open(destination, 'wb')
f.write(request.data)
f.close()
# Update file info
setFileMeta(ff.id, destination)
r.status = 'ready'
replicas_.save(r)
return {'guid': ff.guid}
else:
raise WebpandaError('Replica status: %s' % r.status)
replica = Replica()
if os.path.isfile(destination):
raise WebpandaError('Unable to upload: File exists')
try:
os.makedirs(path)
except (Exception):
_logger.debug('Path exists: %s' % path)
f = open(destination, 'wb')
f.write(request.data)
f.close()
# Update file info
setFileMeta(ff.id, destination)
# Create/change replica
replica.se = site.se
replica.status = 'ready'
replica.lfn = replfn
replica.token = ''
replica.original = ff
replicas_.save(replica)
return {'guid': ff.guid}
def upload():
form = request.form
# Create a unique container quid for this particular batch of uploads.
cguid = 'job.' + commands.getoutput('uuidgen')
# Is the upload using Ajax, or a direct POST by the form?
is_ajax = False
if form.get("__ajax", None) == "true":
is_ajax = True
# Create new container
container = Container()
container.guid = cguid
container.status = 'open'
conts_.save(container)
# Process files in request
for upload in request.files.getlist("file"):
# Define file params
lfn = upload.filename.rsplit("/")[0]
scope = getScope(g.user.username)
guid = getGUID(scope, lfn)
site = sites_.first(se=current_app.config['DEFAULT_SE'])
# Target folder for these uploads.
dir = '/' + os.path.join('system', scope, guid)
target = site.datadir + dir
try:
os.makedirs(target)
except:
if is_ajax:
return ajax_response(
False, "Couldn't create upload directory: %s" % target)
else:
return "Couldn't create upload directory: %s" % target
replfn = os.path.join(dir, lfn)
destination = os.path.join(target, lfn)
upload.save(destination)
if os.path.isfile(destination):
# Check file existence in catalog
adler = adler32(destination)
md5 = md5sum(destination)
size = fsize(destination)
file_id = ddm_checkifexists(lfn, size, adler, md5)
if file_id:
# If file exists
file = files_.get(file_id)
else:
# Otherwise create new
file = File()
file.scope = scope
file.guid = guid
file.type = 'input'
file.lfn = lfn
file.token = ''
file.status = 'defined'
files_.save(file)
setFileMeta(file.id, destination)
replica = Replica()
replica.se = site.se
replica.status = 'ready'
replica.lfn = replfn
replica.original = file
replicas_.save(replica)
# Register file in container
fc.reg_file_in_cont(file, container, 'input')
else:
return ajax_response(False, "Couldn't save file: %s" % target)
if is_ajax:
return ajax_response(True, cguid)
else:
return redirect(url_for("jobs.jobs"))
def linkReplica(replicaid, dir):
replica = replicas_.get(replicaid)
site = sites_.first(se=replica.se)
lfn = replica.lfn
linkdata(site.plugin, {}, lfn, dir)
return 0
def registerLocalFile(arg, dirname, names, scope):
"""Register files from local dir to container
:param arg: Container guid
:param dirname: Abs dir
:param names: File name
:param scope: Scope to upload files in
:return:
"""
site = sites_.first(se=client_config.DEFAULT_SE)
_logger.debug(str(arg))
cont = conts_.first(guid=arg)
files = cont.files
for name in names:
fpath = os.path.join(dirname, name)
fobj = None
# Check in container
for file in files:
if file.lfn == name:
fobj = file
# Check in catalog
if not fobj:
destination = os.path.join(dirname, name)
adler = adler32(destination)
md5 = md5sum(destination)
size = fsize(destination)
file_id = ddm_checkifexists(name, size, adler, md5)
if file_id:
# If file exists
fobj = files_.get(file_id)
if not fobj:
fobj = File()
fobj.scope = scope
fobj.lfn = name
fobj.guid = getGUID(fobj.scope, fobj.lfn)
fobj.type = 'input'
fobj.status = 'defined'
files_.save(fobj)
setFileMeta(fobj.id, fpath)
# Register file in catalog
fc.reg_file_in_cont(fobj, cont, "input")
replicas = fobj.replicas
replica = None
for r in replicas:
if r.se == site.se and r.status == 'ready':
replica = r
if not replica:
ldir = '/' + os.path.join('system', fobj.scope, fobj.guid)
ddm_localmakedirs(ldir)
ddm_localcp(fpath[len(site.datadir):], ldir)
replica = Replica()
replica.se = site.se
replica.status = 'ready'
replica.token = ''
replica.lfn = os.path.join(ldir, fobj.lfn)
replica.original = fobj
replicas_.save(replica)