def link_file(container_guid, lfn):
"""
GET: /pilot/file/<container_guid>/<lfn>/link
Returns ftp link to file
:param container_guid: Guid of container
:type container_guid: str
:param lfn: Local FileName
:type lfn: str
:return: id/guid/ftp
:rtype: json
"""
if ':' in container_guid:
container_guid = container_guid.split(':')[-1]
container = conts_.first(guid=container_guid)
site = sites_.first(se=current_app.config['DEFAULT_SE'])
cc = container.files
for c in cc:
f = c.file
if f.lfn == lfn:
replicas = f.replicas
for r in replicas:
if r.se == site.se and r.status == 'ready':
data = {}
data['lfn'] = f.lfn
data['guid'] = f.guid
data['ftp'] = getFtpLink(r.lfn)
return make_response(jsonify(data), 200)
raise WebpandaError('File not found')
def file_info(container_guid, lfn):
"""
GET: /pilot/file/<container_guid>/<lfn>/info
Returns file metadata
:param container_guid: Guid of container
:type container_guid: str
:param lfn: Local FileName
:type lfn: str
:return: lfn/guid/modtime/fsize/adler32/md5sum
:rtype: json
"""
if ':' in container_guid:
container_guid = container_guid.split(':')[-1]
container = conts_.first(guid=container_guid)
cc = container.files
for c in cc:
f = c.file
if f.lfn == lfn:
data = {}
data['lfn'] = f.lfn
data['guid'] = f.guid
data['modification_time'] = str(f.modification_time)
data['fsize'] = f.fsize
data['adler32'] = f.checksum
data['md5sum'] = f.md5sum
return make_response(jsonify(data), 200)
raise WebpandaError('File not found')
def file_info(container_guid, lfn):
"""
GET: /pilot/file/<container_guid>/<lfn>/info
Returns file metadata
:param container_guid: Guid of container
:type container_guid: str
:param lfn: Local FileName
:type lfn: str
:return: lfn/guid/modtime/fsize/adler32/md5sum
:rtype: json
"""
if ':' in container_guid:
container_guid = container_guid.split(':')[-1]
container = conts_.first(guid=container_guid)
cc = container.files
for c in cc:
f = c.file
if f.lfn == lfn:
data = {}
data['lfn'] = f.lfn
data['guid'] = f.guid
data['modification_time'] = str(f.modification_time)
data['fsize'] = f.fsize
data['adler32'] = f.checksum
data['md5sum'] = f.md5sum
return make_response(jsonify(data), 200)
raise WebpandaError('File not found')
def link_file(container_guid, lfn):
"""
GET: /pilot/file/<container_guid>/<lfn>/link
Returns ftp link to file
:param container_guid: Guid of container
:type container_guid: str
:param lfn: Local FileName
:type lfn: str
:return: id/guid/ftp
:rtype: json
"""
if ':' in container_guid:
container_guid = container_guid.split(':')[-1]
container = conts_.first(guid=container_guid)
site = sites_.first(se=current_app.config['DEFAULT_SE'])
cc = container.files
for c in cc:
f = c.file
if f.lfn == lfn:
replicas = f.replicas
for r in replicas:
if r.se == site.se and r.status == 'ready':
data = {}
data['lfn'] = f.lfn
data['guid'] = f.guid
data['ftp'] = getFtpLink(r.lfn)
return make_response(jsonify(data), 200)
raise WebpandaError('File not found')
def file_fetch(container_guid, lfn):
"""
GET: /pilot/file/<container_guid>/<lfn>/fetch
Returns file in response
:param container_guid: Guid of container
:type container_guid: str
:param lfn: Local FileName
:type lfn: str
:return: File
:rtype: application/octet-stream
"""
if ':' in container_guid:
container_guid = container_guid.split(':')[-1]
container = conts_.first(guid=container_guid)
cc = container.files
for c in cc:
f = c.file
if f.lfn == lfn:
replicas = f.replicas
for replica in replicas:
if replica.se == current_app.config['DEFAULT_SE']:
fullpath = current_app.config['DATA_PATH'] + replica.lfn
f = open(fullpath, 'r')
rr = Response(f.read(), status=200, content_type='application/octet-stream')
rr.headers['Content-Disposition'] = 'inline; filename="%s"' % f.lfn
rr.headers['Content-MD5'] = f.md5sum
f.downloaded += 1
files_.save(f)
return rr
raise WebpandaError('File not found')
def cont_open(guid):
"""
POST: /pilot/container/<guid>/open
Changes container status to 'open'.
:param guid: Container guid
:type guid: str
"""
cont = conts_.first(guid=guid)
if cont is None:
raise WebpandaError("Container not found")
cont.status = 'open'
conts_.save(cont)
return {'response': 'Container status: open'}
def cont_open(guid):
"""
POST: /pilot/container/<guid>/open
Changes container status to 'open'.
:param guid: Container guid
:type guid: str
"""
cont = conts_.first(guid=guid)
if cont is None:
raise WebpandaError("Container not found")
cont.status = 'open'
conts_.save(cont)
return {'response': 'Container status: open'}
def new_replica(container_guid, lfn, se):
"""
POST: /pilot/file/<container_guid>/<lfn>/makereplica/<se>
Creates task to make new file replica
:param container_guid: Guid of container
:type container_guid: str
:param lfn: Local FileName
:type lfn: str
:param se: SE codename
:type se: str
:return: Id of task
:rtype: json
"""
nsite = sites_.find(se=se).count()
if nsite == 0:
return make_response(jsonify({'error': 'SE not found'}), 400)
if ':' in container_guid:
container_guid = container_guid.split(':')[-1]
container = conts_.first(guid=container_guid)
cc = container.files
for c in cc:
f = c.file
if f.lfn == lfn:
rep_num = f.replicas.count()
replicas = f.replicas
if rep_num == 0:
raise WebpandaError('No replicas available')
ready_replica = None
for r in replicas:
if r.se == se:
return {'status': r.status}
if r.se == current_app.config[
'DEFAULT_SE']: # and r.status == 'ready'
ready_replica = r
if ready_replica is None:
ready_replica = replicas[0]
task = async_cloneReplica.delay(ready_replica.id, se)
return {'task_id': task.id}
raise WebpandaError('File not found')
def uploadContainer(ftp_dir, scope, cont_guid):
container = conts_.first(guid=cont_guid)
old_status = container.status
if old_status != 'open':
raise Exception("Invalid container status (not open): {}".format(old_status))
# Set uploading container status
container.status = 'uploading'
conts_.save(container)
# Register ftp filesr
register_ftp_files(ftp_dir, scope, container.guid)
# Set open container status
container.status = old_status
conts_.save(container)
return 0
def new_replica(container_guid, lfn, se):
"""
POST: /pilot/file/<container_guid>/<lfn>/makereplica/<se>
Creates task to make new file replica
:param container_guid: Guid of container
:type container_guid: str
:param lfn: Local FileName
:type lfn: str
:param se: SE codename
:type se: str
:return: Id of task
:rtype: json
"""
nsite = sites_.find(se=se).count()
if nsite == 0:
return make_response(jsonify({'error': 'SE not found'}), 400)
if ':' in container_guid:
container_guid = container_guid.split(':')[-1]
container = conts_.first(guid=container_guid)
cc = container.files
for c in cc:
f = c.file
if f.lfn == lfn:
rep_num = f.replicas.count()
replicas = f.replicas
if rep_num == 0:
raise WebpandaError('No replicas available')
ready_replica = None
for r in replicas:
if r.se == se:
return {'status': r.status}
if r.se == current_app.config['DEFAULT_SE']: # and r.status == 'ready'
ready_replica = r
if ready_replica is None:
ready_replica = replicas[0]
task = async_cloneReplica.delay(ready_replica.id, se)
return {'task_id': task.id}
raise WebpandaError('File not found')
def container_list(guid):
"""Returns list of files registered in container"""
g.user = request.oauth.user
cont = conts_.first(guid=guid)
files = cont.files
datalist = []
for file in files:
data = {}
data['lfn'] = file.lfn
data['guid'] = file.guid
data['modification_time'] = str(file.modification_time)
data['fsize'] = file.fsize
data['adler32'] = file.checksum
data['md5sum'] = file.md5sum
data['scope'] = file.scope
datalist.append(data)
return datalist
def uploadContainer(ftp_dir, scope, cont_guid):
container = conts_.first(guid=cont_guid)
old_status = container.status
if old_status != 'open':
raise Exception(
"Invalid container status (not open): {}".format(old_status))
# Set uploading container status
container.status = 'uploading'
conts_.save(container)
# Register ftp filesr
register_ftp_files(ftp_dir, scope, container.guid)
# Set open container status
container.status = old_status
conts_.save(container)
return 0
def container_list(guid):
"""Returns list of files registered in container"""
g.user = request.oauth.user
cont = conts_.first(guid=guid)
files = cont.files
datalist = []
for file in files:
data = {}
data['lfn'] = file.lfn
data['guid'] = file.guid
data['modification_time'] = str(file.modification_time)
data['fsize'] = file.fsize
data['adler32'] = file.checksum
data['md5sum'] = file.md5sum
data['scope'] = file.scope
datalist.append(data)
return datalist
def cont_close(guid):
"""
POST: /pilot/container/<guid>/close
Changes container status to 'close'
:param guid: Container guid
:type guid: str
"""
cont = conts_.first(guid=guid)
if cont is None:
raise WebpandaError("Container not found")
path = os.path.join(current_app.config['UPLOAD_FOLDER'], getScope(g.user.username), cont.guid)
os.path.walk(path, registerLocalFile, cont.guid)
cont.status = 'close'
conts_.save(cont)
return {'response': 'Container status: close'}
def cont_info(guid):
"""
POST: /pilot/container/<guid>/info
Returns container metadata
:param guid: Container guid
:type guid: str
:return: id/guid/status/nfiles
:rtype: json
"""
cont = conts_.first(guid=guid)
if cont is None:
raise WebpandaError("Container not found")
data = dict()
data['id'] = cont.id
data['guid'] = cont.guid
data['status'] = cont.status
data['nfiles'] = len(cont.files)
return data
def cont_close(guid):
"""
POST: /pilot/container/<guid>/close
Changes container status to 'close'
:param guid: Container guid
:type guid: str
"""
cont = conts_.first(guid=guid)
if cont is None:
raise WebpandaError("Container not found")
path = os.path.join(current_app.config['UPLOAD_FOLDER'],
getScope(g.user.username), cont.guid)
os.path.walk(path, registerLocalFile, cont.guid)
cont.status = 'close'
conts_.save(cont)
return {'response': 'Container status: close'}
def cont_info(guid):
"""
POST: /pilot/container/<guid>/info
Returns container metadata
:param guid: Container guid
:type guid: str
:return: id/guid/status/nfiles
:rtype: json
"""
cont = conts_.first(guid=guid)
if cont is None:
raise WebpandaError("Container not found")
data = dict()
data['id'] = cont.id
data['guid'] = cont.guid
data['status'] = cont.status
data['nfiles'] = len(cont.files)
return data
def new_pipeline_from_cont():
form = RunForm(request.form)
if request.method == 'POST':
icont = conts_.first(guid=form.guid.data)
if icont is None:
raise WebpandaError("Container not found")
current_user = g.user
# Prepare pipeline
pp = Pipeline()
pp.status = 'running'
pp.type_id = pipeline_types_.get(1).id
pp.owner_id = current_user.id
pipelines_.save(pp)
# Prepare container
pp_cont = Container()
pp_cont.guid = 'pipeline.' + commands.getoutput('uuidgen')
conts_.save(pp_cont)
# Add guids to container
for item in icont.files:
f = item.file
# Register file in catalog
fc.reg_file_in_cont(f, pp_cont, 'input')
# Set current task
start_task = pclient.get_start_task(pp)
start_task.input = pp_cont.id
start_task.output = pp_cont.id
tasks_.save(start_task)
return redirect(url_for('pipelines.list_all'))
return render_template('dashboard/pp/new.html', form=form)
def stage_in(container_guid, lfn):
"""
POST: /pilot/file/<container_guid>/<lfn>/copy
Creates task to copy file in path on se
:param container_guid: Guid of container
:type container_guid: str
:param lfn: Local FileName
:type lfn: str
:return: Id of task
:rtype: json
"""
args = request.form
if not ('to_se' in args.keys() and 'to_path' in args.keys()):
raise WebpandaError('Please specify correct request params')
to_se = args.get('to_se', type=str)
to_path = args.get('to_path', type=str)
if ':' in container_guid:
container_guid = container_guid.split(':')[-1]
container = conts_.first(guid=container_guid)
cc = container.files
for c in cc:
f = c.file
if f.lfn == lfn:
replicas = f.replicas
for r in replicas:
if r.status == 'ready':
task = async_copyReplica.delay(r.id, to_se, to_path)
return {'task_id': task.id}
raise WebpandaError('No replicas available')
raise WebpandaError('File not found')
def stage_in(container_guid, lfn):
"""
POST: /pilot/file/<container_guid>/<lfn>/copy
Creates task to copy file in path on se
:param container_guid: Guid of container
:type container_guid: str
:param lfn: Local FileName
:type lfn: str
:return: Id of task
:rtype: json
"""
args = request.form
if not ('to_se' in args.keys() and 'to_path' in args.keys()):
raise WebpandaError('Please specify correct request params')
to_se = args.get('to_se', type=str)
to_path = args.get('to_path', type=str)
if ':' in container_guid:
container_guid = container_guid.split(':')[-1]
container = conts_.first(guid=container_guid)
cc = container.files
for c in cc:
f = c.file
if f.lfn == lfn:
replicas = f.replicas
for r in replicas:
if r.status == 'ready':
task = async_copyReplica.delay(r.id, to_se, to_path)
return {'task_id': task.id}
raise WebpandaError('No replicas available')
raise WebpandaError('File not found')
def file_fetch(container_guid, lfn):
"""
GET: /pilot/file/<container_guid>/<lfn>/fetch
Returns file in response
:param container_guid: Guid of container
:type container_guid: str
:param lfn: Local FileName
:type lfn: str
:return: File
:rtype: application/octet-stream
"""
if ':' in container_guid:
container_guid = container_guid.split(':')[-1]
container = conts_.first(guid=container_guid)
cc = container.files
for c in cc:
f = c.file
if f.lfn == lfn:
replicas = f.replicas
for replica in replicas:
if replica.se == current_app.config['DEFAULT_SE']:
fullpath = current_app.config['DATA_PATH'] + replica.lfn
f = open(fullpath, 'r')
rr = Response(f.read(),
status=200,
content_type='application/octet-stream')
rr.headers[
'Content-Disposition'] = 'inline; filename="%s"' % f.lfn
rr.headers['Content-MD5'] = f.md5sum
f.downloaded += 1
files_.save(f)
return rr
raise WebpandaError('File not found')
def job():
"""
New job form view
:return: Response obj
"""
form = NewJobForm(request.form)
if request.method == 'POST':
site = sites_.get(int(form.site.data))
distr_name, distr_release = form.distr.data.split(':')
distr = distrs_.first(name=distr_name, release=int(distr_release))
container_guid = form.container.data
try:
container = conts_.first(guid=container_guid)
except(Exception):
_logger.error(Exception.message)
return make_response(jsonify({'error': 'Container not found'}), 404)
if site.encode_commands:
# By default frontend encodes with base64 job script parts separated by ";"
# It requires script wrapper on cluster side
jparams = form.params.data
else:
# Set site.encode_commands as False if you want to send command string without base64 encoding
jparams = ';'.join([b64decode(command) for command in form.params.data.split(';')])
ifiles = request.form.getlist('ifiles[]')
iguids = request.form.getlist('iguids[]')
iconts = request.form.getlist('iconts[]')
ofiles = ['{guid}.out.tgz'.format(guid=container.guid)]
scope = getScope(g.user.username)
# Process ftp files
ftp_dir = form.ftpdir.data
register_ftp_files(ftp_dir, scope, container.guid)
# Process guid list
for f in iguids:
if f != '':
file = files_.first(guid=f)
if file is not None:
# Register files in container
fc.reg_file_in_cont(file, container, 'input')
else:
return make_response(jsonify({'error': "GUID {} not found".format(f)}))
# Process containers
for c in iconts:
if c != '':
try:
form_cont = conts_.first(guid=c)
except(Exception):
_logger.error(Exception.message)
return make_response(jsonify({'error': 'Container in form not found'}), 404)
for f in form_cont.files:
# Register file in catalog
fc.reg_file_in_cont(f.file, container, 'input')
# Processes urls
for f in ifiles:
if f != '':
from_se, path, token = getUrlInfo(f)
replfn = ':/'.join([from_se, path])
# Check if used before
file_id = ddm_checkexternalifexists('', replfn)
if file_id:
file = files_.get(file_id)
else:
lfn = path.split('/')[-1]
guid = getGUID(scope, lfn)
file = File()
file.scope = scope
file.guid = guid
file.type = 'input'
file.lfn = lfn
file.status = 'defined'
files_.save(file)
replica = Replica()
replica.se = from_se
replica.status = 'link'
# Separate url & token
replica.lfn = replfn
replica.token = token
replica.original = file
replicas_.save(replica)
# Register file in container
fc.reg_file_in_cont(file, container, 'input')
# Starts cloneReplica tasks
ftasks = prepareInputFiles(container.id, site.se)
# Saves output files meta
for lfn in ofiles:
file = File()
file.scope = scope
file.guid = getGUID(scope, lfn)
file.type = 'output'
file.lfn = lfn
file.status = 'defined'
files_.save(file)
# Register file in container
fc.reg_file_in_cont(file, container, 'output')
# Counts files
allfiles = container.files
nifiles = 0
nofiles = 0
for f in allfiles:
if f.type == 'input':
nifiles += 1
if f.type == 'output':
nofiles += 1
# Defines job meta
job = Job()
job.pandaid = None
job.status = 'pending'
job.owner = g.user
job.params = jparams
job.distr = distr
job.container = container
job.creation_time = datetime.utcnow()
job.modification_time = datetime.utcnow()
job.ninputfiles = nifiles
job.noutputfiles = nofiles
job.corecount = form.corecount.data
job.tags = form.tags.data if form.tags.data != "" else None
jobs_.save(job)
# Async sendjob
res = chord(ftasks)(async_send_job.s(jobid=job.id, siteid=site.id))
return redirect(url_for('jobs.jobs'))
form.distr.choices = [("%s:%s" % (distr.name, distr.release), "%s: %s" % (distr.name, distr.version)) for distr in distrs_.find().order_by('name').order_by('version')]
form.site.choices = [(site.id, "{ce}".format(ce=site.ce)) for site in sites_.find(active=1).order_by('ce')]
return render_template("dashboard/jobs/new.html", form=form)
def file_save(container_guid, lfn):
"""
POST: /pilot/file/<container_guid>/<lfn>/save
Saves file from request, returns file guid
:param container_guid: Guid of container
:type container_guid: str
:param lfn: Local FileName
:type lfn: str
:return: guid
:rtype: json
"""
site = sites_.first(se=current_app.config['DEFAULT_SE'])
if ':' in container_guid:
container_guid = container_guid.split(':')[-1]
container = conts_.first(guid=container_guid)
if container.status != 'open':
raise WebpandaError('Unable to upload: Container is not open')
cc = container.files
ff = None
for c in cc:
f = c.file
if f.lfn == lfn:
ff = f
if not ff:
ff = File()
ff.scope = getScope(g.user.username)
ff.lfn = lfn
ff.guid = getGUID(ff.scope, ff.lfn)
ff.status = 'defined'
files_.save(ff)
# Register file in container
fc.reg_file_in_cont(ff, container, 'input')
path = os.path.join(site.datadir, getScope(g.user.username), container.guid)
replfn = '/' + os.path.join(getScope(g.user.username), container.guid, ff.lfn)
destination = os.path.join(path, ff.lfn)
for r in ff.replicas:
if r.se == site.se:
destination = site.datadir + r.lfn
file_dir = '/'.join(destination.split('/')[:-1])
if r.status == 'ready':
if os.path.isfile(destination): # Check fsize, md5 or adler
raise WebpandaError('Replica exists')
else:
r.status = 'broken'
replicas_.save(r)
raise WebpandaError('Broken replica')
elif r.status == 'defined':
try:
os.makedirs(file_dir)
except(Exception):
pass
f = open(destination, 'wb')
f.write(request.data)
f.close()
# Update file info
setFileMeta(ff.id, destination)
r.status = 'ready'
replicas_.save(r)
return {'guid': ff.guid}
else:
raise WebpandaError('Replica status: %s' % r.status)
replica = Replica()
if os.path.isfile(destination):
raise WebpandaError('Unable to upload: File exists')
try:
os.makedirs(path)
except(Exception):
_logger.debug('Path exists: %s' % path)
f = open(destination, 'wb')
f.write(request.data)
f.close()
# Update file info
setFileMeta(ff.id, destination)
# Create/change replica
replica.se = site.se
replica.status = 'ready'
replica.lfn = replfn
replica.token = ''
replica.original = ff
replicas_.save(replica)
return {'guid': ff.guid}
def registerLocalFile(arg, dirname, names, scope):
"""Register files from local dir to container
:param arg: Container guid
:param dirname: Abs dir
:param names: File name
:param scope: Scope to upload files in
:return:
"""
site = sites_.first(se=client_config.DEFAULT_SE)
_logger.debug(str(arg))
cont = conts_.first(guid=arg)
files = cont.files
for name in names:
fpath = os.path.join(dirname, name)
fobj = None
# Check in container
for file in files:
if file.lfn == name:
fobj = file
# Check in catalog
if not fobj:
destination = os.path.join(dirname, name)
adler = adler32(destination)
md5 = md5sum(destination)
size = fsize(destination)
file_id = ddm_checkifexists(name, size, adler, md5)
if file_id:
# If file exists
fobj = files_.get(file_id)
if not fobj:
fobj = File()
fobj.scope = scope
fobj.lfn = name
fobj.guid = getGUID(fobj.scope, fobj.lfn)
fobj.type = 'input'
fobj.status = 'defined'
files_.save(fobj)
setFileMeta(fobj.id, fpath)
# Register file in catalog
fc.reg_file_in_cont(fobj, cont, "input")
replicas = fobj.replicas
replica = None
for r in replicas:
if r.se == site.se and r.status == 'ready':
replica = r
if not replica:
ldir = '/' + os.path.join('system', fobj.scope, fobj.guid)
ddm_localmakedirs(ldir)
ddm_localcp(fpath[len(site.datadir):], ldir)
replica = Replica()
replica.se = site.se
replica.status = 'ready'
replica.token = ''
replica.lfn = os.path.join(ldir, fobj.lfn)
replica.original = fobj
replicas_.save(replica)
def file_save(container_guid, lfn):
"""
POST: /pilot/file/<container_guid>/<lfn>/save
Saves file from request, returns file guid
:param container_guid: Guid of container
:type container_guid: str
:param lfn: Local FileName
:type lfn: str
:return: guid
:rtype: json
"""
site = sites_.first(se=current_app.config['DEFAULT_SE'])
if ':' in container_guid:
container_guid = container_guid.split(':')[-1]
container = conts_.first(guid=container_guid)
if container.status != 'open':
raise WebpandaError('Unable to upload: Container is not open')
cc = container.files
ff = None
for c in cc:
f = c.file
if f.lfn == lfn:
ff = f
if not ff:
ff = File()
ff.scope = getScope(g.user.username)
ff.lfn = lfn
ff.guid = getGUID(ff.scope, ff.lfn)
ff.status = 'defined'
files_.save(ff)
# Register file in container
fc.reg_file_in_cont(ff, container, 'input')
path = os.path.join(site.datadir, getScope(g.user.username),
container.guid)
replfn = '/' + os.path.join(getScope(g.user.username), container.guid,
ff.lfn)
destination = os.path.join(path, ff.lfn)
for r in ff.replicas:
if r.se == site.se:
destination = site.datadir + r.lfn
file_dir = '/'.join(destination.split('/')[:-1])
if r.status == 'ready':
if os.path.isfile(destination): # Check fsize, md5 or adler
raise WebpandaError('Replica exists')
else:
r.status = 'broken'
replicas_.save(r)
raise WebpandaError('Broken replica')
elif r.status == 'defined':
try:
os.makedirs(file_dir)
except (Exception):
pass
f = open(destination, 'wb')
f.write(request.data)
f.close()
# Update file info
setFileMeta(ff.id, destination)
r.status = 'ready'
replicas_.save(r)
return {'guid': ff.guid}
else:
raise WebpandaError('Replica status: %s' % r.status)
replica = Replica()
if os.path.isfile(destination):
raise WebpandaError('Unable to upload: File exists')
try:
os.makedirs(path)
except (Exception):
_logger.debug('Path exists: %s' % path)
f = open(destination, 'wb')
f.write(request.data)
f.close()
# Update file info
setFileMeta(ff.id, destination)
# Create/change replica
replica.se = site.se
replica.status = 'ready'
replica.lfn = replfn
replica.token = ''
replica.original = ff
replicas_.save(replica)
return {'guid': ff.guid}
def registerLocalFile(arg, dirname, names, scope):
"""Register files from local dir to container
:param arg: Container guid
:param dirname: Abs dir
:param names: File name
:param scope: Scope to upload files in
:return:
"""
site = sites_.first(se=client_config.DEFAULT_SE)
_logger.debug(str(arg))
cont = conts_.first(guid=arg)
files = cont.files
for name in names:
fpath = os.path.join(dirname, name)
fobj = None
# Check in container
for file in files:
if file.lfn == name:
fobj = file
# Check in catalog
if not fobj:
destination = os.path.join(dirname, name)
adler = adler32(destination)
md5 = md5sum(destination)
size = fsize(destination)
file_id = ddm_checkifexists(name, size, adler, md5)
if file_id:
# If file exists
fobj = files_.get(file_id)
if not fobj:
fobj = File()
fobj.scope = scope
fobj.lfn = name
fobj.guid = getGUID(fobj.scope, fobj.lfn)
fobj.type = 'input'
fobj.status = 'defined'
files_.save(fobj)
setFileMeta(fobj.id, fpath)
# Register file in catalog
fc.reg_file_in_cont(fobj, cont, "input")
replicas = fobj.replicas
replica = None
for r in replicas:
if r.se == site.se and r.status == 'ready':
replica = r
if not replica:
ldir = '/' + os.path.join('system', fobj.scope, fobj.guid)
ddm_localmakedirs(ldir)
ddm_localcp(fpath[len(site.datadir):], ldir)
replica = Replica()
replica.se = site.se
replica.status = 'ready'
replica.token = ''
replica.lfn = os.path.join(ldir, fobj.lfn)
replica.original = fobj
replicas_.save(replica)
