def link_file(container_guid, lfn): """ GET: /pilot/file/<container_guid>/<lfn>/link Returns ftp link to file :param container_guid: Guid of container :type container_guid: str :param lfn: Local FileName :type lfn: str :return: id/guid/ftp :rtype: json """ if ':' in container_guid: container_guid = container_guid.split(':')[-1] container = conts_.first(guid=container_guid) site = sites_.first(se=current_app.config['DEFAULT_SE']) cc = container.files for c in cc: f = c.file if f.lfn == lfn: replicas = f.replicas for r in replicas: if r.se == site.se and r.status == 'ready': data = {} data['lfn'] = f.lfn data['guid'] = f.guid data['ftp'] = getFtpLink(r.lfn) return make_response(jsonify(data), 200) raise WebpandaError('File not found')
def file_info(container_guid, lfn): """ GET: /pilot/file/<container_guid>/<lfn>/info Returns file metadata :param container_guid: Guid of container :type container_guid: str :param lfn: Local FileName :type lfn: str :return: lfn/guid/modtime/fsize/adler32/md5sum :rtype: json """ if ':' in container_guid: container_guid = container_guid.split(':')[-1] container = conts_.first(guid=container_guid) cc = container.files for c in cc: f = c.file if f.lfn == lfn: data = {} data['lfn'] = f.lfn data['guid'] = f.guid data['modification_time'] = str(f.modification_time) data['fsize'] = f.fsize data['adler32'] = f.checksum data['md5sum'] = f.md5sum return make_response(jsonify(data), 200) raise WebpandaError('File not found')
def file_fetch(container_guid, lfn): """ GET: /pilot/file/<container_guid>/<lfn>/fetch Returns file in response :param container_guid: Guid of container :type container_guid: str :param lfn: Local FileName :type lfn: str :return: File :rtype: application/octet-stream """ if ':' in container_guid: container_guid = container_guid.split(':')[-1] container = conts_.first(guid=container_guid) cc = container.files for c in cc: f = c.file if f.lfn == lfn: replicas = f.replicas for replica in replicas: if replica.se == current_app.config['DEFAULT_SE']: fullpath = current_app.config['DATA_PATH'] + replica.lfn f = open(fullpath, 'r') rr = Response(f.read(), status=200, content_type='application/octet-stream') rr.headers['Content-Disposition'] = 'inline; filename="%s"' % f.lfn rr.headers['Content-MD5'] = f.md5sum f.downloaded += 1 files_.save(f) return rr raise WebpandaError('File not found')
def cont_open(guid): """ POST: /pilot/container/<guid>/open Changes container status to 'open'. :param guid: Container guid :type guid: str """ cont = conts_.first(guid=guid) if cont is None: raise WebpandaError("Container not found") cont.status = 'open' conts_.save(cont) return {'response': 'Container status: open'}
def new_replica(container_guid, lfn, se): """ POST: /pilot/file/<container_guid>/<lfn>/makereplica/<se> Creates task to make new file replica :param container_guid: Guid of container :type container_guid: str :param lfn: Local FileName :type lfn: str :param se: SE codename :type se: str :return: Id of task :rtype: json """ nsite = sites_.find(se=se).count() if nsite == 0: return make_response(jsonify({'error': 'SE not found'}), 400) if ':' in container_guid: container_guid = container_guid.split(':')[-1] container = conts_.first(guid=container_guid) cc = container.files for c in cc: f = c.file if f.lfn == lfn: rep_num = f.replicas.count() replicas = f.replicas if rep_num == 0: raise WebpandaError('No replicas available') ready_replica = None for r in replicas: if r.se == se: return {'status': r.status} if r.se == current_app.config[ 'DEFAULT_SE']: # and r.status == 'ready' ready_replica = r if ready_replica is None: ready_replica = replicas[0] task = async_cloneReplica.delay(ready_replica.id, se) return {'task_id': task.id} raise WebpandaError('File not found')
def uploadContainer(ftp_dir, scope, cont_guid): container = conts_.first(guid=cont_guid) old_status = container.status if old_status != 'open': raise Exception("Invalid container status (not open): {}".format(old_status)) # Set uploading container status container.status = 'uploading' conts_.save(container) # Register ftp filesr register_ftp_files(ftp_dir, scope, container.guid) # Set open container status container.status = old_status conts_.save(container) return 0
def new_replica(container_guid, lfn, se): """ POST: /pilot/file/<container_guid>/<lfn>/makereplica/<se> Creates task to make new file replica :param container_guid: Guid of container :type container_guid: str :param lfn: Local FileName :type lfn: str :param se: SE codename :type se: str :return: Id of task :rtype: json """ nsite = sites_.find(se=se).count() if nsite == 0: return make_response(jsonify({'error': 'SE not found'}), 400) if ':' in container_guid: container_guid = container_guid.split(':')[-1] container = conts_.first(guid=container_guid) cc = container.files for c in cc: f = c.file if f.lfn == lfn: rep_num = f.replicas.count() replicas = f.replicas if rep_num == 0: raise WebpandaError('No replicas available') ready_replica = None for r in replicas: if r.se == se: return {'status': r.status} if r.se == current_app.config['DEFAULT_SE']: # and r.status == 'ready' ready_replica = r if ready_replica is None: ready_replica = replicas[0] task = async_cloneReplica.delay(ready_replica.id, se) return {'task_id': task.id} raise WebpandaError('File not found')
def container_list(guid): """Returns list of files registered in container""" g.user = request.oauth.user cont = conts_.first(guid=guid) files = cont.files datalist = [] for file in files: data = {} data['lfn'] = file.lfn data['guid'] = file.guid data['modification_time'] = str(file.modification_time) data['fsize'] = file.fsize data['adler32'] = file.checksum data['md5sum'] = file.md5sum data['scope'] = file.scope datalist.append(data) return datalist
def uploadContainer(ftp_dir, scope, cont_guid): container = conts_.first(guid=cont_guid) old_status = container.status if old_status != 'open': raise Exception( "Invalid container status (not open): {}".format(old_status)) # Set uploading container status container.status = 'uploading' conts_.save(container) # Register ftp filesr register_ftp_files(ftp_dir, scope, container.guid) # Set open container status container.status = old_status conts_.save(container) return 0
def cont_close(guid): """ POST: /pilot/container/<guid>/close Changes container status to 'close' :param guid: Container guid :type guid: str """ cont = conts_.first(guid=guid) if cont is None: raise WebpandaError("Container not found") path = os.path.join(current_app.config['UPLOAD_FOLDER'], getScope(g.user.username), cont.guid) os.path.walk(path, registerLocalFile, cont.guid) cont.status = 'close' conts_.save(cont) return {'response': 'Container status: close'}
def cont_info(guid): """ POST: /pilot/container/<guid>/info Returns container metadata :param guid: Container guid :type guid: str :return: id/guid/status/nfiles :rtype: json """ cont = conts_.first(guid=guid) if cont is None: raise WebpandaError("Container not found") data = dict() data['id'] = cont.id data['guid'] = cont.guid data['status'] = cont.status data['nfiles'] = len(cont.files) return data
def new_pipeline_from_cont(): form = RunForm(request.form) if request.method == 'POST': icont = conts_.first(guid=form.guid.data) if icont is None: raise WebpandaError("Container not found") current_user = g.user # Prepare pipeline pp = Pipeline() pp.status = 'running' pp.type_id = pipeline_types_.get(1).id pp.owner_id = current_user.id pipelines_.save(pp) # Prepare container pp_cont = Container() pp_cont.guid = 'pipeline.' + commands.getoutput('uuidgen') conts_.save(pp_cont) # Add guids to container for item in icont.files: f = item.file # Register file in catalog fc.reg_file_in_cont(f, pp_cont, 'input') # Set current task start_task = pclient.get_start_task(pp) start_task.input = pp_cont.id start_task.output = pp_cont.id tasks_.save(start_task) return redirect(url_for('pipelines.list_all')) return render_template('dashboard/pp/new.html', form=form)
def stage_in(container_guid, lfn): """ POST: /pilot/file/<container_guid>/<lfn>/copy Creates task to copy file in path on se :param container_guid: Guid of container :type container_guid: str :param lfn: Local FileName :type lfn: str :return: Id of task :rtype: json """ args = request.form if not ('to_se' in args.keys() and 'to_path' in args.keys()): raise WebpandaError('Please specify correct request params') to_se = args.get('to_se', type=str) to_path = args.get('to_path', type=str) if ':' in container_guid: container_guid = container_guid.split(':')[-1] container = conts_.first(guid=container_guid) cc = container.files for c in cc: f = c.file if f.lfn == lfn: replicas = f.replicas for r in replicas: if r.status == 'ready': task = async_copyReplica.delay(r.id, to_se, to_path) return {'task_id': task.id} raise WebpandaError('No replicas available') raise WebpandaError('File not found')
def file_fetch(container_guid, lfn): """ GET: /pilot/file/<container_guid>/<lfn>/fetch Returns file in response :param container_guid: Guid of container :type container_guid: str :param lfn: Local FileName :type lfn: str :return: File :rtype: application/octet-stream """ if ':' in container_guid: container_guid = container_guid.split(':')[-1] container = conts_.first(guid=container_guid) cc = container.files for c in cc: f = c.file if f.lfn == lfn: replicas = f.replicas for replica in replicas: if replica.se == current_app.config['DEFAULT_SE']: fullpath = current_app.config['DATA_PATH'] + replica.lfn f = open(fullpath, 'r') rr = Response(f.read(), status=200, content_type='application/octet-stream') rr.headers[ 'Content-Disposition'] = 'inline; filename="%s"' % f.lfn rr.headers['Content-MD5'] = f.md5sum f.downloaded += 1 files_.save(f) return rr raise WebpandaError('File not found')
def job(): """ New job form view :return: Response obj """ form = NewJobForm(request.form) if request.method == 'POST': site = sites_.get(int(form.site.data)) distr_name, distr_release = form.distr.data.split(':') distr = distrs_.first(name=distr_name, release=int(distr_release)) container_guid = form.container.data try: container = conts_.first(guid=container_guid) except(Exception): _logger.error(Exception.message) return make_response(jsonify({'error': 'Container not found'}), 404) if site.encode_commands: # By default frontend encodes with base64 job script parts separated by ";" # It requires script wrapper on cluster side jparams = form.params.data else: # Set site.encode_commands as False if you want to send command string without base64 encoding jparams = ';'.join([b64decode(command) for command in form.params.data.split(';')]) ifiles = request.form.getlist('ifiles[]') iguids = request.form.getlist('iguids[]') iconts = request.form.getlist('iconts[]') ofiles = ['{guid}.out.tgz'.format(guid=container.guid)] scope = getScope(g.user.username) # Process ftp files ftp_dir = form.ftpdir.data register_ftp_files(ftp_dir, scope, container.guid) # Process guid list for f in iguids: if f != '': file = files_.first(guid=f) if file is not None: # Register files in container fc.reg_file_in_cont(file, container, 'input') else: return make_response(jsonify({'error': "GUID {} not found".format(f)})) # Process containers for c in iconts: if c != '': try: form_cont = conts_.first(guid=c) except(Exception): _logger.error(Exception.message) return make_response(jsonify({'error': 'Container in form not found'}), 404) for f in form_cont.files: # Register file in catalog fc.reg_file_in_cont(f.file, container, 'input') # Processes urls for f in ifiles: if f != '': from_se, path, token = getUrlInfo(f) replfn = ':/'.join([from_se, path]) # Check if used before file_id = ddm_checkexternalifexists('', replfn) if file_id: file = files_.get(file_id) else: lfn = path.split('/')[-1] guid = getGUID(scope, lfn) file = File() file.scope = scope file.guid = guid file.type = 'input' file.lfn = lfn file.status = 'defined' files_.save(file) replica = Replica() replica.se = from_se replica.status = 'link' # Separate url & token replica.lfn = replfn replica.token = token replica.original = file replicas_.save(replica) # Register file in container fc.reg_file_in_cont(file, container, 'input') # Starts cloneReplica tasks ftasks = prepareInputFiles(container.id, site.se) # Saves output files meta for lfn in ofiles: file = File() file.scope = scope file.guid = getGUID(scope, lfn) file.type = 'output' file.lfn = lfn file.status = 'defined' files_.save(file) # Register file in container fc.reg_file_in_cont(file, container, 'output') # Counts files allfiles = container.files nifiles = 0 nofiles = 0 for f in allfiles: if f.type == 'input': nifiles += 1 if f.type == 'output': nofiles += 1 # Defines job meta job = Job() job.pandaid = None job.status = 'pending' job.owner = g.user job.params = jparams job.distr = distr job.container = container job.creation_time = datetime.utcnow() job.modification_time = datetime.utcnow() job.ninputfiles = nifiles job.noutputfiles = nofiles job.corecount = form.corecount.data job.tags = form.tags.data if form.tags.data != "" else None jobs_.save(job) # Async sendjob res = chord(ftasks)(async_send_job.s(jobid=job.id, siteid=site.id)) return redirect(url_for('jobs.jobs')) form.distr.choices = [("%s:%s" % (distr.name, distr.release), "%s: %s" % (distr.name, distr.version)) for distr in distrs_.find().order_by('name').order_by('version')] form.site.choices = [(site.id, "{ce}".format(ce=site.ce)) for site in sites_.find(active=1).order_by('ce')] return render_template("dashboard/jobs/new.html", form=form)
def file_save(container_guid, lfn): """ POST: /pilot/file/<container_guid>/<lfn>/save Saves file from request, returns file guid :param container_guid: Guid of container :type container_guid: str :param lfn: Local FileName :type lfn: str :return: guid :rtype: json """ site = sites_.first(se=current_app.config['DEFAULT_SE']) if ':' in container_guid: container_guid = container_guid.split(':')[-1] container = conts_.first(guid=container_guid) if container.status != 'open': raise WebpandaError('Unable to upload: Container is not open') cc = container.files ff = None for c in cc: f = c.file if f.lfn == lfn: ff = f if not ff: ff = File() ff.scope = getScope(g.user.username) ff.lfn = lfn ff.guid = getGUID(ff.scope, ff.lfn) ff.status = 'defined' files_.save(ff) # Register file in container fc.reg_file_in_cont(ff, container, 'input') path = os.path.join(site.datadir, getScope(g.user.username), container.guid) replfn = '/' + os.path.join(getScope(g.user.username), container.guid, ff.lfn) destination = os.path.join(path, ff.lfn) for r in ff.replicas: if r.se == site.se: destination = site.datadir + r.lfn file_dir = '/'.join(destination.split('/')[:-1]) if r.status == 'ready': if os.path.isfile(destination): # Check fsize, md5 or adler raise WebpandaError('Replica exists') else: r.status = 'broken' replicas_.save(r) raise WebpandaError('Broken replica') elif r.status == 'defined': try: os.makedirs(file_dir) except(Exception): pass f = open(destination, 'wb') f.write(request.data) f.close() # Update file info setFileMeta(ff.id, destination) r.status = 'ready' replicas_.save(r) return {'guid': ff.guid} else: raise WebpandaError('Replica status: %s' % r.status) replica = Replica() if os.path.isfile(destination): raise WebpandaError('Unable to upload: File exists') try: os.makedirs(path) except(Exception): _logger.debug('Path exists: %s' % path) f = open(destination, 'wb') f.write(request.data) f.close() # Update file info setFileMeta(ff.id, destination) # Create/change replica replica.se = site.se replica.status = 'ready' replica.lfn = replfn replica.token = '' replica.original = ff replicas_.save(replica) return {'guid': ff.guid}
def registerLocalFile(arg, dirname, names, scope): """Register files from local dir to container :param arg: Container guid :param dirname: Abs dir :param names: File name :param scope: Scope to upload files in :return: """ site = sites_.first(se=client_config.DEFAULT_SE) _logger.debug(str(arg)) cont = conts_.first(guid=arg) files = cont.files for name in names: fpath = os.path.join(dirname, name) fobj = None # Check in container for file in files: if file.lfn == name: fobj = file # Check in catalog if not fobj: destination = os.path.join(dirname, name) adler = adler32(destination) md5 = md5sum(destination) size = fsize(destination) file_id = ddm_checkifexists(name, size, adler, md5) if file_id: # If file exists fobj = files_.get(file_id) if not fobj: fobj = File() fobj.scope = scope fobj.lfn = name fobj.guid = getGUID(fobj.scope, fobj.lfn) fobj.type = 'input' fobj.status = 'defined' files_.save(fobj) setFileMeta(fobj.id, fpath) # Register file in catalog fc.reg_file_in_cont(fobj, cont, "input") replicas = fobj.replicas replica = None for r in replicas: if r.se == site.se and r.status == 'ready': replica = r if not replica: ldir = '/' + os.path.join('system', fobj.scope, fobj.guid) ddm_localmakedirs(ldir) ddm_localcp(fpath[len(site.datadir):], ldir) replica = Replica() replica.se = site.se replica.status = 'ready' replica.token = '' replica.lfn = os.path.join(ldir, fobj.lfn) replica.original = fobj replicas_.save(replica)
def file_save(container_guid, lfn): """ POST: /pilot/file/<container_guid>/<lfn>/save Saves file from request, returns file guid :param container_guid: Guid of container :type container_guid: str :param lfn: Local FileName :type lfn: str :return: guid :rtype: json """ site = sites_.first(se=current_app.config['DEFAULT_SE']) if ':' in container_guid: container_guid = container_guid.split(':')[-1] container = conts_.first(guid=container_guid) if container.status != 'open': raise WebpandaError('Unable to upload: Container is not open') cc = container.files ff = None for c in cc: f = c.file if f.lfn == lfn: ff = f if not ff: ff = File() ff.scope = getScope(g.user.username) ff.lfn = lfn ff.guid = getGUID(ff.scope, ff.lfn) ff.status = 'defined' files_.save(ff) # Register file in container fc.reg_file_in_cont(ff, container, 'input') path = os.path.join(site.datadir, getScope(g.user.username), container.guid) replfn = '/' + os.path.join(getScope(g.user.username), container.guid, ff.lfn) destination = os.path.join(path, ff.lfn) for r in ff.replicas: if r.se == site.se: destination = site.datadir + r.lfn file_dir = '/'.join(destination.split('/')[:-1]) if r.status == 'ready': if os.path.isfile(destination): # Check fsize, md5 or adler raise WebpandaError('Replica exists') else: r.status = 'broken' replicas_.save(r) raise WebpandaError('Broken replica') elif r.status == 'defined': try: os.makedirs(file_dir) except (Exception): pass f = open(destination, 'wb') f.write(request.data) f.close() # Update file info setFileMeta(ff.id, destination) r.status = 'ready' replicas_.save(r) return {'guid': ff.guid} else: raise WebpandaError('Replica status: %s' % r.status) replica = Replica() if os.path.isfile(destination): raise WebpandaError('Unable to upload: File exists') try: os.makedirs(path) except (Exception): _logger.debug('Path exists: %s' % path) f = open(destination, 'wb') f.write(request.data) f.close() # Update file info setFileMeta(ff.id, destination) # Create/change replica replica.se = site.se replica.status = 'ready' replica.lfn = replfn replica.token = '' replica.original = ff replicas_.save(replica) return {'guid': ff.guid}