def submit(self): """ Submits a new job It returns the information about the new submitted job. To know the format for the submission, /api-docs/schema/submit gives the expected format encoded as a JSON-schema. It can be used to validate (i.e in Python, jsonschema.validate) """ # First, the request has to be valid JSON submitted_dict = get_input_as_dict(request) # The auto-generated delegation id must be valid user = request.environ['fts3.User.Credentials'] credential = Session.query(Credential).get((user.delegation_id, user.user_dn)) if credential is None: raise HTTPAuthenticationTimeout('No delegation found for "%s"' % user.user_dn) if credential.expired(): remaining = credential.remaining() seconds = abs(remaining.seconds + remaining.days * 24 * 3600) raise HTTPAuthenticationTimeout( 'The delegated credentials expired %d seconds ago (%s)' % (seconds, user.delegation_id) ) if user.method != 'oauth2' and credential.remaining() < timedelta(hours=1): raise HTTPAuthenticationTimeout( 'The delegated credentials has less than one hour left (%s)' % user.delegation_id ) # Populate the job and files populated = JobBuilder(user, **submitted_dict) log.info("%s (%s) is submitting a transfer job" % (user.user_dn, user.vos[0])) # Insert the job try: try: Session.execute(Job.__table__.insert(), [populated.job]) except IntegrityError: raise HTTPConflict('The sid provided by the user is duplicated') if len(populated.files): Session.execute(File.__table__.insert(), populated.files) if len(populated.datamanagement): Session.execute(DataManagement.__table__.insert(), populated.datamanagement) Session.flush() Session.commit() except IntegrityError as err: Session.rollback() raise HTTPConflict('The submission is duplicated '+ str(err)) except: Session.rollback() raise # Send messages # Need to re-query so we get the file ids job = Session.query(Job).get(populated.job_id) for i in range(len(job.files)): try: submit_state_change(job, job.files[i], populated.files[0]['file_state']) except Exception, e: log.warning("Failed to write state message to disk: %s" % e.message)
def hosts_activity(self): """ What are the hosts doing """ staging = Session.execute("SELECT COUNT(*), agent_dn " " FROM t_file " " WHERE file_state = 'STARTED' " " GROUP BY agent_dn") response = dict() for (count, host) in staging: response[host] = dict(staging=count) active = Session.execute("SELECT COUNT(*), transferHost " " FROM t_file " " WHERE file_state = 'ACTIVE' " " GROUP BY transferHost") for (count, host) in active: if host not in response: response[host] = dict() response[host]['active'] = count return response
raise HTTPBadRequest('Can not specify reuse and multiple replicas at the same time') job['reuse_job'] = 'R' # Update the optimizer unique_pairs = set(map(lambda f: (f['source_se'], f['dest_se']), files)) for (source_se, dest_se) in unique_pairs: optimizer_active = OptimizerActive() optimizer_active.source_se = source_se optimizer_active.dest_se = dest_se optimizer_active.ema = 0 optimizer_active.datetime = datetime.utcnow() Session.merge(optimizer_active) # Update the database try: Session.execute(Job.__table__.insert(), [job]) if len(files): Session.execute(File.__table__.insert(), files) if len(datamanagement): Session.execute(DataManagement.__table__.insert(), datamanagement) Session.commit() except: Session.rollback() raise if len(files): log.info("Job %s submitted with %d transfers" % (job['job_id'], len(files))) else: log.info("Job %s submitted with %d data management operations" % (job['job_id'], len(datamanagement))) return {'job_id': job['job_id']}
def _cancel_transfers(storage=None, vo_name=None): """ Cancels the transfers that have the given storage either in source or destination, and belong to the given VO. Returns the list of affected jobs ids. """ affected_job_ids = set() files = Session.query(File.file_id).filter( and_((File.source_se == storage) | (File.dest_se == storage), File.file_state.in_(FileActiveStates + ['NOT_USED']))) if vo_name and vo_name != '*': files = files.filter(File.vo_name == vo_name) now = datetime.utcnow() try: for row in files: file_id = row[0] job_id, file_index = Session.query( File.job_id, File.file_index).filter(File.file_id == file_id).one() affected_job_ids.add(job_id) # Cancel the affected file Session.query(File).filter(File.file_id == file_id)\ .update({ 'file_state': 'CANCELED', 'reason': 'Storage banned', 'finish_time': now, 'dest_surl_uuid': None }, synchronize_session=False) # If there are alternatives, enable them if Session.bind.dialect.name == 'mysql': limit = " LIMIT 1" else: limit = '' Session.execute( "UPDATE t_file SET" " file_state = 'SUBMITTED' " "WHERE" " job_id = :job_id AND file_index = :file_index AND file_state = 'NOT_USED' " + limit, dict(job_id=job_id, file_index=file_index)) Session.commit() Session.expire_all() except Exception: Session.rollback() raise # Set each job terminal state if needed try: for job_id in affected_job_ids: n_files = Session.query(func.count(distinct( File.file_id))).filter(File.job_id == job_id).all()[0][0] n_canceled = Session.query(func.count(distinct(File.file_id)))\ .filter(File.job_id == job_id).filter(File.file_state == 'CANCELED').all()[0][0] n_finished = Session.query(func.count(distinct(File.file_id)))\ .filter(File.job_id == job_id).filter(File.file_state == 'FINISHED').all()[0][0] n_failed = Session.query(func.count(distinct(File.file_id)))\ .filter(File.job_id == job_id).filter(File.file_state == 'FAILED').all()[0][0] n_terminal = n_canceled + n_finished + n_failed # Job finished! if n_terminal == n_files: reason = None Session.query(Job).filter(Job.job_id == job_id).update({ 'job_state': 'CANCELED', 'job_finished': now, 'reason': reason }) Session.commit() except Exception: Session.rollback() raise return affected_job_ids