Beispiel #1
0
def submit(proxy, toTrans, source, destination):

    # prepare rest job with 200 files per job
    transfers = []
    for files in chunks(toTrans, 200):

        c = pycurl.Curl()
        # create destination and source pfns for job
        for lfn in files:
            print(lfn)
            transfers.append(
                fts3.new_transfer(apply_tfc_to_lfn(source, lfn, c),
                                  apply_tfc_to_lfn(destination, lfn, c)))

        c.close()

        # Submit fts job
        context = fts3.Context('https://fts3.cern.ch:8446',
                               proxy,
                               proxy,
                               verify=True)
        print(fts3.delegate(context, lifetime=timedelta(hours=48),
                            force=False))

        job = fts3.new_job(transfers)

        #print("Monitor link: https://fts3.cern.ch:8449/fts3/ftsmon/#/job/"+fts3.submit(context, job))
        jobid = fts3.submit(context, job)

        #for file in (fts3.get_job_status(context, jobid, list_files=True))["files"]:
        for key, value in (fts3.get_job_status(context, jobid,
                                               list_files=True)).iteritems():
            print key
def _fts_submit_job(source_url, dest_url, src_filenames, dst_filenames,
                    checksum, overwrite, testing_folder, context, metadata):
    """
    https://gitlab.cern.ch/fts/fts-rest/-/blob/develop/src/fts3/rest/client/easy/submission.py#L106
    """

    transfers = []
    for i in xrange(len(src_filenames)):
        source_file = os.path.join(source_url, testing_folder, "src",
                                   src_filenames[i])
        dest_file = os.path.join(dest_url, testing_folder, "dest",
                                 dst_filenames[i])
        transfer = fts3.new_transfer(source=source_file, destination=dest_file)
        transfers.append(transfer)

    # create job
    job = fts3.new_job(transfers,
                       verify_checksum=checksum,
                       overwrite=overwrite,
                       timeout=3600,
                       metadata=metadata)

    # submit job
    while True:
        try:
            job_id = fts3.submit(context, job)
            break
        except fts3_client_exceptions.ClientError as e:
            _flush_logging_msg(e)
            return -1

    return job_id
Beispiel #3
0
def submitTheFTSJob(ftsFile):
  ### First way : Random choice of two servers
  # ftsServ = random.choice([ftsServ1, ftsServ2])
  ### Second way : Weighted choice of two servers
  # rndValue = random.uniform(0.0,1.0)
  # ftsServ = ftsServ1
  # if rndValue < 0.7 : ftsServ = ftsServ2
  ### Third way : Random choice of three servers
  fList = [ftsServ1, ftsServ2, ftsServ3]
  ftsServ = random.choice(fList)
  #
  context = fts3.Context(ftsServ)
  # Open the file and stop the processing.
  listOfPairs[:] = open(ceBase + "DOING/" + ftsFile).read().split("\n")
  # listOfPairs[:] = open(ceBase + "TODO/" + ftsFile).read().split("\n")
  # All the threading bit is here to check in parallel whether the files we are looking are okay in castor
  # Once the function is done, the list "okayFiles" should be filled
  transfers = []
  if checkStatus:
    checkStatusOnCastor()
  else:
    okayFiles[:] = []
    for onePair in listOfPairs:
      if len(onePair)<10: continue
      (sourceSURL, targetSURL) = onePair.split("  ")
      okayFiles.append((sourceSURL, targetSURL))
  if len(okayFiles)>0:
    for oneSet in okayFiles:
      transf = fts3.new_transfer(oneSet[0], oneSet[1])
      transfers.append(transf)
    job = fts3.new_job(transfers=transfers, overwrite=True, verify_checksum=True, reuse=False, retry=5) # requested by Andrea Manzi
    ftsJobID = fts3.submit(context, job)
    return ftsJobID, ftsServ
  else: # None of the files in this lot were good!
    return "-1", "-1"
Beispiel #4
0
  def submitFTS3( self, pinTime = False ):
    """ submit fts job using FTS3 rest API """

    if self.FTSGUID:
      return S_ERROR( "FTSJob already has been submitted" )

    transfers = []

    for ftsFile in self:
      trans = fts3.new_transfer( ftsFile.SourceSURL,
                                ftsFile.TargetSURL,
                                checksum = ftsFile.Checksum,
                                filesize = ftsFile.Size )
      transfers.append( trans )

    source_spacetoken = self.SourceToken if self.SourceToken else None
    dest_spacetoken = self.TargetToken if self.TargetToken else None
    copy_pin_lifetime = pinTime if pinTime else None
    bring_online = 86400 if pinTime else None

    job = fts3.new_job( transfers = transfers, overwrite = True,
            source_spacetoken = source_spacetoken, spacetoken = dest_spacetoken,
            bring_online = bring_online, copy_pin_lifetime = copy_pin_lifetime, retry = 3 )

    try:
      if not self._fts3context:
        self._fts3context = fts3.Context( endpoint = self.FTSServer )
      context = self._fts3context
      self.FTSGUID = fts3.submit( context, job )

    except Exception, e:
      return S_ERROR( "Error at submission: %s" % e )
Beispiel #5
0
    def submitFTS3(self, pinTime=False):
        """ submit fts job using FTS3 rest API """

        if self.FTSGUID:
            return S_ERROR("FTSJob already has been submitted")

        transfers = []

        for ftsFile in self:
            trans = fts3.new_transfer(ftsFile.SourceSURL,
                                      ftsFile.TargetSURL,
                                      checksum=ftsFile.Checksum,
                                      filesize=ftsFile.Size)
            transfers.append(trans)

        source_spacetoken = self.SourceToken if self.SourceToken else None
        dest_spacetoken = self.TargetToken if self.TargetToken else None
        copy_pin_lifetime = pinTime if pinTime else None
        bring_online = 86400 if pinTime else None

        job = fts3.new_job(transfers=transfers,
                           overwrite=True,
                           source_spacetoken=source_spacetoken,
                           spacetoken=dest_spacetoken,
                           bring_online=bring_online,
                           copy_pin_lifetime=copy_pin_lifetime,
                           retry=3)

        try:
            context = fts3.Context(self.FTSServer)
            self.FTSGUID = fts3.submit(context, job)

        except Exception, e:
            return S_ERROR("Error at submission: %s" % e)
Beispiel #6
0
    def run(self):
        """

        """

        self.threadLock.acquire()
        self.log.info("Processing transfers from: %s" % self.source)

        # create destination and source pfns for job
        transfers = []
        for lfn in self.files:
            transfers.append(fts3.new_transfer(lfn[0],
                                               lfn[1],
                                               metadata={'oracleId': lfn[2]}
                                               )
                             )
        self.log.info("Submitting %s transfers to FTS server" % len(self.files))

        # Submit fts job
        job = fts3.new_job(transfers,
                           overwrite=True,
                           verify_checksum=True,
                           metadata={"issuer": "ASO",
                                     "userDN": self.files[0][4],
                                     "taskname": self.files[0][5]},
                           copy_pin_lifetime=-1,
                           bring_online=None,
                           source_spacetoken=None,
                           spacetoken=None,
                           # max time for job in the fts queue in seconds.
                           # Usually, it should take O(s) for healthy situations
                           max_time_in_queue=600,
                           retry=3,
                           # seconds after which the transfer is retried
                           # reduced under FTS suggestion w.r.t. the 3hrs of asov1
                           retry_delay=600
                           # timeout on the single transfer process
                           # TODO: not clear if we may need it
                           # timeout = 1300
                           )

        jobid = fts3.submit(self.ftsContext, job)

        self.jobids.append(jobid)

        # TODO: manage exception here, what we should do?
        fileDoc = dict()
        fileDoc['asoworker'] = 'asoless'
        fileDoc['subresource'] = 'updateTransfers'
        fileDoc['list_of_ids'] = [x[2] for x in self.files]
        fileDoc['list_of_transfer_state'] = ["SUBMITTED" for _ in self.files]
        fileDoc['list_of_fts_instance'] = ['https://fts3.cern.ch:8446/' for _ in self.files]
        fileDoc['list_of_fts_id'] = [jobid for _ in self.files]

        self.log.info("Marking submitted %s files" % (len(fileDoc['list_of_ids'])))

        self.toUpdate.append(fileDoc)
        self.threadLock.release()
Beispiel #7
0
def submitTheFTSJob(ftsFile):
    ### First way : Random choice of two servers
    # ftsServ = random.choice([ftsServ1, ftsServ2])
    ### Second way : Weighted choice of two servers
    # rndValue = random.uniform(0.0,1.0)
    # ftsServ = ftsServ1
    # if rndValue < 0.7 : ftsServ = ftsServ2
    ### Third way : Random choice of three servers
    fList = [ftsServ1, ftsServ2, ftsServ3]
    ftsServ = random.choice(fList)
    #
    context = fts3.Context(ftsServ)
    filecontent = open(ceBase + "DOING/" + ftsFile).read().split("\n")
    transfers = []
    for ftra in filecontent:
        if len(ftra) < 10: continue
        (sourceSURL, targetSURL) = ftra.split("  ")
        comm = "gfal-stat " + sourceSURL
        runComm = subprocess.Popen(comm,
                                   shell=True,
                                   stdin=subprocess.PIPE,
                                   stdout=subprocess.PIPE,
                                   stderr=subprocess.PIPE,
                                   close_fds=True)
        theInfo = runComm.communicate()[1].strip()
        if theInfo.startswith(
                "gfal-stat error: 2 (No such file or directory)"):
            bFTS = open(ceBase + "DONE/badFileList.txt", "a")
            bFTS.write(ftra + "\n")
            bFTS.close()
        else:
            transf = fts3.new_transfer(sourceSURL, targetSURL)
            transfers.append(transf)
        # transf = fts3.new_transfer(sourceSURL, targetSURL)
        # transfers.append(transf)
    if len(transfers) > 0:
        # job = fts3.new_job(transfers=transfers, overwrite=True, verify_checksum=True, reuse=True, retry=5)
        # job = fts3.new_job(transfers=transfers, overwrite=True, verify_checksum=True, reuse=False, retry=5) # requested by Andrea Manzi
        job = fts3.new_job(
            transfers=transfers,
            overwrite=True,
            verify_checksum=True,
            reuse=False,
            retry=0)  # To avoid deleted files snarling up the system for hours
        ftsJobID = fts3.submit(context,
                               job,
                               delegation_lifetime=fts3.timedelta(hours=72))
        return ftsJobID, ftsServ
    else:
        return "-1", "-1"
Beispiel #8
0
    def submitFTS3(self, pinTime=False):
        """ submit fts job using FTS3 rest API """

        if self.FTSGUID:
            return S_ERROR("FTSJob already has been submitted")

        transfers = []

        for ftsFile in self:
            trans = fts3.new_transfer(ftsFile.SourceSURL,
                                      ftsFile.TargetSURL,
                                      checksum='ADLER32:%s' % ftsFile.Checksum,
                                      filesize=ftsFile.Size)
            transfers.append(trans)

        source_spacetoken = self.SourceToken if self.SourceToken else None
        dest_spacetoken = self.TargetToken if self.TargetToken else None
        copy_pin_lifetime = pinTime if pinTime else None
        bring_online = 86400 if pinTime else None

        job = fts3.new_job(transfers=transfers,
                           overwrite=True,
                           source_spacetoken=source_spacetoken,
                           spacetoken=dest_spacetoken,
                           bring_online=bring_online,
                           copy_pin_lifetime=copy_pin_lifetime,
                           retry=3)

        try:
            if not self._fts3context:
                self._fts3context = fts3.Context(endpoint=self.FTSServer,
                                                 request_class=ftsSSLRequest,
                                                 verify=False)
            context = self._fts3context
            self.FTSGUID = fts3.submit(context, job)

        except Exception as e:
            return S_ERROR("Error at submission: %s" % e)

        self.Status = "Submitted"
        self._log = gLogger.getSubLogger(
            "req_%s/FTSJob-%s" % (self.RequestID, self.FTSGUID), True)
        for ftsFile in self:
            ftsFile.FTSGUID = self.FTSGUID
            ftsFile.Status = "Submitted"
        return S_OK()
Beispiel #9
0
  def submitFTS3( self, pinTime = False ):
    """ submit fts job using FTS3 rest API """

    if self.FTSGUID:
      return S_ERROR( "FTSJob already has been submitted" )

    transfers = []

    for ftsFile in self:
      trans = fts3.new_transfer( ftsFile.SourceSURL,
                                ftsFile.TargetSURL,
                                checksum = 'ADLER32:%s'%ftsFile.Checksum,
                                filesize = ftsFile.Size )
      transfers.append( trans )

    source_spacetoken = self.SourceToken if self.SourceToken else None
    dest_spacetoken = self.TargetToken if self.TargetToken else None
    copy_pin_lifetime = pinTime if pinTime else None
    bring_online = 86400 if pinTime else None

    job = fts3.new_job( transfers = transfers, overwrite = True,
            source_spacetoken = source_spacetoken, spacetoken = dest_spacetoken,
            bring_online = bring_online, copy_pin_lifetime = copy_pin_lifetime, retry = 3 )

    try:
      if not self._fts3context:
        self._fts3context = fts3.Context( endpoint = self.FTSServer, request_class = ftsSSLRequest, verify = False )
      context = self._fts3context
      self.FTSGUID = fts3.submit( context, job )

    except Exception as e:
      return S_ERROR( "Error at submission: %s" % e )


    self.Status = "Submitted"
    self._log = gLogger.getSubLogger( "req_%s/FTSJob-%s" % ( self.RequestID, self.FTSGUID ) , True )
    for ftsFile in self:
      ftsFile.FTSGUID = self.FTSGUID
      ftsFile.Status = "Submitted"
    return S_OK()
Beispiel #10
0
def submitTheFTSJob(context, ftsFile):
    ftsServ = random.choice([ftsServ1, ftsServ2])
    with open(ceBase + "DOING/" + ftsFile) as fF:
        filecontent = fF.read().split("\n")
    # filecontent = open(ceBase + "DOING/" + ftsFile).read().split("\n")
    transfers = []
    for ftra in filecontent:
        if len(ftra) < 10: continue
        (sourceSURL, targetSURL) = ftra.split("  ")
        transf = fts3.new_transfer(sourceSURL, targetSURL)
        transfers.append(transf)
    if len(transfers) > 0:
        # job = fts3.new_job(transfers=transfers, overwrite=True, verify_checksum=True, reuse=True, retry=5)
        job = fts3.new_job(transfers=transfers,
                           overwrite=True,
                           verify_checksum=True,
                           reuse=False,
                           retry=5)  # requested by Andrea Manzi
        ftsJobID = fts3.submit(context, job)
        return ftsJobID, ftsServ
    else:
        return "-1", "-1"
Beispiel #11
0
            # get the last occurrence of auger in the path
            lfcURI = '/'.join(fs[len(fs) - 1 - fs[::-1].index('auger'):])
        # note that this path construction is not particularly liable and should be upgraded in the future

        transfer = fts3.new_transfer(filename, options.lfcHost + lfcURI)
        transferList.append(transfer)

    if len(transferList) > 0:
        job = fts3.new_job(
            transferList,
            metadata="Registration of the files transfered by job " + job_id,
            overwrite=True)
        if options.register == 1 or options.register == 0 and query_yes_no(
                "The registration job is ready now, do you wish to submit it?"
        ):
            jobID = fts3.submit(reg_context,
                                job) if options.reg_endpoint else fts3.submit(
                                    context, job)
            print "The registration job ID is " + jobID
            with open(options.regJobIdFile, 'a') as f:
                f.write('\t' + jobID + '\n')
    notFinishedTransfers = [
        fts3.new_transfer(f['source_surl'], f['dest_surl'])
        for f in job_status['files']
        if f['file_state'] in ['FAILED', 'CANCELED'] and f['reason'] !=
        'DESTINATION file already exists and overwrite is not enabled'
    ]

    if len(notFinishedTransfers) < 1:
        sys.exit(0)

    jobChunks = [
Beispiel #12
0
    def submit(self,
               context=None,
               ftsServer=None,
               ucert=None,
               pinTime=36000,
               protocols=None):
        """ submit the job to the FTS server

        Some attributes are expected to be defined for the submission to work:
          * type (set by FTS3Operation)
          * sourceSE (only for Transfer jobs)
          * targetSE
          * activity (optional)
          * priority (optional)
          * username
          * userGroup
          * filesToSubmit
          * operationID (optional, used as metadata for the job)

        We also expect the FTSFiles have an ID defined, as it is given as transfer metadata

        :param pinTime: Time the file should be pinned on disk (used for transfers and staging)
                        Used only if he source SE is a tape storage
        :param context: fts3 context. If not given, it is created (see ftsServer & ucert param)
        :param ftsServer: the address of the fts server to submit to. Used only if context is
                          not given. if not given either, use the ftsServer object attribute

        :param ucert: path to the user certificate/proxy. Might be inferred by the fts cli (see its doc)
        :param protocols: list of protocols from which we should choose the protocol to use

        :returns S_OK([FTSFiles ids of files submitted])
    """

        log = gLogger.getSubLogger(
            "submit/%s/%s_%s" %
            (self.operationID, self.sourceSE, self.targetSE), True)

        if not context:
            if not ftsServer:
                ftsServer = self.ftsServer
            context = fts3.Context(endpoint=ftsServer,
                                   ucert=ucert,
                                   request_class=ftsSSLRequest,
                                   verify=False)

        # Construct the target SURL
        res = self.__fetchSpaceToken(self.targetSE)
        if not res['OK']:
            return res
        target_spacetoken = res['Value']

        allLFNs = [ftsFile.lfn for ftsFile in self.filesToSubmit]

        if self.type == 'Transfer':
            res = self._constructTransferJob(pinTime,
                                             allLFNs,
                                             target_spacetoken,
                                             protocols=protocols)
        elif self.type == 'Staging':
            res = self._constructStagingJob(pinTime, allLFNs,
                                            target_spacetoken)
        # elif self.type == 'Removal':
        #   res = self._constructRemovalJob(context, allLFNs, failedLFNs, target_spacetoken)

        if not res['OK']:
            return res

        job, fileIDsInTheJob = res['Value']
        setFileIdsInTheJob = set(fileIDsInTheJob)

        try:
            self.ftsGUID = fts3.submit(context, job)
            log.info("Got GUID %s" % self.ftsGUID)

            # Only increase the amount of attempt
            # if we succeeded in submitting -> no ! Why did I do that ??
            for ftsFile in self.filesToSubmit:
                ftsFile.attempt += 1
                if ftsFile.fileID in setFileIdsInTheJob:
                    ftsFile.status = 'Submitted'

            now = datetime.datetime.utcnow().replace(microsecond=0)
            self.submitTime = now
            self.lastUpdate = now
            self.lastMonitor = now

        except FTS3ClientException as e:
            log.exception("Error at submission", repr(e))
            return S_ERROR("Error at submission: %s" % e)

        return S_OK(fileIDsInTheJob)
        fs = filename.split('/')
        try:
            lfcURI = '/'.join(fs[fs.index('grid'):])
        except ValueError:  # substring 'grid' not found
            # get the last occurrence of auger in the path
            lfcURI = '/'.join(fs[len(fs) - 1 - fs[::-1].index('auger'):])
        # note that this path construction is not particularly liable and should be upgraded in the future

        transfer = fts3.new_transfer(filename, options.lfcHost + lfcURI)
        transferList.append(transfer)

    if len(transferList) > 0:
        job = fts3.new_job(transferList, metadata="Registration of the files transfered by job " + job_id,
                           overwrite=True)
        if options.register == 1 or options.register == 0 and query_yes_no("The registration job is ready now, do you wish to submit it?"):
            jobID = fts3.submit(reg_context, job) if options.reg_endpoint else fts3.submit(context, job)
            print "The registration job ID is " + jobID
            with open(options.regJobIdFile, 'a') as f:
                f.write('\t' + jobID + '\n')
    notFinishedTransfers = [fts3.new_transfer(f['source_surl'], f['dest_surl']) for f in job_status['files']
                            if f['file_state'] in ['FAILED', 'CANCELED']
                            and f['reason'] != 'DESTINATION file already exists and overwrite is not enabled']

    if len(notFinishedTransfers) < 1:
        sys.exit(0)

    jobChunks = [notFinishedTransfers[i:i + MAX_NUM_OF_TRANSFERS] for i in
                 xrange(0, len(notFinishedTransfers), MAX_NUM_OF_TRANSFERS)]

    jobs = []
    for chunkOfTransfers in jobChunks:
Beispiel #14
0
                transfer = fts3.new_transfer(sourceURI, destinationURI)
                if len(lineArr) > 2:
                    for lineIndex in range(1, len(lineArr)-1):
                        fts3.add_alternative_source(transfer, lineArr[lineIndex])

            transferList.append(transfer)
            numOfTransfers += 1

            if numOfTransfers > MAX_NUM_OF_TRANSFERS:
                numOfTransfers = 0
                job = createTransferJob(transferList, filename, str(numOfJobs), options.overwriteFlag)
                if options.dryRun:
                    print json.dumps(job, indent=2)
                    jobID = 'abc'
                else:
                    jobID = fts3.submit(context, job)
                transferJobs.append(jobID)
                numOfJobs += 1
                transferList = []

        # submit the last job
        if transferList:
            job = createTransferJob(transferList, filename, str(numOfJobs) + " last", options.overwriteFlag)
            if options.dryRun:
                print json.dumps(job, indent=2)
                jobID = 'abc'
            else:
                jobID = fts3.submit(context, job)
            transferJobs.append(jobID)

        # if some jobs were submitted, note their IDs
Beispiel #15
0
def submitToFTS(logger, ftsContext, files, jobids, toUpdate):
    """
    actual FTS job submission
    INPUT PARAMS:
    :param logger: logging object
    :param ftsContext: FTS context
    :param files: [
           [source_pfn,
            dest_pfn,
            file oracle id,
            source site,
            username,
            taskname,
            file size,
            checksum],
           ...]
    OUTPUT PARAMS:
    :param jobids: collect the list of job ids when submitted
    :param toUpdate: list of oracle ids to update

    RETURNS: the submitted jobid (a string)
    """

    transfers = []
    for lfn in files:
        transfers.append(
            fts3.new_transfer(lfn[0],
                              lfn[1],
                              filesize=lfn[6],
                              metadata={'oracleId': lfn[2]}))
    logger.info("Submitting %s transfers to FTS server" % len(files))
    #SB#
    for lfn in files:
        logger.info("%s %s %s %s", lfn[0], lfn[1], lfn[6],
                    {'oracleId': lfn[2]})
    #SB#

    # Submit fts job
    job = fts3.new_job(
        transfers,
        overwrite=True,
        verify_checksum=True,
        metadata={
            "issuer": "ASO",
            "userDN": files[0][4],
            "taskname": files[0][5]
        },
        copy_pin_lifetime=-1,
        bring_online=None,
        source_spacetoken=None,
        spacetoken=None,
        # max time for job in the FTS queue in hours. From FTS experts in
        # https://cern.service-now.com/service-portal?id=ticket&table=incident&n=INC2776329
        # The max_time_in_queue applies per job, not per retry.
        # The max_time_in_queue is a timeout for how much the job can stay in
        # a SUBMITTED, ACTIVE or STAGING state.
        # When a job's max_time_in_queue is reached, the job and all of its
        # transfers not yet in a terminal state are marked as CANCELED
        # StefanoB: I see that we hit this at times with 6h, causing job resubmissions,
        # so will try to make it longer to give FTS maximum chances within our
        # 24h overall limit (which takes care also of non-FTS related issues !)
        # ASO transfers never require STAGING so jobs can spend max_time_in_queue only
        # as SUBMITTED (aka queued) or ACTIVE (at least one transfer has been activated)
        max_time_in_queue=10,
        # from same cern.service-now.com ticket as above:
        # The number of retries applies to each transfer within that job.
        # A transfer is granted the first execution + number_of_retries.
        # E.g.: retry=3 --> first execution + 3 retries
        # so retry=3 means each transfer has 4 chances at most during the 6h
        # max_time_in_queue
        retry=3,
        reuse=ftsReuse,
        # seconds after which the transfer is retried
        # this is a transfer that fails, gets put to SUBMITTED right away,
        # but the scheduler will avoid it until NOW() > last_retry_finish_time + retry_delay
        # reduced under FTS suggestion w.r.t. the 3hrs of asov1
        # StefanoB: indeed 10 minutes makes much more sense for storage server glitches
        retry_delay=600
        # timeout on the single transfer process
        # TODO: not clear if we may need it
        # timeout = 1300
    )

    jobid = fts3.submit(ftsContext, job)

    jobids.append(jobid)

    fileDoc = dict()
    fileDoc['asoworker'] = asoworker
    fileDoc['subresource'] = 'updateTransfers'
    fileDoc['list_of_ids'] = [x[2] for x in files]
    fileDoc['list_of_transfer_state'] = ["SUBMITTED" for _ in files]
    fileDoc['list_of_fts_instance'] = [FTS_ENDPOINT for _ in files]
    fileDoc['list_of_fts_id'] = [jobid for _ in files]

    logger.info("Will mark as submitted %s files" %
                (len(fileDoc['list_of_ids'])))
    toUpdate.append(fileDoc)

    return jobid
Beispiel #16
0
  def submit(self, context=None, ftsServer=None, ucert=None, pinTime=36000, protocols=None):
    """ submit the job to the FTS server

        Some attributes are expected to be defined for the submission to work:
          * type (set by FTS3Operation)
          * sourceSE (only for Transfer jobs)
          * targetSE
          * activity (optional)
          * priority (optional)
          * username
          * userGroup
          * filesToSubmit
          * operationID (optional, used as metadata for the job)

        We also expect the FTSFiles have an ID defined, as it is given as transfer metadata

        :param pinTime: Time the file should be pinned on disk (used for transfers and staging)
                        Used only if he source SE is a tape storage
        :param context: fts3 context. If not given, it is created (see ftsServer & ucert param)
        :param ftsServer: the address of the fts server to submit to. Used only if context is
                          not given. if not given either, use the ftsServer object attribute

        :param ucert: path to the user certificate/proxy. Might be inferred by the fts cli (see its doc)
        :param protocols: list of protocols from which we should choose the protocol to use

        :returns S_OK([FTSFiles ids of files submitted])
    """

    log = gLogger.getSubLogger("submit/%s/%s_%s" %
                               (self.operationID, self.sourceSE, self.targetSE), True)

    if not context:
      if not ftsServer:
        ftsServer = self.ftsServer
      context = fts3.Context(
          endpoint=ftsServer,
          ucert=ucert,
          request_class=ftsSSLRequest,
          verify=False)

    # Construct the target SURL
    res = self.__fetchSpaceToken(self.targetSE)
    if not res['OK']:
      return res
    target_spacetoken = res['Value']

    allLFNs = [ftsFile.lfn for ftsFile in self.filesToSubmit]

    if self.type == 'Transfer':
      res = self._constructTransferJob(
          pinTime,
          allLFNs,
          target_spacetoken,
          protocols=protocols)
    elif self.type == 'Staging':
      res = self._constructStagingJob(
          pinTime,
          allLFNs,
          target_spacetoken)
    # elif self.type == 'Removal':
    #   res = self._constructRemovalJob(context, allLFNs, failedLFNs, target_spacetoken)

    if not res['OK']:
      return res

    job, fileIDsInTheJob = res['Value']
    setFileIdsInTheJob = set(fileIDsInTheJob)

    try:
      self.ftsGUID = fts3.submit(context, job)
      log.info("Got GUID %s" % self.ftsGUID)

      # Only increase the amount of attempt
      # if we succeeded in submitting -> no ! Why did I do that ??
      for ftsFile in self.filesToSubmit:
        ftsFile.attempt += 1

        # This should never happen because a file should be "released"
        # first by the previous job.
        # But we just print a warning
        if ftsFile.ftsGUID is not None:
          log.warn(
              "FTSFile has a non NULL ftsGUID at job submission time",
              "FileID: %s existing ftsGUID: %s" %
              (ftsFile.fileID,
               ftsFile.ftsGUID))

        # `assign` the file to this job
        ftsFile.ftsGUID = self.ftsGUID
        if ftsFile.fileID in setFileIdsInTheJob:
          ftsFile.status = 'Submitted'

      now = datetime.datetime.utcnow().replace(microsecond=0)
      self.submitTime = now
      self.lastUpdate = now
      self.lastMonitor = now

    except FTS3ClientException as e:
      log.exception("Error at submission", repr(e))
      return S_ERROR("Error at submission: %s" % e)

    return S_OK(fileIDsInTheJob)
Beispiel #17
0
  def submit(self, context=None, ftsServer=None, ucert=None, pinTime=36000, ):
    """ submit the job to the FTS server

        Some attributes are expected to be defined for the submission to work:
          * type (set by FTS3Operation)
          * sourceSE (only for Transfer jobs)
          * targetSE
          * activity (optional)
          * priority (optional)
          * username
          * userGroup
          * filesToSubmit
          * operationID (optional, used as metadata for the job)

        We also expect the FTSFiles have an ID defined, as it is given as transfer metadata

        :param pinTime: Time the file should be pinned on disk (used for transfers and staging)
                        Used only if he source SE is a tape storage
        :param context: fts3 context. If not given, it is created (see ftsServer & ucert param)
        :param ftsServer: the address of the fts server to submit to. Used only if context is
                          not given. if not given either, use the ftsServer object attribute

        :param ucert: path to the user certificate/proxy. Might be inferred by the fts cli (see its doc)

        :returns S_OK([FTSFiles ids of files submitted])
    """

    log = gLogger.getSubLogger("submit/%s/%s_%s" %
                               (self.operationID, self.sourceSE, self.targetSE), True)

    if not context:
      if not ftsServer:
        ftsServer = self.ftsServer
      context = fts3.Context(
          endpoint=ftsServer,
          ucert=ucert,
          request_class=ftsSSLRequest,
          verify=False)

    # Construct the target SURL
    res = self.__fetchSpaceToken(self.targetSE)
    if not res['OK']:
      return res
    target_spacetoken = res['Value']

    allLFNs = [ftsFile.lfn for ftsFile in self.filesToSubmit]

    failedLFNs = set()

    # getting all the target surls
    res = StorageElement(self.targetSE, vo=self.vo).getURL(allLFNs, protocol='srm')
    if not res['OK']:
      return res

    for lfn, reason in res['Value']['Failed'].iteritems():
      failedLFNs.add(lfn)
      log.error("Could not get target SURL", "%s %s" % (lfn, reason))

    allTargetSURLs = res['Value']['Successful']

    if self.type == 'Transfer':
      res = self._constructTransferJob(
          context,
          pinTime,
          allTargetSURLs,
          failedLFNs,
          target_spacetoken)
    elif self.type == 'Staging':
      res = self._constructStagingJob(
          context,
          pinTime,
          allTargetSURLs,
          failedLFNs,
          target_spacetoken)
    elif self.type == 'Removal':
      res = self._constructRemovalJob(context, allTargetSURLs, failedLFNs, target_spacetoken)

    if not res['OK']:
      return res

    job, fileIDsInTheJob = res['Value']
    setFileIdsInTheJob = set(fileIDsInTheJob)

    try:
      self.ftsGUID = fts3.submit(context, job)
      log.info("Got GUID %s" % self.ftsGUID)

      # Only increase the amount of attempt
      # if we succeeded in submitting -> no ! Why did I do that ??
      for ftsFile in self.filesToSubmit:
        ftsFile.attempt += 1
        if ftsFile.fileID in setFileIdsInTheJob:
          ftsFile.status = 'Submitted'

      now = datetime.datetime.utcnow().replace(microsecond=0)
      self.submitTime = now
      self.lastUpdate = now
      self.lastMonitor = now

    except FTS3ClientException as e:
      log.exception("Error at submission", repr(e))
      return S_ERROR("Error at submission: %s" % e)

    return S_OK(fileIDsInTheJob)
Beispiel #18
0
    def run(self):
        """

        """

        self.threadLock.acquire()
        self.log.info("Processing transfers from: %s" % self.source)

        # create destination and source pfns for job
        transfers = []
        for lfn in self.files:
            transfers.append(fts3.new_transfer(lfn[0],
                                               lfn[1],
                                               filesize=lfn[6],
                                               metadata={'oracleId': lfn[2]}
                                               )
                             )
        self.log.info("Submitting %s transfers to FTS server" % len(self.files))

        # Submit fts job
        job = fts3.new_job(transfers,
                           overwrite=True,
                           verify_checksum=True,
                           metadata={"issuer": "ASO",
                                     "userDN": self.files[0][4],
                                     "taskname": self.files[0][5]},
                           copy_pin_lifetime=-1,
                           bring_online=None,
                           source_spacetoken=None,
                           spacetoken=None,
                           # max time for job in the FTS queue in hours. From FTS experts in
                           # https://cern.service-now.com/service-portal?id=ticket&table=incident&n=INC2776329
                           # The max_time_in_queue applies per job, not per retry.
                           # The max_time_in_queue is a timeout for how much the job can stay in
                           # a SUBMITTED, ACTIVE or STAGING state.
                           # When a job's max_time_in_queue is reached, the job and all of its
                           # transfers not yet in a terminal state are marked as CANCELED
                           # StefanoB: I see that we hit this at times with 6h, causing job resubmissions,
                           # so will try to make it longer to give FTS maximum chances within our
                           # 24h overall limit (which takes care also of non-FTS related issues !)
                           # ASO transfers never require STAGING so jobs can spend max_time_in_queue only
                           # as SUBMITTED (aka queued) or ACTIVE (at least one transfer has been activated)
                           max_time_in_queue=10,
                           # from same cern.service-now.com ticket as above:
                           # The number of retries applies to each transfer within that job.
                           # A transfer is granted the first execution + number_of_retries.
                           # E.g.: retry=3 --> first execution + 3 retries
                           # so retry=3 means each transfer has 4 chances at most during the 6h
                           # max_time_in_queue
                           retry=3,
                           reuse=ftsReuse,
                           # seconds after which the transfer is retried
                           # this is a transfer that fails, gets put to SUBMITTED right away,
                           # but the scheduler will avoid it until NOW() > last_retry_finish_time + retry_delay
                           # reduced under FTS suggestion w.r.t. the 3hrs of asov1
                           # StefanoB: indeed 10 minutes makes much more sense for storage server glitches
                           retry_delay=600
                           # timeout on the single transfer process
                           # TODO: not clear if we may need it
                           # timeout = 1300
                           )

        jobid = fts3.submit(self.ftsContext, job)

        self.jobids.append(jobid)

        # TODO: manage exception here, what we should do?
        fileDoc = dict()
        fileDoc['asoworker'] = asoworker
        fileDoc['subresource'] = 'updateTransfers'
        fileDoc['list_of_ids'] = [x[2] for x in self.files]
        fileDoc['list_of_transfer_state'] = ["SUBMITTED" for _ in self.files]
        fileDoc['list_of_fts_instance'] = [FTS_ENDPOINT for _ in self.files]
        fileDoc['list_of_fts_id'] = [jobid for _ in self.files]

        self.log.info("Marking submitted %s files" % (len(fileDoc['list_of_ids'])))

        self.toUpdate.append(fileDoc)
        self.threadLock.release()
Beispiel #19
0
    def submit(self, files, job_params, timeout=None):
        """
        Submit a transfer to FTS3 via JSON.
        :param external_host: FTS server as a string.
        :param files: List of dictionary which for a transfer.
        :param job_params: Dictionary containing key/value pairs, for all transfers.
        :param user_transfer: boolean for user tranfer submission
        :returns: FTS transfer identifier.
        """
        usercert = self.prepare_credentials(files)

        if not usercert:
            logging.error('Unable to prepare credentials.')
            return None

        logging.info('Proxy %s is ready.', usercert)

        # FTS3 expects 'davs' as the scheme identifier instead of https
        for file in files:
            if not file['sources'] or file['sources'] == []:
                os.unlink(usercert)
                raise Exception('No sources defined')

            new_src_urls = []
            new_dst_urls = []
            for url in file['sources']:
                if url.startswith('https'):
                    new_src_urls.append(':'.join(['davs'] +
                                                 url.split(':')[1:]))
                else:
                    new_src_urls.append(url)
            for url in file['destinations']:
                if url.startswith('https'):
                    new_dst_urls.append(':'.join(['davs'] +
                                                 url.split(':')[1:]))
                else:
                    new_dst_urls.append(url)

            file['sources'] = new_src_urls
            file['destinations'] = new_dst_urls

        transfer_id = None
        expected_transfer_id = None
        if self.deterministic:
            job_params = job_params.copy()
            job_params["id_generator"] = "deterministic"
            job_params["sid"] = files[0]['metadata']['request_id']
            expected_transfer_id = self.__get_deterministic_id(
                job_params["sid"])
            logging.debug(
                "Submit bulk transfers in deterministic mode, sid %s, expected transfer id: %s",
                job_params["sid"], expected_transfer_id)

        # bulk submission
        jobID = None
        transfers = []
        for _file in files:
            for source, destination in zip(_file["sources"],
                                           _file["destinations"]):
                transfers.append(
                    fts.new_transfer(source,
                                     destination,
                                     activity=_file['activity'],
                                     metadata=_file['metadata'],
                                     filesize=_file['filesize'],
                                     checksum=_file['checksum']))

        try:
            context = fts.Context(self.external_host,
                                  ucert=usercert,
                                  ukey=usercert,
                                  verify=True,
                                  capath='/etc/grid-security/certificates/')

            job = fts.new_job(
                transfers,
                overwrite=job_params['overwrite'],
                verify_checksum=job_params['verify_checksum'],
                metadata=job_params['job_metadata'],
                copy_pin_lifetime=job_params['copy_pin_lifetime'],
                bring_online=job_params['bring_online'],
                source_spacetoken=None,
                spacetoken=None,
                priority=job_params['priority'])

            # submission for bindings checks the delegation, so the usual delegation part is useless here
            jobID = fts.submit(context,
                               job,
                               delegation_lifetime=timedelta(hours=12),
                               delegate_when_lifetime_lt=timedelta(hours=8))
        except ServerError:
            logging.error("Server side exception during FTS job submission.")
            return None
        except ClientError:
            logging.error("Client side exception during FTS job submission.")
            return None
        except BadEndpoint:
            logging.error("Wrong FTS endpoint: %s", self.external_host)
            return None

        if jobID:
            record_counter(
                'transfertool.fts3myproxy.%s.submission.success' %
                self.__extract_host(self.external_host), len(files))
            transfer_id = jobID
        else:
            transfer_id = jobID
            logging.error("Unexpected failure during FTS job submission.")
            record_counter(
                'transfertool.fts3myproxy.%s.submission.failure' %
                self.__extract_host(self.external_host), len(files))

        os.unlink(usercert)

        return transfer_id
Beispiel #20
0
                    for lineIndex in range(1, len(lineArr) - 1):
                        fts3.add_alternative_source(transfer,
                                                    lineArr[lineIndex])

            transferList.append(transfer)
            numOfTransfers += 1

            if numOfTransfers > MAX_NUM_OF_TRANSFERS:
                numOfTransfers = 0
                job = createTransferJob(transferList, filename, str(numOfJobs),
                                        options.overwriteFlag)
                if options.dryRun:
                    print json.dumps(job, indent=2)
                    jobID = 'abc'
                else:
                    jobID = fts3.submit(context, job)
                transferJobs.append(jobID)
                numOfJobs += 1
                transferList = []

        # submit the last job
        if transferList:
            job = createTransferJob(transferList, filename,
                                    str(numOfJobs) + " last",
                                    options.overwriteFlag)
            if options.dryRun:
                print json.dumps(job, indent=2)
                jobID = 'abc'
            else:
                jobID = fts3.submit(context, job)
            transferJobs.append(jobID)
Beispiel #21
0
opts = OptionParser()
opts.add_option('-s', '--endpoint', dest='endpoint', default='https://fts3-pilot.cern.ch:8446')
opts.add_option('--dry-run', dest='dry_run', default=False, action='store_true')

(options, args) = opts.parse_args()
if len(args) < 2:
    raise Exception('Need a source and a destination')

source = args[0]
destination = args[1]
checksum = None

if len(args) > 2:
    checksum = args[2]

logging.getLogger('fts3.rest.client').setLevel(logging.DEBUG)

# Build the job
transfer = fts3.new_transfer(source, destination,
                             checksum=checksum, filesize=None,
                             metadata='Test submission')
job = fts3.new_job([transfer], verify_checksum=True, metadata='Test job', retry=1, priority =3)

# Submit or just print
if options.dry_run:
    print json.dumps(job, indent=2)
else:
    context = fts3.Context(options.endpoint)
    print fts3.submit(context, job)