Example #1
0
 def test_proc_attr_order(self):
     """get_num_procs() has an order of preferred attributes
     """
     jobad = classad.ClassAd()
     for attr, val in [(x, condor_meter.PROC_ATTRS.index(x))
                       for x in reversed(condor_meter.PROC_ATTRS)]:
         jobad[attr] = val
         self.assertEquals(condor_meter.get_num_procs(jobad), val)
Example #2
0
 def test_proc_expr(self):
     """get_num_procs() should be able to handle attributes set to ClassAd expressions
     """
     for attr in condor_meter.PROC_ATTRS:
         jobad = classad.ClassAd()
         jobad[attr] = classad.ExprTree('2 + 2')
         procs = condor_meter.get_num_procs(jobad)
         self.assertEquals(procs, 4)
Example #3
0
 def test_proc_int(self):
     """The Processors field should always return an integer
     """
     for attr in condor_meter.PROC_ATTRS:
         jobad = classad.ClassAd()
         jobad[attr] = 'broken attribute'
         procs = condor_meter.get_num_procs(jobad)
         self.assertIsInstance(procs, int)
Example #4
0
def print_capabilities():
    capabilities = {
        "PluginType": "FileTransfer",
        "PluginVersion": "0.1",
        "MultipleFileSupport": True,
        "SupportedMethods": "htmap",
    }
    sys.stdout.write(classad.ClassAd(capabilities).printOld())
Example #5
0
def print_capabilities():
    capabilities = {
        'MultipleFileSupport': True,
        'PluginType': 'FileTransfer',
        'SupportedMethods': 'gluster',
        'Version': GLUSTER_PLUGIN_VERSION,
    }
    sys.stdout.write(classad.ClassAd(capabilities).printOld())
Example #6
0
def get_schedd():
    """Find and return the HTCondor sched.
    :returns: htcondor schedd object."""

    # Getting remote scheduler
    schedd_ad = classad.ClassAd()
    schedd_ad["MyAddress"] = os.environ.get("HTCONDOR_ADDR", None) 
    schedd = htcondor.Schedd(schedd_ad)
    return schedd
Example #7
0
 def test_pickle(self):
     ad = classad.ClassAd({"one": 1})
     expr = classad.ExprTree("2+2")
     pad = pickle.dumps(ad)
     pexpr = pickle.dumps(expr)
     ad2 = pickle.loads(pad)
     expr2 = pickle.loads(pexpr)
     self.assertEquals(ad2.__repr__(), "[ one = 1 ]")
     self.assertEquals(expr2.__repr__(), "2 + 2")
Example #8
0
def condor_stats(timeout=TIMEOUT):
    """
    Return the Condor Schedd stats as a ClassAd instance.
    """
    ad = classad.ClassAd(MyType='Scheduler',
                 Name='Error communicating with the Condor Schedd.')
    args = [utils.which('condor_status'), '-long', '-schedd']
    res = _run_condor_cmd(args, [ad, ], timeout=timeout)
    return(res[0])
Example #9
0
 def test_ad_assignment(self):
     ad = classad.ClassAd()
     ad["foo"] = 2.1
     self.assertEqual(ad["foo"], 2.1)
     ad["foo"] = 2
     self.assertEqual(ad["foo"], 2)
     ad["foo"] = "bar"
     self.assertEqual(ad["foo"], "bar")
     self.assertRaises(TypeError, ad.__setitem__, {})
Example #10
0
 def test_temp_scope(self):
     expr = classad.ExprTree("foo")
     self.assertEquals("bar", expr.eval({"foo": "bar"}))
     ad = classad.ClassAd({"foo": "baz", "test": classad.ExprTree("foo")})
     expr = ad["test"]
     self.assertEquals("baz", expr.eval())
     self.assertEquals("bar", expr.eval({"foo": "bar"}))
     self.assertEquals("bar", expr.eval({"foo": "bar"}))
     self.assertEquals("baz", expr.eval())
Example #11
0
def match(target_classad, requirement_str):
    """
    checks if a single target satifies a given requirement.
    :param classad target_classad:
    :param str requirement_str:
    :return bool:
    """
    requirement_classad = classad.ClassAd()
    requirement_classad['Requirements'] = classad.ExprTree(requirement_str)
    return target_classad.matches(requirement_classad)
Example #12
0
    def get_urls(self, host, args):

        urls = []

        # Parse the input arguments
        for arg in args:
            attr, value = arg.split("=")
            if attr == "observatory":
                observatory = value
            if attr == "type":
                type = value
            if attr == "s":
                start_frame = value
            if attr == "e":
                end_frame = value

        # If any input arguments are missing, report error and exit
        try:
            observatory, type, end_frame, start_frame
        except NameError:
            with open(self.outfile, 'w') as outfile:
                outfile_dict = get_error_dict(
                    "gwdata:// urls must include arguments 'observatory', 'type', 's' (start frame), 'e' (end frame)"
                )
                outfile = open(self.outfile_path, 'w')
                outfile.write(str(classad.ClassAd(outfile_dict)))
                sys.exit(-1)

        # Retrieve the list of URLs
        try:
            urls = gwdatafind.find_urls(host=host,
                                        site=observatory,
                                        frametype=type,
                                        gpsstart=int(start_frame),
                                        gpsend=int(end_frame))
        except Exception as e:
            outfile_dict = get_error_dict(
                f"Error retrieving gwdatafind URLs: {sys.exc_info()[0]} ({e})")
            outfile = open(self.outfile_path, 'w')
            outfile.write(str(classad.ClassAd(outfile_dict)))
            sys.exit(-1)

        return urls
Example #13
0
    def test_list_conversion(self):
        ad = dict(classad.ClassAd("[a = {1,2,3}]"))
        self.assertTrue(isinstance(ad["a"], types.ListType))
        self.assertTrue(isinstance(ad["a"][0], types.LongType))

        def listAdd(a, b):
            return a + b

        classad.register(listAdd)
        self.assertEqual(classad.ExprTree("listAdd({1,2}, {3,4})")[0], 1)
Example #14
0
    def getClusterAd(self):
        """
        _getClusterAd_

        Return common cluster classad

        scriptFile & Output/Error/Log filenames shortened to
        avoid condorg submission errors from >256 chars paths

        """
        ad = classad.ClassAd()

        # ad['universe'] = "vanilla"
        ad['ShouldTransferFiles'] = "YES"
        ad['WhenToTransferOutput'] = "ON_EXIT"
        ad['UserLogUseXML'] = True
        ad['JobNotification'] = 0
        ad['Cmd'] = self.scriptFile
        # Investigate whether we should pass the absolute path for Out and Err ads,
        # just as we did for UserLog. There may be issues, more info on WMCore #7362
        ad['Out'] = classad.ExprTree('strcat("condor.", ClusterId, ".", ProcId, ".out")')
        ad['Err'] = classad.ExprTree('strcat("condor.", ClusterId, ".", ProcId, ".err")')
        ad['UserLog'] = classad.ExprTree('strcat(Iwd, "/condor.", ClusterId, ".", ProcId, ".log")')

        ad['WMAgent_AgentName'] = self.agent

        ad['JobLeaseDuration'] = classad.ExprTree('isUndefined(MachineAttrMaxHibernateTime0) ? 1200 : MachineAttrMaxHibernateTime0')

        ad['PeriodicRemove'] = classad.ExprTree('( JobStatus =?= 5 ) && ( time() - EnteredCurrentStatus > 10 * 60 )')
        removeReasonExpr = 'PeriodicRemove ? "Job automatically removed for being in Held status" : ""'
        ad['PeriodicRemoveReason'] = classad.ExprTree(removeReasonExpr)

        # Required for global pool accounting
        ad['AcctGroup'] = self.acctGroup
        ad['AcctGroupUser'] = self.acctGroupUser
        ad['AccountingGroup'] = "%s.%s" % (self.acctGroup, self.acctGroupUser)

        # Customized classAds for this plugin
        ad['DESIRED_Archs'] = "INTEL,X86_64"

        ad['Rank'] = 0.0
        ad['TransferIn'] = False

        ad['JobMachineAttrs'] = "GLIDEIN_CMSSite"
        ad['JobAdInformationAttrs'] = ("JobStatus,QDate,EnteredCurrentStatus,JobStartDate,DESIRED_Sites,"
                                       "ExtDESIRED_Sites,WMAgent_JobID,MachineAttrGLIDEIN_CMSSite0")

        # TODO: remove when 8.5.7 is deployed (seems to be still needed as of 8.6.11 ...)
        paramsToAdd = htcondor.param['SUBMIT_ATTRS'].split() + htcondor.param['SUBMIT_EXPRS'].split()
        paramsToSkip = ['accounting_group', 'use_x509userproxy', 'PostJobPrio2', 'JobAdInformationAttrs']
        for param in paramsToAdd:
            if (param not in ad) and (param in htcondor.param) and (param not in paramsToSkip):
                ad[param] = classad.ExprTree(htcondor.param[param])

        return ad
Example #15
0
    def test_abstime(self):
        expr = classad.ExprTree('absTime("2013-11-12T07:50:23")')
        dt = expr.eval()
        self.assertTrue(isinstance(dt, datetime.datetime))
        self.assertEquals(dt.year, 2013)
        self.assertEquals(dt.month, 11)
        self.assertEquals(dt.day, 12)
        self.assertEquals(dt.hour, 7)
        self.assertEquals(dt.minute, 50)
        self.assertEquals(dt.second, 23)

        ad = classad.ClassAd({"foo": dt})
        dt2 = ad["foo"]
        self.assertTrue(isinstance(dt2, datetime.datetime))
        self.assertEquals(dt, dt2)

        ad = classad.ClassAd({"foo": datetime.datetime.now()})
        td = (datetime.datetime.now() - ad["foo"])
        self.assertEquals(td.days, 0)
        self.assertTrue(td.seconds < 300)
Example #16
0
def request_token(pool, resource, scopes=None, local_dir=None):
    if ":" in pool:
        alias, port = pool.split(":")
    else:
        alias = pool
        port = DEFAULT_PORT
    ip, port = socket.getaddrinfo(alias, int(port), socket.AF_INET)[0][4]

    if not scopes:
        scopes = DEFAULT_TOKEN_SCOPES

    coll_ad = classad.ClassAd({
        "MyAddress":
        "<{}:{}?alias={}>".format(ip, port, alias),
        "MyType":
        "Collector",
    })
    logger.debug("Constructed collector ad: {}".format(repr(coll_ad)))

    htcondor.param["SEC_TOKEN_DIRECTORY"] = "/etc/condor/tokens.d"

    token = request_token_and_wait_for_approval(resource, alias, coll_ad,
                                                scopes)

    if token is None:
        return False

    print("Token request approved!")

    token_dir = htcondor.param["SEC_TOKEN_DIRECTORY"]
    token_name = "50-{}-{}-registration".format(alias, resource)
    token_path = os.path.join(token_dir, token_name)

    # We tell users to run register.py through the container and volume mount
    # "$PWD/tokens" into /etc/condor/tokens.d so our messages need to reflect
    # the host dir whenever they specify --local-dir (SOFTWARE-4372)
    if local_dir:
        # '/' is an accepted path separator across operating systems
        msg_path = os.path.join(local_dir, token_name).replace('\\', '/')
    else:
        msg_path = token_path

    logger.debug("Writing token to disk (in {})".format(msg_path))
    token.write(token_name)
    logger.debug("Wrote token to disk (at {})".format(msg_path))

    logger.debug("Correcting token file permissions...")
    shutil.chown(token_path, user=TOKEN_OWNER_USER, group=TOKEN_OWNER_GROUP)
    logger.debug("Corrected token file permissions...")
    print("Token was written to {}".format(msg_path))
    if not is_admin():
        print(NONROOT_TOKEN_MSG.format(path=msg_path, name=token_name))

    return True
Example #17
0
def ad_to_json(ad):
    result = classad.ClassAd()
    for (key, val) in ad.items():
        # Evaluate Condor expressions
        if isinstance(val, classad.ExprTree):
            val = val.eval()
        result[key] = val
    # Unfortunately, classad.Value.Undefined is of type int,
    # and json.dumps() converts it to "2".
    # Use HTCondor JSON conversion, then back to a Python object instead.
    return json.loads(result.printJson())
Example #18
0
def print_capabilities():
    capabilities = {
        'MultipleFileSupport': True,
        'PluginType': 'FileTransfer',
        # SupportedMethods indicates which URL methods/types this plugin supports
        #CHANGE ME HERE
        'SupportedMethods': 'example',
        #END CHANGE
        'Version': PLUGIN_VERSION,
    }
    sys.stdout.write(classad.ClassAd(capabilities).printOld())
Example #19
0
 def test_ad_iterator(self):
     ad = classad.ClassAd()
     ad["foo"] = 1
     ad["bar"] = 2
     self.assertEqual(len(ad), 2)
     self.assertEqual(len(list(ad)), 2)
     self.assertEqual(list(ad)[1], "foo")
     self.assertEqual(list(ad)[0], "bar")
     self.assertEqual(list(ad.items())[1][1], 1)
     self.assertEqual(list(ad.items())[0][1], 2)
     self.assertEqual(list(ad.values())[1], 1)
     self.assertEqual(list(ad.values())[0], 2)
Example #20
0
    def who(self) -> classad.ClassAd:
        """
        Return the result of ``condor_who -quick``,
        as a :class:`classad.ClassAd`.
        If ``condor_who -quick`` fails, or the output can't be parsed into
        a sensible who ad, this method returns an empty ad.
        """
        who = self.run_command(["condor_who", "-quick"])

        try:
            parsed = classad.parseOne(who.stdout)

            # If there's no MASTER key in the parsed ad, it indicates
            # that we actually got the special post-shutdown message
            # from condor_who and should act like there's nothing there.
            if "MASTER" not in parsed:
                return classad.ClassAd()

            return parsed
        except Exception:
            return classad.ClassAd()
    def submitDirect(self, schedd, cmd, arg, info): #pylint: disable=R0201
        """
        Submit directly to the schedd using the HTCondor module
        """
        dagAd = classad.ClassAd()
        addCRABInfoToClassAd(dagAd, info)

        # NOTE: Changes here must be synchronized with the job_submit in DagmanCreator.py in CAFTaskWorker
        dagAd["CRAB_Attempt"] = 0
        dagAd["JobUniverse"] = 12
        dagAd["HoldKillSig"] = "SIGUSR1"
        dagAd["Out"] = os.path.join(info['scratch'], "request.out")
        dagAd["Err"] = os.path.join(info['scratch'], "request.err")
        dagAd["Cmd"] = cmd
        dagAd['Args'] = arg
        dagAd["TransferInput"] = info['inputFilesString']
        dagAd["LeaveJobInQueue"] = classad.ExprTree("(JobStatus == 4) && ((StageOutFinish =?= UNDEFINED) || (StageOutFinish == 0))")
        dagAd["TransferOutput"] = info['outputFilesString']
        dagAd["OnExitRemove"] = classad.ExprTree("( ExitSignal =?= 11 || (ExitCode =!= UNDEFINED && ExitCode >=0 && ExitCode <= 2))")
        dagAd["OtherJobRemoveRequirements"] = classad.ExprTree("DAGManJobId =?= ClusterId")
        dagAd["RemoveKillSig"] = "SIGUSR1"
        dagAd["Environment"] = classad.ExprTree('strcat("PATH=/usr/bin:/bin CONDOR_ID=", ClusterId, ".", ProcId)')
        dagAd["RemoteCondorSetup"] = info['remote_condor_setup']
        dagAd["Requirements"] = classad.ExprTree('true || false')
        dagAd["TaskType"] = "ROOT"
        dagAd["X509UserProxy"] = info['userproxy']

        r, w = os.pipe()
        rpipe = os.fdopen(r, 'r')
        wpipe = os.fdopen(w, 'w')
        if os.fork() == 0:
            #pylint: disable=W0212
            try:
                rpipe.close()
                try:
                    resultAds = []
                    htcondor.SecMan().invalidateAllSessions()
                    os.environ['X509_USER_PROXY'] = info['userproxy']
                    schedd.submit(dagAd, 1, True, resultAds)
                    schedd.spool(resultAds)
                    wpipe.write("OK")
                    wpipe.close()
                    os._exit(0)
                except Exception: #pylint: disable=W0703
                    wpipe.write(str(traceback.format_exc()))
            finally:
                os._exit(1)
        wpipe.close()
        results = rpipe.read()
        if results != "OK":
            raise Exception("Failure when submitting HTCondor task: %s" % results)

        schedd.reschedule()
Example #22
0
    def _submit_task(self, job_id, target):
        cluster_ad = classad.ClassAd()
        proc_ad = classad.ClassAd()
        props = self.base_job_properties.copy()
        props['Environment'] = environ_to_condor()
        for k, v in props.items():
            cluster_ad[k] = v
            #     classad.ExprTree(
        reqs = '%s && %s' % (self.base_reqs, 'TARGET.Machine == "%s"' % target)
        cluster_ad['Requirements'] = classad.ExprTree(reqs)

        workdir_i = '%s/%i/' % (self.workdir, job_id)
        proc_ad['UserLog'] = '%s/log' % workdir_i
        proc_ad['Out'] = '%s/out' % workdir_i
        proc_ad['Err'] = '%s/err' % workdir_i
        proc_ad['Arguments'] = str(job_id)

        cluster_id = self.schedd.submitMany(cluster_ad, [(proc_ad, 1)])
        logger.info('Submitted job %i to cluster %i on %s.' %
                    (job_id, cluster_id, target))
        self.running_clusters[cluster_id] = (target, job_id)
        self.available_machines[target] = False
Example #23
0
def makeSortAds():
    anAd = classad.ClassAd()
    anAd["GridResource"] = "condor localhost localhost"
    anAd["TargetUniverse"] = 5
    anAd["Name"] = "Sort Ads"
    anAd["Requirements"] = classad.ExprTree(
        "(sortStringSet(\"\") isnt error) && (target.HasBeenRouted is false) && (target.HasBeenSorted isnt true)"
    )
    anAd["copy_DESIRED_Sites"] = "Prev_DESIRED_Sites"
    anAd["eval_set_DESIRED_Sites"] = classad.ExprTree(
        "debug(sortStringSet(Prev_DESIRED_Sites))")
    anAd["set_HasBeenSorted"] = True
    anAd['set_HasBeenRouted'] = False
 def testCollectorAdvertise(self):
     self.launch_daemons(["COLLECTOR"])
     coll = htcondor.Collector()
     now = time.time()
     ad = classad.ClassAd('[MyType="GenericAd"; Name="Foo"; Foo=1; Bar=%f; Baz="foo"]' % now)
     coll.advertise([ad])
     for i in range(5):
         ads = coll.query(htcondor.AdTypes.Any, 'Name =?= "Foo"', ["Bar"])
         if ads: break
         time.sleep(1)
     self.assertEquals(len(ads), 1)
     self.assertEquals(ads[0]["Bar"], now)
     self.assertTrue("Foo" not in ads[0])
def collect_cache_stats(url, rootdir, cache_max_fs_fraction=1.0):
    """ Collect stats on the cache server """
    start_time = time.time()

    parsed_url = urllib.parse.urlparse(url)

    # Python 2.6's urlparse returns a ParseResult object whereas
    # Python 2.4's urlparse returns a tuple that doesn't handle
    # root:// properly
    try:
        if parsed_url.scheme not in ('root', 'xroot'):
            raise Exception("URL '%s' is not an xrootd url" % url)

        hostname = parsed_url.netloc
    except AttributeError:
        if parsed_url[0] not in ('root', 'xroot'):
            raise Exception("URL '%s' is not an xrootd url" % url)

        hostname = parsed_url[2][2:]  # Avoid the '//' prefix

    result = {
        'MyType': 'Machine',
        'Name': 'xrootd@%s' % hostname,
        'stats_time': int(start_time)
    }
    result.update(test_xrootd_server(url))
    result.update(get_cache_info(rootdir, cache_max_fs_fraction))

    stats_per_vo = scan_cache_dirs(rootdir)
    # add up the sizes
    totals = dict()
    most_recent_access = 0
    result['VO'] = {}
    for vo, vostats in stats_per_vo.items():
        for k, v in vostats.items():
            if k == "most_recent_access_time":
                most_recent_access = max(most_recent_access, v)
            else:
                try:
                    totals[k] += v
                except KeyError:
                    totals[k] = v
        result['VO'][vo] = vostats
    result['used_cache_bytes'] = totals.pop("used_bytes", 0)
    for k, v in totals.items():
        result["total_" + k] = v
    if most_recent_access > 0:
        result["most_recent_access_time"] = most_recent_access

    result['time_to_collect_stats'] = time.time() - start_time
    return classad.ClassAd(result)
Example #26
0
def makeDrainAds(config=None):
    anAd = classad.ClassAd()
    anAd["GridResource"] = "condor localhost localhost"
    anAd["TargetUniverse"] = 5
    set_To = 200000
    draining_agents = config.get('speed_drain', [])
    for agent in draining_agents:
        anAd["Name"] = str("Drain agent %s" % agent)
        exp = 'regexp("%s",GlobalJobId) && JobStatus == 1 && JobPrio<%d' % (
            str(agent), set_To)
        anAd["Requirements"] = classad.ExprTree(str(exp))
        anAd["set_JobPrio"] = set_To
        anAd["set_HasBeenRouted"] = False
        print anAd
Example #27
0
 def killJobs(self, ids):
     ad = classad.ClassAd()
     ad['foo'] = ids
     const = "CRAB_ReqName =?= %s && member(CRAB_Id, %s)" % (
         HTCondorUtils.quote(self.workflow), ad.lookup("foo").__repr__())
     with HTCondorUtils.AuthenticatedSubprocess(self.proxy) as (parent,
                                                                rpipe):
         if not parent:
             self.schedd.act(htcondor.JobAction.Remove, const)
     results = rpipe.read()
     if results != "OK":
         raise TaskWorkerException("The CRAB3 server backend could not kill jobs [%s]. because the Grid scheduler answered with an error\n" % ", ".join(ids)+\
                                   "This is probably a temporary glitch, please try it again and contact an expert if the error persist\n"+\
                                   "Error reason %s" % results)
Example #28
0
File: main.py Project: saleph/HTSim
def perform_submit(work_no):
    this_path = os.path.dirname(os.path.realpath(__file__))
    initial_dir = os.path.join(this_path, "sim_{}".format(work_no))
    work = dict(Cmd=os.path.join(this_path, "batch.py"),
                Iwd=initial_dir,
                UserLog="sim.log",
                UserOutput="sim.out",
                TransferInput="simdata",
                Err="sim.err",
                Arguments="simdata",
                ShouldTransferFiles="YES")
    work_descriptor = classad.ClassAd(work)
    cluster_id = htcondor.Schedd().submit(work_descriptor)
    return cluster_id
Example #29
0
 def submit_with_htcondor(self, job):
     '''
         === In development ===
         Submits all registered jobs to the local HTCondor scheduler using
         the HTCondor python bindings
     '''
     import htcondor
     import classad
     # following
     # http://osgtech.blogspot.co.uk/2014/03/submitting-jobs-to-htcondor-using-python.html
     schedd = htcondor.Schedd()
     job_ad = classad.ClassAd()
     job_ad['executable'] = 'condor/run_job'
     # TODO
     schedd.submit(job_ad)
Example #30
0
def makeReleaseAds(config):
    """
    Create a set of rules to release a task to match
    """
    for task, where in config.get('release', {}).items():
        anAd = classad.ClassAd()
        anAd["Name"] = str("Releasing task %s" % (task))
        anAd["GridResource"] = "condor localhost localhost"
        exp = '(HasBeenSetHeld is true) && (target.WMAgent_SubTaskName =?= %s)' % classad.quote(
            str(task))
        anAd["Requirements"] = classad.ExprTree(str(exp))
        anAd["copy_Held_DESIRED_Sites"] = "DESIRED_Sites"
        anAd["set_HasBeenRouted"] = False
        anAd["set_HasBeenSetHeld"] = False
        print anAd