Beispiel #1
0
 def __init__(self, config, dasmgr):
     DASRepresentation.__init__(self, config)
     self.dasmgr     = dasmgr
     self.dasmapping = self.dasmgr.mapping
     if  'dbs' in config:
         self.dbs_global = self.dasmapping.dbs_global_instance()
     else:
         self.dbs_global = None
     self.colors = {'das':gen_color('das')}
     for system in self.dasmgr.systems:
         self.colors[system] = gen_color(system)
Beispiel #2
0
 def __init__(self, config, dasmgr):
     DASRepresentation.__init__(self, config)
     self.dasmgr     = dasmgr
     self.dasmapping = self.dasmgr.mapping
     if  'dbs' in config:
         self.dbs_global = self.dasmapping.dbs_global_instance()
     else:
         self.dbs_global = None
     self.colors = {'das':gen_color('das')}
     for system in self.dasmgr.systems:
         self.colors[system] = gen_color(system)
Beispiel #3
0
    def init(self):
        """Init DAS web server, connect to DAS Core"""
        try:
            self.reqmgr = RequestManager(lifetime=self.lifetime)
            self.dasmgr = DASCore(engine=self.engine)
            self.repmgr = CMSRepresentation(self.dasconfig, self.dasmgr)
            self.daskeys = self.dasmgr.das_keys()
            self.gfs = db_gridfs(self.dburi)
            self.daskeys.sort()
            self.dasmapping = self.dasmgr.mapping
            self.dbs_url = self.dasmapping.dbs_url()
            self.dbs_global = self.dasmapping.dbs_global_instance()
            self.dbs_instances = self.dasmapping.dbs_instances()
            self.dasmapping.init_presentationcache()
            self.colors = {"das": gen_color("das")}
            for system in self.dasmgr.systems:
                self.colors[system] = gen_color(system)
            # get SiteDB from global scope
            self.sitedbmgr = SERVICES.get("sitedb2", None)
            # Start DBS daemon
            if self.dataset_daemon:
                self.dbs_daemon(self.dasconfig["web_server"])
            if not self.daskeyslist:
                keylist = [r for r in self.dasmapping.das_presentation_map()]
                keylist.sort(key=lambda r: r["das"])
                self.daskeyslist = keylist

        except ConnectionFailure as _err:
            tstamp = dastimestamp("")
            mythr = threading.current_thread()
            print "### MongoDB connection failure thread=%s, id=%s, time=%s" % (mythr.name, mythr.ident, tstamp)
        except Exception as exc:
            print_exc(exc)
            self.dasmgr = None
            self.reqmgr = None
            self.dbs_url = None
            self.dbs_global = None
            self.dbs_instances = []
            self.daskeys = []
            self.colors = {}
            self.q_rewriter = None
            return

        # KWS and Query Rewriting failures are not fatal
        try:
            # init query rewriter, if needed
            if self.dasconfig["query_rewrite"]["pk_rewrite_on"]:
                self.q_rewriter = CMSQueryRewrite(self.repmgr, self.templatepage)
        except Exception as exc:
            print_exc(exc)
            self.q_rewriter = None
Beispiel #4
0
    def init(self):
        """Init DAS web server, connect to DAS Core"""
        try:
            self.reqmgr     = RequestManager(lifetime=self.lifetime)
            self.dasmgr     = DASCore(engine=self.engine)
            self.repmgr     = CMSRepresentation(self.dasconfig, self.dasmgr)
            self.daskeys    = self.dasmgr.das_keys()
            self.gfs        = db_gridfs(self.dburi)
            self.daskeys.sort()
            self.dasmapping = self.dasmgr.mapping
            self.dbs_url    = self.dasmapping.dbs_url()
            self.dbs_global = self.dasmapping.dbs_global_instance()
            self.dbs_instances = self.dasmapping.dbs_instances()
            self.dasmapping.init_presentationcache()
            self.colors = {'das':gen_color('das')}
            for system in self.dasmgr.systems:
                self.colors[system] = gen_color(system)
            if  not self.daskeyslist:
                keylist = [r for r in self.dasmapping.das_presentation_map()]
                keylist.sort(key=lambda r: r['das'])
                self.daskeyslist = keylist

        except ConnectionFailure as _err:
            tstamp = dastimestamp('')
            mythr  = threading.current_thread()
            print("### MongoDB connection failure thread=%s, id=%s, time=%s" \
                    % (mythr.name, mythr.ident, tstamp))
        except Exception as exc:
            print_exc(exc)
            self.dasmgr  = None
            self.reqmgr  = None
            self.dbs_url = None
            self.dbs_global = None
            self.dbs_instances = []
            self.daskeys = []
            self.colors  = {}
            self.q_rewriter = None
            return

        # KWS and Query Rewriting failures are not fatal
        try:
            # init query rewriter, if needed
            if self.dasconfig['query_rewrite']['pk_rewrite_on']:
                self.q_rewriter = CMSQueryRewrite(self.repmgr,
                                                  self.templatepage)
        except Exception as exc:
            print_exc(exc)
            self.q_rewriter = None
Beispiel #5
0
 def init(self):
     """Init DAS web server, connect to DAS Core"""
     try:
         self.logcol     = DASLogdb(self.dasconfig)
         self.reqmgr     = RequestManager(self.dburi, lifetime=self.lifetime)
         self.dasmgr     = DASCore(engine=self.engine)
         self.repmgr     = CMSRepresentation(self.dasconfig, self.dasmgr)
         self.daskeys    = self.dasmgr.das_keys()
         self.gfs        = db_gridfs(self.dburi)
         self.daskeys.sort()
         self.dasmapping = self.dasmgr.mapping
         self.dasmapping.init_presentationcache()
         self.colors = {}
         for system in self.dasmgr.systems:
             self.colors[system] = gen_color(system)
         self.sitedbmgr   = SiteDBService(self.dasconfig)
     except Exception as exc:
         print_exc(exc)
         self.dasmgr = None
         self.daskeys = []
         self.colors = {}
         return
     # Start Onhold_request daemon
     if  self.dasconfig['web_server'].get('onhold_daemon', False):
         self.process_requests_onhold()
Beispiel #6
0
 def __init__(self, config, dasmgr):
     DASRepresentation.__init__(self, config)
     self.dasmgr     = dasmgr
     self.dasmapping = self.dasmgr.mapping
     if  config.has_key('dbs'):
         self.dbs_global = config['dbs'].get('dbs_global_instance', None)
     else:
         self.dbs_global = None
     self.colors     = {}
     for system in self.dasmgr.systems:
         self.colors[system] = gen_color(system)
Beispiel #7
0
def help_cards(base):
    """
    Return list of help cards used on web UI
    """
    hide = """
<div style="text-align: right">
<a href="javascript:HideTag('das_cards')">hide</a>
</div>
"""
    first = """
<h3 class="big">Help: DAS queries</h3>
DAS queries are formed by
<b>key=value</b> pairs, for example
<ul>
<li>dataset=*RelVal*</li>
<li>release=CMSSW_2_0_*</li>
<li>run=148126</li>
</ul>

The wild-card can be used to specify the pattern.
The list of supported DAS 
<b>keys</b> can be found in 
<b><a href="%s/services">Services</a></b> section.
For more details please read DAS 
<b><a href="%s/faq">Frequently Asked Questions</a></b>.
    """ % (base, base)
    first += hide

    card_list = []

    card = """
<h3 class="big">Help: DAS filters</h3>
DAS supports <b>grep</b>, <b>sort</b> and <b>unique</b> filters. You can use
<b>grep</b> filter to select part of the DAS record. For example,
<pre>
dataset=*RelVal* | grep dataset.nevents
</pre>
To sort your results use sort filter
<pre>
dataset=*RelVal* | sort dataset.nevents
</pre>
Multiple filters can be applied together, for instance
<pre>
dataset=*RelVal* | unique | grep dataset.name, dataset.nevents
</pre>
    """
    card += hide
    card_list.append(card)

    card = """
<h3 class="big">Help: DAS filters and conditions</h3>
DAS supports the following condition operators in filters:
&lt;, &gt;, != and =. Their usage is trivial:
<pre>
file dataset=/a/b/c | grep file.name, file.size>3000000, file.size<6000000
</pre>
The usage of wild-card
is allowed for string patterns. For example, you can select record
attribute and apply a wild-card condition at the same time:
<pre>
dataset=*RelVal* | grep dataset.name, dataset.name=*RECO
</pre>
    """
    card += hide
    card_list.append(card)

    card = """
<h3 class="big">Help: DAS aggregators</h3>
DAS supports variety of aggregator functions, such as:
<b>min, max, sum, count, avg, median</b>. They can be applied in any
order to any DAS record attribute. For example:
<pre>
file dataset=/a/b/c |  max(file.size), min(file.size),avg(file.size),median(file.size)
</pre>
Custom map-reduce function are also supported. Please contact DAS 
<b><a href="https://svnweb.cern.ch/trac/CMSDMWM/newticket?component=DAS&summary=Request map reduce function&owner=valya">support</a></b> if you need one.
    """
    card += hide
    card_list.append(card)

    card = """
<h3 class="big">Help: DAS date usage</h3>
DAS has special <b>date</b> keyword. It accepts values either in
YYYYMMDD format or via <b>last</b> operator. For example:
<pre>
run date=20110316
run date last 24h
jobsummary date last 60m
</pre>
Supported units for <b>last</b> operator are <b>d</b> (days),
<b>h</b> (hours) and <b>m</b>(minutes).
    """
    card += hide
    card_list.append(card)

    card = """
<h3 class="big">Help: DAS special keywords</h3>
DAS has special <b>records</b> and <b>queries</b> keywords.
The former can be used to look-up all records in DAS cache, while
later shows most recent queries placed in DAS. The DAS filters
can be used with them too. For example, if you want to see if there
are any file records in DAS cache you can simply type:
<pre>
records | grep file.name
</pre>
    """
    card += hide
    card_list.append(card)

    card = """
<h3 class="big">Help: DAS command line interface</h3>
DAS CLI tools is available at <b><a href="%s/cli">CLI</a></b>.
You may download and save it on your disk, e.g. as <em>das_cli</em>,
and use it in the following way:
<pre>
python das_cli --query="dataset=/ExpressPhysics*"
</pre>
Please use <b>--help</b> for more options.
The default output format is JSON. But you can apply filter and use plain data format, e.g.:
<pre>
python das_cli --query="dataset=/ExpressPhysics* | grep dataset.name, dataset.nevents" --format=plain
/ExpressPhysics/Run2011A-Express-v2/FEVT 13321290
...
</pre>
    """ % base
    card += hide
    card_list.append(card)

    card = """
<h3 class="big">Help: DAS colors</h3>
Colored little box, e.g.
<cite style="background-color:%s;padding-left:7px; padding-right:7px">&nbsp;</cite>
next to the Record output 
represents specific CMS data-services.<br />
Please follow DAS color map: <u style="color:#8f1f08;">show</u>
<u style="color:#8f1f08;">hide</u>
links to find out which system it corresponds to.

<br/>
<br/>

If you see several colored boxes in a row, e.g.
<cite style="background-color:%s;padding-left:7px; padding-right:7px">&nbsp;</cite>
<cite style="background-color:%s;padding-left:7px; padding-right:7px">&nbsp;</cite>,
it means that this
record is aggregated across several CMS data-services.
    """ % (gen_color('phedex'), gen_color('dbs'), gen_color('phedex'))
    card += hide
    card_list.append(card)

    card = """
<h3 class="big">Help: Commonly used dataset queries</h3>
Most physicists are interested to find their desired dataset. Here is an incomplete
list of queries which cover this use-case:
<pre>
dataset=*Zee*
dataset release=CMSSW_4*
dataset release=CMSSW_4* datatype=mc
dataset dataset=*Zee* datatype=mc release=CMSSW_4*
dataset primary_dataset=ZJetToEE_Pt* tier=*GEN*
dataset group=Top datatype=mc
dataset run=148126
dataset dataset=/Cosmics/Run2010B* site=T1_US_FNAL
</pre>
"""
    card += hide
    card_list.append(card)

    card = """
<h3 class="big">Help: Free text based queries</h3>
If you type free text based query, e.g.
<pre>
Zee CMSSW_3
</pre>
DAS will match provided keywords with
appropriate DAS <b>key=value</b> pairs which will displayed in the input field,
in this case 
<pre>dataset=*Zee* release=CMSSW_3*</pre>
<b>Please note:</b> DAS queries are case-sensitive due to underlying 
restrictions from participated data-services, but in
<b>autocompletion</b> mode DAS will apply case-insensitive searches for
all valid datasets to find your match.
"""
    card += hide
    card_list.append(card)

    random.shuffle(card_list)
    cards = [first] + card_list
    return cards