Exemple #1
0
 def get(self, filename):
     util.getLogger(__name__).debug("CSS name: %s" % filename)
     css = db.Query(CSSFile).filter('default =', True).filter('filename =', filename).fetch(1)
     if len(css) > 0:
         util.getLogger(__name__).debug("CSS found in DS")
         self.generate('css.html', { 'css' : css[0] })
     else:
         self.generate('css.html', {})
Exemple #2
0
def GetMysqlCursor():
    conn = cursor = None
    conn = GetMysqlConn()
    if conn is None:
        return None, None
    try:
        cursor = conn.cursor()
    except MySQLdb.MySQLError, e:
        util.getLogger().critical( "error(%d:%s): cursor()"%(e.args[0], e.args[1]))
Exemple #3
0
def main():
    logger = util.getLogger(__name__)
    out_hdlr = logging.StreamHandler(sys.stdout)
    out_hdlr.setFormatter(util.ColorLevelFormatter('%(message)s'))
    out_hdlr.setLevel(logging.INFO)
    logger.addHandler(out_hdlr)

    modulePath = os.path.dirname(os.path.abspath(__file__))
    config = configfile.getClusterSnapshotConfig(
        os.path.join(modulePath, 'snapshot.conf'), sys.argv[1:])

    logger.info("Starting Cluster Snaphot...")
    sourceDir = config.get('SOURCE_DIR', '')
    sourceFile = ''
    if os.path.isfile(os.path.join(sourceDir, "profile.platform")):
        sourceFile = util.sourceEnv(os.path.join(sourceDir,
                                                 'profile.platform'))
    elif os.path.isfile(os.path.join(sourceDir, "profile.lsf")):
        sourceFile = util.sourceEnv(os.path.join(sourceDir,
                                                 'profile.platform'))

    snapFile = runClsSnapshot(config)
    if snapFile:
        logger.info('Cluster snapshot %s has been created.', snapFile)
    else:
        logger.info('Failed to generate a cluster snapshot.')
        exit(-1)
Exemple #4
0
def save(config, lsfConfig, path='/tmp'):
    logger = util.getLogger(__name__)

    if 'LSF_LOGDIR' in lsfConfig:
        logDir = lsfConfig.get('LSF_LOGDIR', '')
        if os.path.isdir(logDir):
            lsfDC = Data_Collect(path, __name__)
            tempFolder = tempfile.mkdtemp(prefix='lc_')
            args = [
                os.path.join(sys.path[0], 'log_collector.py'), 'deploy',
                tempFolder,
                socket.gethostname(), 'lsf'
            ]
            log_collector = imp.load_source(
                __name__ + '.log_collector',
                os.path.join(os.path.dirname(os.path.realpath(__file__)),
                             'log_collector.py'))
            log_collector.main(args)
            for fname in iglob(os.path.join(tempFolder,
                                            '*.lsf.logs.*.tar.gz')):
                lsfDC.moveit(fname)
            util.rmtree(tempFolder)
        else:
            logger.error("LSF_LOGDIR directory '%s' does not exist.", logDir)
    else:
        logger.warn(
            'LSF_LOGDIR is not set in lsf.conf. No logs will be collected.')
Exemple #5
0
 def __init__(self, server_address, RequestHandlerClass, servernumber, 
              bind_and_activate=True):
     HTTPServer.__init__(self, server_address, RequestHandlerClass,
                         bind_and_activate=bind_and_activate)
     self.log = util.getLogger()
     self.servernumber = servernumber
     self.cloudiness_cache = {}
     self.cloudiness_cache_misses = 0
     self.cloudiness_cache_hits = 0
     self.image_cache = {}
     self.image_cache_size = 0
     
     self._cpu_usage = 0.0
     self._cpu_prev_idle = 0.0
     self._cpu_prev_total = 0.0
     self._cpu_usage_last_100_samples = []
     
     # Shared object is expected to reside in lib/libcloudiness.so
     LIB_FILENAME = "libcloudiness.so"
     LIBPATH = os.path.join(constants.ROOT_DIR, "lib", LIB_FILENAME)
     
     # Load the shared object. This object has been compiled with some
     # particular flags. Look at the Makefile accompanying it.
     self.libcloudiness = ctypes.cdll.LoadLibrary(LIBPATH)
     
     # This is one of the functions declared in the library
     self.calc_cloudiness = self.libcloudiness.calc_cloudiness
     self.calc_cloudiness.restype = ctypes.c_double
Exemple #6
0
def save(config, lsfConfig, path='/tmp'):

    logger = util.getLogger(__name__)

    if 'LSB_SHAREDIR' not in lsfConfig:
        logger.error("LSB_SHAREDIR is not defined in lsf.conf!")
        return False
    lsfShareDir = lsfConfig['LSB_SHAREDIR']
    lsfData = Data_Collect(path, __name__)
    lsid_out = lsfData.runit('lsid')
    ClName = ''
    for line in lsid_out.splitlines():
        if "My cluster name is" in line:
            ClName = line.split(' ')[4]
            break
    if not ClName:
        logger.error("No cluster can be found!")
        return False
    lsfWorkDir = os.path.join(lsfShareDir, ClName, 'logdir')

    eventNum = config.get('EVENTS_NUM', 0)
    if eventNum:
        eventNum = int(eventNum)
    else:
        eventNum = 0
    lsfData.copyit(os.path.join(lsfWorkDir, 'lsb.events'), 'work/')
    i = 0
    while i < eventNum:
        i += 1
        lsfData.copyit(os.path.join(lsfWorkDir, 'lsb.events' + '.' + str(i)),
                       'work/')

    lsfData.copyit(os.path.join(lsfWorkDir, 'lsb.acct'), 'work/')
Exemple #7
0
class Ra3GsLoginServerFactory(ServerFactory):
    protocol = Ra3GsLoginServer
    log = util.getLogger('gamespy.ra3Serv', self)

    def buildProtocol(self, addr):
        p = ServerFactory.buildProtocol(self, addr)
        p.theater = Theater.getTheater(gameId)
        return p
Exemple #8
0
def save(config, path='/tmp/S_SHUT'):
    modName = os.path.splitext(os.path.basename(__file__))[0]
    savePath = os.path.join(path, modName)
    logFileName = modName + '.log'
    logger = util.getLogger(__name__)

    if os.path.exists(savePath):
        dt = strftime("%Y-%m-%d:%H:%M:%S", gmtime())
        os.rename(savePath, savePath + '.' + dt)
    try:
        os.makedirs(savePath)
    except IOError as e:
        if e.errno == errno.EEXIST:
            pass
        else:
            logger.exception("Could not create directory path '%s'.", savePath)
            return

    thisModPath = os.path.dirname(os.path.abspath(__file__))
    sys.path.append(thisModPath)

    debugLevel = config.get('SYM_LOG_LEVEL')
    LOGFILE = os.path.join(savePath, logFileName)
    formatter = logging.Formatter(
        '%(asctime)s - %(levelname)s - %(name)s.%(funcName)s() - %(message)s')
    fHandle = logging.FileHandler(LOGFILE)
    fHandle.setFormatter(formatter)
    if debugLevel:
        try:
            fHandle.setLevel(getattr(logging, debugLevel.strip().upper()))
        except TypeError:
            pass
    logger.addHandler(fHandle)

    symModules = configfile.getModuleList(config.get('SYM_MDLS', allMods),
                                          allMods)
    sym_mods = []
    myScriptPath = os.path.dirname(os.path.realpath(__file__))
    for mod in symModules:
        try:
            sym_mods.append(
                imp.load_source(__name__ + '.' + mod,
                                os.path.join(myScriptPath, mod + '.py')))
        except ImportError:
            logger.exception(
                "Error loading module as defined in CLI parameter '--sym_mdls' or snapshot.conf SYM_MDLS "
                "parameter. Check that the module %s is valid.", mod)

    if not os.getenv('SOAM_HOME', ''):
        logger.warn(
            'Cannot find $SOAM_HOME in the environment. Data collection will be incomplete.'
        )

    for mod in sym_mods:
        mod.save(savePath)
    fHandle.close()
    logger.removeHandler(fHandle)
Exemple #9
0
    def __init__(self, tweets):
        self.twitterHandles = []
        self.trends = []
        self.categories = []
        self.indexLists = []
        self.logger = util.getLogger("populate_feed.InvertedIndex")
        self.totalTweets = 0

        self._populate(tweets)
Exemple #10
0
    def __init__(self, peer_port = PeerServer.DEFAULT_PORT, \
                 data_port = DataServer.DEFAULT_PORT, \
                 data_transfer_protocal = 'tcp',
                 p2p_prefix = None, callbacks = {}):
        '''
        Constructor
        *peer_port*: int. Peer server service port.
        *data_port*: int. Data server service port.
        *data_transfer_protocal*: string. data transfer protocal. supports 'tcp', 'http', 'udp'
        *p2p_prefix*: str. the protocal prefix to identify the peer data package.
        *callbacks*: the external callback functions to handle events.
                you need to parse the data depends your business.

                "register" - register callback. Be invoked after a peer registered to me.
                "query" - query callback. to parse query string and perform query.
                "message" - message callback. to populate the message when received.
                "action" - action callback. to parse and perform the action.
        *callback returns*:
                *"query"* callback should return (resource, service_protocal, service_port).
                        "resource" identify how to get the resource.
                        "service_protocal" is the transfer protocal(http,tcp,udp) to serving the resource.
                        "service_port" is the port to serving the resource.

        '''
        self.log = util.getLogger('ConnManager(%d)' % peer_port)
        if p2p_prefix:
            P.setPrefix(p2p_prefix)

        self.peer_port = peer_port
        self.data_port = data_port
        self.peerServer = PeerServer(('0.0.0.0', peer_port), PeerServerHandler)
        self.ip = self.peerServer.server_address[0]

        # initialize internal and external callbacks
        self.callbacks = callbacks  #external callbacks
        
        # init peer callbacks
        peerCallbacks = {
            'register'      : self._on_register,
            'query'         : self._on_query,
            'message'       : self._on_message,
            'action'        : self._on_action,
        }
        self.peerServer.init(peerCallbacks) #internal callbacks
        self.log.info("P2P Sever initialized on %s:%d:%d" % (self.ip, self.peer_port, self.data_port))
        
        
        dataCallbacks = {
            'connect'       : self._on_connect,
            'transfer'      : self._on_transfer,
            'disconnect'    : self._on_disconnect,
            'resource'      : self._on_resource,
            'signature'     : self._on_signature,
        
        }
        self.dataServer = DataServer(port=data_port, protocal=data_transfer_protocal, callbacks=dataCallbacks)
Exemple #11
0
def GetMysqlConn():
    try:
        cfg = ConfigParser.ConfigParser()
        cfg.read("/var/www/webot/blog/config.conf")
        db_host = cfg.get("db", "host")
        db_user = cfg.get("db", "user")
        db_passwd = cfg.get("db", "passwd")
        db_default = cfg.get("db", "default_db")

        conn = MySQLdb.connect(host=db_host,
                               user=db_user,
                               passwd=db_passwd,
                               db=db_default)
        conn.autocommit(True)
        conn.query("set names 'utf8'")

    except MySQLdb.MySQLError, e:
        util.getLogger().critical("error(%d:%s): connect"%(e.args[0], e.args[1]))
        return None
Exemple #12
0
def save(config, lsfConfig, path='/tmp'):
    logger = util.getLogger(__name__)

    lsfConfCp = Data_Collect(path, __name__)
    lsfConfDir = lsfConfig.get('LSF_CONFDIR', '')
    if lsfConfDir:
        lsfConfCp.copyit(lsfConfig['LSF_CONFDIR'])
    else:
        logger.error("Could not determine the LSF configuration dir. Failed to collect LSF configuration files.")
    lsfConfCp.copyit('/etc/lsf.sudoers')
    for fname in iglob('/etc/init.d/*lsf*'):
        lsfConfCp.copyit(fname, 'initd/')
Exemple #13
0
def save(path='/tmp'):
    logger = util.getLogger(__name__)
    egoConfDir = os.getenv("EGO_CONFDIR")
    if not egoConfDir:
        logger.error('Could not determine the EGO conf directory. Failed to collect EGO work directory files.')
        return

    egoWorkDir = os.path.join(egoConfDir, '..', 'work')
    if not os.path.exists(egoWorkDir):
        logger.error('The EGO work directory %s does not exist. Failed to collect EGO work directory files.',
                     egoWorkDir)
        return
    egoConfCp = Data_Collect(path, __name__)
    egoConfCp.copyit(egoWorkDir)
Exemple #14
0
def save(path='/tmp'):
    logger = util.getLogger(__name__)
    symDC = Data_Collect(path, __name__)
    output = symDC.runit('soamview app -s enabled')

    if output:
        lines = output.splitlines()
        if lines[0].startswith('APPLICATION'):
            lines.pop(0)
            for line in lines:
                app = line.split()[0]
                for task in appTasks:
                    symTasks.append(task % app)

    for cmd in symTasks:
        logger.debug("Calling %s ..." % cmd)
        symDC.saveit(cmd)
Exemple #15
0
    def __init__(self, port=DEFAULT_PORT, protocal='tcp', callbacks={}):
        '''
        Constructor.
        *port*: int. service port for DataPeer
        *protocal*: string. data transfer protocal. supports 'tcp', 'http', 'udp'
        *callbacks*: map. callback functions to process the data your self.
        '''
        self.log = util.getLogger('DataServer(%d)' % port)
        if not protocal in DataServer.SUPPORT_PROTOCALS:
            self.log.error = '%s is not a supported data server protocal'
            raise 'Not supported data transfer protocal'
        pass

        self.protocal = protocal
        self.port = port
        self.callbacks = callbacks
        self.log.info('Data server (%s) created.' % protocal.upper())
Exemple #16
0
def save(path='/tmp'):
    logger = util.getLogger(__name__)
    hosts = set()
    egoConfDir = os.getenv('EGO_CONFDIR', '')
    if not egoConfDir:
        logger.error(
            'Could not determine the EGO conf directory. Failed to collect Symphony log files.'
        )
        return
    egoConfFile = os.path.join(egoConfDir, 'ego.conf')
    if not os.path.isfile(egoConfFile):
        logger.error(
            "Could not find ego.conf. Failed to collect Symphony log files.")
        return
    config = configfile.getConfigFromFile(egoConfFile)
    try:
        for candidate in config['EGO_MASTER_LIST'].strip('"').split():
            hosts.add(socket.getfqdn(candidate))
    except:
        pass

    egoDC = Data_Collect(path, __name__)
    out = egoDC.runit('egosh rg ManagementHosts')
    if out:
        for line in out.splitlines():
            if line.startswith('Resource List:'):
                for host in line.split(':', 1)[-1].split():
                    hosts.add(socket.getfqdn(host))
                break

    tempFolder = tempfile.mkdtemp(prefix='lc_')
    hostList = ' '.join(hosts)
    args = [
        os.path.join(sys.path[0], 'log_collector.py'), 'deploy', tempFolder,
        hostList, 'ego'
    ]
    log_collector = imp.load_source(
        __name__ + '.log_collector',
        os.path.join(os.path.dirname(os.path.realpath(__file__)),
                     'log_collector.py'))
    log_collector.main(args)
    for filePattern in iglob(os.path.join(tempFolder, '*.ego.logs.*.tar.gz')):
        egoDC.moveit(filePattern)
    util.rmtree(tempFolder)
Exemple #17
0
def save(path='/tmp'):
    logger = util.getLogger(__name__)
    egoConfDir = os.getenv('EGO_CONFDIR')
    if not egoConfDir:
        logger.error(
            'Could not determine the EGO conf directory. Failed to collect EGO log files.'
        )
        return
    elif not os.path.exists(egoConfDir):
        logger.error('The EGO conf directory %s does not exist.', egoConfDir)
    egoConfCp = Data_Collect(path, __name__)
    # for fname in iglob(os.path.join(egoConfDir,'*.conf')):
    #	egoConfCp.copyit(fname)
    # for fname in iglob(os.path.join(egoConfDir,'*.xml')):
    #	egoConfCp.copyit(fname)
    # for fname in iglob(os.path.join(egoConfDir,'*.shared')):
    #	egoConfCp.copyit(fname)
    egoConfCp.copyit(egoConfDir)

    egoServiceConfDir = os.path.join(egoConfDir, '..', '..', 'eservice', 'esc',
                                     'conf', 'services')
    # for fname in iglob(os.path.join(egoServiceConfDir,'*.xml')):
    #	egoConfCp.copyit(fname)
    egoConfCp.copyit(egoServiceConfDir)
Exemple #18
0
def save(path='/tmp'):
    logger = util.getLogger(__name__)
    symConfCp = Data_Collect(path, __name__)
    egoConfDir = os.getenv('EGO_CONFDIR')
    if not egoConfDir:
        logger.error(
            "Could not determine the EGO conf directory. Failed to collect Symphony configuration files."
        )
        return
    profileDir = os.path.join(egoConfDir, '..', '..', 'soam', 'profiles')
    egoServiceConfDir = os.path.join(egoConfDir, '..', '..', 'eservice', 'esc',
                                     'conf', 'services')
    if os.path.exists(profileDir):
        symConfCp.copyit(profileDir)
    else:
        logger.error(
            "The Symphony application profile directory %s does not exist. "
            "Failed to copy application profiles.", profileDir)
    if os.path.exists(egoServiceConfDir):
        symConfCp.copyit(egoServiceConfDir)
    else:
        logger.error(
            "The EGO service profile directory %s does not exist. Failed to copy EGO service profiles.",
            egoServiceConfDir)
Exemple #19
0
import yum
from yum.misc import to_xml
from yum.update_md import UpdateMetadata, UpdateNotice

import util

log = util.getLogger(__name__)
#
# yum 3.2.22 compat:  UpdateMetadata.add_notice() not
# supported in 3.2.22.
#
if yum.__version__ < (3, 2, 28):

    def add_notice(self, un):
        if not un or not un["update_id"] or un['update_id'] in self._notices:
            return
        self._notices[un['update_id']] = un
        pkglist = un['pkglist'] or []
        for pkg in pkglist:
            for filedata in pkg['packages']:
                self._cache['%s-%s-%s' %
                            (filedata['name'], filedata['version'],
                             filedata['release'])] = un
                no = self._no_cache.setdefault(filedata['name'], set())
                no.add(un)
        return True

    UpdateMetadata.add_notice = add_notice


# Work around for:  https://bugzilla.redhat.com/show_bug.cgi?id=886240#c13
Exemple #20
0
def runClsSnapshot(config):
    logger = util.getLogger(__name__)

    modulePath = os.path.dirname(os.path.abspath(__file__))
    modname = os.path.splitext(os.path.basename(__file__))[0]
    savePath = config.get('TOP', '')
    if not savePath:
        savePath = os.path.join(modulePath, 'snapshots')
        if not os.path.exists(savePath):
            try:
                os.makedirs(savePath)
            except OSError as e:
                if e.errno == errno.EEXIST:
                    pass
                else:
                    logger.error("Could not create save path %s.", savePath)
                    return None
    elif not os.path.exists(savePath):
        logger.error("The save path %s does not exist.", savePath)
        return None

    logFileName = modname + '.log'

    logFile = os.path.join(savePath, logFileName)
    stdErr = config.get('LOG_STDERR', '')
    dbglevel = getattr(logging, config.get('LOG_LEVEL', 'WARN').upper())

    formatter = logging.Formatter(
        '%(asctime)s - %(levelname)s - %(name)s.%(funcName)s() - %(message)s')
    logfh = logging.FileHandler(logFile, mode='a')
    logfh.setFormatter(formatter)

    logger.addHandler(logfh)
    logger.setLevel(dbglevel)
    if stdErr:
        logstderr = logging.StreamHandler(sys.stderr)
        logstderr.setLevel(dbglevel)
        logstderr.setFormatter(
            util.LogFileFormatter('%(name)s: %(levelname)s- %(message)s'))
        logger.addHandler(logstderr)

    pKey = config.get('pKey', '')

    if pKey and os.path.isfile(pKey):
        os.putenv('SSH_PRIVATE_KEY', pKey)

    modules = configfile.getModuleList(config.get('MODULES', ['all']), allMods)

    mList = list()
    myScriptPath = os.path.dirname(os.path.realpath(__file__))

    for m in modules:
        try:
            mList.append(
                imp.load_source(__name__ + '.' + m,
                                os.path.join(myScriptPath, m + '.py')))
        except ImportError as e:
            logger.exception(
                "Error loading module as defined in CLI parameter '--modules' or snapshot.conf MODULES"
                " parameter. Check that the module %s is valid.", m)

    savePath = os.path.abspath(savePath)
    t_list = []
    for mod in mList:
        t_list.append(Thread(target=mod.save, args=[config, savePath]))
        t_list[-1].start()
    for thread in t_list:
        thread.join()
        logger.debug('%s is complete.', thread)

    logger.info('Compressing cluster snapshot...')
    timestr = datetime.now().strftime("%Y-%m-%d-%H%M%S")
    cwd = os.getcwd()
    os.chdir(savePath)
    sstarfile = 'CL_Snapshot.%s.tar.gz' % timestr
    tar = tarfile.open(sstarfile, 'w:gz')
    os.chdir(cwd)
    removeDir = []
    for name in os.listdir(savePath):
        if name.endswith('.tar.gz') or name.endswith('.log'):
            continue
        fullname = os.path.join(savePath, name)
        tar.add(fullname, arcname=name)
        removeDir.append(fullname)
    tar.close()

    for f in removeDir:
        if not util.rmtree(f, 5):
            logger.error("Unable to remove '%s'", f)

    logger.info("Compressed to '%s'", sstarfile)

    return os.path.join(savePath, sstarfile)
Exemple #21
0
#!/usr/bin/env python3.9
from uuid import uuid4

from flask import Flask, request
from flask_socketio import SocketIO, emit, send
import flask_socketio
import yaml

from util import getLogger, manager

async_mode = "gevent"
app = Flask(__name__)
app.config['SECRET_KEY'] = str(uuid4())

logger = getLogger('server', saveName='server.log')
slogger = logger.getChild('sIO')
elogger = logger.getChild('eIO')

socketio = SocketIO(app,
                    logger=slogger,
                    engineio_logger=elogger,
                    async_mode=async_mode,
                    ping_interval=60)

with open('event_list.yaml') as f:
    event_dict = yaml.safe_load(f)

with open('user.yaml') as f:
    users = manager(yaml.safe_load(f))

Exemple #22
0
#-*- coding: UTF-8 -*-


import logging
import logging.config
import settings
import util
import engine

logging.config.dictConfig(settings.logging)
logger = util.getLogger()

class Application:
	"""docstring for Application"""
	def __init__(self):
		logger.info('initializing applicaiton')

	def buildEnv(self):
		logger.info('buiding environment')

	def run(self):
		searcher = engine.Searcher()
		searcher.query('list append')


def main():
	app = Application()
	logger.info('application is running')
	app.run()

Exemple #23
0
import util
from consumer import Consumer
import environment
import cache

log = util.getLogger('ss.wizard')


def get_payload(endpoint):
    ttl = cache.TIME_HOUR / 2
    endpoint = util.sources_endpoint(endpoint)

    return environment.json_from_url(endpoint, expires=ttl)


def translate(foreign):
    final_url = foreign.get('final_url')

    if final_url:
        return final_url
    else:
        endpoint = util.translate_endpoint(foreign['original_url'],
                                           foreign['foreign_url'])
        response = environment.json_from_url(endpoint)

        return util.translated_from(response)


class Wizard(object):
    def __init__(self, endpoint, avoid_flv=False, start_at=0):
        self.endpoint = endpoint
Exemple #24
0
def save(config, lsfConfig, path='/tmp/'):
    logger = getLogger(__name__)
    sysDataCollector = Data_Collect(path, __name__)
    for syscmd in Sys_Tasks:
        logger.debug("Calling %s ..." % syscmd)
        sysDataCollector.saveit(syscmd)
#!/usr/bin/env python
import unittest
import util
import subprocess
from certConfig import Resource
from certConfig import CertTestConfig

logger = util.getLogger('certTest')


class CallbackScaleTest(unittest.TestCase):
    res = Resource("CallbackScaleTest")

    @classmethod
    def setUpClass(cls):
        try:
            logger.info("------------------------------------------")
            logger.info("set up the fixture for Callback Scale Test")
            cls.res.preTestValidation()
            cls.res.configure()
            cls.res.deployInfrastucture()
            cls.res.deployVM()
            cls.res.powerVM()
            logger.info("Press ENTER to continue...")
            input()
        except:
            cls.res.cleaner.pop_all().close()
            cls.res.saveLog()
            raise

    @classmethod
Exemple #26
0
__author__ = 'strong'

import splunk.entity as entity
import splunk.admin as admin
import util, time
import splunklib.client as client

logger = util.getLogger()

CONF_TARGET = 'addon_target'
CONF_WEB = 'configs/conf-web'

TARGET_PROPERTIES = ['port', 'scheme', 'username', 'password']

TARGET_APP = 'Splunk_TA_snow'
TARGET_OWNER = 'nobody'
TIME_FORMAT = "%Y-%m-%d %H:%M:%S"


class TargetManager(object):
    def __init__(self, app=None, owner=None, session_key=None):
        self._app = app
        self._owner = 'nobody'  # so that conf file will be saved in app
        self._sessionKey = session_key
        splunkd_host_port = self._get_entity(CONF_WEB, 'settings').get(
            'mgmtHostPort', '127.0.0.1:8089')
        host_and_port = splunkd_host_port.split(':')
        self.local_splunk_host = host_and_port[0]
        self.local_splunk_port = host_and_port[1]
        logger.info('app %s, owner %s, host %s, port %s' %
                    (self._app, self._owner, self.local_splunk_host,
Exemple #27
0
def save(config, path='/tmp/S_SHUT'):
    modName = os.path.splitext(os.path.basename(__file__))[0]
    savePath = os.path.join(path, modName)

    logger = util.getLogger(__name__)

    if os.path.exists(savePath):
        dt = strftime("%Y-%m-%d:%H:%M:%S", gmtime())
        os.rename(savePath, savePath + '.' + dt)
    try:
        os.makedirs(savePath)
    except Exception as e:
        if e.errno == errno.EEXIST:
            pass
        else:
            logger.exception("Could not create directory path '%s'.", savePath)
            return

    logFileName = modName + '.log'
    debugLevel = config.get('LSF_LOG_LEVEL')
    LOGFILE = os.path.join(savePath, logFileName)
    formatter = logging.Formatter(
        '%(asctime)s - %(levelname)s - %(name)s.%(funcName)s() - %(message)s')
    fHandle = logging.FileHandler(LOGFILE)
    fHandle.setFormatter(formatter)
    # fHandle.addFilter(util.ReplaceMainFilter())
    if debugLevel:
        try:
            fHandle.setLevel(getattr(logging, debugLevel.strip().upper()))
        except:
            pass
    logger.addHandler(fHandle)

    lsfModules = configfile.getModuleList(config.get('LSF_MDLS', allMods),
                                          allMods)
    lsf_mods = []
    myScriptPath = os.path.dirname(os.path.realpath(__file__))

    for mod in lsfModules:
        try:
            lsf_mods.append(
                imp.load_source(__name__ + '.' + mod,
                                os.path.join(myScriptPath, mod + '.py')))
        except ImportError:
            logger.error(
                "Error loading module as defined in CLI parameter '--lsf_mdls' or snapshot.conf LSF_MDLS "
                "parameter. Check that the module %s is valid.", mod)

    if not os.getenv('LSF_ENVDIR'):
        logger.warn(
            'Cannot find $LSF_ENVDIR in the environment. Data collection will be incomplete.'
        )

    configFilePath = os.path.join(os.getenv('LSF_ENVDIR', '/etc'), 'lsf.conf')
    lsfConfig = {}
    try:
        lsfConfig = configfile.getConfigFromFile(configFilePath)
    except IOError:
        logger.error(
            "Could not load %s. LSF data collection will be incomplete.",
            configFilePath)

    for mod in lsf_mods:
        mod.save(config, lsfConfig, savePath)
    fHandle.close()
    logger.removeHandler(fHandle)
Exemple #28
0
def save(path='/tmp'):
    logger = util.getLogger(__name__)
    egoDC = Data_Collect(path, __name__)
    for cmd in egoTasks:
        logger.debug("Calling %s ..." % cmd)
        egoDC.saveit(cmd)
Exemple #29
0
def getClusterSnapshotConfig(configFile=None,
                             arguments=None,
                             preferConfigFile=False):

    logger = getLogger(__name__)
    logLevels = ['critical', 'error', 'warning', 'info', 'debug']
    configFromFile = {}
    if configFile:
        if os.path.isfile(configFile):
            configFromFile = getConfigFromFile(configFile)
        else:
            logger.warn(
                "The snapshot configuration file %s could not be found.",
                configFile)
    configFromArg = {}
    if arguments:
        parser = argparse.ArgumentParser(
            description='Spectrum Cluster Snapshot Tool')
        # General arguments.
        parser.add_argument("--source_dir",
                            dest="SOURCE_DIR",
                            metavar="<path>",
                            help="Cluster source file location.")
        parser.add_argument("--save_path",
                            dest="TOP",
                            metavar="<path>",
                            help="Top dir to save snapshot data in.")
        parser.add_argument("--log_level",
                            dest="LOG_LEVEL",
                            default='warning',
                            choices=logLevels,
                            help="debug level")
        parser.add_argument(
            "--log_stderr",
            dest="verbose",
            default='n',
            choices=['y', 'n'],
            help="Run in verbose mode by printing logging message to stderr.")
        choiceList = getChoiceList(clsnapshot.allMods)
        parser.add_argument(
            "--modules",
            dest="MODULES",
            default=['all'],
            nargs="+",
            choices=choiceList,
            help=
            "List of cluster data to collect. Items marked with ~ are explicitly not collected."
        )
        # LSF arguments.
        choiceList = getChoiceList(lsf.allMods)
        parser.add_argument(
            "--lsf_mdls",
            dest="LSF_MDLS",
            default=['all'],
            nargs="+",
            choices=choiceList,
            help=
            "List of LSF data to collect. Items marked with ~ are explicitly "
            "not collected.")
        parser.add_argument("--lsf_log_level",
                            default='warning',
                            dest="LSF_LOG_LEVEL",
                            choices=logLevels,
                            help="Debug level for LSF data collection.")
        parser.add_argument("--lsf_events_num",
                            type=int,
                            dest="EVENTS_NUM",
                            metavar="<number of lsb.events files>",
                            help="The number of lsb.events* to save")
        # EGO arguments
        choiceList = getChoiceList(ego.allMods)
        parser.add_argument(
            "--ego_mdls",
            dest="EGO_MDLS",
            nargs="+",
            choices=choiceList,
            default=["all"],
            help=
            "The comma delimited list of EGO data to collect. Items marked with ~ are never "
            "collected.")
        parser.add_argument("--ego_log_level",
                            dest="EGO_LOG_LEVEL",
                            default='warning',
                            choices=logLevels,
                            help="Debug level for EGO data collection.")
        # SYM arguments
        choiceList = getChoiceList(sym.allMods)
        parser.add_argument(
            "--sym_mdls",
            dest="SYM_MDLS",
            default=['all'],
            choices=choiceList,
            nargs="+",
            help=
            "The comma delimited list of SYM data to collect. Items marked with ~ are never"
            " collected.")
        parser.add_argument("--sym_log_level",
                            default='warning',
                            dest="SYM_LOG_LEVEL",
                            choices=logLevels,
                            help="Debug level for SYM data collection.")
        parser.add_argument(
            "--pkey",
            action="store",
            dest="pKey",
            metavar="<path>",
            help="Private key file path for logging into remote hosts.")

        configFromArg = vars(parser.parse_args(arguments))

    if preferConfigFile:
        totalConfig = configFromArg
        totalConfig.update(configFromFile)
    else:
        totalConfig = configFromFile
        totalConfig.update(configFromArg)

    return totalConfig
Exemple #30
0
import shutil
import cProfile
import pstats

from mtoken import Character, Morph, LToken, NPC
from macro import CssMacro, TMacro, LibMacro, SMacro
from mtable import Table, Entry
from cmpgn import Campaign, CProp, PSet
from zone import Zone
from util import lName, getLogger, configureLogger, parse_args
from data import content

#host = "192.168.200.7:5123"
host = "localhost:5123"

log = getLogger(lName)

morph_props = '''[
{"name": "pools", "showOnSheet": true, "value": "Ins {insight} | Mox {moxie} |Vig {vigor} | Flex {flex}"}
]
'''
pc_props = '''[
{"name": "aptitudes", "showOnSheet": true, "value": "COG {cognition} | INT {intuition} | REF {reflexes} | SAV {savvy} | SOM {somatics} | WIL {willpower}"},
{"name": "pools", "showOnSheet": true, "value": "Ins {insight} | Mox {moxie} |Vig {vigor} | Flex {flex}"},
{"name": "initiative", "showOnSheet": true, "value": "{(reflexes + intuition)/5}"},
{"name": "lucidity", "showOnSheet": true, "value": "{willpower*2}"},
{"name": "insanity", "showOnSheet": true, "value": "{lucidity*2}"},
{"name": "trauma", "showOnSheet": true, "value": "{lucidity/5}"},
{"name": "infection", "showOnSheet": true, "value": "{psi*10}"},
{"name": "morph", "showOnSheet": true, "value": "{morph}"}
]
import sendgrid
import util, configparser, getopt, subprocess, sys, time, psutil
#from models import ProcessTimeSeries
from models import Database

start_time = time.time()

# initial setup
config = util.readConfig()
logger = util.getLogger(config)
sg = util.getSendGrid(config)
db = Database(config)

# parse arguments
try:
    opts, args = getopt.getopt(sys.argv[1:], "hvrt", ["--restarts"])
except getopt.GetoptError:
    print ("Not enough options")
    sys.exit(2)

for opt, arg in opts:
    if opt == '-h':
        util.printhelp()
        sys.exit(0)
    elif opt in ("-r", "--restarts"):
        util.printrestarts(config)
        sys.exit(0)
    elif opt in ("-t", "--time"):
        util.printtimeseries(config)
        sys.exit(0)
    elif opt == "-v":
Exemple #32
0
class QueryMasterFactory(ServerFactory):
    protocol = gamespy.master.QueryMaster
    log = util.getLogger('gamespy.ra3master', self)
    gameName = gameId
Exemple #33
0
 def post(self):
   action = self.request.get('action')
   request_ = self.request
   result = getattr(self, action)(request_)
   util.getLogger(__name__).debug('ajax action "%s"return value is %s', action,simplejson.dumps(result))
   self.response.out.write(simplejson.dumps(result))
__author__ = 'strong'

import httplib
import urllib
import base64
import util as u
import splunklib.client as client
import splunklib.binding as binding
import splunk.admin as admin
from urlparse import urlparse

logger = u.getLogger()

SETUP_ACCOUNT_URL = "service_now_setup/snow_account"
TABLE_NAME = "cmn_department"


class SnowAccount(object):
    def __init__(self,
                 snow_url,
                 release,
                 username,
                 password,
                 name='snow_account'):
        self.name = name
        self.snow_url = snow_url
        self.release = release and release.strip() or ''
        self.username = username and username.strip() or ''
        self.password = password and password.strip() or ''

Exemple #35
0
import gzip
import os
import shutil
import xml.dom.minidom

import yum.comps

import util
log = util.getLogger(__name__)


def yum_group_to_model_group(obj):
    """
    Translate a yum.comps.Group to a dict
    @param obj: yum.comps.Group object
    @return: dict
    """
    grp = {}
    grp['id'] = obj.groupid
    grp['name'] = obj.name
    grp['description'] = obj.description
    grp['user_visible'] = obj.user_visible
    grp['display_order'] = obj.display_order
    grp['default'] = obj.default
    grp['langonly'] = obj.langonly
    grp['mandatory_package_names'] = []
    grp['mandatory_package_names'].extend(obj.mandatory_packages.keys())
    grp['optional_package_names'] = []
    grp['optional_package_names'].extend(obj.optional_packages.keys())
    grp['default_package_names'] = []
    grp['default_package_names'].extend(obj.default_packages.keys())
"""
python version:3.5
"""

__author__ = "Kantha Girish", "Pankaj Uchil Vasant", "Samana Katti"

import json

import twitter

from inverted_index import InvertedIndex
from database import getDBInstance
from util import getLogger, config

logger = getLogger("populate_feed")


def getTweets(woeid):
    """
    :param woeid : Where On Earth ID of the place of interest for which trends and related
        tweets are to be extracted.
    :return: A python dictionary containing keys as trends and the values as the list of tweets
        fetched from the twitter API call.
        tweets = {
                    "trend1": [tweet1, tweet2, ...],
                    "trend2": [tweet3, tweet4, ...]
                }

    This function reads config file `config.ini` containing the access keys for twitter API
    calls and creates a connection to the twitter API. The trends are fetched for the
    specified `woeid` and the tweets are fetched for each trend. The result is returned as a
"""
python version:3.5
"""

__author__ = "Kantha Girish", "Pankaj Uchil Vasant", "Samana Katti"

from os import listdir
from os.path import isdir, isfile, join

from naivebayes import NaiveBayes
from util import config, getLogger

logger = getLogger("train_classifiers")


def getData(folder):
    """
    :param folder: full path of the folder read files from for training the model
    :return: documents(list of documents), labels(list of labels)

    This function will put all the contents of the documents in each category into a list
    and creates a list of labels which are the names of the category folders
    """
    data = {}
    for f in listdir(folder):
        if isdir(join(folder, f)):
            data[f] = []
            for file in listdir(join(folder, f)):
                if isfile(join(folder, f, file)) and ".txt" in file:
                    with open(join(folder, f, file)) as fobj:
                        data[f].append(fobj.read().replace('\n', ''))
Exemple #38
0
# GNU General Public License for more details.
#
#You should have received a copy of the GNU General Public License
#along with django-facebookconnect.  If not, see <http://www.gnu.org/licenses/>.

from django.http import HttpResponse, HttpResponseRedirect, Http404
from django.template import RequestContext
from django.core.urlresolvers import reverse
from django.shortcuts import render_to_response
from django.contrib.auth import authenticate, login, logout, REDIRECT_FIELD_NAME
from django.contrib.auth.forms import AuthenticationForm, UserCreationForm
from django.contrib.auth.models import User
from django.conf import settings

import logging, util
log = util.getLogger('facebookconnect.views')

from facebook.djangofb import require_login as require_fb_login

from facebookconnect.models import FacebookProfile
from facebookconnect.forms import FacebookUserCreationForm

def facebook_login(request, redirect_url=None,
                   template_name='facebook/login.html',
                   extra_context=None):

    """
    facebook_login
    ===============================
    
    Handles logging in a facebook user. Usually handles the django side of
Exemple #39
0
from util import getLogger
from _default import cache_store as _default_store
log = getLogger('ss.cache')

TIME_MINUTE = 60
TIME_HOUR   = 60  * TIME_MINUTE
TIME_DAY    = 24  * TIME_HOUR
TIME_WEEK   = 7   * TIME_DAY
TIME_MONTH  = 30  * TIME_DAY
TIME_YEAR   = 365 * TIME_DAY

def reset():
    store.reset()

def set(key, value, **kwargs):
    if not 'expires' in kwargs:
        kwargs['expires'] = 10

    store.set(key, value, **kwargs)

def get(key):
    try:
        return store.get(key)
    except: pass

def fetch(key, cb, **kwargs):
    if expired(key):
        log.debug('%s is stale' % key)
        set(key, cb(), **kwargs)

    return get(key)
Exemple #40
0
import util
from consumer import Consumer
import environment
import cache

log = util.getLogger('ss.wizard')

def get_payload(endpoint):
    ttl      = cache.TIME_HOUR / 2
    endpoint = util.sources_endpoint(endpoint)

    return environment.json_from_url(endpoint, expires = ttl)

def translate(foreign):
    final_url = foreign.get('final_url')

    if final_url:
        return final_url
    else:
        endpoint = util.translate_endpoint( foreign['original_url'], foreign['foreign_url'])
        response = environment.json_from_url(endpoint)

        return util.translated_from(response)

class Wizard(object):
    def __init__(self, endpoint, avoid_flv = False, start_at = 0):
        self.endpoint    = endpoint
        self.file_hint   = None
        self.avoid_flv   = avoid_flv
        self.consumer    = None
        self.start_at    = start_at
Exemple #41
0
import util
from wizard         import Wizard
from downloadstatus import DownloadStatus

log = util.getLogger('ss.downloader')

def curl_strategy_command(dl):
    command = [
        'curl',
        '--location',
        '--referer',    dl.consumer.url,
        '--cookie',     dl.consumer.cookie_string,
        '--user-agent', dl.consumer.ua,
        '--stderr',     dl.status_file,
        '--output',     dl.local_partfile
    ]

    if int(dl.limit) > 0:
        command.append('--limit-rate')
        command.append('%sK' % dl.limit)

    command.append(dl.consumer.asset_url)

    return command

def wget_strategy_command(dl):
    command = [
        'wget',
        '--no-cookies',
        '--continue',
        '--referer',         dl.consumer.url,
Exemple #42
0
from lcs import *
from lcdb import Nation, Code, Text, store
from util import Cache, getLogger, utf8, convertRelativeLink
from bs4 import BeautifulSoup
from bs4.element import Tag
from zipfile import ZipFile
from datetime import datetime
import requests, os, logging, shutil
from clint.textui import progress
from multiprocessing import Pool

log = getLogger(__name__)

now = datetime.now
BS = lambda xml: BeautifulSoup(xml, 'lxml')

dedup = lambda l: list(set(l))



# The UK presents a unique challenge in that, while it has great
# data accessibility for changes, its legal publication is ongoing
# and does not have a general revision identified for the entirety
# this essentially means that each scrape will have its own revision
# and that revision will only be related to when it was scraped.
# Essentially this program was architected to report on publications
# of entire legal codes, and not real-time changes. Modifications
# could be made to specify point-in-time reporting, which would 
# introduce significant complexity but easily handle this instance
# and allow the consumption of the UK's RSS feeds for changes. 
Exemple #43
0
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import json
import jinja2
import re
from util import jenv, getLogger

log = getLogger(__name__)

cachedMacro = {}

class Macro(object):
	def __init__(self, label, tmpl, group, colors, others=None):
		self.tmpl = tmpl
		self._template = None
		self._label = label
		self.colors = colors
		self.cattr = ['cognition', 'intuition', 'reflex', 'savvy', 'somatics', 'willpower'] # XXX eclipse specific in lib
		self.group = group
		self.others = others if (others is not None) else {}
		self.allowPlayerEdits = 'false'
		self.autoExecute = 'true'
		self._command = None

	def __str__(self): return '%s<%s,grp=%s>' % (self.__class__.__name__, self.label, self.group)
	def __repr__(self): return str(self)

	def to_dict(self): return dict(self.__dict__)

	@property
	def label(self): return self._label
Exemple #44
0
 def connectionMade(self):
     EaServer.connectionMade(self)
     self.log = util.getLogger('login.cnc4', self)
     self.msgFactory = MessageFactory(self.transport, EaLoginMessage)
     self.hlrFactory = MessageHandlerFactory(
         self, 'ea.games.redalert3.Ra3MsgHlr')
Exemple #45
0
#!/usr/bin/env python 

from bottle import *
from redis import Redis
import util, json
from time import strftime,gmtime,time,sleep

KEY_MAPPING="arduino:keymapping"
KEY_POSITIONS="arduino:keypositions"
KEY_POSITION_FMT="arduino:keyposition:%s"
ARDUINO_COMMAND="arduino:remote-command"
LOG=util.getLogger('arduino_remote_server')
AUTH_COOKIE='remote-auth'
COOKIE_SECRET=util.CONFIG['web']['cookie']
PASSWORD=util.CONFIG['web']['password']

def auth_required():
    def decorator(func):
        def wrapper(*args, **kwargs):
            authd=request.get_cookie(AUTH_COOKIE,secret=COOKIE_SECRET)
            if authd !=PASSWORD:
              redirect('/login')
            return func(*args, **kwargs)
        return wrapper
    return decorator

def getRelativeCookieTime(days):
  return strftime("%a, %d-%b-%Y %H:%M:%S GMT", gmtime(days*24*60*60+time()))

from bottle import get, post, request
Exemple #46
0
  def work(self):
    while True:
      item=self.redis.blpop("arduino:remote-command",timeout=5);
      if item:
        self.misses=0
        self.handleItem(item[1]) #0=> key
      else:
        self.misses+=1
        if self.misses > self.misslimit and self.serial:
          self.serial.close()
          self.serial=None
          log.debug("closed serial port")

if __name__ =="__main__":
  log=util.getLogger('arduino_remote_worker')
  #set up redis
  rdb=Redis()

  #setup serial
  PORT=util.CONFIG['hardware']['port'] #kevin is on COM4
  MISSLIMIT=10000000000
  getserial=lambda: serial.Serial(port=PORT, baudrate=115200,timeout=2)

  while True:
    try:
      worker(rdb,log,MISSLIMIT,getserial).work()
    except KeyboardInterrupt:
      log.info("KeyboardInterrupt>>shutdown")
      break
    except:
Exemple #47
0
import urllib, zlib, re, gzip, urlparse

import Cookie
import mechanize
import util
import environment
import cache

import datetime
import time

log = util.getLogger('ss.consumer')

def browser_agent(user_agent):
    br = mechanize.Browser()
    br.set_handle_robots(False)

    br.addheaders = [
        ('User-agent', user_agent),
        ('Accept', '*/*'),
        ('Accept-Encoding', 'gzip,deflate,identity'),
        ('Accept-Charset', 'ISO-8859-1,utf-8;q=0.7,*;q=0.7'),
        ('Accept-Language', 'en-us,en;q=0.5'),
    ]

    return br

def all_procedures():
    return environment.json_from_url(util.procedures_endpoint(), expires = cache.TIME_DAY)

def procedure_for_url(url):
Exemple #48
0
 def connectionMade(self):
     EaServer.connectionMade(self)
     self.log = util.getLogger("login.nfsps2", self)
     self.msgFactory = MessageFactory(self.transport, EaLoginMessage)
     self.hlrFactory = MessageHandlerFactory(self, "ea.games.redalert3.Ra3MsgHlr")