def __init__(self, parse_conf=True):
"""Initialize a configuration from a conf directory and conf file."""
super(TestConfigPrivate, self).__init__()
config_files = []
failsafe_path = self.DEFAULT_CONFIG_FILE
# Environment variables override defaults...
conf_dir = os.environ.get('TEST_CONFIG_DIR', self.DEFAULT_CONFIG_DIR)
conf_file = os.environ.get('TEST_CONFIG_FILE',
self.DEFAULT_CONFIG_FILE)
path = os.path.join(conf_dir, conf_file)
if not os.path.isfile(path):
path = failsafe_path
# only parse the config file if we expect one to exist. This is needed
# to remove an issue with the config file up to date checker.
if parse_conf:
config_files.append(path)
cfg.CONF([], project='contrailtest', default_config_files=config_files)
logging.setup('contrailtest')
LOG = logging.getLogger('contrailtest')
LOG.info("Using contrailtest config file %s" % path)
register_opts()
self._set_attrs()
if parse_conf:
cfg.CONF.log_opt_values(LOG, std_logging.DEBUG)
def start_service():
import traceback
from twisted.internet import reactor
from twisted.web import server
from service.ovs_service import DeviceCommService
from service.monitor import MonitorService
from service.compute import ComputeService, ComputeManager
from service.vpn_tools import VPNService
from db.models import Domain
from etc import config
from virt.libvirt_event import domainEventThread
from common import log as logging
LOG = logging.getLogger("agent.virt")
if config.compute_service:
service = server.Site(ComputeService())
reactor.listenTCP(config.compute_service_port, service)
if config.ovs_service:
service = server.Site(DeviceCommService())
reactor.listenTCP(config.ovs_service_port, service)
if config.monitor_service:
service = server.Site(MonitorService())
reactor.listenTCP(config.monitor_service_port, service)
if config.vpn_service:
service = server.Site(VPNService())
reactor.listenTCP(config.vpn_service_port, service)
try:
domainEventThread()
Domain.start_vms(ComputeManager.start_vms)
reactor.run()
except Exception:
LOG.error(traceback.print_exc())
def __init__(self, parse_conf=True):
"""Initialize a configuration from a conf directory and conf file."""
super(TestConfigPrivate, self).__init__()
config_files = []
failsafe_path = self.DEFAULT_CONFIG_FILE
# Environment variables override defaults...
conf_dir = os.environ.get('TEST_CONFIG_DIR',
self.DEFAULT_CONFIG_DIR)
conf_file = os.environ.get('TEST_CONFIG_FILE', self.DEFAULT_CONFIG_FILE)
path = os.path.join(conf_dir, conf_file)
if not os.path.isfile(path):
path = failsafe_path
# only parse the config file if we expect one to exist. This is needed
# to remove an issue with the config file up to date checker.
if parse_conf:
config_files.append(path)
cfg.CONF([], project='contrailtest', default_config_files=config_files)
logging.setup('contrailtest')
LOG = logging.getLogger('contrailtest')
LOG.info("Using contrailtest config file %s" % path)
register_opts()
self._set_attrs()
if parse_conf:
cfg.CONF.log_opt_values(LOG, std_logging.DEBUG)
def __init__(self, name, app, host='0.0.0.0', port=0, pool_size=None,
protocol=eventlet.wsgi.HttpProtocol, backlog=128,
use_ssl=False, max_url_len=None):
"""Initialize, but do not start, a WSGI server.
:param name: Pretty name for logging.
:param app: The WSGI application to serve.
:param host: IP address to serve the application.
:param port: Port number to server the application.
:param pool_size: Maximum number of eventlets to spawn concurrently.
:param backlog: Maximum number of queued connections.
:param max_url_len: Maximum length of permitted URLs.
:returns: None
:raises: cloudemulator.exception.InvalidInput
"""
# Allow operators to customize http requests max header line size.
eventlet.wsgi.MAX_HEADER_LINE = CONF.max_header_line
self.name = name
self.app = app
self._server = None
self._protocol = protocol
self.pool_size = pool_size or self.default_pool_size
self._pool = eventlet.GreenPool(self.pool_size)
self._logger = logging.getLogger(
"cloudemulator.%s.wsgi.server" % self.name)
self._wsgi_logger = logging.WritableLogger(self._logger)
self._use_ssl = use_ssl
self._max_url_len = max_url_len
self.client_socket_timeout = CONF.client_socket_timeout or None
if backlog < 1:
raise exception.InvalidInput(
reason='The backlog must be more than 1')
bind_addr = (host, port)
# TODO(dims): eventlet's green dns/socket module does not actually
# support IPv6 in getaddrinfo(). We need to get around this in the
# future or monitor upstream for a fix
try:
info = socket.getaddrinfo(bind_addr[0],
bind_addr[1],
socket.AF_UNSPEC,
socket.SOCK_STREAM)[0]
family = info[0]
bind_addr = info[-1]
except Exception:
family = socket.AF_INET
try:
self._socket = eventlet.listen(bind_addr, family, backlog=backlog)
except EnvironmentError:
LOG.error(_("Could not bind to %(host)s:%(port)s"),
{'host': host, 'port': port})
raise
(self.host, self.port) = self._socket.getsockname()[0:2]
LOG.info(_("%(name)s listening on %(host)s:%(port)s"),
{'name': self.name, 'host': self.host, 'port': self.port})
def testFileAutoCreate(self):
"""
测试log文件是否自动创建
:return:
"""
from common import log
logger1 = log.get_default_logger()
logger1.info("1222")
logger2 = log.getLogger("334555")
logger2.info("4555")
def run():
mLog = log.getLogger()
mLog.info('Starting EMR Scrapper')
mLog.debug('Parsing of command line args %s', sys.argv)
parser = argparse.ArgumentParser()
parser.add_argument('-cu',
'--cures-username',
required=True,
help="CURES website username")
parser.add_argument('-cp',
'--cures-password',
required=True,
help="CURES website password")
parser.add_argument('-eu',
'--emr-username',
required=True,
help="EMR website username")
parser.add_argument('-ep',
'--emr-password',
required=True,
help="EMR website password")
args = parser.parse_args()
mLog.debug('Args %s', args)
mCred = cred.Credentials(args.cures_username, args.cures_password, args.emr_username, args.emr_password)
mLog.debug('Credentials object created')
patients = db.Dbtools().load_patients()
mLog.debug('Found [' + str(len(patients)) + '] patients.')
# mCures = cures.Curestools(mCred)
# mCures.login()
mLog.debug('Starting EMR Scrapper Logging in.')
mEmr = emr.Emrtools(mCred)
mEmr.login()
time.sleep(5)
for patient in patients:
mLog.debug('Looking up patient ' + patient.getFirstname() + ' ' + patient.getLastname())
mLog.debug('MRN for lookup ' + patient.getMrn())
mEmr.search(patient.getMrn())
mLog.info('Finished logging out.')
mEmr.logout()
time.sleep(5)
mLog.info('Finished EMR Scrapper')
def createClusterSnapshot(snapshotHome='',
saveTop='',
lsfTop='',
egoTop='',
loglevel='error',
modules=None,
**kwargs):
global clsnapshotModule
logger = getLogger(__name__)
initSnapshotModule(snapshotHome)
generatedSnapFile = None
if not clsnapshotModule:
logger.error("Could not load the Cluster Snapshot module.")
return generatedSnapFile
origArgv = list(sys.argv)
error = False
if not modules:
modules = ['all']
else:
modules = list(modules)
try:
if not snapshotHome:
snapshotHome = os.path.join(sys.path[0], 'clsnapshot')
if not saveTop:
saveTop = os.path.join(sys.path[0], 'snapshots')
params = {'TOP': saveTop, 'LOG_LEVEL': loglevel, 'MODULES': modules}
for component in ['ego', 'sym', 'lsf']:
if component in modules or 'all' in modules:
componentArg = component + '_mdls'
if componentArg in kwargs:
subModules = list(kwargs[componentArg])
params[componentArg.upper()] = subModules
componentLog = component + '_log_level'
if componentLog in kwargs:
params[componentLog.upper()] = kwargs[componentLog]
generatedSnapFile = clsnapshotModule.runClsSnapshot(params)
except Exception as e:
error = True
logger.exception(e)
finally:
sys.argv = origArgv
if 'queue' in kwargs:
kwargs['queue'].put(not error, True)
return generatedSnapFile
def initSnapshotModule(snapshotHome=None):
global clsnapshotModule
logger = getLogger(__name__)
if not clsnapshotModule:
if not snapshotHome:
snapshotHome = 'snapshot'
if not os.path.isabs(snapshotHome):
snapshotHome = os.path.join(sys.path[0], snapshotHome)
try:
sys.path.append(snapshotHome)
clsnapshotModule = imp.load_source(
'clsnapshot', os.path.join(snapshotHome, 'clsnapshot.py'))
except:
logger.exception("Could not load the Cluster Snapshot module.")
return bool(clsnapshotModule)
def cleanSnapshots(retentionNum, directory=None):
logger = getLogger(__name__)
logger.info("Cleaning up old snapshots...")
error = False
if not retentionNum:
return False
files = getSnapshotFiles(directory)
files.sort()
while len(files) > retentionNum:
f = files.pop(0)
try:
os.remove(f)
logger.info("Deleted snapshot file %s.", f)
except EnvironmentError as e:
logger.exception(
"Could not delete snapshot file '%s'. Error no %d.", f,
e.errno)
error = True
return not error
import tornado.httpclient
import tornado.web
import tornado.gen
from base import BaseHandler
from common.log import getLogger
log = getLogger('business/views')
class IndexHandler(BaseHandler):
def get(self):
self.write('zhangsanlaile')
class TestSyncHandler(BaseHandler):
def get(self):
client = tornado.httpclient.HTTPClient()
client.fetch('http://192.168.1.189:8001')
self.write('success')
class TestAsync1Handler(BaseHandler):
@tornado.web.asynchronous
def get(self):
client = tornado.httpclient.AsyncHTTPClient()
client.fetch('http://192.168.1.189:8001', callback=self.on_response)
def on_response(self, response):
self.write('success async')
self.finish()
"large tokens (typically those generated by the "
"Keystone v3 API with big service catalogs)."),
cfg.BoolOpt('wsgi_keep_alive',
default=True,
help="If False, closes the client socket connection "
"explicitly."),
cfg.IntOpt('client_socket_timeout', default=0,
help="Timeout for client connections' socket operations. "
"If an incoming connection is idle for this number of "
"seconds it will be closed. A value of '0' means "
"wait forever."),
]
CONF = cfg.CONF
CONF.register_opts(wsgi_opts)
LOG = logging.getLogger(__name__)
SUPPORTED_CONTENT_TYPES = (
'application/json',
'application/xml',
)
# These are typically automatically created by routes as either defaults
# collection or member methods.
_ROUTES_METHODS = [
'create',
'delete',
'show',
'update',
]
import json
from base import BaseHandler
from common.log import getLogger
from common.utils import owner_authenticated
from blog import dao
log = getLogger('blog/views')
class BlogListHandler(BaseHandler):
def get(self):
log.debug('BlogList get')
blog_list = dao.get_blog_list()
self.render('blog.html', blog_list=blog_list)
class AddBlogHandler(BaseHandler):
def get(self):
self.render('add_blog.html')
def post(self):
log.debug('AddBlog post')
user = self.get_current_user()
title = self.get_argument('title', '')
content = self.get_argument('content', '')
dao.add_blog(user.oid, title, content)
self.write(json.dumps({'status': 'success'}))
class UpdateBlogHandler(BaseHandler):
@owner_authenticated
def __init__(self,
name,
app,
host='0.0.0.0',
port=0,
pool_size=None,
protocol=eventlet.wsgi.HttpProtocol,
backlog=128,
use_ssl=False,
max_url_len=None):
"""Initialize, but do not start, a WSGI server.
:param name: Pretty name for logging.
:param app: The WSGI application to serve.
:param host: IP address to serve the application.
:param port: Port number to server the application.
:param pool_size: Maximum number of eventlets to spawn concurrently.
:param backlog: Maximum number of queued connections.
:param max_url_len: Maximum length of permitted URLs.
:returns: None
:raises: cloudemulator.exception.InvalidInput
"""
# Allow operators to customize http requests max header line size.
eventlet.wsgi.MAX_HEADER_LINE = CONF.max_header_line
self.name = name
self.app = app
self._server = None
self._protocol = protocol
self.pool_size = pool_size or self.default_pool_size
self._pool = eventlet.GreenPool(self.pool_size)
self._logger = logging.getLogger("cloudemulator.%s.wsgi.server" %
self.name)
self._wsgi_logger = logging.WritableLogger(self._logger)
self._use_ssl = use_ssl
self._max_url_len = max_url_len
self.client_socket_timeout = CONF.client_socket_timeout or None
if backlog < 1:
raise exception.InvalidInput(
reason='The backlog must be more than 1')
bind_addr = (host, port)
# TODO(dims): eventlet's green dns/socket module does not actually
# support IPv6 in getaddrinfo(). We need to get around this in the
# future or monitor upstream for a fix
try:
info = socket.getaddrinfo(bind_addr[0], bind_addr[1],
socket.AF_UNSPEC, socket.SOCK_STREAM)[0]
family = info[0]
bind_addr = info[-1]
except Exception:
family = socket.AF_INET
try:
self._socket = eventlet.listen(bind_addr, family, backlog=backlog)
except EnvironmentError:
LOG.error(_("Could not bind to %(host)s:%(port)s"), {
'host': host,
'port': port
})
raise
(self.host, self.port) = self._socket.getsockname()[0:2]
LOG.info(_("%(name)s listening on %(host)s:%(port)s"), {
'name': self.name,
'host': self.host,
'port': self.port
})
"Keystone v3 API with big service catalogs)."),
cfg.BoolOpt('wsgi_keep_alive',
default=True,
help="If False, closes the client socket connection "
"explicitly."),
cfg.IntOpt('client_socket_timeout',
default=0,
help="Timeout for client connections' socket operations. "
"If an incoming connection is idle for this number of "
"seconds it will be closed. A value of '0' means "
"wait forever."),
]
CONF = cfg.CONF
CONF.register_opts(wsgi_opts)
LOG = logging.getLogger(__name__)
SUPPORTED_CONTENT_TYPES = (
'application/json',
'application/xml',
)
# These are typically automatically created by routes as either defaults
# collection or member methods.
_ROUTES_METHODS = [
'create',
'delete',
'show',
'update',
]
# Date:Sun Jul 07 01:14:35 CST 2013 # Author:Pengbo Li # E-mail:[email protected] import time import libvirt import traceback import json from lxml import etree from virt.libvirtConn import LibvirtConnection from virt.images import qemu_img_info import psutil from common import log as logging import re import platform NET_DEV_PATTERN = re.compile('^v(net|br|base|peer).*') LOG = logging.getLogger("agent.monitor") class HostMonitor(object): def __init__(self): super(HostMonitor, self).__init__() self.conn = None def get_info(self): try: info = {} if self.conn is None: self.conn = LibvirtConnection()._conn conn_info = self.conn.getInfo() arch = conn_info[0]
Add to ThreadGroup a add_period_timer()-like function. Then add framework like:
timer = <Init looping call here>
timer.start(<arguments here>)
self.timers.append(timer)
See each object for details on usage.
"""
import sys
import datetime
from eventlet import greenlet
from eventlet import greenpool
from eventlet import greenthread
from eventlet import event
import common.log as logging
logger = logging.getLogger()
class LoopingCallDone(Exception):
"""Exception to break out and stop a LoopingCall.
The poll-function passed to LoopingCall can raise this exception to
break out of the loop normally.
An optional return-value can be included as the argument to the exception;
this return-value will be returned by LoopingCall.wait()
"""
def __init__(self, retvalue=True):
""":param retvalue: Value that LoopingCall.wait() should return."""
self.retvalue = retvalue
#-*-coding:UTF-8-*-
#__author__ = 'Administrator'
from api.public import DsmPublicApi
from common import log as logging
from common.gettextutils import _
import service
if __name__ == '__main__':
wsdl = 'https://192.168.0.168:4119/webservice/Manager?WSDL'
username = '******'
password = '******'
LOG = logging.getLogger('nova.all')
api = DsmPublicApi(wsdl, username, password)
print api.getApiVersion()
print api.getManagerTime()
launcher = service.process_launcher()
for binary in ['mod1', 'mod2', 'mod3']:
topic = None
manager = None
try:
launcher.launch_service(service.Service.create(binary=binary,
topic=topic,
manager=manager))
default=20),
cfg.IntOpt(KEY_DASHBOARD_UPDATE_GRAPH_DATA_CHECK_INTERVAL_IN_SECOND,
default=20)
]
conf = cfg.CONF
conf.register_opts(opts)
conf(["--config-file", path])
def get_home_dir():
home_dir = "."
try:
opts, args = getopt.getopt(sys.argv, "", ["home_dir"])
except getopt.GetoptError:
print 'main.py --home_dir <home dir path>'
sys.exit(2)
for opt, arg in opts:
if opt == 'home_dir':
home_dir = arg
return home_dir
logger = log.getLogger("configs")
CONF = cfg.CONF
config_ini = get_home_dir() + "/conf/deployment_master.conf"
loadConfig(path=config_ini)
CONF.log_opt_values(logger, logging.INFO)
#!/usr/bin/env python # -*- coding: utf-8 -*- # Filename:exception.py # Date:Mon Oct 21 23:59:33 CST 2013 # Author:Pengbo Li # E-mail:[email protected] from common import log as logging LOG = logging.getLogger("agent.virt") class AgentException(Exception): """Base Agent Exception To correctly use this class, inherit from it and define a 'msg_fmt' property. That msg_fmt will get printf'd with the keyword arguments provided to the constructor. """ msg_fmt = "An unknown exception occurred." code = 500 headers = {} safe = False def __init__(self, message=None, **kwargs): self.kwargs = kwargs if 'code' not in self.kwargs: try: self.kwargs['code'] = self.code except AttributeError: pass
# -*- coding: utf-8 -*-
import json
from base import BaseHandler
from common.log import getLogger
from common.utils import md5
from admin import dao
log = getLogger('admin/views')
class LoginHandler(BaseHandler):
def get(self):
log.debug('Login get')
if self.get_current_user():
self.redirect('/blog/list')
self.render('login.html')
def post(self):
log.debug('Login post')
username = self.get_argument('username')
password = self.get_argument('password')
user = dao.get_user_by_username(username)
if user and user.password == md5(password):
self.set_cookie('username', username)
self.write(json.dumps({'status': 'success'}))
else:
self.write(json.dumps({'status': 'fail', 'err_msg': u'用户名和密码不匹配'}))
class RegisterHandler(BaseHandler):
#!/usr/bin/env python # -*- coding: utf-8 -*- # Filename:api.py # Date:Mon Oct 21 10:17:50 CST 2013 # Author:Pengbo Li # E-mail:[email protected] from common.utils import import_module, import_object from common import log as logging LOG = logging.getLogger('agent') class VFS(object): @staticmethod def instance_for_image(imgfile, imgfmt, partition): LOG.debug("Instance for image imgfile=%(imgfile)s imgfmt=%(imgfmt)s partition=%(partition)s" % {'imgfile': imgfile, 'imgfmt': imgfmt, 'partition': partition}) hasGuestfs = False try: LOG.debug("Trying to import guestfs") import_module("guestfs") hasGuestfs = True except Exception: pass if hasGuestfs: LOG.debug("Using primary VFSGuestFS") return import_object( "virt.disk.vfs.guestfs.VFSGuestFS", imgfile, imgfmt, partition)
#!/usr/bin/env python # -*- coding: utf-8 -*- # Filename:table_define.py # Date:三 4月 02 14:10:24 CST 2014 # Author:Pengbo Li # E-mail:[email protected] import os import random from sqlalchemy.ext.declarative import declarative_base #from sqlalchemy import ForeignKey #from sqlalchemy.orm import relationship, backref from sqlalchemy import Column, String, Integer, create_engine from sqlalchemy.orm import sessionmaker, scoped_session #from etc.config import disks_mountpoint from common import log as logging LOG = logging.getLogger("agent.db") engine_file = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'agent.db')) engine = create_engine('sqlite:////%s' % engine_file) Base = declarative_base() Session = scoped_session(sessionmaker(bind=engine)) def init_repository(): Base.metadata.create_all(engine) session = Session() #for dev, mountpoint in disks_mountpoint: #record = Disk(dev=dev, mount_point=mountpoint) #session.add(record) session.commit()
Initial Date: July 2014
Module Description:
TBD
"""
import os
import re
import string
import inspect
import common.log as logging
import ConfigParser
import argparse
from types import *
logger = logging.getLogger()
def _handle_file_(x):
if not os.path.isfile(x):
return open(x, 'w')
else:
return open(x, 'rw')
# to use this module, please use
# from Parameters import *
class Opt(object):
def __init__(self,
from lner import lnclient
from lner.serializers import LightningNodeSerializer
from lner.serializers import InvoiceSerializer
from lner.serializers import InvoiceRequestSerializer
from lner.serializers import CheckPaymentSerializer
from lner.serializers import VerifyMessageResponseSerializer
from lner.serializers import PayAwardResponseSerializer
from common import log
from common import validators
from common import json_util
from common.const import MEMO_RE
logger = log.getLogger("lner.views")
class LightningNodeViewSet(viewsets.ModelViewSet):
"""
List all available lightning nodes
"""
queryset = LightningNode.objects.all()
serializer_class = LightningNodeSerializer
class CreateInvoiceError(Exception):
pass
class CreateInvoiceViewSet(viewsets.ModelViewSet):
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
from datetime import datetime
from common import constant
from common import log
from common.dao import PerformanceDAO
from common.db.db_analyzer import DatabaseAnalyzer, \
calculate_active_tenant_mppdbs_count_in_one_epoch
logger = log.getLogger(__name__)
class Performance:
def __init__(self):
self.service_level_checker = DatabaseAnalyzer(
DatabaseAnalyzer.DASHBOARD_PERFORMANCE_GRAPH_UPDATING)
def calculate_performance(self, session):
try:
tenant_mppdb_groups = self.service_level_checker.get_current_tenant_mppdb_group_list_in_dashboard(
)
for tenant_mppdb_group in tenant_mppdb_groups:
tenant_mppdb_set = tenant_mppdb_group.tenant_mppdb_members_list
# Filename:compute.py # Date:Tue Sep 03 13:44:37 CST 2013 # Author:Pengbo Li # E-mail:[email protected] import traceback import threading from virt.libvirtConn import LibvirtConnection from virt.disk.api import inject_data #from virt.disk.vfs import api as vfs from common.ccf_client import CCFClient from etc import config from etc import constants from virt.images import fetch_with_wget from common import log as logging #from db.models import create_instance LOG = logging.getLogger("agent") class ComputeManager(object): def __init__(self, *args, **kwargs): super(ComputeManager, self).__init__(*args, **kwargs) self.conn = LibvirtConnection() @staticmethod def create_domain(vmInfo, key=None): """ vmInfo: { 'name': name, 'mem': mem,