# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
This script can be used for testing purposes, adding data passed in as the only argument to the
body of a new entry on the queue.
"""
import sys
import pika
from ovs.log.logHandler import LogHandler
logger = LogHandler('extensions', name='sender')
if __name__ == '__main__':
data = sys.argv[1] if len(sys.argv) >= 2 else '{}'
queue = sys.argv[2] if len(sys.argv) >= 3 else 'default'
connection = pika.BlockingConnection(pika.ConnectionParameters(host='localhost'))
channel = connection.channel()
channel.queue_declare(queue=queue, durable=True)
logger.debug('Sending to {0}: {1}'.format(queue, data))
channel.basic_publish(exchange='',
routing_key=queue,
body=data,
properties=pika.BasicProperties(delivery_mode=2))
# See the License for the specific language governing permissions and
# limitations under the License.
import sys
import os
import time
import subprocess
import shutil
from ovs.extensions.db.arakoon.EnsureArakoonWorks import EnsureArakoonWorks
from ArakoonManagement import ArakoonManagementEx
from ovs.log.logHandler import LogHandler
from ovs.plugin.provider.process import Process
manager = ArakoonManagementEx()
ensurearakoonworks = EnsureArakoonWorks()
logger = LogHandler('arakoon', name='tlogchcker')
class CheckArakoonError(Exception):
def __init__(self, message):
self.message = message
def __speak__(self):
print '{0}'.format(self.message)
class CheckArakoonTlogMark():
"""
check if tlogs need marking
mark tlogs that are unmarked
start arakoon and check if running
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import re
from ovs.log.logHandler import LogHandler
logger = LogHandler('extensions', name='fstab')
class Fstab(object):
"""
/etc/fstab manager
"""
def __init__(self):
"""
Init
"""
self.fstab_file = '/etc/fstab'
def _slurp(self):
"""
Read from /etc/fstab
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#!/usr/bin/env python
"""
Migration module
"""
import ConfigParser
from model import Model
from brander import Brander
from ovs.log.logHandler import LogHandler
logger = LogHandler('extensions', name='migrations')
logger.logger.propagate = False # No need to propagate this
class Migration(object):
"""
Handles all migrations between versions
"""
@staticmethod
def migrate():
"""
Executes all migrations. It keeps track of an internal "migration version" which is
a always increasing one digit version for now.
"""
def execute(function, start, end):
"""
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
File mutex module
"""
import time
import fcntl
import os
import stat
from ovs.log.logHandler import LogHandler
logger = LogHandler('extensions', 'file mutex')
class FileMutex(object):
"""
This is mutex backed on the filesystem. It's cross thread and cross process. However
its limited to the boundaries of a filesystem
"""
def __init__(self, name):
"""
Creates a file mutex object
"""
self.name = name
self._has_lock = False
self._start = 0
from ovs.dal.lists.storagedriverlist import StorageDriverList
from ovs.dal.lists.storagerouterlist import StorageRouterList
from ovs.dal.lists.backendtypelist import BackendTypeList
from ovs.dal.lists.vmachinelist import VMachineList
from ovs.extensions.generic.system import System
from ovs.extensions.generic.osdist import Osdist
from ovs.extensions.db.arakoon.ArakoonManagement import ArakoonManagementEx
from ovs.extensions.generic.sshclient import SSHClient
from ovs.extensions.storage.persistentfactory import PersistentFactory
from ovs.plugin.provider.configuration import Configuration
from ovs.plugin.provider.package import Package
from volumedriver.storagerouter.storagerouterclient import ClusterRegistry, ArakoonNodeConfig, ClusterNodeConfig, LocalStorageRouterClient
from ovs.log.logHandler import LogHandler
from ovs.extensions.openstack.cinder import OpenStackCinder
logger = LogHandler('lib', name='storagerouter')
class StorageRouterController(object):
"""
Contains all BLL related to StorageRouter
"""
@staticmethod
@celery.task(name='ovs.storagerouter.get_physical_metadata')
def get_physical_metadata(files, storagerouter_guid):
"""
Gets physical information about the machine this task is running on
"""
from ovs.lib.vpool import VPoolController
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Memcache store module
"""
import memcache
import re
from threading import Lock
from ovs.log.logHandler import LogHandler
logger = LogHandler('extensions', 'memcache store')
def locked():
"""
Locking decorator.
"""
def wrap(f):
"""
Returns a wrapped function
"""
def new_function(self, *args, **kwargs):
"""
Executes the decorated function in a locked context
"""
lock = kwargs.get('lock', True)
from ovs.celery_run import celery
from ovs.dal.hybrids.pmachine import PMachine
from ovs.dal.hybrids.vmachine import VMachine
from ovs.dal.hybrids.vdisk import VDisk
from ovs.dal.lists.vmachinelist import VMachineList
from ovs.dal.lists.pmachinelist import PMachineList
from ovs.dal.lists.vdisklist import VDiskList
from ovs.dal.lists.storagerouterlist import StorageRouterList
from ovs.dal.lists.storagedriverlist import StorageDriverList
from ovs.extensions.hypervisor.factory import Factory
from ovs.lib.vdisk import VDiskController
from ovs.lib.messaging import MessageController
from ovs.log.logHandler import LogHandler
from ovs.extensions.generic.volatilemutex import VolatileMutex
logger = LogHandler('lib', name='vmachine')
class VMachineController(object):
"""
Contains all BLL related to VMachines
"""
@staticmethod
@celery.task(name='ovs.machine.create_multiple_from_template')
def create_multiple_from_template(name,
machineguid,
pmachineguids,
start,
amount,
description=None):
pmachine_pointer = 0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import sys
import time
import uuid
import os
import logging
from ovs.log.logHandler import LogHandler
logging.basicConfig()
logger = LogHandler('extensions', name='watcher')
def _log(log_target, entry, level):
"""
Logs an entry
"""
if level > 0: # 0 = debug, 1 = info, 2 = error
logger.debug('[{0}] {1}'.format(log_target, entry))
def services_running(target):
try:
key = 'ovs-watcher-{0}'.format(str(uuid.uuid4()))
value = str(time.time())
"""
Celery beat module
"""
import time
import cPickle
from celery.beat import Scheduler
from celery import current_app
from ovs.extensions.storage.persistentfactory import PersistentFactory
from ovs.extensions.storage.exceptions import KeyNotFoundException
from ovs.extensions.generic.volatilemutex import VolatileMutex
from ovs.plugin.provider.configuration import Configuration
from ovs.log.logHandler import LogHandler
logger = LogHandler('celery', name='celery beat')
class DistributedScheduler(Scheduler):
"""
Distributed scheduler that can run on multiple nodes at the same time.
"""
TIMEOUT = 60 * 30
def __init__(self, *args, **kwargs):
"""
Initializes the distributed scheduler
"""
self._persistent = PersistentFactory.get_client()
MgmtCenter module
"""
from rest_framework import viewsets
from rest_framework.permissions import IsAuthenticated
from ovs.dal.lists.mgmtcenterlist import MgmtCenterList
from ovs.dal.hybrids.mgmtcenter import MgmtCenter
from ovs.extensions.hypervisor.factory import Factory
from backend.serializers.serializers import FullSerializer
from rest_framework.exceptions import NotAcceptable
from rest_framework.response import Response
from rest_framework import status
from backend.decorators import required_roles, load, return_object, return_list, log
from ovs.log.logHandler import LogHandler
logger = LogHandler('api', 'mgmtcenters')
class MgmtCenterViewSet(viewsets.ViewSet):
"""
Information about mgmtCenters
"""
permission_classes = (IsAuthenticated,)
prefix = r'mgmtcenters'
base_name = 'mgmtcenters'
@log()
@required_roles(['read'])
@return_list(MgmtCenter)
@load()
def list(self):
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Module containing certain helper classes providing various logic
"""
import inspect
import os
import imp
import copy
import re
import hashlib
from ovs.extensions.storage.volatilefactory import VolatileFactory
from ovs.extensions.storage.persistentfactory import PersistentFactory
from ovs.log.logHandler import LogHandler
logger = LogHandler('api', name='debug')
class Descriptor(object):
"""
The descriptor class contains metadata to instanciate objects that can be serialized.
It points towards the sourcefile, class name and class type
"""
object_cache = {}
def __init__(self, object_type=None, guid=None):
"""
Initializes a descriptor for a given type. Optionally already providing a guid for the
instanciator
"""
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Generic persistent factory.
"""
from ovs.plugin.provider.configuration import Configuration
from ovs.log.logHandler import LogHandler
logger = LogHandler('extensions', name='persistent factory')
class PersistentFactory(object):
"""
The PersistentFactory will generate certain default clients.
"""
@staticmethod
def get_client(client_type=None):
"""
Returns a persistent storage client
"""
if not hasattr(PersistentFactory,
'store') or PersistentFactory.store is None:
if client_type is None:
"""
Metadata views
"""
import time
from ovs.log.logHandler import LogHandler
from django.views.generic import View
from django.views.decorators.csrf import csrf_exempt
from django.http import HttpResponse
from django.conf import settings
from oauth2.decorators import json_response, limit
from ovs.dal.lists.bearertokenlist import BearerTokenList
from ovs.dal.lists.storagerouterlist import StorageRouterList
from ovs.dal.lists.backendtypelist import BackendTypeList
logger = LogHandler('api', name='metadata')
class MetadataView(View):
"""
Implements retrieval of generic metadata about the services
"""
@json_response()
@limit(amount=60, per=60, timeout=60)
def get(self, request, *args, **kwargs):
"""
Fetches metadata
"""
_ = args, kwargs
data = {
'authenticated':
import traceback
from time import mktime
from datetime import datetime
from ovs.plugin.provider.configuration import Configuration
from ovs.celery_run import celery
from ovs.lib.vmachine import VMachineController
from ovs.lib.vdisk import VDiskController
from ovs.dal.lists.vmachinelist import VMachineList
from ovs.dal.lists.vdisklist import VDiskList
from ovs.dal.lists.loglist import LogList
from ovs.extensions.db.arakoon.ArakoonManagement import ArakoonManagementEx
from volumedriver.scrubber.scrubber import Scrubber
from ovs.log.logHandler import LogHandler
_storagedriver_scrubber = Scrubber()
logger = LogHandler('lib', name='scheduled tasks')
def ensure_single(tasknames):
"""
Decorator ensuring a new task cannot be started in case a certain task is
running, scheduled or reserved.
The task using this decorator on, must be a bound task (with bind=True argument). Keep also in
mind that validation will be executed by the worker itself, so if the task is scheduled on
a worker currently processing a "duplicate" task, it will only get validated after the first
one completes, which will result in the fact that the task will execute normally.
@param tasknames: list of names to check
@type tasknames: list
"""
import pika
import os
import imp
import inspect
import time
import sys
import logging
from configobj import ConfigObj
import pyinotify
from ovs.extensions.rabbitmq.processor import process
from ovs.extensions.generic.system import System
from ovs.plugin.provider.configuration import Configuration
from ovs.log.logHandler import LogHandler
logger = LogHandler('extensions', name='consumer')
logging.basicConfig()
KVM_ETC = '/etc/libvirt/qemu/'
KVM_RUN = '/run/libvirt/qemu/'
mapping = {}
def run_kvm_watcher():
"""
Check whether to run the KVM file watcher
"""
return System.get_my_storagerouter().pmachine.hvtype == 'KVM'
def run_event_consumer():
from ovs.dal.hybrids.vdisk import VDisk
from ovs.dal.hybrids.vmachine import VMachine
from ovs.dal.hybrids.pmachine import PMachine
from ovs.dal.hybrids.storagedriver import StorageDriver
from ovs.dal.lists.vdisklist import VDiskList
from ovs.dal.lists.storagedriverlist import StorageDriverList
from ovs.dal.lists.vpoollist import VPoolList
from ovs.dal.lists.pmachinelist import PMachineList
from ovs.dal.hybrids.vpool import VPool
from ovs.extensions.hypervisor.factory import Factory
from ovs.extensions.storageserver.storagedriver import StorageDriverClient
from ovs.log.logHandler import LogHandler
from ovs.extensions.generic.sshclient import SSHClient
from ovs.extensions.generic.volatilemutex import VolatileMutex
logger = LogHandler('lib', name='vdisk')
class VDiskController(object):
"""
Contains all BLL regarding VDisks
"""
@staticmethod
@celery.task(name='ovs.disk.list_volumes')
def list_volumes(vpool_guid=None):
"""
List all known volumes on a specific vpool or on all
"""
if vpool_guid is not None:
vpool = VPool(vpool_guid)
storagedriver_client = StorageDriverClient().load(vpool)
# limitations under the License.
"""
This module contains all code for using the KVM libvirt api
"""
from xml.etree import ElementTree
from threading import Lock
import subprocess
import os
import glob
import re
import time
from ovs.log.logHandler import LogHandler
logger = LogHandler('extensions', name='kvm sdk')
ROOT_PATH = '/etc/libvirt/qemu/' # Get static info from here, or use dom.XMLDesc(0)
RUN_PATH = '/var/run/libvirt/qemu/' # Get live info from here
# Helpers
def _recurse(treeitem):
result = {}
for key, item in treeitem.items():
result[key] = item
for child in treeitem.getchildren():
result[child.tag] = _recurse(child)
for key, item in child.items():
result[child.tag][key] = item
return result
import time
from ovs.dal.lists.userlist import UserList
from rest_framework.response import Response
from toolbox import Toolbox
from rest_framework.exceptions import PermissionDenied, NotAuthenticated, NotAcceptable, Throttled
from rest_framework import status
from django.http import Http404
from django.conf import settings
from ovs.dal.exceptions import ObjectNotFoundException
from backend.serializers.serializers import FullSerializer
from ovs.log.logHandler import LogHandler
from ovs.extensions.storage.volatilefactory import VolatileFactory
from ovs.extensions.generic.volatilemutex import VolatileMutex
from ovs.dal.hybrids.log import Log
logger = LogHandler('api')
regex = re.compile('^(.*; )?version=(?P<version>([0-9]+|\*)?)(;.*)?$')
def required_roles(roles):
"""
Role validation decorator
"""
def wrap(f):
"""
Wrapper function
"""
def new_function(*args, **kw):
"""
Wrapped function
"""
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import re
import subprocess
from ovs.log.logHandler import LogHandler
logger = LogHandler('extensions', name='exportfs')
class Nfsexports(object):
"""
Basic management for /etc/exports
"""
def __init__(self):
self._exportsFile = '/etc/exports'
self._cmd = ['/usr/bin/sudo', '-u', 'root', '/usr/sbin/exportfs']
self._restart = ['/usr/bin/sudo', '-u', 'root', '/usr/sbin/exportfs', '-ra']
self._rpcmountd_stop = ['/usr/bin/sudo', '-u', 'root', 'pkill', 'rpc.mountd']
self._rpcmountd_start = ['/usr/bin/sudo', '-u', 'root', '/usr/sbin/rpc.mountd', '--manage-gids']
def _slurp(self):
"""
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import sys
import time
import uuid
import os
import shutil
from ovs.extensions.storage.persistentfactory import PersistentFactory
from ovs.log.logHandler import LogHandler
logger = LogHandler('arakoon', name='validator')
class EnsureArakoonError(Exception):
def __init__(self, message):
self.message = message
def __speak__(self):
print '{0}'.format(self.message)
class EnsureArakoonWorks():
"""
Wait for the following operation to be possible:
1) Set a value
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Volatile mutex module
"""
import time
from ovs.extensions.storage.volatilefactory import VolatileFactory
from ovs.log.logHandler import LogHandler
logger = LogHandler('extensions', 'volatile mutex')
class VolatileMutex(object):
"""
This is a volatile, distributed mutex to provide cross thread, cross process and cross node
locking. However, this mutex is volatile and thus can fail. You want to make sure you don't
lock for longer than a few hundred milliseconds to prevent this.
"""
def __init__(self, name):
"""
Creates a volatile mutex object
"""
self._volatile = VolatileFactory.get_client()
self.name = name
self._has_lock = False
"""
This module contains all code for using the VMware SOAP API/SDK
"""
from time import sleep
import re
import os
import shutil
from suds.client import Client, WebFault
from suds.cache import ObjectCache
from suds.sudsobject import Property
from suds.plugin import MessagePlugin
from ovs.log.logHandler import LogHandler
logger = LogHandler('extensions', name='vmware sdk')
class NotAuthenticatedException(BaseException):
pass
def authenticated(force=False):
"""
Decorator to make that a login is executed in case the current session isn't valid anymore
@param force: Force a (re)login, as some methods also work when not logged in
"""
def wrapper(function):
def new_function(self, *args, **kwargs):
self.__doc__ = function.__doc__
try:
# > Executes every day at 00:30
'arakoon-collapse': {
'task': 'ovs.scheduled.collapse_arakoon',
'schedule': crontab(minute='30', hour='0'),
'args': []
},
# Clean audit trail logs after 30 days
# > Executes every day at 00:30
'logs-cleanup': {
'task': 'ovs.scheduled.clean_logs',
'schedule': crontab(minute='30', hour='0'),
'args': []
}
}
loghandler = LogHandler('celery', name='celery')
@task_postrun.connect
def task_postrun_handler(sender=None,
task_id=None,
task=None,
args=None,
kwargs=None,
**kwds):
"""
Hook for celery postrun event
"""
_ = sender, task, args, kwargs, kwds
MessageController.fire(MessageController.Type.TASK_COMPLETE, task_id)
"""
KVM XML watcher
"""
from ovs.extensions.generic.system import System
from xml.etree import ElementTree
from ovs.log.logHandler import LogHandler
from ovs.dal.lists.vmachinelist import VMachineList
import glob
import pyinotify
import os
import re
import shutil
logger = LogHandler('extensions', name='xml processor')
class Kxp(pyinotify.ProcessEvent):
def __init__(self):
"""
dummy constructor
"""
pass
def _recurse(self, treeitem):
result = {}
for key, item in treeitem.items():
result[key] = item
for child in treeitem.getchildren():
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Contains various decorator
"""
import json
import time
from django.http import HttpResponse
from ovs.extensions.storage.volatilefactory import VolatileFactory
from ovs.extensions.generic.volatilemutex import VolatileMutex
from ovs.log.logHandler import LogHandler
from ovs.dal.hybrids.log import Log
logger = LogHandler('api', 'oauth2')
def json_response():
"""
Json response wrapper
"""
def wrap(f):
"""
Wrapper function
"""
def new_function(*args, **kw):
"""
Wrapped function
"""
results = f(*args, **kw)
