Exemplo n.º 1
0
    def __init__(self, config, prefix, producer = True):

        # base class init
        Queue.__init__(self, config, prefix, producer)

        # get other attributes
        self._folder     = config.getMulti(prefix, "folder")
        self._ext        = "." + config.getMulti(prefix, "extension", "bin")
        self._period     = config.getMulti(prefix, "rescan", 5)
        self._pid        = config.getMulti(prefix, "lockfile", "queue.pid")
        self._keep       = config.getMulti(prefix, "keep", False)
        self._errFolder  = config.getMulti(prefix, "errFolder",  "err")
        self._newFolder  = config.getMulti(prefix, "newFolder",  "new")
        self._doneFolder = config.getMulti(prefix, "doneFolder", "done")
        self._workFolder = config.getMulti(prefix, "workFolder", "work")
        self._errFolder  = self._folder + os.sep + self._errFolder
        self._newFolder  = self._folder + os.sep + self._newFolder
        self._doneFolder = self._folder + os.sep + self._doneFolder
        self._workFolder = self._folder + os.sep + self._workFolder
        self._pattern    = re.compile("^[0-9]{20}-[0-9a-fA-F]{12}-[0-9a-fA-F]{8}" + self._ext + "$")

        # only for consumer queue instances, but define for all
        self._seq      = Sequencer(0)
        self._queue    = [ ]

        # check we have a folder
        if self._folder is None:
            raise QueueConfigurationException("Missing folder path for queue [%s]" % prefix)

        # be sure all folders exist
        self._ensureFolder(self._folder)
        if not self._producer:
            self._ensureFolder(self._errFolder)
            self._ensureFolder(self._newFolder)
            self._ensureFolder(self._doneFolder)
            self._ensureFolder(self._workFolder)

        # if a consumer, lock the queue
        if not self._producer:
            # the pid file
            pidfile = self._folder + os.sep + self._pid

            # try locking
            self._lockFile = None
            try:
                self._lockFile = zc.lockfile.LockFile(pidfile)
            except zc.lockfile.LockError:
                raise QueueFolderLockException("Cannot lock queue, pid file [%s]" % pidfile)

        # if there are entries in work, move them to ready
        #
        # NOTE: only if we are consumers
        if not self._producer:
            self._undoWork()

        # if not a producer, scan the queue
        if not self._producer:
            self._rescan()
Exemplo n.º 2
0
    def __init__(self, config = None, prefix = None):

        # base class init
        DatasetLoader.__init__(self, config, prefix)

        # prepare internal variables
        self._folders = defaultFolders
        if config is not None:
            self._folders = config.getMulti(prefix, "paths", defaultFolders)
        if self._folders is None:
            raise DBDatasetConfigurationException(
                    "Dataset Folder loader folders not defined")

        # find the folder
        self._folder = None
        for f in self._folders:
            self._folder = self._findFolder(f)
            if self._folder is not None:
                break

        # we better have a folder now or else... raise!
        if self._folder is None:
            raise DBDatasetConfigurationException(
                    "Dataset Folder loader cannot find a suitable folder from list %s" %
                    self._folders)
Exemplo n.º 3
0
def createLoader(application = "default", config = None):
    """Create a Dataset Loader as specified by params"""

    global _applicationLoaders

    # sanity check
    if application is None:
        application = "default"

    # be sure we have a config
    if config is None:
        config = ecommerce.config.getConfig()
    if config is None:
        raise DBDatasetRuntimeException("Cannot initialize ecommerce.db.dataset: missing config")

    # figure out the prefix
    prefix = ("db" if application == "default" else application) + ".dataset"

    # get the loader
    lname = config.getMulti(prefix, "loader", "folder")
    if lname not in _loaderDef:
        raise DBDatasetConfigurationException("Dataset loader [%s] does not exists" % lname)

    # instantiate the appropriate loader
    _applicationLoaders[application] = _loaderDef[lname](config, prefix)

    # return the created loader
    return _applicationLoaders[application]
Exemplo n.º 4
0
def _createFilesystemStorage(config, prefix):

    # Mount directory with authorization to the named drive.
    drive = config.getMulti(prefix, "drive")
    folder = config.getMulti(prefix, "folder")
    usr = config.getMulti(prefix, "usr")
    pwd = config.getMulti(prefix, "pwd")
    
    if drive is not None: 
        creds = UncCredentials(usr, pwd)
        authz_unc = UncDirectory(folder, creds)
    
        mnt = UncDirectoryMount(authz_unc, DiskDrive(drive))
    
        if not mnt.is_mounted():
            mnt.mount()
            print 'Mounted?', mnt.is_mounted()
    
    # fetch attributes and create
    dir = config.getMulti(prefix, "path")
    return FilesystemStorage(dir)
Exemplo n.º 5
0
def getStorage(config, prefix, name):
    """Get a storage as indicated by configuration"""

    # sanity checks
    if config is None or prefix is None or name is None:
        raise ValueError("ecommerce.storage.getStorage called with None parameter(s)")

    # try to get the type (default is "folder")
    type = config.getMulti(prefix + "." + name, "type", "folder")

    # check is a type we know
    if type not in _types:
        raise ValueError("Storage type [%s] now known" % type)

    # create the storage
    return _types[type](config, prefix + "." + name)
Exemplo n.º 6
0
def _createS3Storage(config, prefix, name):

    # fetch attributes and create
    bucket     = config.getMulti(prefix, "bucket")
    dir        = config.getMulti(prefix, "dir")
    gzip       = config.getMulti(prefix, "gzip", True)
    cache      = config.getMulti(prefix, "cache", "public,max-age=3600")
    accessKey  = config.getMulti(prefix, "access-key")
    secretKey  = config.getMulti(prefix, "secret-key")

    return S3Storage(bucket, dir, gzip, cache, accessKey, secretKey)
Exemplo n.º 7
0
def task_load():
    """Loads the task configuration file"""

    # get a config file
    config = ecommerce.config.getConfig()

    # get the task file
    fname = config.getMulti(configPrefix, application + ".tasks.file")
    if fname is None:
        fname = "/etc/ecommerce/tasks.yaml"
        if platform.system() == "Windows":
            fname = 'c:\\etc\\ecommerce\\tasks.yaml'

    # if it is still none => return a default tasks file
    if fname is None:
        return {
            "exclude" : { },
            "tasks"   : { }
        }

    # if the file does not exists => error => return None
    if not os.path.exists(fname):
        return None
    try:
        f = open(fname, 'r')
        tasksText = f.read()
        f.close()
    except:
        return None

    # try to parse the file, if error, return None
    tasks = None
    try:
        # read and parse the configuration
        tasks = safe_load(tasksText)
    except:
        return None

    # be sure that we have an "exclude" and a "tasks" section
    if "tasks" not in tasks or not isinstance(tasks["tasks"], dict):
        return None
    if "exclude" not in tasks or not isinstance(tasks["exclude"], dict):
        return None

    return tasks
Exemplo n.º 8
0
def _init(config = None):
    """Initialize the module

    Steps executed:
    1: get the default database
    2: process each module in "db.modules"
    3: process each database in "db.databases" (set default if none)
    """

    # get the configuration
    if config is None:
        config = ecommerce.config.getConfig()

    # 1: get the default database
    defaultDB = config.get("db.default")

    # 2: process each module in "db.modules"
    modules = { }
    try:
        mDef = config.get("db.python", { })
    except:
        raise DBConfigurationException("Cannot find 'db.python' configuration entry")
    for m in mDef:
        modules[m] = {
            "params"  : mDef[m],
            "module"  : None,
            "connect" : None
        }

    # 3: process each database in "db.databases" (set default if none)
    databases = { }
    try:
        dblist = config.get("db.databases", { })
    except:
        raise DBConfigurationException("Cannot find 'db.databases' configuration entry")
    for db in dblist:

        # ignore errors
        try:
            # get db conf
            dbconf = config.getMulti("db", db)
            if dbconf is None:     # ignore if not present
                continue

            # get the module
            module = dbconf["python"]

            # prepare the params according to the module
            params = { }
            mDef = modules[module]["params"]
            for key in mDef:
                if key in dbconf:
                    if key == 'password':
                        params[key] = config.keychain.fetch(dbconf[key])
                    else:
                        params[key] = dbconf[key]

            # create the db entry
            databases[db] = {
                "def"      : dbconf,
                "module"   : module,
                "params"   : params
            }

            # if no default, use this one
            if defaultDB is None:
                defaultDB = db
        except:
            raise DBConfigurationException("Configuration error processing 'db.%s'" % db)

    # return the config
    return ( defaultDB, databases, modules )