def __init__(self, config, prefix, producer = True): # base class init Queue.__init__(self, config, prefix, producer) # get other attributes self._folder = config.getMulti(prefix, "folder") self._ext = "." + config.getMulti(prefix, "extension", "bin") self._period = config.getMulti(prefix, "rescan", 5) self._pid = config.getMulti(prefix, "lockfile", "queue.pid") self._keep = config.getMulti(prefix, "keep", False) self._errFolder = config.getMulti(prefix, "errFolder", "err") self._newFolder = config.getMulti(prefix, "newFolder", "new") self._doneFolder = config.getMulti(prefix, "doneFolder", "done") self._workFolder = config.getMulti(prefix, "workFolder", "work") self._errFolder = self._folder + os.sep + self._errFolder self._newFolder = self._folder + os.sep + self._newFolder self._doneFolder = self._folder + os.sep + self._doneFolder self._workFolder = self._folder + os.sep + self._workFolder self._pattern = re.compile("^[0-9]{20}-[0-9a-fA-F]{12}-[0-9a-fA-F]{8}" + self._ext + "$") # only for consumer queue instances, but define for all self._seq = Sequencer(0) self._queue = [ ] # check we have a folder if self._folder is None: raise QueueConfigurationException("Missing folder path for queue [%s]" % prefix) # be sure all folders exist self._ensureFolder(self._folder) if not self._producer: self._ensureFolder(self._errFolder) self._ensureFolder(self._newFolder) self._ensureFolder(self._doneFolder) self._ensureFolder(self._workFolder) # if a consumer, lock the queue if not self._producer: # the pid file pidfile = self._folder + os.sep + self._pid # try locking self._lockFile = None try: self._lockFile = zc.lockfile.LockFile(pidfile) except zc.lockfile.LockError: raise QueueFolderLockException("Cannot lock queue, pid file [%s]" % pidfile) # if there are entries in work, move them to ready # # NOTE: only if we are consumers if not self._producer: self._undoWork() # if not a producer, scan the queue if not self._producer: self._rescan()
def __init__(self, config = None, prefix = None): # base class init DatasetLoader.__init__(self, config, prefix) # prepare internal variables self._folders = defaultFolders if config is not None: self._folders = config.getMulti(prefix, "paths", defaultFolders) if self._folders is None: raise DBDatasetConfigurationException( "Dataset Folder loader folders not defined") # find the folder self._folder = None for f in self._folders: self._folder = self._findFolder(f) if self._folder is not None: break # we better have a folder now or else... raise! if self._folder is None: raise DBDatasetConfigurationException( "Dataset Folder loader cannot find a suitable folder from list %s" % self._folders)
def createLoader(application = "default", config = None): """Create a Dataset Loader as specified by params""" global _applicationLoaders # sanity check if application is None: application = "default" # be sure we have a config if config is None: config = ecommerce.config.getConfig() if config is None: raise DBDatasetRuntimeException("Cannot initialize ecommerce.db.dataset: missing config") # figure out the prefix prefix = ("db" if application == "default" else application) + ".dataset" # get the loader lname = config.getMulti(prefix, "loader", "folder") if lname not in _loaderDef: raise DBDatasetConfigurationException("Dataset loader [%s] does not exists" % lname) # instantiate the appropriate loader _applicationLoaders[application] = _loaderDef[lname](config, prefix) # return the created loader return _applicationLoaders[application]
def _createFilesystemStorage(config, prefix): # Mount directory with authorization to the named drive. drive = config.getMulti(prefix, "drive") folder = config.getMulti(prefix, "folder") usr = config.getMulti(prefix, "usr") pwd = config.getMulti(prefix, "pwd") if drive is not None: creds = UncCredentials(usr, pwd) authz_unc = UncDirectory(folder, creds) mnt = UncDirectoryMount(authz_unc, DiskDrive(drive)) if not mnt.is_mounted(): mnt.mount() print 'Mounted?', mnt.is_mounted() # fetch attributes and create dir = config.getMulti(prefix, "path") return FilesystemStorage(dir)
def getStorage(config, prefix, name): """Get a storage as indicated by configuration""" # sanity checks if config is None or prefix is None or name is None: raise ValueError("ecommerce.storage.getStorage called with None parameter(s)") # try to get the type (default is "folder") type = config.getMulti(prefix + "." + name, "type", "folder") # check is a type we know if type not in _types: raise ValueError("Storage type [%s] now known" % type) # create the storage return _types[type](config, prefix + "." + name)
def _createS3Storage(config, prefix, name): # fetch attributes and create bucket = config.getMulti(prefix, "bucket") dir = config.getMulti(prefix, "dir") gzip = config.getMulti(prefix, "gzip", True) cache = config.getMulti(prefix, "cache", "public,max-age=3600") accessKey = config.getMulti(prefix, "access-key") secretKey = config.getMulti(prefix, "secret-key") return S3Storage(bucket, dir, gzip, cache, accessKey, secretKey)
def task_load(): """Loads the task configuration file""" # get a config file config = ecommerce.config.getConfig() # get the task file fname = config.getMulti(configPrefix, application + ".tasks.file") if fname is None: fname = "/etc/ecommerce/tasks.yaml" if platform.system() == "Windows": fname = 'c:\\etc\\ecommerce\\tasks.yaml' # if it is still none => return a default tasks file if fname is None: return { "exclude" : { }, "tasks" : { } } # if the file does not exists => error => return None if not os.path.exists(fname): return None try: f = open(fname, 'r') tasksText = f.read() f.close() except: return None # try to parse the file, if error, return None tasks = None try: # read and parse the configuration tasks = safe_load(tasksText) except: return None # be sure that we have an "exclude" and a "tasks" section if "tasks" not in tasks or not isinstance(tasks["tasks"], dict): return None if "exclude" not in tasks or not isinstance(tasks["exclude"], dict): return None return tasks
def _init(config = None): """Initialize the module Steps executed: 1: get the default database 2: process each module in "db.modules" 3: process each database in "db.databases" (set default if none) """ # get the configuration if config is None: config = ecommerce.config.getConfig() # 1: get the default database defaultDB = config.get("db.default") # 2: process each module in "db.modules" modules = { } try: mDef = config.get("db.python", { }) except: raise DBConfigurationException("Cannot find 'db.python' configuration entry") for m in mDef: modules[m] = { "params" : mDef[m], "module" : None, "connect" : None } # 3: process each database in "db.databases" (set default if none) databases = { } try: dblist = config.get("db.databases", { }) except: raise DBConfigurationException("Cannot find 'db.databases' configuration entry") for db in dblist: # ignore errors try: # get db conf dbconf = config.getMulti("db", db) if dbconf is None: # ignore if not present continue # get the module module = dbconf["python"] # prepare the params according to the module params = { } mDef = modules[module]["params"] for key in mDef: if key in dbconf: if key == 'password': params[key] = config.keychain.fetch(dbconf[key]) else: params[key] = dbconf[key] # create the db entry databases[db] = { "def" : dbconf, "module" : module, "params" : params } # if no default, use this one if defaultDB is None: defaultDB = db except: raise DBConfigurationException("Configuration error processing 'db.%s'" % db) # return the config return ( defaultDB, databases, modules )