コード例 #1
0
ファイル: oxide.py プロジェクト: SamuelMulder/oxide
def initialize_all_modules():
    logger.debug("initialize_all_modules (%s)", module_types)
    for mod_type in module_types:
        mod_dir = os.path.join(config.dir_modules, mod_type)
        sys_utils.assert_dir_exists(mod_dir)
        dev_dir = mod_dir + "_dev"
        sys_utils.assert_dir_exists(dev_dir)
        mod_list = os.listdir(mod_dir)
        if config.dev_mode_enable:
           mod_list.extend(os.listdir(dev_dir)) 
        for mod_name in mod_list:
            this_mod_dir = os.path.join(mod_dir, mod_name)
            if not os.path.isdir(this_mod_dir):
                this_mod_dir = os.path.join(dev_dir, mod_name)
            init_file = os.path.join(this_mod_dir,"__init__.py")
            interface_file = os.path.join(this_mod_dir, "module_interface.py")
            if (os.path.isdir(this_mod_dir) and os.path.isfile(init_file) and
                 os.path.isfile(interface_file)):
                if initialize_module(mod_name, os.path.split(this_mod_dir)[0]):
                    modules_available[mod_type].append(mod_name)
                else:
                    logger.debug("Not able to initalize module %s",mod_name)
    
    # ugly hack to make source module lookup faster, places collections and files first in the list
    modules_available['source'].remove('collections')
    modules_available['source'].remove('files')
    modules_available['source'].insert(0, 'collections')
    modules_available['source'].insert(1, 'files')
コード例 #2
0
def store(mod_name, oid, data, opts, block=True):
    mod_dir = get_mod_dir(mod_name)
    sys_utils.assert_dir_exists(mod_dir)

    acquire_file_lock(mod_name, oid, opts, write=True)

    tempfile = os.path.join(datastore_dir, "TMP"+str(os.getpid())+mod_name)
    filename = get_fullpath(mod_dir, mod_name, oid, opts)
    logger.debug("Storing data at %s", filename)

    return_val = True

    if not sys_utils.write_object_to_file(tempfile, data):
        logger.error("Not able to store data at %s", filename)
        return_val = False
    try:
        # Need to do this for windows becuase you cannot rename to an existing file.
        # FIXME: is there any way to do this only if we know we are on windows?
        #if os.path.isfile(filename):
        #    os.remove(filename)
        os.rename(tempfile, filename)
    except:
        logger.error("Not able to rename tempfile to %s", filename)
        return_val = False
    logger.debug("Releasing " + filename)
    release_file_lock(mod_name, oid, opts)

    return return_val
コード例 #3
0
ファイル: local_datastore.py プロジェクト: SamuelMulder/oxide
def local_store(plugin_name, data_name, data):
    plugin_dir = os.path.join(localstore_dir, plugin_name)
    sys_utils.assert_dir_exists(plugin_dir)
    filename = os.path.join(plugin_dir, data_name)
    logger.debug("Storing data at %s", filename)
    if not sys_utils.write_object_to_file(filename, data):
        logger.error("Not able to store data at %s", filename)
        return False
    return True
コード例 #4
0
ファイル: local_datastore.py プロジェクト: SamuelMulder/oxide
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
"""

import os, logging
import config, sys_utils

name = "localstore"
logger = logging.getLogger(name)

localstore_dir = config.dir_localstore
sys_utils.assert_dir_exists(localstore_dir)

def local_store(plugin_name, data_name, data):
    plugin_dir = os.path.join(localstore_dir, plugin_name)
    sys_utils.assert_dir_exists(plugin_dir)
    filename = os.path.join(plugin_dir, data_name)
    logger.debug("Storing data at %s", filename)
    if not sys_utils.write_object_to_file(filename, data):
        logger.error("Not able to store data at %s", filename)
        return False
    return True

def local_exists(plugin_name, data_name):
    plugin_dir = os.path.join(localstore_dir, plugin_name)
    if not os.path.isdir(plugin_dir):
        return False
コード例 #5
0
ファイル: oxide.py プロジェクト: SamuelMulder/oxide
from tags import get_tags, apply_tags, tag_filter
import datastore_filesystem as datastore

name = "oxide"
logger = logging.getLogger(name)
logger.debug("Starting oxide4")

try:
    import multiproc as mp
except ImportError:
    config.multiproc_on  = False
    config.multiproc_max = 1
    logger.info("Not able to import multiproc, multiprocessing will be disabled")

for d in config.get_section("dir").values():
    sys_utils.assert_dir_exists(d)

initialized_modules = {} # Used to call modules
module_types = ["source", "extractors", "analyzers", "map_reducers"]
modules_available = {}
for mod_type in module_types:
    modules_available[mod_type] = []

################### CORE FUNCTIONS #############################################       
def get_oid_from_data(data):
    """ Given the a blob of data return the <oid>. 
    """
    return hashlib.sha1(data).hexdigest()

def documentation(mod_name):
    """ Return the documentaton of a module
コード例 #6
0
THE SOFTWARE.
"""

import os, shutil, cPickle, logging, time, errno
from glob import glob

name = "ds_filesystem"
import ologger
logger = logging.getLogger(name)

import sys_utils, config, api
from options import build_suffix
from options import parse_suffix

datastore_dir = config.dir_datastore
sys_utils.assert_dir_exists(datastore_dir)
scratch_dir = config.dir_scratch
sys_utils.assert_dir_exists(scratch_dir)

COMPONENT_DELIM = '.' # separates oid from mangle opts


############# MAIN FUNCTIONS ###################################################

def store(mod_name, oid, data, opts, block=True):
    mod_dir = get_mod_dir(mod_name)
    sys_utils.assert_dir_exists(mod_dir)

    acquire_file_lock(mod_name, oid, opts, write=True)

    tempfile = os.path.join(datastore_dir, "TMP"+str(os.getpid())+mod_name)