def initialize_all_modules():
logger.debug("initialize_all_modules (%s)", module_types)
for mod_type in module_types:
mod_dir = os.path.join(config.dir_modules, mod_type)
sys_utils.assert_dir_exists(mod_dir)
dev_dir = mod_dir + "_dev"
sys_utils.assert_dir_exists(dev_dir)
mod_list = os.listdir(mod_dir)
if config.dev_mode_enable:
mod_list.extend(os.listdir(dev_dir))
for mod_name in mod_list:
this_mod_dir = os.path.join(mod_dir, mod_name)
if not os.path.isdir(this_mod_dir):
this_mod_dir = os.path.join(dev_dir, mod_name)
init_file = os.path.join(this_mod_dir,"__init__.py")
interface_file = os.path.join(this_mod_dir, "module_interface.py")
if (os.path.isdir(this_mod_dir) and os.path.isfile(init_file) and
os.path.isfile(interface_file)):
if initialize_module(mod_name, os.path.split(this_mod_dir)[0]):
modules_available[mod_type].append(mod_name)
else:
logger.debug("Not able to initalize module %s",mod_name)
# ugly hack to make source module lookup faster, places collections and files first in the list
modules_available['source'].remove('collections')
modules_available['source'].remove('files')
modules_available['source'].insert(0, 'collections')
modules_available['source'].insert(1, 'files')
def store(mod_name, oid, data, opts, block=True):
mod_dir = get_mod_dir(mod_name)
sys_utils.assert_dir_exists(mod_dir)
acquire_file_lock(mod_name, oid, opts, write=True)
tempfile = os.path.join(datastore_dir, "TMP"+str(os.getpid())+mod_name)
filename = get_fullpath(mod_dir, mod_name, oid, opts)
logger.debug("Storing data at %s", filename)
return_val = True
if not sys_utils.write_object_to_file(tempfile, data):
logger.error("Not able to store data at %s", filename)
return_val = False
try:
# Need to do this for windows becuase you cannot rename to an existing file.
# FIXME: is there any way to do this only if we know we are on windows?
#if os.path.isfile(filename):
# os.remove(filename)
os.rename(tempfile, filename)
except:
logger.error("Not able to rename tempfile to %s", filename)
return_val = False
logger.debug("Releasing " + filename)
release_file_lock(mod_name, oid, opts)
return return_val
def local_store(plugin_name, data_name, data):
plugin_dir = os.path.join(localstore_dir, plugin_name)
sys_utils.assert_dir_exists(plugin_dir)
filename = os.path.join(plugin_dir, data_name)
logger.debug("Storing data at %s", filename)
if not sys_utils.write_object_to_file(filename, data):
logger.error("Not able to store data at %s", filename)
return False
return True
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
"""
import os, logging
import config, sys_utils
name = "localstore"
logger = logging.getLogger(name)
localstore_dir = config.dir_localstore
sys_utils.assert_dir_exists(localstore_dir)
def local_store(plugin_name, data_name, data):
plugin_dir = os.path.join(localstore_dir, plugin_name)
sys_utils.assert_dir_exists(plugin_dir)
filename = os.path.join(plugin_dir, data_name)
logger.debug("Storing data at %s", filename)
if not sys_utils.write_object_to_file(filename, data):
logger.error("Not able to store data at %s", filename)
return False
return True
def local_exists(plugin_name, data_name):
plugin_dir = os.path.join(localstore_dir, plugin_name)
if not os.path.isdir(plugin_dir):
return False
from tags import get_tags, apply_tags, tag_filter
import datastore_filesystem as datastore
name = "oxide"
logger = logging.getLogger(name)
logger.debug("Starting oxide4")
try:
import multiproc as mp
except ImportError:
config.multiproc_on = False
config.multiproc_max = 1
logger.info("Not able to import multiproc, multiprocessing will be disabled")
for d in config.get_section("dir").values():
sys_utils.assert_dir_exists(d)
initialized_modules = {} # Used to call modules
module_types = ["source", "extractors", "analyzers", "map_reducers"]
modules_available = {}
for mod_type in module_types:
modules_available[mod_type] = []
################### CORE FUNCTIONS #############################################
def get_oid_from_data(data):
""" Given the a blob of data return the <oid>.
"""
return hashlib.sha1(data).hexdigest()
def documentation(mod_name):
""" Return the documentaton of a module
THE SOFTWARE.
"""
import os, shutil, cPickle, logging, time, errno
from glob import glob
name = "ds_filesystem"
import ologger
logger = logging.getLogger(name)
import sys_utils, config, api
from options import build_suffix
from options import parse_suffix
datastore_dir = config.dir_datastore
sys_utils.assert_dir_exists(datastore_dir)
scratch_dir = config.dir_scratch
sys_utils.assert_dir_exists(scratch_dir)
COMPONENT_DELIM = '.' # separates oid from mangle opts
############# MAIN FUNCTIONS ###################################################
def store(mod_name, oid, data, opts, block=True):
mod_dir = get_mod_dir(mod_name)
sys_utils.assert_dir_exists(mod_dir)
acquire_file_lock(mod_name, oid, opts, write=True)
tempfile = os.path.join(datastore_dir, "TMP"+str(os.getpid())+mod_name)