def retrieve(mod_name, oid_list, opts=None, lock=False):
""" Returns the results of calling a module over an oid_list.
"""
logger.debug("retrieve %s %s", mod_name, oid_list)
if not opts: opts = {}
# Clean up and validate inputs
mod_type = get_mod_type(mod_name)
if not mod_type:
logger.error("Module %s not found", mod_name)
return None
oid_list = cleanup_oid_list(mod_name, oid_list)
# Validate only mangle options unless we have to actually call the module
if not options.validate_opts(mod_name, opts, True):
logger.warning("Failed to validate opts for %s : %s", mod_name, opts)
return None
try:
if not config.multiproc_on or mod_type in ["analyzers"]:
if mod_type in ["extractors", "source"]:
if len(oid_list) == 1:
return single_retrieve(mod_name, oid_list[0], opts, lock)
else:
return multi_retrieve(mod_name, oid_list, opts, lock)
else:
if len(oid_list) == 1:
if datastore.exists(mod_name, oid_list[0], opts):
return datastore.retrieve(mod_name, oid_list[0], opts)
if not options.validate_opts(mod_name, opts):
logger.warning("Failed to validate opts for %s : %s", mod_name, opts)
return False
return single_call_module(mod_type, mod_name, oid_list, opts)
else: # Multiprocessing is on and not an analysis module
if mod_type in ["extractors", "source"]:
if len(oid_list) == 1:
return single_retrieve(mod_name, oid_list[0], opts, lock)
else:
new_list = []
for oid in oid_list:
if not exists(mod_name, oid, opts):
new_list.append(oid)
if new_list and not options.validate_opts(mod_name, opts):
logger.warning("Failed to validate opts for %s : %s", mod_name, opts)
return None
func = initialized_modules[mod_name].process
mp.multi_map(func, new_list, opts, True)
return multi_retrieve(mod_name, oid_list, opts, lock)
else: # Map Reducer module
if len(oid_list) == 1:
if datastore.exists(mod_name, oid_list[0], opts):
return datastore.retrieve(mod_name, oid_list[0], opts)
if not options.validate_opts(mod_name, opts):
logger.warning("Failed to validate opts for %s : %s", mod_name, opts)
return False
jobid = get_cid_from_oid_list(oid_list)
map_func = initialized_modules[mod_name].mapper
reduce_func = initialized_modules[mod_name].reducer
return mp.multi_mapreduce(map_func, reduce_func, oid_list, opts, jobid)
except:
datastore.cleanup()
raise
def prune_collection_by_cid(cid, oid_prune_list):
source_set_dict = get_set_names()
if cid not in source_set_dict:
logger.error("Cannot prune this collection, cid not found:%s", cid)
return False
d = datastore.retrieve("collections", cid)
md = datastore.retrieve("collections_meta", cid)
oid_list = d["oid_list"]
for oid in oid_prune_list:
if oid in oid_list:
oid_list.remove(oid)
if delete_collection_by_cid(cid):
return create_collection(md["name"], oid_list, md["notes"])
create_collection(md["name"], oid_list, notes)
return False
def rename_collection_by_cid(cid, new_name):
source_set_dict = get_set_names()
if cid not in source_set_dict:
logger.error("Cannot rename this collection, cid not found:%s", cid)
return False
d = datastore.retrieve("collections", cid)
md = datastore.retrieve("collections_meta", cid)
oid_list = d["oid_list"]
notes = md["notes"]
col_names = collection_names()
if new_name in col_names:
logger.error("Collection by that name already exist.")
return False
if delete_collection_by_cid(cid):
return create_collection(new_name, oid_list, notes)
create_collection(md["name"], oid_list, notes)
return False
def single_retrieve(mod_name, oid, opts, lock):
if not datastore.exists(mod_name, oid, opts):
if not options.validate_opts(mod_name, opts):
logger.warning("Failed to validate opts for %s : %s", mod_name, opts)
return None
process(mod_name, oid, opts)
if lock:
return datastore.retrieve_lock(mod_name, oid, opts)
return datastore.retrieve(mod_name, oid, opts)
def get_colname_from_oid(oid):
""" Given an oid for a collection search the source modules and return
the name belonging to that oid
"""
logger.debug("Getting name for collection oid:%s", oid)
for s in modules_available["source"]:
if s == "collections":
s = "collections_meta"
ds = datastore.retrieve(s,oid)
if not ds or not isinstance(ds, dict):
continue
if "name" in ds:
return ds["name"]
return set()
