def retrieve(mod_name, oid_list, opts=None, lock=False): """ Returns the results of calling a module over an oid_list. """ logger.debug("retrieve %s %s", mod_name, oid_list) if not opts: opts = {} # Clean up and validate inputs mod_type = get_mod_type(mod_name) if not mod_type: logger.error("Module %s not found", mod_name) return None oid_list = cleanup_oid_list(mod_name, oid_list) # Validate only mangle options unless we have to actually call the module if not options.validate_opts(mod_name, opts, True): logger.warning("Failed to validate opts for %s : %s", mod_name, opts) return None try: if not config.multiproc_on or mod_type in ["analyzers"]: if mod_type in ["extractors", "source"]: if len(oid_list) == 1: return single_retrieve(mod_name, oid_list[0], opts, lock) else: return multi_retrieve(mod_name, oid_list, opts, lock) else: if len(oid_list) == 1: if datastore.exists(mod_name, oid_list[0], opts): return datastore.retrieve(mod_name, oid_list[0], opts) if not options.validate_opts(mod_name, opts): logger.warning("Failed to validate opts for %s : %s", mod_name, opts) return False return single_call_module(mod_type, mod_name, oid_list, opts) else: # Multiprocessing is on and not an analysis module if mod_type in ["extractors", "source"]: if len(oid_list) == 1: return single_retrieve(mod_name, oid_list[0], opts, lock) else: new_list = [] for oid in oid_list: if not exists(mod_name, oid, opts): new_list.append(oid) if new_list and not options.validate_opts(mod_name, opts): logger.warning("Failed to validate opts for %s : %s", mod_name, opts) return None func = initialized_modules[mod_name].process mp.multi_map(func, new_list, opts, True) return multi_retrieve(mod_name, oid_list, opts, lock) else: # Map Reducer module if len(oid_list) == 1: if datastore.exists(mod_name, oid_list[0], opts): return datastore.retrieve(mod_name, oid_list[0], opts) if not options.validate_opts(mod_name, opts): logger.warning("Failed to validate opts for %s : %s", mod_name, opts) return False jobid = get_cid_from_oid_list(oid_list) map_func = initialized_modules[mod_name].mapper reduce_func = initialized_modules[mod_name].reducer return mp.multi_mapreduce(map_func, reduce_func, oid_list, opts, jobid) except: datastore.cleanup() raise
def prune_collection_by_cid(cid, oid_prune_list): source_set_dict = get_set_names() if cid not in source_set_dict: logger.error("Cannot prune this collection, cid not found:%s", cid) return False d = datastore.retrieve("collections", cid) md = datastore.retrieve("collections_meta", cid) oid_list = d["oid_list"] for oid in oid_prune_list: if oid in oid_list: oid_list.remove(oid) if delete_collection_by_cid(cid): return create_collection(md["name"], oid_list, md["notes"]) create_collection(md["name"], oid_list, notes) return False
def rename_collection_by_cid(cid, new_name): source_set_dict = get_set_names() if cid not in source_set_dict: logger.error("Cannot rename this collection, cid not found:%s", cid) return False d = datastore.retrieve("collections", cid) md = datastore.retrieve("collections_meta", cid) oid_list = d["oid_list"] notes = md["notes"] col_names = collection_names() if new_name in col_names: logger.error("Collection by that name already exist.") return False if delete_collection_by_cid(cid): return create_collection(new_name, oid_list, notes) create_collection(md["name"], oid_list, notes) return False
def single_retrieve(mod_name, oid, opts, lock): if not datastore.exists(mod_name, oid, opts): if not options.validate_opts(mod_name, opts): logger.warning("Failed to validate opts for %s : %s", mod_name, opts) return None process(mod_name, oid, opts) if lock: return datastore.retrieve_lock(mod_name, oid, opts) return datastore.retrieve(mod_name, oid, opts)
def get_colname_from_oid(oid): """ Given an oid for a collection search the source modules and return the name belonging to that oid """ logger.debug("Getting name for collection oid:%s", oid) for s in modules_available["source"]: if s == "collections": s = "collections_meta" ds = datastore.retrieve(s,oid) if not ds or not isinstance(ds, dict): continue if "name" in ds: return ds["name"] return set()