def __init__(
self,
group_to_object,
object_to_group,
sample_to_response,
n_processes,
parse_object_string=parse_object_string_sample,
):
"""
Builds a ProblemData object which is responsible for providing an interface to all aspects of a dataset.
Args: object_to_group, group_to_object,
"""
if not isinstance(group_to_object, types.ListType):
raise InputTypeError("group_to_object should be a list type")
if not isinstance(object_to_group, types.ListType):
raise InputTypeError("object_to_group should be a list type")
if not len(object_to_group) == len(group_to_object):
raise InputTypeError("object_to_group and group_to_object should be the same length")
if not all([isinstance(o_to_g, types.DictType) for o_to_g in object_to_group]):
raise InputTypeError("object_to_group should be a list of dict types")
if not all([isinstance(g_to_o, types.DictType) for g_to_o in group_to_object]):
raise InputTypeError("group_to_object should be a list of dict types")
if not isinstance(sample_to_response, types.DictType):
raise InputTypeError("sample_to_response should be a dict type")
if not isinstance(n_processes, types.IntType) or n_processes < 0:
raise InputTypeError("n_processes should be a non-negative int")
if not isinstance(parse_object_string, types.FunctionType):
raise InputTypeError("parse_object_sample should be a function")
if len(inspect.getargspec(parse_object_string)[0]) < 1:
raise InputTypeError("parse_object_sample should take at least one argument")
self.response_variables = np.array([sample_to_response[sample] for sample in sample_to_response.keys()])
self.n_scopes = len(object_to_group)
self.n_unmasked_samples = len(sample_to_response)
# Keep a stack of masks for various levels of data partitions
self.mask_stack = MaskStack(self.n_unmasked_samples)
sample_indices = dict(
[(sample_to_response.keys()[index], index) for index in range(len(sample_to_response.keys()))]
)
assert all(
[
self.response_variables[sample_indices[sample]] == sample_to_response[sample]
for sample in sample_to_response.keys()
]
), "sample_indices are not able to map correctly back to the response variable"
process_definitions = []
results = []
for scope in range(len(group_to_object)):
process_definitions.append(
ProcessDefinition(
build_group_records,
positional_arguments=(scope, group_to_object[scope], sample_indices, parse_object_string),
tag=scope,
)
)
multiprocess_functions(process_definitions, results.append, n_processes)
self.group_records = [None] * self.n_scopes
for scope, result in results:
self.group_records[scope] = result
for scope in range(self.n_scopes):
for group in self.group_records[scope]:
self.build_scope_map(self.group_records[scope][group], group_to_object, object_to_group)
for scope in range(self.n_scopes):
for key in self.group_records[scope]:
assert (
self.group_records[scope][key].feature_record.get_id() == key
), "feature_record had mismatched id to it's key"
assert (
self.group_records[scope][key].feature_record.get_scope() == scope
), "feature_record had mismatched scope to it's key"
class ProblemData:
def __init__(
self,
group_to_object,
object_to_group,
sample_to_response,
n_processes,
parse_object_string=parse_object_string_sample,
):
"""
Builds a ProblemData object which is responsible for providing an interface to all aspects of a dataset.
Args: object_to_group, group_to_object,
"""
if not isinstance(group_to_object, types.ListType):
raise InputTypeError("group_to_object should be a list type")
if not isinstance(object_to_group, types.ListType):
raise InputTypeError("object_to_group should be a list type")
if not len(object_to_group) == len(group_to_object):
raise InputTypeError("object_to_group and group_to_object should be the same length")
if not all([isinstance(o_to_g, types.DictType) for o_to_g in object_to_group]):
raise InputTypeError("object_to_group should be a list of dict types")
if not all([isinstance(g_to_o, types.DictType) for g_to_o in group_to_object]):
raise InputTypeError("group_to_object should be a list of dict types")
if not isinstance(sample_to_response, types.DictType):
raise InputTypeError("sample_to_response should be a dict type")
if not isinstance(n_processes, types.IntType) or n_processes < 0:
raise InputTypeError("n_processes should be a non-negative int")
if not isinstance(parse_object_string, types.FunctionType):
raise InputTypeError("parse_object_sample should be a function")
if len(inspect.getargspec(parse_object_string)[0]) < 1:
raise InputTypeError("parse_object_sample should take at least one argument")
self.response_variables = np.array([sample_to_response[sample] for sample in sample_to_response.keys()])
self.n_scopes = len(object_to_group)
self.n_unmasked_samples = len(sample_to_response)
# Keep a stack of masks for various levels of data partitions
self.mask_stack = MaskStack(self.n_unmasked_samples)
sample_indices = dict(
[(sample_to_response.keys()[index], index) for index in range(len(sample_to_response.keys()))]
)
assert all(
[
self.response_variables[sample_indices[sample]] == sample_to_response[sample]
for sample in sample_to_response.keys()
]
), "sample_indices are not able to map correctly back to the response variable"
process_definitions = []
results = []
for scope in range(len(group_to_object)):
process_definitions.append(
ProcessDefinition(
build_group_records,
positional_arguments=(scope, group_to_object[scope], sample_indices, parse_object_string),
tag=scope,
)
)
multiprocess_functions(process_definitions, results.append, n_processes)
self.group_records = [None] * self.n_scopes
for scope, result in results:
self.group_records[scope] = result
for scope in range(self.n_scopes):
for group in self.group_records[scope]:
self.build_scope_map(self.group_records[scope][group], group_to_object, object_to_group)
for scope in range(self.n_scopes):
for key in self.group_records[scope]:
assert (
self.group_records[scope][key].feature_record.get_id() == key
), "feature_record had mismatched id to it's key"
assert (
self.group_records[scope][key].feature_record.get_scope() == scope
), "feature_record had mismatched scope to it's key"
# Note: this is a pretty good test of the scope_map that
# was helpful in debugging, but it has nasty complexity
# ==============================================================
# for t in final_groups:
# assert isinstance(t, types.TupleType) and len(t) == 2,\
# "scope_map includes an entry that is not a scope/group pair"
# final_scope, final_group = t
# final_objs = group_to_object[final_scope][final_group]
# objs = group_to_object[scope][key[0]]
# assert any([final_obj in objs for final_obj in final_objs]),\
# "scope_map has an entry that lists a group which shares no objects"
# ==============================================================
def build_scope_map(self, group_record, group_to_object, object_to_group):
scope_map = [None] * self.n_scopes
scope = group_record.feature_record.get_scope()
group = group_record.feature_record.get_id()
objects = group_to_object[scope][group]
for final_scope in range(self.n_scopes):
if final_scope == scope:
scope_map[final_scope] = [(scope, group)]
continue
final_group_set = set()
for obj in objects:
try:
# make sure not to reference the object_to_group string, or object_to_string won't be collected
final_group_id = object_to_group[final_scope][obj]
final_group = self.group_records[final_scope][final_group_id].feature_record.get_id()
final_group_set.add((final_scope, final_group))
except KeyError:
continue
scope_map[final_scope] = list(final_group_set)
group_record.scope_map = scope_map
def push_mask(self, mask):
self.mask_stack.push_mask(mask)
def pop_mask(self):
self.mask_stack.pop_mask()
def get_feature_abundance(self, scope, group):
try:
# is sum(self.get_feature_column(scope, group)) faster?
mask = self.mask_stack.get_aggregate_mask()
return sum(
[
abundance
for index, abundance in self.group_records[scope][group].sparce_sample_abundances
if mask[index]
]
)
except KeyError:
raise KeyError("group (%s) not found at scope (%s)" % (group, scope))
def get_n_unmasked_samples(self):
return self.n_unmasked_samples
def get_max_scope(self):
return self.n_scopes - 1
def get_group_ids(self, scope):
return [group_id for group_id in self.group_records[scope]]
def get_feature_column(self, scope, group):
if not isinstance(scope, types.IntType) or scope < 0 or scope > self.get_max_scope():
raise InputTypeError("scope (%s) is not a valid scope index" % scope)
try:
group_record = self.group_records[scope][group]
# convert from sparce to dense format
feature_column = np.zeros(self.n_unmasked_samples)
for index, abundance in group_record.sparce_sample_abundances:
feature_column[index] = abundance
mask = self.mask_stack.get_aggregate_mask()
return feature_column[mask]
except KeyError:
raise KeyError("group (%s) not found at scope (%s)" % (group, scope))
def get_feature_record(self, scope, group):
if not isinstance(scope, types.IntType) or scope < 0 or scope > self.get_max_scope():
raise InputTypeError("scope (%s) is not a valid scope index" % scope)
try:
return self.group_records[scope][group].feature_record
except KeyError:
raise KeyError("group (%s) not found at scope (%s)" % (group, scope))
def get_split_groups(self, scope, group, final_scope):
if not isinstance(scope, types.IntType) or scope < 0 or scope > self.get_max_scope():
raise InputTypeError("scope (%s) is not a valid scope index" % scope)
if not isinstance(final_scope, types.IntType) or final_scope < 0 or final_scope > self.get_max_scope():
raise InputTypeError("final_scope (%s) is not a valid scope index" % final_scope)
try:
return self.group_records[scope][group].scope_map[final_scope]
except KeyError:
raise KeyError("(group, final_scope) (%s, %s) not found at scope (%s)" % (group, final_scope, scope))
def get_response_variables(self):
mask = self.mask_stack.get_aggregate_mask()
return self.response_variables[mask]