def main():
debug = Debugger()
chrono = Chrono()
universe = Universe(debug)
source = Source(debug).get_source()
bucket_chain = BucketChain(debug, chrono, universe, source)
clusters = Clusters(debug, chrono, universe)
algorithm = OnlineClustering(debug, universe, bucket_chain, clusters)
while True:
operation_time = time.time()
if bucket_chain.is_updated():
universe.compute_log_n_df()
bucket_chain.compute_universal_counts()
bucket_chain.compute_universal_tfidf()
clusters.update_centroid_counts()
clusters.update_centroid_tfidf()
algorithm.pre_clustering_work()
algorithm.online_clustering()
clusters.remove_old_clusters()
universe.prune_terms(clusters)
debug.log("BUCKET FINISHED IN: " +
str(time.time() - operation_time))
clusters.debug_active_clusters()
clusters.save_active_clusters()
def test_merge_df(self):
cluster = Clusters(
"../tests/data/test_1k_cluster_data.csv",
"dec0dedfeed1111111111111",
"addedfeed000000000000000",
"testCluster",
)
cell_metadata_df = Annotations(
self.CELL_METADATA_PATH,
["text/csv", "text/plain", "text/tab-separated-values"],
)
cell_metadata_df.preprocess()
cell_names_cell_metadata_df = np.asarray(cell_metadata_df.file["NAME"])
cell_names_cluster_df = np.asarray(cluster.file["NAME"])
# Cell names found in both cluster and metadata files
common_cell_names = cell_names_cluster_df[
np.isin(cell_names_cluster_df, cell_names_cell_metadata_df)
]
print(f"common cell names: {common_cell_names}")
# Perform merge
print(cluster.file[["NAME", "x", "y", "z"]])
cluster.merge_df(cluster.file[["NAME", "x", "y", "z"]], cell_metadata_df.file)
# Ensure ONLY common cell names found in cell metadata file and cluster file
# are in the newly merged df
result = all(
cell[0] in common_cell_names for cell in cluster.file["NAME"].values
)
self.assertTrue(
result,
f"Merge was not performed correctly. Merge should be performed on 'NAME'",
)
def main():
parallel = True
num_processes = 16
functions = Functions()
connect = functions.get_config()
c = Clusters(connect, num_processes, parallel)
TOTAL_TASKS.observe(2)
c.process_clusters()
def fit(self):
i = self.start_lvl - self.curr_lvl
X = np.asarray(list(zip(self.cur_cA, self.cD[i])))
if self.centers is not None:
if self.union_level:
self.centers = centnters_update(self.centers,
self.__split_mask)
else:
self.centers = centnters_update(self.centers)
self.k = len(self.centers)
kmeans = KMeans(n_clusters=self.k, init=self.centers)
else:
kmeans = KMeans(n_clusters=self.k)
kmeans.fit(X)
self.labels = kmeans.predict(X)
self.centers = kmeans.cluster_centers_
self.clasters = Clusters(self.data, self.labels, self.recunstruct)
return Clusters(self.data, self.labels, self.recunstruct)
def test_validate_header_for_coordinate_values_false(self):
"""Ensures validate_header_for_coordinate_values returns false when
coordinate is missing in header
"""
cluster = Clusters(
"../tests/data/cluster_bad_missing_coordinate.txt",
"5d276a50421aa9117c982845",
"5dd5ae25421aa910a723a337",
"testCluster",
)
self.assertFalse(cluster.validate_header_for_coordinate_values())
def test_validate_header_for_coordinate_values_true(self):
"""Ensures validate_header_for_coordinate_values returns true when
coordintate value is in cluster file
"""
cluster = Clusters(
"../tests/data/cluster_example.txt",
"5d276a50421aa9117c982845",
"5dd5ae25421aa910a723a337",
"testCluster",
)
self.assertTrue(cluster.validate_header_for_coordinate_values())
def diveg(f="data/weather.csv",loud=False): the=about.defaults() t=Tab(file=f) all = sorted(t.rows) for x in all: print(x.y()) return 1 c=Clusters(t,the,cols=t.cols.y, loud=loud) print([col.txt for col in t.cols.y]) all = sorted(c.all) for t1 in c.all: print(rs(t1.y())) for span in all[0].bins(all[-1],the): print(span) print(the)
def test_non_coordinate_case_integrity(self):
# if headers of this file change, the reference for the assert needs to be updated
cluster = Clusters(
"../tests/data/test_1k_cluster_data.csv",
"dec0dedfeed1111111111111",
"addedfeed000000000000000",
"testCluster",
)
assert cluster.headers == [
"NAME",
"x",
"y",
"z",
"CLUSTER",
"SUBCLUSTER",
"1",
], "cluster instantiation should only downcase coordinate header columns"
def cluster_events(self,
clustering_type,
features=None,
**clustering_parameters):
"""Create a NILM.Clusters object as Meter attribute.
Parameters
----------
clustering_type: string
Name of a clustering function. This function will be used to
cluster the events. Needs to be one of the keys of the dictionnary
'clustering_types' of NILM.Clusters object.
clustering_parameters: dict (optional)
Arguments to be passed as argument of the function which will be
used to cluster the events. Arguments not informed will take the
default value defined in the dictionnary 'clustering_types' of
NILM.Clusters object.
"""
clusters = Clusters(clustering_type, **clustering_parameters)
clusters.clustering(self, features)
self.clusters_ = clusters
print "Meter: events clustered!"
def test_cluster_type_inference(self):
"""Confirm consistency of type inference behavior
in instantiated data frame
Note: metadata has similar set of tests
"""
cluster = Clusters(
"../tests/data/cluster_NA.txt",
"addedfeed000000000000000",
"dec0dedfeed1111111111111",
"testCluster",
)
# integers, empty cell and string as inputs for numeric annotation
assert isinstance(cluster.file["NA_i_n_s__grp"]["group"][3],
str), "empty cell -> NaN, expect coercion to string"
# integers and empty cell as inputs for numeric annotation
assert isinstance(cluster.file["NA_i_n_grp"]["group"][3],
str), "empty cell -> NaN, expect coercion to string"
# floats, empty cell and string as inputs for numeric annotation
assert isinstance(cluster.file["NA_f_n_s__grp"]["group"][3],
str), "empty cell -> NaN, expect coercion to string"
# floats and empty cell as inputs for numeric annotation
assert isinstance(cluster.file["NA_f_n_grp"]["group"][3],
str), "empty cell -> NaN, expect coercion to string"
# integers, empty cell and string as inputs for group annotation
assert isinstance(
cluster.file["NA_i_n_s__num"]["numeric"][3],
float), "empty cell -> NaN that remains float (not coerced)"
# floats, empty cell and string as inputs for group annotation
assert isinstance(
cluster.file["NA_f_n_s__num"]["numeric"][3],
float), "empty cell -> NaN that remains float (not coerced)"
def loadClusters():
return Clusters.Clusters()
def __init__(self, coordinates, number_of_clusters):
self.clusters = Clusters(number_of_clusters, coordinates.shape[1])
self.data = Data(coordinates, self.clusters)
self.clusters.set_data(self.data)
def __init__(self):
self.elements = []
self.clusters = Clusters()