def worker_setup_node():
while True:
node = node_q.get()
# copy ivs pkg to node
Helper.copy_pkg_scripts_to_remote(node)
# deploy node
Helper.safe_print("Start to deploy %(hostname)s\n" %
{'hostname': node.hostname})
if node.role == const.ROLE_NEUTRON_SERVER:
Helper.run_command_on_remote(node, (
r'''/bin/bash %(dst_dir)s/%(hostname)s_ospurge.sh >> %(log)s 2>&1'''
% {
'dst_dir': node.dst_dir,
'hostname': node.hostname,
'log': node.log
}))
Helper.run_command_on_remote(
node,
(r'''/bin/bash %(dst_dir)s/%(hostname)s.sh >> %(log)s 2>&1''' % {
'dst_dir': node.dst_dir,
'hostname': node.hostname,
'log': node.log
}))
Helper.safe_print("Finish deploying %(hostname)s\n" %
{'hostname': node.hostname})
node_q.task_done()
def _create_reachability_graph(self):
queue = [(0, 0, self.transitions)]
states_list = [(0, 0, self._get_network_state(self.transitions), {}, None)]
stop_condition = 20
state_id = 0
while queue and state_id < stop_condition:
state = queue.pop(0)
parent_state_id = state[0]
transitions = state[2]
transitions_ids_to_do = []
for transition in transitions:
if transition.is_doable():
priority_queue = Helper.get_competitive_transitions_priority_queue(transitions, transition)
if priority_queue.get() is transition:
transitions_ids_to_do.append(transition.id)
for transition_id in transitions_ids_to_do:
new_transitions_state = deepcopy(transitions)
transition = Helper.find_transition_by_id(new_transitions_state, transition_id)
transition.run_transition()
new_network_state = self._get_network_state(new_transitions_state)
found_state = self._find_state_based_on_network_state(states_list, new_network_state)
if found_state is not None:
id = found_state[0]
parent_network_state = self._find_state_by_id(states_list, parent_state_id)
parent_network_state[3][id] = transition.id
else:
state_id += 1
states_list.append((state_id, parent_state_id, new_network_state, {}, transition.id))
queue.append((state_id, parent_state_id, new_transitions_state))
return states_list
def split_only_sequential(self, URM, URM_df):
helper = Helper()
sequential_playlists = helper.get_target_playlists_list()[:5000]
selected_playlists = np.array([])
self.target_playlists = sequential_playlists
grouped = URM_df.groupby(
'playlist_id', as_index=True).apply(lambda x: list(x['track_id']))
relevant_items = defaultdict(list)
for playlist_id in sequential_playlists:
# Tracks = lista delle tracks prese dalla URM
tracks = np.array(grouped[playlist_id])
to_be_removed = int(len(tracks) * 0.2)
# Torna le #to_be_removed tracks ordinate sequenzialmente. e le toglie dalla lista delle tracks
to_be_removed_tracks = helper.get_sorted_tracks_in_playlist(
playlist_id)[-to_be_removed:]
for track in to_be_removed_tracks:
relevant_items[playlist_id].append(track)
tracks = np.delete(tracks, np.where(tracks == track))
grouped[playlist_id] = tracks
all_tracks = self.tracks_df["track_id"].unique()
matrix = MultiLabelBinarizer(classes=all_tracks,
sparse_output=True).fit_transform(grouped)
self.URM_train = matrix.tocsr()
self.URM_train = self.URM_train.astype(np.float64)
self.dict_test = relevant_items
def split_randomic_exactly_last(self, URM, URM_df):
# splitting URM in test set e train set
selected_playlists = np.array([])
helper = Helper()
self.target_playlists = helper.get_target_playlists_list()[5000:]
selected_playlists = self.target_playlists
grouped = URM_df.groupby(
'playlist_id', as_index=True).apply(lambda x: list(x['track_id']))
relevant_items = defaultdict(list)
count = 0
for playlist_id in selected_playlists:
tracks = np.array(grouped[playlist_id])
to_be_removed = int(len(tracks) * 0.2)
for i in range(to_be_removed):
index = randint(0, len(tracks) - 1)
removed_track = tracks[index]
relevant_items[playlist_id].append(removed_track)
tracks = np.delete(tracks, index)
grouped[playlist_id] = tracks
count += 1
all_tracks = self.tracks_df["track_id"].unique()
matrix = MultiLabelBinarizer(classes=all_tracks,
sparse_output=True).fit_transform(grouped)
self.URM_train = matrix.tocsr()
self.URM_train = self.URM_train.astype(np.float64)
self.dict_test = relevant_items
def recommend(self, playlist_id, at=10):
playlist_id = int(playlist_id)
helper = Helper()
### DUE TO TIME CONSTRAINT THE CODE STRUCTURE HERE IS REDUNTANT
### TODO exploit inheritance to reduce code duplications and simple extract ratings, combine them, simply by iterate over a list of recommenders
### COMMON CODE ###
self.hybrid_ratings = None #BE CAREFUL, MAGIC INSIDE :)
### COMBINE RATINGS IN DIFFERENT WAYS (seq, random short, random long)
if (helper.is_sequential(playlist_id)):
self.userCF_ratings = self.userCF_sequential.get_expected_ratings(
playlist_id)
self.itemCF_ratings = self.itemCF_sequential.get_expected_ratings(
playlist_id)
self.cbf_ratings = self.cbf_sequential.get_expected_ratings(
playlist_id)
self.slim_elastic_ratings = self.slim_elastic_sequential.get_expected_ratings(
playlist_id)
# self.svd_icm_ratings = self.svd_icm_sequential.get_expected_ratings(playlist_id)
self.ALS_ratings = self.ALS_sequential.get_expected_ratings(
playlist_id)
self.slim_ratings = self.slim_sequential.get_expected_ratings(
playlist_id)
w_right = self.w_seq
else:
self.userCF_ratings = self.userCF.get_expected_ratings(playlist_id)
self.itemCF_ratings = self.itemCF.get_expected_ratings(playlist_id)
self.cbf_ratings = self.cbf.get_expected_ratings(playlist_id)
self.slim_elastic_ratings = self.slim_elastic.get_expected_ratings(
playlist_id)
# self.svd_icm_ratings = self.svd_icm.get_expected_ratings(playlist_id)
self.ALS_ratings = self.ALS.get_expected_ratings(playlist_id)
self.slim_ratings = self.slim_random.get_expected_ratings(
playlist_id)
if len(self.URM[playlist_id].indices) > 10:
w_right = self.w_long
else:
w_right = self.w_short
self.hybrid_ratings = self.userCF_ratings * w_right["user_cf"]
self.hybrid_ratings += self.itemCF_ratings * w_right["item_cf"]
self.hybrid_ratings += self.cbf_ratings * w_right["cbf"]
self.hybrid_ratings += self.slim_ratings * w_right["slim"]
# self.hybrid_ratings += self.svd_icm_ratings * w_right["svd_icm"]
self.hybrid_ratings += self.ALS_ratings * w_right["als"]
self.hybrid_ratings += self.slim_elastic_ratings * w_right["elastic"]
recommended_items = np.flip(np.argsort(self.hybrid_ratings), 0)
# REMOVING SEEN
unseen_items_mask = np.in1d(recommended_items,
self.URM[playlist_id].indices,
assume_unique=True,
invert=True)
recommended_items = recommended_items[unseen_items_mask]
return recommended_items[0:at]
def certify_node_setup(q):
while True:
node = q.get()
if node.certificate_dir:
if not os.path.isfile("%s/ca.cert" % node.certificate_dir):
safe_print("Missing ca.cert in %s\n" % node.certificate_dir)
break
Helper.certify_node(node)
q.task_done()
def certify_node_setup(q):
while True:
node = q.get()
if node.certificate_dir:
if not os.path.isfile("%s/ca.cert" % node.certificate_dir):
safe_print("Missing ca.cert in %s\n" % node.certificate_dir)
break
Helper.certify_node(node)
q.task_done()
def setup_sriov(node_dic):
for hostname, node in node_dic.iteritems():
if node.skip:
safe_print("skip node %(fqdn)s due to %(error)s\n" % {
'fqdn': node.fqdn,
'error': node.error
})
continue
if node.tag != node.env_tag:
safe_print("skip node %(fqdn)s due to mismatched tag\n" %
{'fqdn': node.fqdn})
continue
if node.role != const.ROLE_SRIOV:
safe_print("Skipping node %(hostname)s because deployment mode is "
"SRIOV and role set for node is not SRIOV. It is "
"%(role)s\n" % {
'hostname': hostname,
'role': node.role
})
continue
if node.os != const.REDHAT:
safe_print("Skipping node %(hostname)s because deployment mode is "
"SRIOV and non REDHAT OS is not supported. OS set for "
"node is %(os)s\n" % {
'hostname': hostname,
'os': node.os
})
continue
# all okay, generate scripts for node
Helper.generate_sriov_scripts_for_redhat(node)
node_q.put(node)
with open(const.LOG_FILE, "a") as log_file:
for hostname, node in node_dic.iteritems():
log_file.write(str(node))
# Use multiple threads to setup nodes
for i in range(const.MAX_WORKERS):
t = threading.Thread(target=worker_upgrade_or_sriov_node,
args=(node_q, ))
t.daemon = True
t.start()
node_q.join()
sorted_time_dict = OrderedDict(
sorted(time_dict.items(), key=lambda x: x[1]))
for fqdn, h_time in sorted_time_dict.items():
safe_print("node: %(fqdn)s, time: %(time).2f\n" % {
'fqdn': fqdn,
'time': h_time
})
safe_print("Big Cloud Fabric deployment finished! "
"Check %(log)s on each node for details.\n" %
{'log': const.LOG_FILE})
def _create_transitions(self, transitions):
for transition in transitions:
links_in = []
links_out = []
for conn_id in transition["links_in_ids"]:
links_in.append(Helper.find_link_by_id(self.links, conn_id))
for conn_id in transition["links_out_ids"]:
links_out.append(Helper.find_link_by_id(self.links, conn_id))
transition_obj = Transition(links_in, links_out, transition["priority"], transition["id"], transition["name"])
self.transitions.append(transition_obj)
def split_cluster_randomic_only_last(self, URM, URM_df):
# splitting URM in test set e train set
segment = 1
# splitting URM in test set e train set
selected_playlists = np.array([])
available_playlists = np.arange(URM.shape[0])
target_analyzer = TargetAnalyzer()
# Gets distribution of only last 5000 playlists
dist = target_analyzer.get_distribution_array_only_last(segment)
helper = Helper()
target_playlists = helper.get_target_playlists_list(
)[:5000] # WILL REMOVE THEM
print("Clustering with segment = " + str(segment))
for key in tqdm(range(len(dist))):
while dist[key] != 0:
random_index = randint(0, len(available_playlists) - 1)
playlist_id = available_playlists[random_index]
target_segment = int(0.8 * len(URM[playlist_id].data))
if target_segment == key and playlist_id not in target_playlists:
available_playlists = np.delete(
available_playlists,
np.where(available_playlists == playlist_id))
selected_playlists = np.append(selected_playlists,
playlist_id)
dist[key] -= 1
self.target_playlists = selected_playlists.astype(int)
grouped = URM_df.groupby(
'playlist_id', as_index=True).apply(lambda x: list(x['track_id']))
relevant_items = defaultdict(list)
for playlist_id in selected_playlists:
# Tracks = lista delle tracks prese dalla URM
tracks = np.array(grouped[playlist_id])
to_be_removed = int(len(tracks) * 0.2)
for i in range(to_be_removed):
index = randint(0, len(tracks) - 1)
removed_track = tracks[index]
relevant_items[playlist_id].append(removed_track)
tracks = np.delete(tracks, index)
grouped[playlist_id] = tracks
all_tracks = self.tracks_df["track_id"].unique()
matrix = MultiLabelBinarizer(classes=all_tracks,
sparse_output=True).fit_transform(grouped)
self.URM_train = matrix.tocsr()
self.URM_train = self.URM_train.astype(np.float64)
self.dict_test = relevant_items
def _create_transitions(self, transitions):
for transition in transitions:
links_in = []
links_out = []
for conn_id in transition["links_in_ids"]:
links_in.append(Helper.find_link_by_id(self.links, conn_id))
for conn_id in transition["links_out_ids"]:
links_out.append(Helper.find_link_by_id(self.links, conn_id))
transition_obj = Transition(links_in, links_out,
transition["priority"],
transition["id"], transition["name"])
self.transitions.append(transition_obj)
def _create_coverability_graph(self):
queue = [(0, 0, self.transitions)]
states_list = [(0, 0, self._get_network_state(self.transitions), {},
None)]
stop_condition = 20
state_id = 0
while queue and state_id < stop_condition:
state = queue.pop(0)
parent_state_id = state[0]
transitions = state[2]
transitions_ids_to_do = []
for transition in transitions:
if transition.is_doable():
priority_queue = Helper.get_competitive_transitions_priority_queue(
transitions, transition)
if priority_queue.get() is transition:
transitions_ids_to_do.append(transition.id)
for transition_id in transitions_ids_to_do:
new_transitions_state = deepcopy(transitions)
transition = Helper.find_transition_by_id(
new_transitions_state, transition_id)
transition.run_transition()
new_network_state = self._get_network_state(
new_transitions_state)
state_id += 1
states_list_element = (state_id, parent_state_id,
new_network_state, {}, transition.id)
predecessors = self._get_predecessors(states_list,
states_list_element)
self._check_infinite_state(predecessors, new_network_state)
found_state = self._find_state_based_on_network_state(
states_list, new_network_state)
if found_state is not None:
state_id -= 1
id = found_state[0]
parent_network_state = self._find_state_by_id(
states_list, parent_state_id)
parent_network_state[3][id] = transition.id
else:
states_list.append(states_list_element)
queue.append(
(state_id, parent_state_id, new_transitions_state))
return states_list
def is_network_reversible(self):
for state in self.states_list:
states_queue = [state[0]]
if self.__is_root(state):
continue
found_root_state = False
reached_states = []
while states_queue:
state_id = states_queue.pop(0)
state_obj = Helper.find_state_by_id(self.states_list, state_id)
if self.__is_root(state_obj):
found_root_state = True
break
reached_states.append(state_id)
print 'Reached state: ', reached_states
print 'Children: ', self.__find_children(state_obj)
filtered_state_children = list(set(self.__find_children(state_obj))-set(reached_states)-set(states_queue))
print 'FilteredStateChildren: ', list(set(self.__find_children(state_obj))-set(reached_states)-set(states_queue))
states_queue.extend(filtered_state_children)
print '-------------------------------'
if not found_root_state:
return False
return True
def _create_links(self, links):
for link in links:
link_obj = Connector(
link["id"],
Helper.find_place_by_id(self.places, link["place_id"]),
link["direction"], link["weight"])
self.links.append(link_obj)
def is_network_reversible(self):
for state in self.states_list:
states_queue = [state[0]]
if self.__is_root(state):
continue
found_root_state = False
reached_states = []
while states_queue:
state_id = states_queue.pop(0)
state_obj = Helper.find_state_by_id(self.states_list, state_id)
if self.__is_root(state_obj):
found_root_state = True
break
reached_states.append(state_id)
print 'Reached state: ', reached_states
print 'Children: ', self.__find_children(state_obj)
filtered_state_children = list(
set(self.__find_children(state_obj)) -
set(reached_states) - set(states_queue))
print 'FilteredStateChildren: ', list(
set(self.__find_children(state_obj)) -
set(reached_states) - set(states_queue))
states_queue.extend(filtered_state_children)
print '-------------------------------'
if not found_root_state:
return False
return True
def __init__(self, **kwargs): super(WindowApp, self).__init__(**kwargs) #create recognizer class self.recognizer = Recognizer() #load templates to recognizer self.templates = Helper.returnTemplates() self.loadTemplates() #store current gesture self.gesture = []
def on_touch_up(self, touch):
self.gesture.append(touch.userdata["trace"])
if len(getCurrentTouches()) == 0:
templatePath = Helper.returnTemplatePath()
output = open(templatePath, "wb")
pickle.dump(self.gesture, output)
output.close()
print templatePath + " saved..."
self.gesture = []
def worker_setup_node(q):
while True:
node = q.get()
# copy ivs pkg to node
Helper.copy_pkg_scripts_to_remote(node)
# deploy node
safe_print("Start to deploy %(fqdn)s\n" % {'fqdn': node.fqdn})
if node.cleanup and node.role == const.ROLE_NEUTRON_SERVER:
Helper.run_command_on_remote(
node, (r'''sudo bash %(dst_dir)s/%(hostname)s_ospurge.sh''' % {
'dst_dir': node.dst_dir,
'hostname': node.hostname,
'log': node.log
}))
# a random delay to smooth apt-get/yum
delay = random.random() * 10.0
time.sleep(delay)
start_time = datetime.datetime.now()
Helper.run_command_on_remote(
node, (r'''sudo bash %(dst_dir)s/%(hostname)s.sh''' % {
'dst_dir': node.dst_dir,
'hostname': node.hostname,
'log': node.log
}))
end_time = datetime.datetime.now()
# parse setup log
diff = Helper.timedelta_total_seconds(end_time - start_time)
node.set_time_diff(diff)
node = Helper.update_last_log(node)
node_dict[node.fqdn] = node
time_dict[node.fqdn] = diff
# when deploying T5 on UBUNTU, reboot compute nodes
Helper.reboot_if_necessary(node)
safe_print("Finish deploying %(fqdn)s, cost time: %(diff).2f\n" % {
'fqdn': node.fqdn,
'diff': node.time_diff
})
q.task_done()
def split_randomic_all_playlists_longer_10000(self,
URM,
URM_df,
threshold_length=10):
# splitting URM in test set e train set
selected_playlists = np.array([])
available_playlists = np.arange(URM.shape[0])
helper = Helper()
target_playlists_kaggle = helper.get_target_playlists_list()
for playlist_id in available_playlists:
if len(selected_playlists) == 10000:
break
if playlist_id not in target_playlists_kaggle and len(
URM[playlist_id].indices) > threshold_length:
selected_playlists = np.append(selected_playlists, playlist_id)
self.target_playlists = selected_playlists.astype(int)
grouped = URM_df.groupby(
'playlist_id', as_index=True).apply(lambda x: list(x['track_id']))
relevant_items = defaultdict(list)
for playlist_id in selected_playlists:
tracks = np.array(grouped[playlist_id])
to_be_removed = int(len(tracks) * 0.2)
for i in range(to_be_removed):
index = randint(0, len(tracks) - 1)
removed_track = tracks[index]
relevant_items[playlist_id].append(removed_track)
tracks = np.delete(tracks, index)
grouped[playlist_id] = tracks
all_tracks = self.tracks_df["track_id"].unique()
matrix = MultiLabelBinarizer(classes=all_tracks,
sparse_output=True).fit_transform(grouped)
self.URM_train = matrix.tocsr()
self.URM_train = self.URM_train.astype(np.float64)
self.dict_test = relevant_items
def worker_upgrade_or_sriov_node(q):
while True:
node = q.get()
# copy ivs pkg to node
Helper.copy_pkg_scripts_to_remote(node)
# deploy node
safe_print("Start to deploy %(fqdn)s\n" %
{'fqdn': node.fqdn})
start_time = datetime.datetime.now()
if node.role == const.ROLE_SRIOV:
Helper.run_command_on_remote(node,
(r'''sudo bash %(dst_dir)s/%(hostname)s_sriov.sh''' %
{'dst_dir': node.dst_dir,
'hostname': node.hostname,
'log': node.log}))
elif node.role in const.DPDK_ROLES:
Helper.run_command_on_remote(node,
(r'''sudo bash %(dst_dir)s/%(hostname)s_dpdk.sh''' %
{'dst_dir': node.dst_dir,
'hostname': node.hostname,
'log': node.log}))
else:
Helper.run_command_on_remote(node,
(r'''sudo bash %(dst_dir)s/%(hostname)s_upgrade.sh''' %
{'dst_dir': node.dst_dir,
'hostname': node.hostname,
'log': node.log}))
end_time = datetime.datetime.now()
# parse setup log
diff = Helper.timedelta_total_seconds(end_time - start_time)
node.set_time_diff(diff)
node = Helper.update_last_log(node)
node_dict[node.fqdn] = node
time_dict[node.fqdn] = diff
safe_print("Finish executing script for node %(fqdn)s, "
"cost time: %(diff).2f\n" %
{'fqdn': node.fqdn, 'diff': node.time_diff})
q.task_done()
def seek(cls, query):
url = cls.get_url(query)
raw_content = cls.get_raw_content(url)
soup = cls.get_soup(raw_content)
songs = cls.parse(soup)
if not Helper.array_of(songs, Song):
raise ValueError('parse() response must be an array of Song objects')
return songs
def worker_setup_node(q):
while True:
node = q.get()
# copy ivs pkg to node
Helper.copy_pkg_scripts_to_remote(node)
# deploy node
safe_print("Start to deploy %(fqdn)s\n" %
{'fqdn': node.fqdn})
if node.cleanup and node.role == const.ROLE_NEUTRON_SERVER:
Helper.run_command_on_remote(node,
(r'''sudo bash %(dst_dir)s/%(hostname)s_ospurge.sh''' %
{'dst_dir': node.dst_dir,
'hostname': node.hostname,
'log': node.log}))
# a random delay to smooth apt-get/yum
delay = random.random() * 10.0
time.sleep(delay)
start_time = datetime.datetime.now()
Helper.run_command_on_remote(node,
(r'''sudo bash %(dst_dir)s/%(hostname)s.sh''' %
{'dst_dir': node.dst_dir,
'hostname': node.hostname,
'log': node.log}))
end_time = datetime.datetime.now()
# parse setup log
diff = Helper.timedelta_total_seconds(end_time - start_time)
node.set_time_diff(diff)
node = Helper.update_last_log(node)
node_dict[node.fqdn] = node
time_dict[node.fqdn] = diff
# when deploying T5 on UBUNTU, reboot compute nodes
Helper.reboot_if_necessary(node)
safe_print("Finish deploying %(fqdn)s, cost time: %(diff).2f\n" %
{'fqdn': node.fqdn, 'diff': node.time_diff})
q.task_done()
def main():
# Check if network is working properly
code = subprocess.call("ping www.bigswitch.com -c1", shell=True)
if code != 0:
Helper.safe_print("Network is not working properly, quit deployment\n")
exit(1)
# Parse configuration
parser = argparse.ArgumentParser()
parser.add_argument("-c", "--config-file", required=True,
help="BCF YAML configuration file")
parser.add_argument('-f', "--fuel-cluster-id", required=False,
help="Fuel cluster ID. Fuel settings may override YAML configuration. Please refer to config.yaml")
parser.add_argument('-t', "--tag", required=False,
help="Deploy to tagged nodes only.")
parser.add_argument('--cleanup', action='store_true', default=False,
help="Clean up existing routers, networks and projects.")
args = parser.parse_args()
with open(args.config_file, 'r') as config_file:
config = yaml.load(config_file)
deploy_bcf(config, args.fuel_cluster_id, args.tag, args.cleanup)
def worker_upgrade_node(q):
while True:
node = q.get()
# copy ivs pkg to node
Helper.copy_pkg_scripts_to_remote(node)
# deploy node
safe_print("Start to deploy %(fqdn)s\n" %
{'fqdn': node.fqdn})
start_time = datetime.datetime.now()
Helper.run_command_on_remote(node,
(r'''sudo bash %(dst_dir)s/%(hostname)s_upgrade.sh''' %
{'dst_dir': node.dst_dir,
'hostname': node.hostname,
'log': node.log}))
end_time = datetime.datetime.now()
# parse setup log
diff = Helper.timedelta_total_seconds(end_time - start_time)
node.set_time_diff(diff)
node = Helper.update_last_log(node)
node_dict[node.fqdn] = node
time_dict[node.fqdn] = diff
safe_print("Finish upgrading %(fqdn)s, cost time: %(diff).2f\n" %
{'fqdn': node.fqdn, 'diff': node.time_diff})
q.task_done()
def simulate(self):
live_transitions = self.__get_live_transitions()
if not live_transitions:
return self.__json_type_wrapper(RequestType.END)
transition = random.sample(live_transitions, 1)[0]
priority_queue = Helper.get_competitive_transitions_priority_queue(self.transitions, transition)
transition = priority_queue.get()
transition.run_transition()
return self.__json_type_wrapper(RequestType.SIMULATE, transition.to_json(), True)
def sign(data: object, private_key: str) -> str:
"""
signs a message
@param message: (object) message to sign
@param private_key: (str) private key to sign with
"""
private_key_bytes = serialization.load_pem_private_key(
bytes.fromhex(private_key),
password=None,
backend=default_backend()
)
data_bytes = Helper.object_to_bytes(data)
signature = private_key_bytes.sign(data_bytes, ec.ECDSA(hashes.SHA256()))
return signature.hex()
def worker_setup_node():
while True:
node = node_q.get()
# copy ivs pkg to node
Helper.copy_pkg_scripts_to_remote(node)
# deploy node
Helper.safe_print("Start to deploy %(hostname)s\n" %
{'hostname' : node.hostname})
if node.role == const.ROLE_NEUTRON_SERVER:
Helper.run_command_on_remote(node,
(r'''/bin/bash %(dst_dir)s/%(hostname)s_ospurge.sh >> %(log)s 2>&1''' %
{'dst_dir' : node.dst_dir,
'hostname' : node.hostname,
'log' : node.log}))
Helper.run_command_on_remote(node,
(r'''/bin/bash %(dst_dir)s/%(hostname)s.sh >> %(log)s 2>&1''' %
{'dst_dir' : node.dst_dir,
'hostname' : node.hostname,
'log' : node.log}))
Helper.safe_print("Finish deploying %(hostname)s\n" %
{'hostname' : node.hostname})
node_q.task_done()
def print_search_content(self,
content,
show_authors=False,
show_type=False,
show_publisher=False,
show_url=False):
base_template = "{score:.2f} - {year:4d} - {cfg_doi}{doi:40}{cfg_end} \
- {cfg_title}{title}{cfg_end}"
template = base_template
if show_authors:
template += "\n {cfg_more}AUTHORS{cfg_end} : {authors}"
if show_type:
template += "\n {cfg_more}TYPE{cfg_end} : {type}"
if show_publisher:
template += "\n {cfg_more}PUBLISHER{cfg_end} : {publisher}"
if show_url:
template += "\n {cfg_more}URL{cfg_end} : {url}"
for result in content.get('items', ()):
sr = SearchResult(result)
payload = {
"score": sr.get_score(),
"year": sr.get_year(),
"doi": sr.get_doi().get_identifier(),
"title": sr.get_title(),
"authors": sr.get_authors(),
"type": sr.get_type(),
"publisher": sr.get_publisher(),
"url": sr.get_url(),
"cfg_more": '',
"cfg_end": '',
"cfg_doi": '',
"cfg_title": '',
}
if self.colored_output:
color_options = [
("cfg_doi", self.color_doi),
("cfg_title", self.color_title),
("cfg_more", self.color_more),
("cfg_end", 'reset'),
]
for key, value in color_options:
payload[key] = Helper.get_fg_colorcode_by_identifier(value)
print(template.format(**payload))
def verify_node_setup(q):
while True:
node = q.get()
all_service_status = 'Service status for node: ' + node.fqdn
# check services are running and IVS version is correct
if node.deploy_dhcp_agent:
dhcp_status = Helper.check_os_service_status(
node, "neutron-dhcp-agent")
all_service_status = (all_service_status +
' | DHCP Agent ' + dhcp_status)
metadata_status = Helper.check_os_service_status(
node, "neutron-metadata-agent")
all_service_status = (all_service_status +
' | Metadata Agent ' + metadata_status)
if node.deploy_l3_agent and node.deploy_mode == const.T5:
l3_status = Helper.check_os_service_status(
node, "neutron-l3-agent")
all_service_status = (all_service_status +
' | L3 Agent ' + l3_status)
# for T5 deployment, check LLDP service status on compute nodes
if node.deploy_mode == const.T5 and node.role != const.ROLE_NEUTRON_SERVER:
lldp_status = Helper.check_os_service_status(node, "send_lldp")
all_service_status = (all_service_status +
' | LLDP Service ' + lldp_status)
# for T6 deployment, check IVS status and version too
if node.deploy_mode == const.T6:
# check ivs status and version
ivs_status = Helper.check_os_service_status(node, "ivs")
if ivs_status == ':-)':
# ivs is OK. check version
ivs_version = Helper.check_ivs_version(node)
all_service_status = (all_service_status +
' | IVS version ' + ivs_version)
else:
# ivs not OK
all_service_status = (all_service_status +
' | IVS ' + ivs_status)
# check neutron-bsn-agent status
bsn_agent_status = Helper.check_os_service_status(
node, "neutron-bsn-agent")
all_service_status = (all_service_status +
' | BSN Agent ' + bsn_agent_status)
# after forming the complete string, put in respective list
if ":-(" not in all_service_status:
node_pass[node.fqdn] = all_service_status
else:
node_fail[node.fqdn] = all_service_status
q.task_done()
def verify_node_setup(q):
while True:
node = q.get()
all_service_status = 'Service status for node: ' + node.fqdn
# check services are running and IVS version is correct
if node.deploy_dhcp_agent:
dhcp_status = Helper.check_os_service_status(
node, "neutron-dhcp-agent")
all_service_status = (all_service_status + ' | DHCP Agent ' +
dhcp_status)
metadata_status = Helper.check_os_service_status(
node, "neutron-metadata-agent")
all_service_status = (all_service_status + ' | Metadata Agent ' +
metadata_status)
if node.deploy_l3_agent and node.deploy_mode == const.T5:
l3_status = Helper.check_os_service_status(node,
"neutron-l3-agent")
all_service_status = (all_service_status + ' | L3 Agent ' +
l3_status)
# for T5 deployment, check LLDP service status on compute nodes
if node.deploy_mode == const.T5 and node.role != const.ROLE_NEUTRON_SERVER:
lldp_status = Helper.check_os_service_status(node, "send_lldp")
all_service_status = (all_service_status + ' | LLDP Service ' +
lldp_status)
# for T6 deployment, check IVS status and version too
if node.deploy_mode == const.T6:
# check ivs status and version
ivs_status = Helper.check_os_service_status(node, "ivs")
if ivs_status == ':-)':
# ivs is OK. check version
ivs_version = Helper.check_ivs_version(node)
all_service_status = (all_service_status + ' | IVS version ' +
ivs_version)
else:
# ivs not OK
all_service_status = (all_service_status + ' | IVS ' +
ivs_status)
# check neutron-bsn-agent status
bsn_agent_status = Helper.check_os_service_status(
node, "neutron-bsn-agent")
all_service_status = (all_service_status + ' | BSN Agent ' +
bsn_agent_status)
# after forming the complete string, put in respective list
if ":-(" not in all_service_status:
node_pass[node.fqdn] = all_service_status
else:
node_fail[node.fqdn] = all_service_status
q.task_done()
def verify(data: object, signature: str, public_key: str) -> bool:
"""
verfies message with digital signature
@param message: (object) message to verify
@param signature: (str) signature to verify message with
@param public_key: (str) public key to verify message with
"""
try:
public_key_bytes = serialization.load_pem_public_key(
bytes.fromhex(public_key),
backend=default_backend()
)
signature_bytes = bytes.fromhex(signature)
data_bytes = Helper.object_to_bytes(data)
public_key_bytes.verify(signature_bytes, data_bytes, ec.ECDSA(hashes.SHA256()))
return True
except InvalidSignature:
return False
def __init__(self, start_time, end_time, is_dup_data):
self.config_options = Helper.get_configurations(os.path.join(os.path.dirname(__file__), os.path.pardir, 'dnl_softswitch', 'conf', 'dnl_softswitch.conf'))
self.connect_database()
self.start_time = start_time if start_time is not None else self.get_start_time(end_time)
self.end_time = end_time
self.is_dup_data = is_dup_data
self.total_start_time = start_time
options = self.get_options()
try:
self.billing_path = options.get('switch_cdr', 'cdr_directory')
except Exception:
self.billing_path = None
if not self.billing_path:
self.billing_path = os.path.join(os.path.dirname(__file__), os.path.pardir, 'dnl_softswitch', 'cdr')
if not os.path.exists(self.billing_path):
os.makedirs(self.billing_path)
def print_search_content(self, content, show_authors=False,
show_type=False, show_publisher=False, show_url=False):
base_template = "{score:.2f} - {year:4d} - {cfg_doi}{doi:40}{cfg_end} \
- {cfg_title}{title}{cfg_end}"
template = base_template
if show_authors:
template += "\n {cfg_more}AUTHORS{cfg_end} : {authors}"
if show_type:
template += "\n {cfg_more}TYPE{cfg_end} : {type}"
if show_publisher:
template += "\n {cfg_more}PUBLISHER{cfg_end} : {publisher}"
if show_url:
template += "\n {cfg_more}URL{cfg_end} : {url}"
for result in content.get('items', ()):
sr = SearchResult(result)
payload = {
"score" : sr.get_score(),
"year" : sr.get_year(),
"doi" : sr.get_doi().get_identifier(),
"title" : sr.get_title(),
"authors" : sr.get_authors(),
"type" : sr.get_type(),
"publisher" : sr.get_publisher(),
"url" : sr.get_url(),
"cfg_more" : '',
"cfg_end" : '',
"cfg_doi" : '',
"cfg_title" : '',
}
if self.colored_output:
color_options = [
("cfg_doi", self.color_doi),
("cfg_title", self.color_title),
("cfg_more", self.color_more),
("cfg_end", 'reset'),
]
for key, value in color_options:
payload[key] = Helper.get_fg_colorcode_by_identifier(value)
print(template.format(**payload))
def upgrade_bcf(node_dic):
for hostname, node in node_dic.iteritems():
if node.skip:
safe_print("skip node %(fqdn)s due to %(error)s\n" % {
'fqdn': node.fqdn,
'error': node.error
})
continue
if node.tag != node.env_tag:
safe_print("skip node %(fqdn)s due to mismatched tag\n" %
{'fqdn': node.fqdn})
continue
if node.os == const.CENTOS:
Helper.generate_upgrade_scripts_for_centos(node)
elif node.os == const.UBUNTU:
Helper.generate_upgrade_scripts_for_ubuntu(node)
elif node.os == const.REDHAT:
Helper.generate_upgrade_scripts_for_redhat(node)
node_q.put(node)
with open(const.LOG_FILE, "a") as log_file:
for hostname, node in node_dic.iteritems():
log_file.write(str(node))
# Use multiple threads to setup compute nodes
for i in range(const.MAX_WORKERS):
t = threading.Thread(target=worker_upgrade_or_sriov_node,
args=(node_q, ))
t.daemon = True
t.start()
node_q.join()
sorted_time_dict = OrderedDict(
sorted(time_dict.items(), key=lambda x: x[1]))
for fqdn, h_time in sorted_time_dict.items():
safe_print("node: %(fqdn)s, time: %(time).2f\n" % {
'fqdn': fqdn,
'time': h_time
})
safe_print("Big Cloud Fabric deployment finished! "
"Check %(log)s on each node for details.\n" %
{'log': const.LOG_FILE})
def upgrade_bcf(node_dic):
for hostname, node in node_dic.iteritems():
if node.skip:
safe_print("skip node %(fqdn)s due to %(error)s\n" %
{'fqdn': node.fqdn, 'error': node.error})
continue
if node.tag != node.env_tag:
safe_print("skip node %(fqdn)s due to mismatched tag\n" %
{'fqdn': node.fqdn})
continue
if node.os == const.CENTOS:
Helper.generate_upgrade_scripts_for_centos(node)
elif node.os == const.UBUNTU:
Helper.generate_upgrade_scripts_for_ubuntu(node)
elif node.os == const.REDHAT:
Helper.generate_upgrade_scripts_for_redhat(node)
node_q.put(node)
with open(const.LOG_FILE, "a") as log_file:
for hostname, node in node_dic.iteritems():
log_file.write(str(node))
# Use multiple threads to setup compute nodes
for i in range(const.MAX_WORKERS):
t = threading.Thread(target=worker_upgrade_node, args=(node_q,))
t.daemon = True
t.start()
node_q.join()
sorted_time_dict = OrderedDict(sorted(time_dict.items(),
key=lambda x: x[1]))
for fqdn, h_time in sorted_time_dict.items():
safe_print("node: %(fqdn)s, time: %(time).2f\n" %
{'fqdn': fqdn, 'time': h_time})
safe_print("Big Cloud Fabric deployment finished! "
"Check %(log)s on each node for details.\n" %
{'log': const.LOG_FILE})
def deploy_bcf(config, fuel_cluster_id):
# Deploy setup node
Helper.safe_print("Start to prepare setup node\n")
env = Environment(config, fuel_cluster_id)
Helper.common_setup_node_preparation(env)
# Generate detailed node information
Helper.safe_print("Start to setup Big Cloud Fabric\n")
nodes_config = None
if 'nodes' in config:
nodes_yaml_config = config['nodes']
node_dic = Helper.load_nodes(nodes_yaml_config, env)
# Generate scripts for each node
for hostname, node in node_dic.iteritems():
if node.os == const.CENTOS:
Helper.generate_scripts_for_centos(node)
elif node.os == const.UBUNTU:
Helper.generate_scripts_for_ubuntu(node)
with open(const.LOG_FILE, "a") as log_file:
log_file.write(str(node))
if node.skip:
Helper.safe_print("skip node %(hostname)s due to %(error)s\n" % {
'hostname': hostname,
'error': node.error
})
continue
node_q.put(node)
# Use multiple threads to setup nodes
for i in range(const.MAX_WORKERS):
t = threading.Thread(target=worker_setup_node)
t.daemon = True
t.start()
node_q.join()
Helper.safe_print(
"Big Cloud Fabric deployment finished! Check %(log)s on each node for details.\n"
% {'log': const.LOG_FILE})
def __get_live_transitions(self):
live_transitions_ids = self.__get_live_transitions_ids()
return [Helper.find_transition_by_id(self.transitions, id) for id in live_transitions_ids]
def file_path(file):
return Helper.app_path() + "/" + Logger.dir + file
def run_selected_transitions(self, transition_id):
transition = Helper.find_transition_by_id(self.transitions, transition_id)
transition.run_transition()
return self.__json_type_wrapper(RequestType.RUN_SELECTED_TRANSITION, transition.to_json(), True)
def split_cluster_randomic(self, URM, URM_df):
# splitting URM in test set e train set
selected_playlists = np.array([])
available_playlists = np.arange(URM.shape[0])
target_analyzer = TargetAnalyzer()
segment_size = 1
min_playlist_len_after_split = 5
dist = target_analyzer.get_distribution_array(
segment_size=segment_size)
# in this way n_target = 10000
helper = Helper()
target_playlists = helper.get_target_playlists_list()[:5000]
n_target = np.sum(dist) # - len(target_playlists)
while n_target > 0:
random_index = randint(0, len(available_playlists) - 1)
playlist_id = available_playlists[random_index]
target_len = len(URM[playlist_id].data) * 0.8
if target_len > min_playlist_len_after_split:
target_segment = int(target_len / segment_size)
while dist[target_segment] <= 0:
random_index = randint(0, len(available_playlists) - 1)
playlist_id = available_playlists[random_index]
target_len = len(URM[playlist_id].data) * 0.8
if target_len > min_playlist_len_after_split:
target_segment = int(target_len / segment_size)
n_target -= 1
dist[target_segment] -= 1
selected_playlists = np.append(selected_playlists, playlist_id)
available_playlists = np.delete(
available_playlists,
np.where(available_playlists == playlist_id))
self.target_playlists = selected_playlists.astype(int)
grouped = URM_df.groupby(
'playlist_id', as_index=True).apply(lambda x: list(x['track_id']))
grouped_test = grouped.copy()
relevant_items = defaultdict(list)
count = 0
for playlist_id in selected_playlists:
tracks = np.array(grouped[playlist_id])
# if playlist_id in target_playlists:
# to_be_removed = int(len(tracks) * 0.2)
# to_be_removed_tracks = helper.get_sorted_tracks_in_playlist(playlist_id)[-to_be_removed:]
# for track in to_be_removed_tracks:
# relevant_items[playlist_id].append(track)
# tracks = np.delete(tracks, np.where(tracks == track))
# for i in range(to_be_removed):
# removed_track = tracks[-1]
# relevant_items[playlist_id].append(removed_track)
# tracks = np.delete(tracks, len(tracks) - 1)
# else:
to_be_removed = int(len(tracks) * 0.2)
for i in range(to_be_removed):
index = randint(0, len(tracks) - 1)
removed_track = tracks[index]
relevant_items[playlist_id].append(removed_track)
tracks = np.delete(tracks, index)
grouped[playlist_id] = tracks
grouped_test[playlist_id] = relevant_items[playlist_id]
count += 1
all_tracks = self.tracks_df["track_id"].unique()
matrix = MultiLabelBinarizer(classes=all_tracks,
sparse_output=True).fit_transform(grouped)
self.URM_train = matrix.tocsr()
self.dict_test = relevant_items
# bib URM
# self.URM_train = helper.get_urm_csr_bib(URM = self.URM_train)
# plotter = TargetAnalyzer()
# plotter.plot_standard_distribution()
# plotter.plot_distribution(self.URM_train, self.target_playlists)
self.URM_test = MultiLabelBinarizer(
classes=all_tracks, sparse_output=True).fit_transform(grouped_test)
self.URM_test = self.URM_test.tocsr()
self.URM_test = self.URM_test.astype(np.float64)
self.URM_train = self.URM_train.astype(np.float64)
def worker_setup_dhcp_agent():
while True:
node = dhcp_node_q.get()
Helper.safe_print("Copy dhcp_agent.ini to %(hostname)s\n" %
{'hostname' : node.hostname})
Helper.copy_file_to_remote(node, r'''%(dir)s/dhcp_agent.ini''' % {'dir' : node.setup_node_dir},
'/etc/neutron', 'dhcp_agent.ini')
Helper.safe_print("Copy metadata_agent.ini to %(hostname)s\n" %
{'hostname' : node.hostname})
Helper.copy_file_to_remote(node, r'''%(dir)s/metadata_agent.ini''' % {'dir': node.setup_node_dir},
'/etc/neutron', 'metadata_agent.ini')
Helper.safe_print("Restart neutron-metadata-agent and neutron-dhcp-agent on %(hostname)s\n" %
{'hostname' : node.hostname})
Helper.run_command_on_remote(node, 'service neutron-metadata-agent restart')
Helper.run_command_on_remote(node, 'service neutron-dhcp-agent restart')
Helper.safe_print("Finish deploying dhcp agent and metadata agent on %(hostname)s\n" %
{'hostname' : node.hostname})
dhcp_node_q.task_done()
controller_node.setup_node_dir)
for i in range(const.MAX_WORKERS):
t = threading.Thread(target=worker_setup_dhcp_agent)
t.daemon = True
t.start()
dhcp_node_q.join()
Helper.safe_print("Big Cloud Fabric deployment finished! Check %(log)s on each node for details.\n" %
{'log' : const.LOG_FILE})
if __name__=='__main__':
# Check if network is working properly
code = subprocess.call("ping www.bigswitch.com -c1", shell=True)
if code != 0:
Helper.safe_print("Network is not working properly, quit deployment\n")
exit(1)
# Parse configuration
parser = argparse.ArgumentParser()
parser.add_argument("-c", "--config-file", required=True,
help="BCF YAML configuration file")
parser.add_argument('-f', "--fuel-cluster-id", required=False,
help="Fuel cluster ID. Fuel settings may override YAML configuration. Please refer to example.yaml")
args = parser.parse_args()
with open(args.config_file, 'r') as config_file:
config = yaml.load(config_file)
deploy_bcf(config, args.fuel_cluster_id)
def deploy_bcf(config, mode, fuel_cluster_id, rhosp, tag, cleanup,
verify, verify_only, skip_ivs_version_check,
certificate_dir, certificate_only, generate_csr,
support, upgrade_dir):
# Deploy setup node
safe_print("Start to prepare setup node\n")
env = Environment(config, mode, fuel_cluster_id, rhosp, tag, cleanup,
skip_ivs_version_check, certificate_dir, upgrade_dir)
Helper.common_setup_node_preparation(env)
controller_nodes = []
# Generate detailed node information
safe_print("Start to setup Big Cloud Fabric\n")
nodes_yaml_config = config['nodes'] if 'nodes' in config else None
node_dic = Helper.load_nodes(nodes_yaml_config, env)
if upgrade_dir:
return upgrade_bcf(node_dic)
if generate_csr:
safe_print("Start to generate csr for virtual switches.\n")
# create ~/csr and ~/key directory
Helper.run_command_on_local("mkdir -p %s" % const.CSR_DIR)
Helper.run_command_on_local("mkdir -p %s" % const.KEY_DIR)
for hostname, node in node_dic.iteritems():
if node.skip:
safe_print("skip node %(fqdn)s due to %(error)s\n" %
{'fqdn': node.fqdn, 'error': node.error})
continue
if node.tag != node.env_tag:
safe_print("skip node %(fqdn)s due to mismatched tag\n" %
{'fqdn': node.fqdn})
continue
if node.deploy_mode == const.T6 and node.role == const.ROLE_COMPUTE:
Helper.generate_csr(node)
safe_print("Finish generating csr for virtual switches.\n")
return
# copy neutron config from neutron server to setup node
for hostname, node in node_dic.iteritems():
if node.role == const.ROLE_NEUTRON_SERVER:
controller_nodes.append(node)
Helper.copy_neutron_config_from_controllers(controller_nodes)
# check if vlan is the tenant network type for fuel environment
if not Helper.check_if_vlan_is_used(controller_nodes):
safe_print("tenant network type is not vlan. Stop deploying.\n")
return
# prepare keystone client from /etc/neutron/api-paste.ini
#Helper.prepare_keystone_client(controller_nodes)
# Generate scripts for each node
for hostname, node in node_dic.iteritems():
if support:
support_node_q.put(node)
if node.skip:
safe_print("skip node %(fqdn)s due to %(error)s\n" %
{'fqdn': node.fqdn, 'error': node.error})
continue
if node.tag != node.env_tag:
safe_print("skip node %(fqdn)s due to mismatched tag\n" %
{'fqdn': node.fqdn})
continue
if node.os == const.CENTOS:
Helper.generate_scripts_for_centos(node)
elif node.os == const.UBUNTU:
Helper.generate_scripts_for_ubuntu(node)
elif node.os == const.REDHAT:
Helper.generate_scripts_for_redhat(node)
if node.role == const.ROLE_NEUTRON_SERVER:
controller_node_q.put(node)
else:
# python doesn't have deep copy for Queue, hence add to all
node_q.put(node)
verify_node_q.put(node)
if node.deploy_mode == const.T6 and node.role == const.ROLE_COMPUTE:
certify_node_q.put(node)
if node.rhosp:
Helper.chmod_node(node)
with open(const.LOG_FILE, "a") as log_file:
version = Helper.run_command_on_local("pip show bosi")
log_file.write(str(version))
for hostname, node in node_dic.iteritems():
log_file.write(str(node))
if support:
safe_print("Start to collect logs.\n")
# copy installer logs to ~/support
Helper.run_command_on_local("mkdir -p %s" % const.SUPPORT_DIR)
Helper.run_command_on_local("cp -r %(src)s %(dst)s" %
{"src": const.LOG_FILE,
"dst": const.SUPPORT_DIR})
Helper.run_command_on_local("cp -r %(setup_node_dir)s/%(generated_script_dir)s %(dst)s" %
{"setup_node_dir": env.setup_node_dir,
"generated_script_dir": const.GENERATED_SCRIPT_DIR,
"dst": const.SUPPORT_DIR})
for i in range(const.MAX_WORKERS):
t = threading.Thread(target=support_node_setup,
args=(support_node_q,))
t.daemon = True
t.start()
support_node_q.join()
# compress ~/support
Helper.run_command_on_local("cd /tmp; tar -czf support.tar.gz support")
safe_print("Finish collecting logs. logs are at /tmp/support.tar.gz.\n")
return
# in case of verify_only or certificate_only, do not deploy
if (not verify_only) and (not certificate_only):
# Use single thread to setup controller nodes
t = threading.Thread(target=worker_setup_node,
args=(controller_node_q,))
t.daemon = True
t.start()
controller_node_q.join()
# Use multiple threads to setup compute nodes
for i in range(const.MAX_WORKERS):
t = threading.Thread(target=worker_setup_node, args=(node_q,))
t.daemon = True
t.start()
node_q.join()
sorted_time_dict = OrderedDict(sorted(time_dict.items(),
key=lambda x: x[1]))
for fqdn, h_time in sorted_time_dict.items():
safe_print("node: %(fqdn)s, time: %(time).2f\n" %
{'fqdn': fqdn, 'time': h_time})
safe_print("Big Cloud Fabric deployment finished! "
"Check %(log)s on each node for details.\n" %
{'log': const.LOG_FILE})
if certificate_dir or certificate_only:
# certify each node
safe_print("Start to certify virtual switches.\n")
for i in range(const.MAX_WORKERS):
t = threading.Thread(target=certify_node_setup,
args=(certify_node_q,))
t.daemon = True
t.start()
certify_node_q.join()
safe_print('Certifying virtual switches done.\n')
if verify or verify_only:
# verify each node and post results
safe_print("Verifying deployment for all compute nodes.\n")
for i in range(const.MAX_WORKERS):
t = threading.Thread(target=verify_node_setup,
args=(verify_node_q,))
t.daemon = True
t.start()
verify_node_q.join()
# print status
# success nodes
safe_print('Deployed successfully to: \n')
for node_element in node_pass:
safe_print(node_element + '\n')
# failed nodes
safe_print('Deployment to following failed: \n')
for node_element in node_fail:
safe_print(str(node_element) + ' : '
+ str(node_fail[node_element]) + '\n')
def support_node_setup(q):
while True:
node = q.get()
Helper.support_node(node)
q.task_done()
def support_node_setup(q):
while True:
node = q.get()
Helper.support_node(node)
q.task_done()
def _create_links(self, links):
for link in links:
link_obj = Connector(link["id"], Helper.find_place_by_id(self.places, link["place_id"]), link["direction"], link["weight"])
self.links.append(link_obj)
def deploy_bcf(config, mode, fuel_cluster_id, rhosp, tag, cleanup, verify,
verify_only, skip_ivs_version_check, certificate_dir,
certificate_only, generate_csr, support, upgrade_dir,
offline_dir, sriov):
# Deploy setup node
safe_print("Start to prepare setup node\n")
env = Environment(config, mode, fuel_cluster_id, rhosp, tag, cleanup,
skip_ivs_version_check, certificate_dir, upgrade_dir,
offline_dir, sriov)
Helper.common_setup_node_preparation(env)
controller_nodes = []
# Generate detailed node information
safe_print("Start to setup Big Cloud Fabric\n")
nodes_yaml_config = config['nodes'] if 'nodes' in config else None
node_dic = Helper.load_nodes(nodes_yaml_config, env)
if upgrade_dir:
return upgrade_bcf(node_dic)
if sriov:
return setup_sriov(node_dic)
if generate_csr:
safe_print("Start to generate csr for virtual switches.\n")
# create ~/csr and ~/key directory
Helper.run_command_on_local("mkdir -p %s" % const.CSR_DIR)
Helper.run_command_on_local("mkdir -p %s" % const.KEY_DIR)
for hostname, node in node_dic.iteritems():
if node.skip:
safe_print("skip node %(fqdn)s due to %(error)s\n" % {
'fqdn': node.fqdn,
'error': node.error
})
continue
if node.tag != node.env_tag:
safe_print("skip node %(fqdn)s due to mismatched tag\n" %
{'fqdn': node.fqdn})
continue
if node.deploy_mode == const.T6 and node.role == const.ROLE_COMPUTE:
Helper.generate_csr(node)
safe_print("Finish generating csr for virtual switches.\n")
return
# copy neutron config from neutron server to setup node
for hostname, node in node_dic.iteritems():
if node.role == const.ROLE_NEUTRON_SERVER:
controller_nodes.append(node)
Helper.copy_neutron_config_from_controllers(controller_nodes)
# check if vlan is the tenant network type for fuel environment
if not Helper.check_if_vlan_is_used(controller_nodes):
safe_print("tenant network type is not vlan. Stop deploying.\n")
return
# prepare keystone client from /etc/neutron/api-paste.ini
#Helper.prepare_keystone_client(controller_nodes)
# Generate scripts for each node
for hostname, node in node_dic.iteritems():
if support:
support_node_q.put(node)
if node.skip:
safe_print("skip node %(fqdn)s due to %(error)s\n" % {
'fqdn': node.fqdn,
'error': node.error
})
continue
if node.tag != node.env_tag:
safe_print("skip node %(fqdn)s due to mismatched tag\n" %
{'fqdn': node.fqdn})
continue
if node.os == const.CENTOS:
Helper.generate_scripts_for_centos(node)
elif node.os == const.UBUNTU:
Helper.generate_scripts_for_ubuntu(node)
elif node.os == const.REDHAT:
Helper.generate_scripts_for_redhat(node)
if node.role == const.ROLE_NEUTRON_SERVER:
controller_node_q.put(node)
else:
# python doesn't have deep copy for Queue, hence add to all
node_q.put(node)
verify_node_q.put(node)
if node.deploy_mode == const.T6 and node.role == const.ROLE_COMPUTE:
certify_node_q.put(node)
if node.rhosp:
Helper.chmod_node(node)
with open(const.LOG_FILE, "a") as log_file:
version = Helper.run_command_on_local("pip show bosi")
log_file.write(str(version))
for hostname, node in node_dic.iteritems():
log_file.write(str(node))
if support:
safe_print("Start to collect logs.\n")
# copy installer logs to ~/support
Helper.run_command_on_local("mkdir -p %s" % const.SUPPORT_DIR)
Helper.run_command_on_local("cp -r %(src)s %(dst)s" % {
"src": const.LOG_FILE,
"dst": const.SUPPORT_DIR
})
Helper.run_command_on_local(
"cp -r %(setup_node_dir)s/%(generated_script_dir)s %(dst)s" % {
"setup_node_dir": env.setup_node_dir,
"generated_script_dir": const.GENERATED_SCRIPT_DIR,
"dst": const.SUPPORT_DIR
})
for i in range(const.MAX_WORKERS):
t = threading.Thread(target=support_node_setup,
args=(support_node_q, ))
t.daemon = True
t.start()
support_node_q.join()
# compress ~/support
Helper.run_command_on_local("cd /tmp; tar -czf support.tar.gz support")
safe_print(
"Finish collecting logs. logs are at /tmp/support.tar.gz.\n")
return
# in case of verify_only or certificate_only, do not deploy
if (not verify_only) and (not certificate_only):
# Use single thread to setup controller nodes
t = threading.Thread(target=worker_setup_node,
args=(controller_node_q, ))
t.daemon = True
t.start()
controller_node_q.join()
# Use multiple threads to setup compute nodes
for i in range(const.MAX_WORKERS):
t = threading.Thread(target=worker_setup_node, args=(node_q, ))
t.daemon = True
t.start()
node_q.join()
sorted_time_dict = OrderedDict(
sorted(time_dict.items(), key=lambda x: x[1]))
for fqdn, h_time in sorted_time_dict.items():
safe_print("node: %(fqdn)s, time: %(time).2f\n" % {
'fqdn': fqdn,
'time': h_time
})
safe_print("Big Cloud Fabric deployment finished! "
"Check %(log)s on each node for details.\n" %
{'log': const.LOG_FILE})
if certificate_dir or certificate_only:
# certify each node
safe_print("Start to certify virtual switches.\n")
for i in range(const.MAX_WORKERS):
t = threading.Thread(target=certify_node_setup,
args=(certify_node_q, ))
t.daemon = True
t.start()
certify_node_q.join()
safe_print('Certifying virtual switches done.\n')
if verify or verify_only:
# verify each node and post results
safe_print("Verifying deployment for all compute nodes.\n")
for i in range(const.MAX_WORKERS):
t = threading.Thread(target=verify_node_setup,
args=(verify_node_q, ))
t.daemon = True
t.start()
verify_node_q.join()
# print status
# success nodes
safe_print('Deployed successfully to: \n')
for node_element in node_pass:
safe_print(node_element + '\n')
# failed nodes
safe_print('Deployment to following failed: \n')
for node_element in node_fail:
safe_print(
str(node_element) + ' : ' + str(node_fail[node_element]) +
'\n')
for i in range(const.MAX_WORKERS):
t = threading.Thread(target=worker_setup_node)
t.daemon = True
t.start()
node_q.join()
Helper.safe_print(
"Big Cloud Fabric deployment finished! Check %(log)s on each node for details.\n"
% {'log': const.LOG_FILE})
if __name__ == '__main__':
# Check if network is working properly
code = subprocess.call("ping www.bigswitch.com -c1", shell=True)
if code != 0:
Helper.safe_print("Network is not working properly, quit deployment\n")
exit(1)
# Parse configuration
parser = argparse.ArgumentParser()
parser.add_argument("-c",
"--config-file",
required=True,
help="BCF YAML configuration file")
parser.add_argument(
'-f',
"--fuel-cluster-id",
required=False,
help=
"Fuel cluster ID. Fuel settings may override YAML configuration. Please refer to example.yaml"
)
currentDir = os.path.basename(os.getcwd()) currentFileName = os.path.basename(__file__) libDir = os.path.abspath(os.path.join(os.path.dirname(__file__), "../")) sys.path.append(libDir) print libDir from lib.router import Router router = Router() ROUTES = router.getRoutes() from lib.loader import Loader loader = Loader(currentFullRoute, ROUTES) from lib.helper import Helper helper = Helper() from lib.filewriter import FileWriter filewriter = FileWriter() from lib.reporter import Reporter reporter = Reporter(ROUTES) # return to current path sys.path.append(currentFullRoute) # ---------------------------------------------------------------------- CUSTOM LIBS from math import * from graphviz import Digraph
from lib.helper import Helper
from lib.encoder import Encoder
import sys
if __name__ == "__main__":
helper = Helper(sys.argv)
helper.ParseConfig().ShowConfigDescription()
encoder = Encoder(helper)
encoder.append_vars(helper.GetConfig("vars"))
template = helper.LoadFile(helper.GetConfig("template"))
data = template
for evasion in helper.GetConfig("evasion"):
data += helper.LoadFile("templates/evasions/" + evasion + ".vba")
data = encoder.replace_var(data, "offset", encoder.get_encoding_offset())
data = encoder.encode_user_vars(data)
data = encoder.append_def_use_tag(data)
data = encoder.rand_vars(data)
data = encoder.rand_int(data)
data = encoder.rand_smallint(data)
encodedvars = helper.GetConfig("encodedvars")
for var in encodedvars:
data = encoder.replace_var(data, var, encodedvars[var], True)
if "-s" in sys.argv or "--split_strings" in sys.argv:
data = encoder.split_strings(data)
if "-x" in sys.argv or "--strings_to_hex" in sys.argv:
def get_icon(self):
if len(self.icon):
return Helper.app_path() + "/icon/" + self.icon
return ''
def deploy_bcf(config, fuel_cluster_id):
# Deploy setup node
Helper.safe_print("Start to prepare setup node\n")
env = Environment(config, fuel_cluster_id)
Helper.common_setup_node_preparation(env)
# Generate detailed node information
Helper.safe_print("Start to setup Big Cloud Fabric\n")
nodes_config = None
if 'nodes' in config:
nodes_yaml_config = config['nodes']
node_dic = Helper.load_nodes(nodes_yaml_config, env)
# Generate scripts for each node
for hostname, node in node_dic.iteritems():
if node.os == const.CENTOS:
Helper.generate_scripts_for_centos(node)
elif node.os == const.UBUNTU:
Helper.generate_scripts_for_ubuntu(node)
with open(const.LOG_FILE, "a") as log_file:
log_file.write(str(node))
if node.skip:
Helper.safe_print("skip node %(hostname)s due to %(error)s\n" %
{'hostname' : hostname,
'error' : node.error})
continue
node_q.put(node)
# Use multiple threads to setup nodes
for i in range(const.MAX_WORKERS):
t = threading.Thread(target=worker_setup_node)
t.daemon = True
t.start()
node_q.join()
Helper.safe_print("Big Cloud Fabric deployment finished! Check %(log)s on each node for details.\n" %
{'log' : const.LOG_FILE})
def certify_node_setup(q):
while True:
node = q.get()
Helper.certify_node(node)
q.task_done()
def deploy_bcf(config, fuel_cluster_id):
# Deploy setup node
Helper.safe_print("Start to prepare setup node\n")
env = Environment(config, fuel_cluster_id)
Helper.common_setup_node_preparation(env)
controller_node = None
# Generate detailed node information
Helper.safe_print("Start to setup Big Cloud Fabric\n")
nodes_config = None
if 'nodes' in config:
nodes_yaml_config = config['nodes']
node_dic = Helper.load_nodes(nodes_yaml_config, env)
# Generate scripts for each node
for hostname, node in node_dic.iteritems():
if node.os == const.CENTOS:
Helper.generate_scripts_for_centos(node)
elif node.os == const.UBUNTU:
Helper.generate_scripts_for_ubuntu(node)
with open(const.LOG_FILE, "a") as log_file:
log_file.write(str(node))
if node.skip:
Helper.safe_print("skip node %(hostname)s due to %(error)s\n" %
{'hostname' : hostname,
'error' : node.error})
continue
node_q.put(node)
if node.role == const.ROLE_NEUTRON_SERVER:
controller_node = node
elif node.deploy_dhcp_agent:
dhcp_node_q.put(node)
# Use multiple threads to setup nodes
for i in range(const.MAX_WORKERS):
t = threading.Thread(target=worker_setup_node)
t.daemon = True
t.start()
node_q.join()
# Use multiple threads to setup up dhcp agent and metadata agent
if controller_node:
Helper.safe_print("Copy dhcp_agent.ini from openstack controller %(controller_node)s\n" %
{'controller_node' : controller_node.hostname})
Helper.copy_file_from_remote(controller_node, '/etc/neutron', 'dhcp_agent.ini',
controller_node.setup_node_dir)
Helper.safe_print("Copy metadata_agent.ini from openstack controller %(controller_node)s\n" %
{'controller_node' : controller_node.hostname})
Helper.copy_file_from_remote(controller_node, '/etc/neutron', 'metadata_agent.ini',
controller_node.setup_node_dir)
for i in range(const.MAX_WORKERS):
t = threading.Thread(target=worker_setup_dhcp_agent)
t.daemon = True
t.start()
dhcp_node_q.join()
Helper.safe_print("Big Cloud Fabric deployment finished! Check %(log)s on each node for details.\n" %
{'log' : const.LOG_FILE})
def split_sequential(self, URM, URM_df):
segment = 1
# splitting URM in test set e train set
selected_playlists = np.array([])
available_playlists = np.arange(URM.shape[0])
target_analyzer = TargetAnalyzer()
#Gets distribution of only last 5000 playlists
dist = target_analyzer.get_distribution_array_only_last(segment)
helper = Helper()
target_playlists = helper.get_target_playlists_list()[:5000]
#n_target = np.sum(dist) - len(target_playlists)
# Removing from the cluster distribution the len of the sequential target
for playlist_id in target_playlists:
playlist_id = int(playlist_id)
available_playlists = np.delete(
available_playlists,
np.where(available_playlists == playlist_id))
selected_playlists = np.append(selected_playlists, playlist_id)
#target_len = len(URM[playlist_id].data)
#dist[target_len] -= 1
print("Clustering with segment = " + str(segment))
for key in tqdm(range(len(dist))):
while dist[key] != 0:
random_index = randint(0, len(available_playlists) - 1)
playlist_id = available_playlists[random_index]
target_segment = int(0.8 * len(URM[playlist_id].data))
if target_segment == key:
available_playlists = np.delete(
available_playlists,
np.where(available_playlists == playlist_id))
selected_playlists = np.append(selected_playlists,
playlist_id)
dist[key] -= 1
self.target_playlists = selected_playlists.astype(int)
grouped = URM_df.groupby(
'playlist_id', as_index=True).apply(lambda x: list(x['track_id']))
relevant_items = defaultdict(list)
for playlist_id in selected_playlists:
#Tracks = lista delle tracks prese dalla URM
tracks = np.array(grouped[playlist_id])
if playlist_id in target_playlists:
to_be_removed = int(len(tracks) * 0.2)
#Torna le #to_be_removed tracks ordinate sequenzialmente. e le toglie dalla lista delle tracks
to_be_removed_tracks = helper.get_sorted_tracks_in_playlist(
playlist_id)[-to_be_removed:]
for track in to_be_removed_tracks:
relevant_items[playlist_id].append(track)
tracks = np.delete(tracks, np.where(tracks == track))
else:
to_be_removed = int(len(tracks) * 0.2)
for i in range(to_be_removed):
index = randint(0, len(tracks) - 1)
removed_track = tracks[index]
relevant_items[playlist_id].append(removed_track)
tracks = np.delete(tracks, index)
grouped[playlist_id] = tracks
all_tracks = self.tracks_df["track_id"].unique()
matrix = MultiLabelBinarizer(classes=all_tracks,
sparse_output=True).fit_transform(grouped)
self.URM_train = matrix.tocsr()
self.URM_train = self.URM_train.astype(np.float64)
self.dict_test = relevant_items
