def plot(title, second_count, frame, ax, is_image, label_connect,
filter_in_nth, cp):
values = FeatureBase.derive_plot_values(title, second_count,
frame).copy()
if values is None:
return None
VAL_COL = 'latest' #cp + 'hp_value'
CYCLE_COL = cp + 'cycle' # TODO: not historic!
min_value = values[VAL_COL].min()
max_cycle = values[CYCLE_COL].max()
values[CYCLE_COL] = min_value + values[
CYCLE_COL] - max_cycle # make the cycle more visible and prevent it from scaling the graph
df_ut = values.iloc[values.index % filter_in_nth == 0][[
'index', cp + 'up_tick_value'
]].dropna() #.loc[values.shift(-1) != values].dropna()
ax.scatter(df_ut.ix[:, 0].tolist(),
df_ut.ix[:, 1].tolist(),
color='green',
s=0.9)
df_dt = values.iloc[values.index % filter_in_nth == 0][[
'index', cp + 'down_tick_value'
]].dropna()
ax.scatter(
df_dt.ix[:, 0].tolist(),
df_dt.ix[:, 1].tolist(),
color='red',
s=0.9,
)
return ax
def plot(title, second_count, frame, ax, is_image, label_connect,
filter_in_nth, cp):
values = FeatureBase.derive_plot_values(title, second_count,
frame.ix[:, 'latest'])
if values is None:
return None
filtered = values[values.index % filter_in_nth == 0]
ax.plot(filtered['index'].tolist(),
filtered['latest'].tolist(),
color='blue',
alpha=0.9,
zorder=-10)
def plot(title, second_count, frame, ax, is_image, label_connect,
filter_in_nth, cp):
trend_col = cp + trend_cf
values = FeatureBase.derive_plot_values(
title, second_count,
frame[[cp + 'slow_avg', cp + 'fast_avg', trend_col]])
if values is None:
return None
assert not values.empty
df_trend_shift = values[~(
values[trend_col] == values[trend_col].shift(-1))][:-1]
assert not df_trend_shift.empty
df_strongvol = df_trend_shift[
df_trend_shift[trend_col] ==
VolumeFeature.STRONG_VOLUME] #.drop_duplicates(trend_col)
assert not df_strongvol.empty
strongvol_y = df_strongvol[cp + 'fast_avg'].tolist()
strongvol_scatter = plot = ax.scatter(df_strongvol['index'].tolist(),
strongvol_y,
color='green',
s=70,
alpha=0.7,
zorder=10)
label_connect(strongvol_scatter, strongvol_y, color='green')
df_weakvol = df_trend_shift[df_trend_shift[trend_col] ==
VolumeFeature.WEAK_VOLUME]
assert not df_weakvol.empty
strongvol_y = df_weakvol[cp + 'fast_avg'].tolist()
weakvol_scatter = ax.scatter(df_weakvol['index'].tolist(),
strongvol_y,
color='red',
s=70,
alpha=0.7,
zorder=10)
label_connect(weakvol_scatter, strongvol_y, color='red')
filtered = values[values.index % filter_in_nth == 0]
indices = filtered['index'].tolist()
df_slow_avg = filtered[cp + 'slow_avg']
ax.plot(indices, df_slow_avg, color='pink', alpha=0.9)
df_fast_avg = filtered[cp + 'fast_avg']
ax.plot(indices, df_fast_avg, color='violet', alpha=0.9)
def plot(title, second_count, frame, ax, is_image, label_connect,
filter_in_nth, cp):
values = FeatureBase.derive_plot_values(title, second_count, frame)
if values is None:
return None
latest_min = values['latest'].min()
mult_factor = values['latest'].max() - latest_min
offset_y = (latest_min - mult_factor)
values[cp + 'd_slow'] = values[cp + 'd_slow'] * mult_factor + offset_y
values[cp + 'd_fast'] = values[cp + 'd_fast'] * mult_factor + offset_y
values[cp + 'k'] = values[cp + 'k'] * mult_factor + offset_y
filtered = values[values.index % filter_in_nth == 0]
assert len(filtered) > 0
Log.d(frame[cp + action_cf].value_counts())
indices = filtered['index'].tolist()
df_k = filtered[cp + 'k']
ax.plot(indices, df_k, color='orange', alpha=0.9, zorder=-5)
df_d_slow = filtered[cp + 'd_slow']
ax.plot(indices, df_d_slow, color='lightblue', alpha=0.9)
for y in [0, lower_k, upper_k, 1]:
ax.plot([indices[0], indices[-1]],
[y * mult_factor + offset_y] * 2,
color='white',
dashes=[6, 2],
alpha=0.5,
zorder=-10)
df_buy = values[values[cp + action_cf] == ActionFeature.BUY]
ax.scatter(df_buy['index'],
df_buy[cp + 'd_slow'],
color='green',
s=70,
zorder=10,
alpha=0.7)
df_sell = values[values[cp + action_cf] == ActionFeature.SELL]
ax.scatter(df_sell['index'],
df_sell[cp + 'd_slow'],
color='red',
s=70,
zorder=10,
alpha=0.7)
def plot(title,
second_count,
frame,
ax,
is_image,
label_connect,
filter_in_nth,
cp,
plot_color_fast='gold',
plot_color_slow='sienna'):
values = FeatureBase.derive_plot_values(
title, second_count,
frame[[cp + 'slow_avg', cp + 'fast_avg', cp + action_cf]])
if values is None:
return None
filtered = values[values.index % filter_in_nth == 0]
indices = filtered['index'].tolist()
df_slow_avg = filtered[cp + 'slow_avg']
ax.plot(indices, df_slow_avg, color=plot_color_slow, alpha=0.9)
df_fast_avg = filtered[cp + 'fast_avg']
ax.plot(indices, df_fast_avg, color=plot_color_fast, alpha=0.9)
df_buy = values[values[cp + action_cf] == ActionFeature.BUY]
df_buy_y = df_buy[cp + 'fast_avg'].tolist()
buy_scatter = plot = ax.scatter(df_buy['index'],
df_buy_y,
color='green',
s=70,
alpha=0.7,
zorder=10)
label_connect(buy_scatter, df_buy_y, color='green')
df_sell = values[values[cp + action_cf] == ActionFeature.SELL]
df_sell_y = df_sell[cp + 'fast_avg'].tolist()
df_sell_scatter = ax.scatter(df_sell['index'],
df_sell_y,
color='red',
s=70,
alpha=0.7,
zorder=10)
label_connect(df_sell_scatter, df_sell_y, color='red')
def __init__(self,
dm_logger=None,
args=None,
zookeeper_client=None,
amqp_client=None):
"""Physical Router init routine."""
DeviceManager._instance = self
self._args = args
self._amqp_client = amqp_client
self.logger = dm_logger or DeviceManagerLogger(args)
self._vnc_amqp = DMAmqpHandle(self.logger, self.REACTION_MAP,
self._args)
PushConfigState.set_push_mode(int(self._args.push_mode))
PushConfigState.set_repush_interval(int(self._args.repush_interval))
PushConfigState.set_repush_max_interval(
int(self._args.repush_max_interval))
PushConfigState.set_push_delay_per_kb(
float(self._args.push_delay_per_kb))
PushConfigState.set_push_delay_max(int(self._args.push_delay_max))
PushConfigState.set_push_delay_enable(
bool(self._args.push_delay_enable))
self._chksum = ""
if self._args.collectors:
self._chksum = hashlib.md5(''.join(
self._args.collectors)).hexdigest()
# Register Plugins
try:
DeviceConf.register_plugins()
except DeviceConf.PluginsRegistrationFailed as e:
self.logger.error("Exception: " + str(e))
except Exception as e:
tb = traceback.format_exc()
self.logger.error("Internal error while registering plugins: " +
str(e) + tb)
# Register Ansible Plugins
try:
AnsibleBase.register_plugins()
except AnsibleBase.PluginsRegistrationFailed as e:
self.logger.error("Exception: " + str(e))
except Exception as e:
tb = traceback.format_exc()
self.logger.error(
"Internal error while registering ansible plugins: " + str(e) +
tb)
# Register Feature Plugins
try:
FeatureBase.register_plugins()
except FeatureBase.PluginRegistrationFailed as e:
self.logger.error("Exception: " + str(e))
except Exception as e:
tb = traceback.format_exc()
self.logger.error(
"Internal error while registering feature plugins: " + str(e) +
tb)
raise e
# Retry till API server is up
connected = False
self.connection_state_update(ConnectionStatus.INIT)
api_server_list = args.api_server_ip.split(',')
while not connected:
try:
self._vnc_lib = VncApi(
args.admin_user,
args.admin_password,
args.admin_tenant_name,
api_server_list,
args.api_server_port,
api_server_use_ssl=args.api_server_use_ssl)
connected = True
self.connection_state_update(ConnectionStatus.UP)
except requests.exceptions.ConnectionError as e:
# Update connection info
self.connection_state_update(ConnectionStatus.DOWN, str(e))
time.sleep(3)
except ResourceExhaustionError: # haproxy throws 503
time.sleep(3)
if PushConfigState.is_push_mode_ansible():
FabricManager.initialize(args, dm_logger, self._vnc_lib)
# Initialize amqp
self._vnc_amqp.establish()
# Initialize cassandra
self._object_db = DMCassandraDB.get_instance(zookeeper_client,
self._args, self.logger)
DBBaseDM.init(self, self.logger, self._object_db)
DBBaseDM._sandesh = self.logger._sandesh
GlobalSystemConfigDM.locate_all()
FeatureDM.locate_all()
PhysicalRoleDM.locate_all()
OverlayRoleDM.locate_all()
RoleDefinitionDM.locate_all()
FeatureConfigDM.locate_all()
NodeProfileDM.locate_all()
RoleConfigDM.locate_all()
GlobalVRouterConfigDM.locate_all()
VirtualNetworkDM.locate_all()
DataCenterInterconnectDM.locate_all()
FabricDM.locate_all()
FabricNamespaceDM.locate_all()
LogicalRouterDM.locate_all()
RoutingInstanceDM.locate_all()
FloatingIpPoolDM.locate_all()
BgpRouterDM.locate_all()
PhysicalInterfaceDM.locate_all()
LogicalInterfaceDM.locate_all()
PhysicalRouterDM.locate_all()
LinkAggregationGroupDM.locate_all()
VirtualPortGroupDM.locate_all()
PortDM.locate_all()
TagDM.locate_all()
NetworkIpamDM.locate_all()
VirtualMachineInterfaceDM.locate_all()
SecurityGroupDM.locate_all()
AccessControlListDM.locate_all()
PortProfileDM.locate_all()
StormControlProfileDM.locate_all()
ServiceInstanceDM.locate_all()
ServiceApplianceSetDM.locate_all()
ServiceApplianceDM.locate_all()
ServiceTemplateDM.locate_all()
PortTupleDM.locate_all()
InstanceIpDM.locate_all()
FloatingIpDM.locate_all()
for vn in VirtualNetworkDM.values():
vn.update_instance_ip_map()
ServiceEndpointDM.locate_all()
ServiceConnectionModuleDM.locate_all()
ServiceObjectDM.locate_all()
NetworkDeviceConfigDM.locate_all()
E2ServiceProviderDM.locate_all()
PeeringPolicyDM.locate_all()
pr_obj_list = PhysicalRouterDM.list_obj()
pr_uuid_set = set([pr_obj['uuid'] for pr_obj in pr_obj_list])
self._object_db.handle_pr_deletes(pr_uuid_set)
dci_obj_list = DataCenterInterconnectDM.list_obj()
dci_uuid_set = set([dci_obj['uuid'] for dci_obj in dci_obj_list])
self._object_db.handle_dci_deletes(dci_uuid_set)
si_obj_list = ServiceInstanceDM.list_obj()
si_uuid_set = set([si_obj['uuid'] for si_obj in si_obj_list])
self._object_db.handle_pnf_resource_deletes(si_uuid_set)
for pr in PhysicalRouterDM.values():
pr.set_config_state()
pr.uve_send()
self._vnc_amqp._db_resync_done.set()
gevent.joinall(self._vnc_amqp._vnc_kombu.greenlets())
def __init__(self, dm_logger=None, args=None, zookeeper_client=None,
amqp_client=None):
DeviceManager._instance = self
self._args = args
self._amqp_client = amqp_client
self.logger = dm_logger or DeviceManagerLogger(args)
self._vnc_amqp = DMAmqpHandle(self.logger, self.REACTION_MAP,
self._args)
PushConfigState.set_push_mode(int(self._args.push_mode))
PushConfigState.set_repush_interval(int(self._args.repush_interval))
PushConfigState.set_repush_max_interval(
int(self._args.repush_max_interval))
PushConfigState.set_push_delay_per_kb(
float(self._args.push_delay_per_kb))
PushConfigState.set_push_delay_max(int(self._args.push_delay_max))
PushConfigState.set_push_delay_enable(
bool(self._args.push_delay_enable))
self._chksum = ""
if self._args.collectors:
self._chksum = hashlib.md5(
''.join(self._args.collectors)).hexdigest()
# Register Plugins
try:
DeviceConf.register_plugins()
except DeviceConf.PluginsRegistrationFailed as e:
self.logger.error("Exception: " + str(e))
except Exception as e:
tb = traceback.format_exc()
self.logger.error(
"Internal error while registering plugins: " + str(e) + tb)
# Register Ansible Plugins
try:
AnsibleBase.register_plugins()
except AnsibleBase.PluginsRegistrationFailed as e:
self.logger.error("Exception: " + str(e))
except Exception as e:
tb = traceback.format_exc()
self.logger.error(
"Internal error while registering ansible plugins: " +
str(e) + tb)
# Register Feature Plugins
try:
FeatureBase.register_plugins()
except FeatureBase.PluginRegistrationFailed as e:
self.logger.error("Exception: " + str(e))
except Exception as e:
tb = traceback.format_exc()
self.logger.error(
"Internal error while registering feature plugins: " +
str(e) + tb)
raise e
# Retry till API server is up
connected = False
self.connection_state_update(ConnectionStatus.INIT)
api_server_list = args.api_server_ip.split(',')
while not connected:
try:
self._vnc_lib = VncApi(
args.admin_user, args.admin_password,
args.admin_tenant_name, api_server_list,
args.api_server_port,
api_server_use_ssl=args.api_server_use_ssl)
connected = True
self.connection_state_update(ConnectionStatus.UP)
except requests.exceptions.ConnectionError as e:
# Update connection info
self.connection_state_update(ConnectionStatus.DOWN, str(e))
time.sleep(3)
except ResourceExhaustionError: # haproxy throws 503
time.sleep(3)
if PushConfigState.is_push_mode_ansible():
FabricManager.initialize(args, dm_logger, self._vnc_lib)
# Initialize amqp
self._vnc_amqp.establish()
# Initialize cassandra
self._object_db = DMCassandraDB.get_instance(zookeeper_client, self._args, self.logger)
DBBaseDM.init(self, self.logger, self._object_db)
DBBaseDM._sandesh = self.logger._sandesh
GlobalSystemConfigDM.locate_all()
FeatureDM.locate_all()
PhysicalRoleDM.locate_all()
OverlayRoleDM.locate_all()
RoleDefinitionDM.locate_all()
FeatureConfigDM.locate_all()
NodeProfileDM.locate_all()
RoleConfigDM.locate_all()
GlobalVRouterConfigDM.locate_all()
VirtualNetworkDM.locate_all()
DataCenterInterconnectDM.locate_all()
FabricDM.locate_all()
FabricNamespaceDM.locate_all()
LogicalRouterDM.locate_all()
RoutingInstanceDM.locate_all()
FloatingIpPoolDM.locate_all()
BgpRouterDM.locate_all()
PhysicalInterfaceDM.locate_all()
LogicalInterfaceDM.locate_all()
PhysicalRouterDM.locate_all()
LinkAggregationGroupDM.locate_all()
VirtualPortGroupDM.locate_all()
PortDM.locate_all()
TagDM.locate_all()
NetworkIpamDM.locate_all()
VirtualMachineInterfaceDM.locate_all()
SecurityGroupDM.locate_all()
AccessControlListDM.locate_all()
ServiceInstanceDM.locate_all()
ServiceApplianceSetDM.locate_all()
ServiceApplianceDM.locate_all()
ServiceTemplateDM.locate_all()
PortTupleDM.locate_all()
InstanceIpDM.locate_all()
FloatingIpDM.locate_all()
for vn in VirtualNetworkDM.values():
vn.update_instance_ip_map()
ServiceEndpointDM.locate_all()
ServiceConnectionModuleDM.locate_all()
ServiceObjectDM.locate_all()
NetworkDeviceConfigDM.locate_all()
E2ServiceProviderDM.locate_all()
PeeringPolicyDM.locate_all()
pr_obj_list = PhysicalRouterDM.list_obj()
pr_uuid_set = set([pr_obj['uuid'] for pr_obj in pr_obj_list])
self._object_db.handle_pr_deletes(pr_uuid_set)
dci_obj_list = DataCenterInterconnectDM.list_obj()
dci_uuid_set = set([dci_obj['uuid'] for dci_obj in dci_obj_list])
self._object_db.handle_dci_deletes(dci_uuid_set)
si_obj_list = ServiceInstanceDM.list_obj()
si_uuid_set = set([si_obj['uuid'] for si_obj in si_obj_list])
self._object_db.handle_pnf_resource_deletes(si_uuid_set)
for pr in PhysicalRouterDM.values():
pr.set_config_state()
pr.uve_send()
self._vnc_amqp._db_resync_done.set()
gevent.joinall(self._vnc_amqp._vnc_kombu.greenlets())