def scale_microservice(service_name, value):
# Read the current config file
micro_config = util.read_config_file(micro_config_name)
print("### Scaling micro_service: " + service_name + " of value: " +
str(value))
current_replica = plan.get_micro_replicas(service_name)
max_replica = int(micro_config.get(service_name, 'max_replica'))
min_replica = int(micro_config.get(service_name, 'min_replica'))
# This represents the total number of services 'after' it has been scaled
total_replica = current_replica + value
if total_replica > max_replica:
print('### Abort micro scaling for microservice: ' + service_name +
' due to max replica limit: ' + str(max_replica) + '.\n')
return
elif total_replica < min_replica:
print('### Abort micro scaling for microservice: ' + service_name +
' due to min replica limit: ' + str(min_replica) + '.\n')
return
else:
print("====> Scaling microservice: " + service_name + " to: " +
str(total_replica) + "\n")
result = util.run_command("sudo docker service scale " + service_name +
"=" + str(total_replica))
return
def db_connect():
'''
Method for database creation and connection using predefined
database settings.
'''
db_url = read_config_file()
return create_engine(db_url['SQLALCHEMY_DATABASE_URI'])
def main():
while True:
micro_config = util.read_config_file(micro_config_file_name)
macro_config = util.read_config_file(macro_config_file_name)
micro_util = monitor.get_microservices_utilization()
#micro_util = monitor.get_container_utilization()
macro_util = monitor.get_macroservices_utilization()
compare_cpu_util(micro_config, micro_util, macro_config, macro_util)
print("****** Completed monitoring! Wait for " +
str(monitoring_interval) + " seconds. *****")
util.progress_bar(monitoring_interval)
def create_vm(new_vm_name, user_name, password, label, value):
# Read the config file and get the Elascale directory
config = util.read_config_file(config_name)
elascale_dir = config.get('swarm', 'elascale_root_dir')
#provision_vm bash script
prov_vm_file = elascale_dir + "./provision_vm.sh"
command = "sudo " + prov_vm_file + " " + new_vm_name + " " + user_name + " " + password + " " + label + " " + value
result = util.provision_vm(command)
def main() -> None:
# pylint: disable=no-member
config = util.read_config_file("quality-control")
parser = load_args_parser(config)
args = parser.parse_args(sys.argv)
if not args.check_working and not args.check_daily and not args.check_history:
logger.info(" [default to all sources]")
args.check_working = True
args.check_daily = True
args.check_history = True
config = QCConfig(
results_dir=args.results_dir,
save_results=args.save_results,
enable_experimental=args.enable_experimental,
images_dir=args.images_dir,
plot_models=args.plot_models,
)
if config.save_results:
logger.warning(f" [save results to {args.results_dir}]")
if config.plot_models:
logger.warning(f" [save forecast curves to {args.images_dir}]")
if len(args.state) != 0:
logger.error(" [states filter not implemented]")
ds = DataSource()
if args.check_working:
logger.info(
"--| QUALITY CONTROL --- GOOGLE WORKING SHEET |---------------------------------------------------"
)
log = check_working(ds, config=config)
log.print()
if args.check_daily:
logger.info(
"--| QUALITY CONTROL --- CURRENT |-----------------------------------------------------------"
)
log = check_current(ds, config=config)
log.print()
if args.check_history:
logger.info(
"--| QUALITY CONTROL --- HISTORY |-----------------------------------------------------------"
)
log = check_history(ds)
log.print()
def reset(self):
self._working = None
self._current = None
self._history = None
config = util.read_config_file("quality-control")
self.config = QCConfig(
results_dir=config["CHECKS"]["results_dir"],
enable_experimental=config["CHECKS"]["enable_experimental"] == "True",
save_results=config["CHECKS"]["save_results"] == "True",
images_dir=config["MODEL"]["images_dir"],
plot_models=config["MODEL"]["plot_models"] == "True",
)
self.ds = DataSource()
def ashcpu():
"""
Generates a graph that shows cpu usage within an Oracle database
by various parts of the application. The graph is configured by lines in
the text file ashcpufile.txt.
Each line is part of a client machine name and a label
for machines with that pattern.
For example:
abcd WEBFARM
Any client machines with the string abcd in their name
will have their database cpu usage grouped into the category WEBFARM.
The graph is hard coded to only look at CPU usage between 8 am and 5 pm
of the day of the week specified. This is intended to look at CPU usage
during the work day.
"""
day = raw_input('Enter day of week: ')
user = util.my_oracle_user
c = perfq.cpubymachine(day, 8, 17)
lines = util.read_config_file(util.config_dir, database + util.ashcpu_file)
for l in lines:
args = l.split()
if len(args) == 2:
c.add_machine(args[0], args[1])
querytext = c.build_query()
user = util.my_oracle_user
password = util.get_oracle_password(database)
d = db.connection(user, password, database)
h = saveawr.day_history(d, day, 'ASHCPUBYMACHINE', querytext)
results = h.save_day_results()
column_names = h.get_column_names()
myplot.plot_cpu_by_day(database, day, results, column_names)
def groupsigs():
"""
This shows the average elapsed time and total number of executions for
a group of SQL statements defined by their force matching signature.
A signature represents a group of queries that are the same except for their
constants. The goal of this query is to pick some group of queries
that we care about such as the main queries the users use every day and
show their performance over time. It does hide the details of the individual
queries but may have value if we choose the best set of signatures.
"""
user = util.my_oracle_user
queryobj = perfq.groupofsignatures()
lines = util.read_config_file(util.config_dir,
database + util.groupsigs_file)
for line in lines:
if len(line) > 0:
queryobj.add_signature(int(line))
querytext = queryobj.build_query()
user = util.my_oracle_user
password = util.get_oracle_password(database)
dbconn = db.connection(user, password, database)
results = dbconn.run_return_flipped_results(querytext)
if results == None:
print "No results returned"
return
# plot query
title = "SQL matching group of signatures on " + database + " database elapsed versus executions"
top_label = "Number of executions"
bottom_label = "Averaged Elapsed Microseconds"
date_time = results[0]
executions = results[1]
avg_elapsed = results[2]
myplot.frequency_average(title, top_label, bottom_label, date_time,
executions, avg_elapsed)
def prepare_for_beats(vm_name):
# Read the current config file
config = util.read_config_file(config_name)
print('\nDeploying beats on the VM: ' + vm_name)
# Get where the config files are located. This can be found on config.ini file
elascale_config_dir = config.get('swarm', 'elascale_config_dir')
# Get where the Elascale root directory are located. This can be found on config.ini file
elascale_certs_dir = config.get('swarm', 'elascale_certs_dir')
# Copy the metricbeat.yml on swarm-master to the new node
result = util.run_command("sudo docker-machine scp " +
elascale_config_dir + "metricbeat.yml " +
vm_name + ":~")
# Change the hostname on the new metricbeat.yml
sed_command = "sed -i \"s/name: \".*\"/name: \"" + vm_name + "\"/g\" ~/metricbeat.yml"
result = util.run_command("sudo docker-machine ssh " + vm_name + " " +
sed_command)
# Copy the dockbeat.yml on swarm-master to the new node
result = util.run_command("sudo docker-machine scp " +
elascale_config_dir + "dockbeat.yml " + vm_name +
":~")
# Copy elasticsearch_certificate.yml to the new machine
result = util.run_command("sudo docker-machine scp " + elascale_certs_dir +
"elasticsearch_certificate.pem " + vm_name +
":~/certs")
# Create /volumes/dockbeat-logs dir on the new node (required for dockbeat to work properly)
result = util.run_command("sudo docker-machine ssh " + vm_name +
" sudo mkdir -p /volumes/dockbeat-logs/")
print(
'dockbeat and metricbeat yml files have been copied/configured successfully.'
)
def scale_macroservice(host_name, value):
# Read the current config file
macro_config = util.read_config_file(macro_config_name)
print("### Scaling macro_service: " + host_name + " of value: " +
str(value))
base_name = plan.get_macro_base_name(host_name)
current_replica = plan.get_macro_replicas(base_name)
max_replica = int(macro_config.get(base_name, 'max_replica'))
min_replica = int(macro_config.get(base_name, 'min_replica'))
# This represents the total number of services 'after' it has been scaled
# 'value' variable tells whether to scale down ( - value) or scale up ( + value)
total_replica = current_replica + value
if total_replica > max_replica:
print('### Abort macro scaling for ' + host_name +
' due to max replica limit: ' + str(max_replica) + '.\n')
return
if total_replica < min_replica:
print('### Abort macro scaling for ' + host_name +
' due to min replica limit: ' + str(min_replica) + '.\n')
return
else:
if value > 0:
print("====> Scaling out the macroservice: " + host_name +
" by 1\n")
docker_machine_scale(host_name, 'scale-out') # scale-out
else:
#value < 0
print("====> Scaling in the macroservice: " + host_name +
" by -1\n")
docker_machine_scale(host_name, 'scale-in') # scale-in
def sigselapctcpu():
"""
Plots elapsed for a group of sql statements based
on their signatures against percent CPU of the host.
"""
user = util.my_oracle_user
queryobj = perfq.groupofsignatures()
lines = util.read_config_file(util.config_dir,
database + util.groupsigs_file)
for line in lines:
if len(line) > 0:
queryobj.add_signature(int(line))
querytext = queryobj.build_query3()
user = util.my_oracle_user
password = util.get_oracle_password(database)
dbconn = db.connection(user, password, database)
results = dbconn.run_return_flipped_results(querytext)
if results == None:
print "No results returned"
return
# plot query
title = "SQL matching group of signatures on " + database + " database elapsed versus cpu"
y_label = "Minutes versus Percentage"
number_of_plots = 2
plot_names = ["CPU % Busy", "Elapsed in Minutes"]
myplot.plotmulti(title, y_label, number_of_plots, plot_names, results)
def sigscpuio():
"""
Plots elapsed, cpu, and io for a group of sql statements based
on their signatures.
"""
user = util.my_oracle_user
queryobj = perfq.groupofsignatures()
lines = util.read_config_file(util.config_dir,
database + util.groupsigs_file)
for line in lines:
if len(line) > 0:
queryobj.add_signature(int(line))
querytext = queryobj.build_query2()
user = util.my_oracle_user
password = util.get_oracle_password(database)
dbconn = db.connection(user, password, database)
results = dbconn.run_return_flipped_results(querytext)
if results == None:
print "No results returned"
return
# plot query
title = "SQL matching group of signatures on " + database + " database elapsed CPU IO"
y_label = "Seconds"
number_of_plots = 3
plot_names = ["Elapsed", "CPU+IO", "IO"]
myplot.plotmulti(title, y_label, number_of_plots, plot_names, results)
def sigfour():
"""
Plots executions, average elapsed, cpu percent
and average single block IO time.
"""
user = util.my_oracle_user
queryobj = perfq.groupofsignatures()
lines = util.read_config_file(util.config_dir,
database + util.groupsigs_file)
for line in lines:
if len(line) > 0:
queryobj.add_signature(int(line))
querytext = queryobj.build_query4()
user = util.my_oracle_user
password = util.get_oracle_password(database)
dbconn = db.connection(user, password, database)
results = dbconn.run_return_flipped_results(querytext)
if results == None:
print "No results returned"
return
# plot query
title = "SQL matching group of signatures on " + database + " database four graphs"
myplot.plot_four(title, "CPU % Busy", "Number of executions (/100000)",
"Average Elapsed Time (microseconds)",
"Average single block read time (ms)", results[0],
results[1], results[2], results[3], results[4])
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import scoped_session, sessionmaker
from sqlalchemy.exc import IntegrityError
from sqlalchemy_utils import database_exists
from datetime import datetime
from util import generate_salt, generate_password_hash, read_config_file
class DuplicateLoginException(Exception):
def __init__(self, message):
super(DuplicateLoginException, self).__init__(message)
_DATABASE = 'sqlite:///platypus.db'
_DEV_DATABASE = 'sqlite:///platypus-dev.db'
database = _DEV_DATABASE if read_config_file('dev') else _DATABASE
setup = database_exists(database)
engine = create_engine(database, echo=True)
session = scoped_session(sessionmaker(
autocommit=False,
autoflush=False,
bind=engine))
Base = declarative_base()
Base.query = session.query_property()
from model import Article, User, Role
def init():
Base.metadata.create_all(bind=engine)
if not setup:
if calendar_type == "Next" and today >= prev_date and today < curr_date:
return True
return False
def get_random_emoji():
all_emoji = sh.get_emoji()["emoji"]
return random.choice(list(all_emoji.keys()))
if __name__ == "__main__":
testing_slack_channel = None
if len(sys.argv) > 1:
testing_slack_channel = sys.argv[1]
config_data = read_config_file("config.env")
CREDENTIALS = get_conf_or_env("CREDENTIALS", config_data)
CREDENTIALS_FILE = get_conf_or_env("CREDENTIALS_FILE", config_data,
"credentials.json")
WORKBOOK = get_conf_or_env("WORKBOOK", config_data)
WORKSHEET_META_TAB = get_conf_or_env("WORKSHEET_META_TAB", config_data)
SLACK_TOKEN = get_conf_or_env("SLACK_TOKEN", config_data)
SLACK_USERNAME = get_conf_or_env("SLACK_USERNAME", config_data)
SLACK_ICON_URL = get_conf_or_env("SLACK_ICON_URL", config_data)
WORKSHEET_PEOPLE_TAB = get_conf_or_env("WORKSHEET_PEOPLE_TAB", config_data)
required_variables = (
"WORKBOOK WORKSHEET_META_TAB SLACK_TOKEN SLACK_USERNAME SLACK_ICON_URL"
.split(" "))
if calendar_type == 'Current' and today >= curr_date and today < next_date:
return True
elif calendar_type == 'Next' and today >= prev_date and today < curr_date:
return True
return False
def get_random_emoji():
all_emoji = sh.get_emoji()['emoji']
return random.choice(list(all_emoji.keys()))
if __name__ == '__main__':
testing_slack_channel = None
if len(sys.argv) > 1:
testing_slack_channel = sys.argv[1]
config_data = read_config_file('config.env')
CREDENTIALS_FILE = get_conf_or_env('CREDENTIALS_FILE', config_data, 'credentials.json')
WORKBOOK = get_conf_or_env('WORKBOOK', config_data)
WORKSHEET_META_TAB = get_conf_or_env('WORKSHEET_META_TAB', config_data)
SLACK_TOKEN = get_conf_or_env('SLACK_TOKEN', config_data)
SLACK_USERNAME = get_conf_or_env('SLACK_USERNAME', config_data)
SLACK_ICON_URL = get_conf_or_env('SLACK_ICON_URL', config_data)
WORKSHEET_PEOPLE_TAB = get_conf_or_env('WORKSHEET_PEOPLE_TAB', config_data)
required_variables = 'CREDENTIALS_FILE WORKBOOK WORKSHEET_META_TAB SLACK_TOKEN SLACK_USERNAME SLACK_ICON_URL'.split(' ')
for variable in required_variables:
if eval(variable) is None:
logger.error('Missing ' + variable)
exit(1)
values = parser.parse_args()
if values.N is not None:
N = values.N
if values.config is not None:
config = values.config
if values.indb is not None:
indb = values.indb
if values.port is not None:
port = values.port
if values.local is not None:
local = values.local
print("Collecting protocol parameters")
C, Bits = util.read_config_file(config)
print("Generating test case")
util.GenerateTestCase(N, C, Bits, indb, config)
if local == 1:
print("Running local test")
client = Popen([
"./tests/ProtocolClient", "./data/info.txt", "./data/client/",
"127.0.0.1",
str(port)
])
server = Popen([
"./tests/ProtocolServer", "./data/info.txt", "./data/server/",
str(port)
],
return self.db_connection.run_return_all_results(query_string)
def get_column_names(self):
return self.db_connection.get_column_names()
# Main program starts here
database,dbconnection = util.script_startup('Database CPU by Application Area')
day=util.input_no_default('Enter day of week: ')
user=util.my_oracle_user
c = cpubymachine(day,8,17)
lines = util.read_config_file(util.config_dir,database+util.ashcpu_file)
for l in lines:
args = l.split()
if len(args) == 2:
c.add_machine(args[0],args[1])
querytext = c.build_query()
h = day_history(dbconnection,day,'ASHCPUBYMACHINE',querytext)
results = h.save_day_results()
column_names = h.get_column_names()
# Load global variables for graph
import util
import signatures
database,dbconnection = util.script_startup('Stats for SQL statments by signature')
start_time=util.input_with_default('Start date and time (DD-MON-YYYY HH24:MI:SS)','01-JAN-1900 12:00:00')
end_time=util.input_with_default('End date and time (DD-MON-YYYY HH24:MI:SS)','01-JAN-2200 12:00:00')
instance_number=util.input_with_default('Database Instance (1 if not RAC)','1')
queryobj = signatures.groupofsignatures()
queryobj.set_start_end_instance(start_time,end_time,instance_number)
lines = util.read_config_file(util.config_dir,database+util.groupsigs_file)
for line in lines:
if len(line) > 0:
queryobj.add_signature(int(line))
querytext = queryobj.build_query()
results = dbconnection.run_return_flipped_results(querytext)
util.exit_no_results(results)
# plot query
myplot.title = "SQL matching group of signatures on "+database+" database, instance "+instance_number+" elapsed versus executions"
myplot.ylabel1 = "Number of executions"
def get_column_names(self):
return self.db_connection.get_column_names()
# Main program starts here
database, dbconnection = util.script_startup(
'Database CPU by Application Area')
day = util.input_no_default('Enter day of week: ')
user = util.my_oracle_user
c = cpubymachine(day, 8, 17)
lines = util.read_config_file(util.config_dir, database + util.ashcpu_file)
for l in lines:
args = l.split()
if len(args) == 2:
c.add_machine(args[0], args[1])
querytext = c.build_query()
h = day_history(dbconnection, day, 'ASHCPUBYMACHINE', querytext)
results = h.save_day_results()
column_names = h.get_column_names()
# Load global variables for graph
