def test_timedelta_total_seconds():
# Take care not to directly compare results of float operations
# for exact values!
assert util.timedelta_total_seconds(timedelta(0, 0, 0)) == 0.0
assert util.timedelta_total_seconds(timedelta(0, 1, 0)) == 1.0
assert abs(
util.timedelta_total_seconds(timedelta(1, 0, 1)) -
((24 * 60 * 60) + 0.000001)) < 0.001
assert abs(
util.timedelta_total_seconds(timedelta(1, 1, 1)) -
((24 * 60 * 60) + 1 + 0.000001)) < 0.001
assert abs(
util.timedelta_total_seconds(timedelta(1, 5, 1)) -
((24 * 60 * 60) + 5 + 0.000001)) < 0.001
assert abs(
util.timedelta_total_seconds(timedelta(1, 5, 100)) -
((24 * 60 * 60) + 5 + 0.0001)) < 0.001
assert abs(
util.timedelta_total_seconds(timedelta(3, 5, 100)) -
((3 * 24 * 60 * 60) + 5 + 0.0001)) < 0.001
assert abs(
util.timedelta_total_seconds(timedelta(0, 5, 100)) -
(5 + 0.0001)) < 0.001
assert abs(
util.timedelta_total_seconds(timedelta(3, 0, 100)) -
((3 * 24 * 60 * 60) + 0.0001)) < 0.001
assert abs(
util.timedelta_total_seconds(timedelta(3, 5, 0)) -
((3 * 24 * 60 * 60) + 5)) < 0.001
def test_timedelta_total_seconds():
# Take care not to directly compare results of float operations
# for exact values!
assert util.timedelta_total_seconds(timedelta(0, 0, 0)) == 0.0
assert util.timedelta_total_seconds(timedelta(0, 1, 0)) == 1.0
assert abs(util.timedelta_total_seconds(timedelta(1, 0, 1)) - ((24 * 60 * 60) + 0.000001)) < 0.001
assert abs(util.timedelta_total_seconds(timedelta(1, 1, 1)) - ((24 * 60 * 60) + 1 + 0.000001)) < 0.001
assert abs(util.timedelta_total_seconds(timedelta(1, 5, 1)) - ((24 * 60 * 60) + 5 + 0.000001)) < 0.001
assert abs(util.timedelta_total_seconds(timedelta(1, 5, 100)) - ((24 * 60 * 60) + 5 + 0.0001)) < 0.001
assert abs(util.timedelta_total_seconds(timedelta(3, 5, 100)) - ((3 * 24 * 60 * 60) + 5 + 0.0001)) < 0.001
assert abs(util.timedelta_total_seconds(timedelta(0, 5, 100)) - (5 + 0.0001)) < 0.001
assert abs(util.timedelta_total_seconds(timedelta(3, 0, 100)) - ((3 * 24 * 60 * 60) + 0.0001)) < 0.001
assert abs(util.timedelta_total_seconds(timedelta(3, 5, 0)) - ((3 * 24 * 60 * 60) + 5)) < 0.001
def _check_duration_exceeded(self, data, db_now_utc, entry):
"""Called when there is a "started_at" value.
There is a started_at, and may or not be a completed_at.
Sends an event if the extract ran too long.
"""
if entry.completed_at:
run_time = timedelta_total_seconds(entry.completed_at,
entry.started_at)
else:
# Compute how long it's been running so far
run_time = int((db_now_utc - entry.started_at).\
total_seconds())
if data[SystemKeys.EXTRACT_DURATION_ERROR] and \
(run_time >=
data[SystemKeys.EXTRACT_DURATION_ERROR]) and \
not entry.notification_state & \
ExtractNotification.EXTRACT_DURATION_ERROR:
data['extract_duration_seconds'] = run_time
self._eventgen(EventControl.EXTRACT_DURATION_ERROR, data, entry)
entry.notification_state |= \
ExtractNotification.EXTRACT_DURATION_ERROR
elif data[SystemKeys.EXTRACT_DURATION_WARN] and \
(run_time >= \
data[SystemKeys.EXTRACT_DURATION_WARN]) and \
not entry.notification_state & \
(ExtractNotification.EXTRACT_DURATION_WARN |
ExtractNotification.EXTRACT_DURATION_ERROR):
# Note: We don't send a WARN event if we already sent an ERROR
data['extract_duration_seconds'] = run_time
self._eventgen(EventControl.EXTRACT_DURATION_WARN, data, entry)
entry.notification_state |= \
ExtractNotification.EXTRACT_DURATION_WARN
def _check_start_delay(self, data, db_now_utc, entry):
"""Check for delayed start time.
Sends an event on start time exceeded.
"""
time_since_created = timedelta_total_seconds(db_now_utc,
entry.created_at)
# Check to see if the extract STARTED late.
if data[SystemKeys.EXTRACT_DELAY_ERROR] and \
(time_since_created >= \
data[SystemKeys.EXTRACT_DELAY_ERROR]) and \
not entry.notification_state & \
ExtractNotification.EXTRACT_DELAY_ERROR:
data['extract_delay_seconds'] = time_since_created
self._eventgen(EventControl.EXTRACT_DELAY_ERROR, data, entry)
entry.notification_state |= \
ExtractNotification.EXTRACT_DELAY_ERROR
elif data[SystemKeys.EXTRACT_DELAY_WARN] and \
(time_since_created >= \
data[SystemKeys.EXTRACT_DELAY_WARN]) and \
not entry.notification_state & \
(ExtractNotification.EXTRACT_DELAY_WARN | \
ExtractNotification.EXTRACT_DELAY_ERROR):
# Note: We don't send a WARN event if we already sent an ERROR
data['extract_delay_seconds'] = time_since_created
self._eventgen(EventControl.EXTRACT_DELAY_WARN, data, entry)
entry.notification_state |= ExtractNotification.EXTRACT_DELAY_WARN
def _eventgen(self, key, data, entry):
"""Fill in information into the data dictionary and send the event."""
# Placeholder to be set by the next steps.
entry.system_user_id = -1
item_entry = self._add_info(entry)
if key == EventControl.EXTRACT_OK:
if item_entry:
self.server.extract_archive.add(item_entry, entry)
else:
logger.info("extract OK: Can't archive workbook/datasource "
"row - not found.")
data = dict(data.items() + entry.todict().items())
if entry.completed_at is not None and entry.started_at is not None:
duration = timedelta_total_seconds(entry.completed_at,
entry.started_at)
data['duration'] = duration
data['duration_hms'] = to_hhmmss(duration)
timestamp = utc2local(entry.completed_at)
else:
timestamp = None
envid = self.server.environment.envid
profile = UserProfile.get_by_system_user_id(envid,
data['system_user_id'])
if profile:
data['username'] = profile.display_name()
userid = profile.userid
else:
data['username'] = "******"
userid = None
self.server.event_control.gen(key,
data,
userid=userid,
site_id=data['site_id'],
timestamp=timestamp)
def info(p_json, p_home, p_repo, print_flag=True):
import os
p_user = util.get_user()
p_is_admin = util.is_admin()
pip_ver = get_pip_ver()
this_os = ""
this_uname = str(platform.system())
host_ip = util.get_host_ip()
wmic_path = os.getenv("SYSTEMROOT", "") + os.sep + "System32" + os.sep + "wbem" + os.sep + "wmic"
if this_uname == "Windows":
import psutil
host_display = os.getenv('LOGONSERVER','') + '\\' + os.getenv('COMPUTERNAME')
system_cpu_cores = os.getenv('NUMBER_OF_PROCESSORS','1')
os_arch = os.getenv('PROCESSOR_ARCHITECTURE', '')
cpu_model = check_output_wmic([wmic_path, "cpu", "get", "name"])
## system_memory_in_gb ######################################
m = psutil.virtual_memory().total
mem_bytes = int(m)
system_memory_in_kbytes = mem_bytes / 1024.0
system_memory_in_gb = str(mem_bytes / (1024.0**3))
else:
os_arch = util.getoutput("uname -m")
host_display = util.get_host_short()
## Check the OS & Resources ########################################
plat = util.get_os()
os_major_ver = ""
if this_uname == "Darwin":
mem_mb = util.get_mem_mb()
system_memory_in_kbytes = mem_mb * 1024
system_memory_in_gb = mem_mb / 1024.0
system_cpu_cores = util.get_cpu_cores()
cpu_model=util.getoutput("/usr/sbin/sysctl -n machdep.cpu.brand_string")
prod_name = util.getoutput("sw_vers -productName")
prod_version = util.getoutput("sw_vers -productVersion")
this_os = prod_name + " " + prod_version
elif this_uname == "Linux":
mem_mb = util.get_mem_mb()
system_memory_in_kbytes = mem_mb * 1024
system_memory_in_gb = mem_mb / 1024.0
system_cpu_cores = util.get_cpu_cores()
cpu_model=util.getoutput("grep 'model name' /proc/cpuinfo | head -1 | cut -d':' -f2")
os_major_ver = util.getoutput("cat /etc/os-release | grep VERSION_ID | cut -d= -f2 | tr -d '\"'")
if cpu_model == "":
cpu_model="ARM"
if os.path.exists("/etc/redhat-release"):
this_os = util.getoutput("cat /etc/redhat-release")
elif os.path.exists("/etc/system-release"):
this_os = util.getoutput("cat /etc/system-release")
elif os.path.exists("/etc/lsb-release"):
this_os = util.getoutput("cat /etc/lsb-release | grep DISTRIB_DESCRIPTION | cut -d= -f2 | tr -d '\"'")
elif os.path.exists("/etc/os-release"):
this_os = util.getoutput("cat /etc/os-release | grep PRETTY_NAME | cut -d= -f2 | tr -d '\"'")
elif this_uname == "Windows":
caption = check_output_wmic([wmic_path, "os", "get", "caption"])
svcpack = check_output_wmic([wmic_path, "os", "get", "servicepackmajorversion"])
if svcpack == "0":
this_os = caption
else:
this_os = caption + ", SP " + svcpack
round_mem = util.pretty_rounder(float(system_memory_in_gb), 1)
mem = str(round_mem) + " GB"
cores = str(system_cpu_cores)
cpu = cpu_model.strip()
cpu = cpu.replace("(R)", "")
cpu = cpu.replace("(TM)", "")
cpu = cpu.replace(" CPU ", " ")
os = this_os.replace(" release ", " ")
os = os.replace(" (Final)", "")
arch = os_arch.replace("x86_64", "AMD")
arch = arch.replace("AMD64", "AMD")
ver = util.get_version()
[last_update_utc, last_update_local, unique_id] = util.read_hosts('localhost')
if last_update_local:
last_upd_dt = datetime.strptime(last_update_local, "%Y-%m-%d %H:%M:%S")
time_diff = int(util.timedelta_total_seconds(datetime.now() - last_upd_dt))
last_update_readable = util.get_readable_time_diff(str(time_diff), precision=2)
versions_sql = util.get_versions_sql()
perl_ver = util.get_perl_ver()
[java_major_ver, java_ver] = util.get_java_ver()
os_pkg_mgr = util.get_pkg_mgr()
jvm_location = util.get_jvm_location()
if p_json:
infoJsonArray = []
infoJson = {}
infoJson['version'] = ver
infoJson['home'] = p_home
infoJson['user'] = p_user
infoJson['host'] = host_display
infoJson['host_short'] = util.get_host_short()
infoJson['host_long'] = util.get_host()
infoJson['host_ip'] = util.get_host_ip()
infoJson['os'] = unicode(str(os),sys.getdefaultencoding(),errors='ignore').strip()
infoJson['os_pkg_mgr'] = os_pkg_mgr
infoJson['os_major_ver'] = os_major_ver
infoJson['platform'] = unicode(str(plat),sys.getdefaultencoding(),errors='ignore').strip()
infoJson['arch'] = arch
infoJson['mem'] = round_mem
infoJson['cores'] = system_cpu_cores
infoJson['cpu'] = cpu
infoJson['last_update_utc'] = last_update_utc
if last_update_local:
infoJson['last_update_readable'] = last_update_readable
infoJson['unique_id'] = unique_id
infoJson['repo'] = p_repo
infoJson['versions_sql'] = versions_sql
infoJson['system_memory_in_kb'] = system_memory_in_kbytes
infoJson['python_ver'] = python_ver
infoJson['python_exe'] = python_exe
if pip_ver != 'None':
infoJson['pip_ver'] = pip_ver
infoJson['perl_ver'] = perl_ver
infoJson['java_ver'] = java_ver
infoJson['java_major_ver'] = java_major_ver
infoJson['jvm_location'] = jvm_location
infoJsonArray.append(infoJson)
if print_flag:
print(json.dumps(infoJson))
return
else:
return infoJson
if p_is_admin:
admin_display = " (Admin)"
else:
admin_display = ""
langs = "Python v" + python_ver
if perl_ver > "":
langs = langs + " | Perl v" + perl_ver
if java_ver > "":
langs = langs + " | Java v" + java_ver
util.validate_distutils_click(False)
print(style_start + ("#" * 70) + style_end)
print(style_start + "# IO: " + style_end + "v" + ver + " " + p_home)
print(style_start + "# User & Host: " + style_end + p_user + admin_display + " " + host_display)
print(style_start + "# Operating System: " + style_end + os.rstrip() + " - " + str(plat))
print(style_start + "# Machine: " + style_end + mem + ", " + cores + " vCPU, " + cpu)
print(style_start + "# Programming Langs: " + style_end + langs)
default_repo = "https://openrds-download.s3.amazonaws.com/REPO"
if p_repo != default_repo:
print(style_start + "# Repo URL: " + style_end + p_repo)
if versions_sql == "versions.sql":
pass
else:
print(style_start + "# Versions SQL: " + style_end + versions_sql)
if not last_update_local:
last_update_local="None"
print(style_start + "# Last Update: " + style_end + str(last_update_local))
print(style_start + ("#" * 70) + style_end)
def _test_for_load_alerts(self, rows, entry, agent, body):
# pylint: disable=too-many-return-statements
# pylint: disable=too-many-branches
# Different event types are described in PD-5352.
# print "action = ", entry.action, "body = ", body
errorlevel = self.system[SystemKeys.HTTP_LOAD_ERROR]
warnlevel = self.system[SystemKeys.HTTP_LOAD_WARN]
if not errorlevel and not warnlevel:
# alerts for http-requests aren't enabled
return
# if entry.action == 'show' and not entry.vizql_session and \
# not entry.currentsheet:
# # Type 5
# self._eventgen(EventControl.HTTP_INITIAL_LOAD_FAILED,
# agent, entry, body=body)
# return
if entry.action in ('performPostLoadOperations', 'sessions',
'get_customized_views'):
# Type 2: Post-Initial View Generation, but Success Not Certain
seconds = int(
timedelta_total_seconds(entry.completed_at, entry.created_at))
body['post_initial_compute'] = seconds
body['duration'] = seconds # fixme: remove when event doesn't use
elif entry.action == 'bootstrapSession':
if entry.currentsheet and entry.vizql_session and \
entry.status == 200:
# Type 1: Initial Successful View Generation
# Remove information from the 'bootstrapSession' row
# as we will be using the better information from the
# 'show' row.
body = {}
seconds_compute = int(
timedelta_total_seconds(entry.completed_at,
entry.created_at))
body['view_compute_duration'] = seconds_compute
show_entry = self._find_vizql_entry(rows, entry, 'show')
if not show_entry:
logger.error(
"http load test Type 1: http_requests "
"For id %d, action %s, vizql_session %d, "
"did not find 'show'", entry.id, entry.action,
entry.vizql_session)
return
seconds_load = int(
timedelta_total_seconds(show_entry.completed_at,
show_entry.created_at))
body['view_load_duration'] = seconds_load
seconds = body['view_compute_duration'] + \
body['view_load_duration']
body['total_view_generation_time'] = seconds
body['duration'] = seconds # fixme: remove when event
# doesn't use
# For the event, use the action='show' row since it has
# more information than the action='bootstrapSession' row.
entry = show_entry
# Update the body repository_url, view, etc. with the
# 'show' row.
self._parseuri(entry.http_request_uri, body)
elif entry.vizql_session and entry.status == 500:
# Type 3: Failed Initial View Generation
show_entry = self._find_vizql_entry(rows, entry, 'show')
if not show_entry:
logger.error(
"http load test Type 3: "
"http_requests "
"For id %d, action %s, vizql_session %d, "
"did not find 'show'", entry.id, entry.action,
entry.vizql_session)
return
# Use the information from the 'show' row instead of
# the 'bootstrapSession' since the 'show' row has more info.
body = {}
entry = show_entry
self._parseuri(entry.http_request_uri, body)
self._eventgen(EventControl.HTTP_LOAD_TYPE_3,
agent,
entry,
body=body)
return
# elif not entry.currentsheet and not entry.vizql_session:
# # Type 4: currentsheet is blank or empty and vizql_session
# # is blank or empty
# self._eventgen(EventControl.HTTP_LOAD_TYPE_4,
# agent, entry, body=body)
# return
else:
logger.debug(
"http load test: "
"http_requests "
"For id %d, action %s, currently unhandled "
"bootstrapSession case", entry.id, entry.action)
return
# print 'found it id:', entry.id, 'action:', entry.action,
# print ', body:', body, ', seconds:', seconds
else:
# print 'ignoring id:', entry.id, 'action:', entry.action,
# print ', body:', body
return
# print 'id:', entry.id, 'action:', entry.action,
# print 'body:', body, 'seconds:', seconds
if errorlevel != 0 and seconds >= errorlevel:
self._eventgen(EventControl.HTTP_LOAD_ERROR,
agent,
entry,
body=body)
elif warnlevel != 0 and seconds >= warnlevel:
self._eventgen(EventControl.HTTP_LOAD_WARN,
agent,
entry,
body=body)
def info(p_json, p_home, p_repo, print_flag=True):
import os
p_user = util.get_user()
p_is_admin = util.is_admin()
this_os = ""
this_uname = str(platform.system())
host_ip = util.get_host_ip()
wmic_path = os.getenv(
"SYSTEMROOT",
"") + os.sep + "System32" + os.sep + "wbem" + os.sep + "wmic"
if this_uname == "Windows":
import psutil
host_display = os.getenv('LOGONSERVER',
'') + '\\' + os.getenv('COMPUTERNAME')
system_cpu_cores = os.getenv('NUMBER_OF_PROCESSORS', '1')
os_arch = os.getenv('PROCESSOR_ARCHITECTURE', '')
cpu_model = subprocess.check_output([wmic_path, "cpu", "get",
"name"]).strip().split("\n")[1]
## system_memory_in_gb ######################################
m = psutil.virtual_memory().total
mem_bytes = int(m)
system_memory_in_kbytes = mem_bytes / 1024.0
system_memory_in_gb = mem_bytes / (1024.0**3)
else:
os_arch = util.getoutput("uname -m")
HOST = util.get_host_short()
host_display = "{0} {1}".format(HOST, host_ip)
## Check the OS & Resources ########################################
plat = util.get_os()
if this_uname == "Darwin":
system_memory_in_bytes = int(
util.getoutput("/usr/sbin/sysctl hw.memsize | awk '{print $2}'"))
system_memory_in_kbytes = system_memory_in_bytes / 1024
system_memory_in_gb = system_memory_in_bytes / 1024 / 1024 / 1024
system_cpu_cores = int(
util.getoutput(
"/usr/sbin/sysctl hw.physicalcpu | awk '{print $2}'"))
cpu_model = util.getoutput(
"/usr/sbin/sysctl -n machdep.cpu.brand_string")
prod_name = util.getoutput("sw_vers -productName")
prod_version = util.getoutput("sw_vers -productVersion")
this_os = prod_name + " " + prod_version
elif this_uname == "Linux":
system_memory_in_kbytes = int(
util.getoutput("cat /proc/meminfo | awk '/MemTotal/ {print $2}'"))
system_memory_in_gb = system_memory_in_kbytes / 1024.0 / 1024.0
system_cpu_cores = int(
util.getoutput(
"egrep -c 'processor([[:space:]]+):.*' /proc/cpuinfo"))
cpu_model = util.getoutput(
"grep 'model name' /proc/cpuinfo | head -1 | cut -d':' -f2")
if os.path.exists("/etc/redhat-release"):
this_os = util.getoutput("cat /etc/redhat-release")
elif os.path.exists("/etc/system-release"):
this_os = util.getoutput("cat /etc/system-release")
elif os.path.exists("/etc/lsb-release"):
this_os = util.getoutput(
"cat /etc/lsb-release | grep DISTRIB_DESCRIPTION | cut -d= -f2 | tr -d '\"'"
)
elif this_uname == "Windows":
caption_result = subprocess.check_output(
[wmic_path, "os", "get", "caption"])
try:
caption = str(caption_result).strip().split("\n")[1]
except UnicodeDecodeError as e:
caption = unicode(caption_result,
sys.getdefaultencoding(),
errors='ignore').strip().split("\n")[1]
svcpack_result = subprocess.check_output(
[wmic_path, "os", "get", "servicepackmajorversion"])
try:
svcpack = str(svcpack_result).strip().split("\n")[1]
except UnicodeDecodeError as e:
svcpack = unicode(svcpack_result,
sys.getdefaultencoding(),
errors='ignore').strip().split("\n")[1]
if svcpack == "0":
this_os = caption
else:
this_os = caption + ", SP " + svcpack
round_mem = util.pretty_rounder(system_memory_in_gb, 1)
mem = str(round_mem) + " GB"
cores = str(system_cpu_cores) + " x"
cpu = cpu_model.strip()
cpu = cpu.replace("(R)", "")
cpu = cpu.replace("(TM)", "")
cpu = cpu.replace(" CPU ", " ")
os = this_os.replace(" release ", " ")
os = os.replace(" (Final)", "")
arch = os_arch.replace("x86_64", "x64")
arch = arch.replace("AMD64", "x64")
ver = util.get_pgc_version()
[
interval, last_update_utc, next_update_utc, last_update_local,
next_update_local, unique_id
] = util.read_hosts('localhost')
if last_update_local:
last_upd_dt = datetime.strptime(last_update_local, "%Y-%m-%d %H:%M:%S")
time_diff = int(
util.timedelta_total_seconds(datetime.now() - last_upd_dt))
last_update_readable = util.get_readable_time_diff(str(time_diff),
precision=2)
versions_sql = util.get_versions_sql()
if p_json:
infoJsonArray = []
infoJson = {}
infoJson['version'] = ver
infoJson['home'] = p_home
infoJson['user'] = p_user
infoJson['host'] = host_display
infoJson['host_short'] = util.get_host_short()
infoJson['host_long'] = util.get_host()
infoJson['host_ip'] = util.get_host_ip()
infoJson['os'] = unicode(str(os),
sys.getdefaultencoding(),
errors='ignore').strip()
infoJson['platform'] = unicode(str(plat),
sys.getdefaultencoding(),
errors='ignore').strip()
infoJson['arch'] = arch
infoJson['mem'] = round_mem
infoJson['cores'] = system_cpu_cores
infoJson['cpu'] = cpu
infoJson['interval'] = interval
infoJson['last_update_utc'] = last_update_utc
if last_update_local:
infoJson['last_update_readable'] = last_update_readable
infoJson['next_update_utc'] = next_update_utc
infoJson['unique_id'] = unique_id
infoJson['repo'] = p_repo
infoJson['versions_sql'] = versions_sql
infoJson['system_memory_in_kb'] = system_memory_in_kbytes
infoJson['python_ver'] = python_ver
infoJson['python_exe'] = python_exe
if pip_ver != 'None':
infoJson['pip_ver'] = pip_ver
infoJsonArray.append(infoJson)
if print_flag:
print(json.dumps(infoJsonArray, sort_keys=True, indent=2))
return
else:
return infoJson
if p_is_admin:
admin_display = " (Admin)"
else:
admin_display = ""
print(style_start + ("#" * 70) + style_end)
print(style_start + "# PGC: " + style_end + "v" + ver + " " +
p_home)
print(style_start + "# User & Host: " + style_end + p_user +
admin_display + " " + host_display)
print(style_start + "# Platform: " + style_end + plat + " | " +
os.rstrip())
if pip_ver != "None":
print(style_start + "# Python: " + style_end + "pip v" + pip_ver + \
" | v" + python_ver + " | " + python_exe)
else:
print(style_start + "# Python: " + style_end + "v" +
python_ver + " | " + python_exe)
print(style_start + "# Hardware: " + style_end + mem + ", " +
cores + " " + cpu)
default_repo = "https://s3.amazonaws.com/pgcentral"
if p_repo != default_repo:
print(style_start + "# Repo URL: " + style_end + p_repo)
if versions_sql == "versions.sql":
pass
else:
print(style_start + "# Versions SQL: " + style_end + versions_sql)
if not last_update_local:
last_update_local = "None"
print(style_start + "# Last Update: " + style_end +
str(last_update_local))
if interval:
print(style_start + "# Update Interval: " + style_end + str(interval))
print(style_start + "# Next Update : " + style_end +
str(next_update_local))
print(style_start + ("#" * 70) + style_end)