def __init__(self, filename: str, debug=False):
self.__html_style = """"""
# Open head and body streams (for reading and writing)
self.__html_head = Buffer()
self.__html_body = Buffer()
self.__debug = debug
# Name of the output file
self.__webpage_filename = filename
def __init__(self, tabs=None, plot_webpage_name: str = "index.html"):
if tabs is None:
tabs = {}
self.tabs: Dict[str, Dict[str, Union[str, List]]] = {
key: {
"data": [],
"directory": value
}
for (key, value) in tabs.items()
}
self.data = Buffer()
self.name: str = plot_webpage_name
self.uploads_days_ago: List[int] = []
self.uploads_z: List[float] = []
self.run_links: List[str] = []
self.N_individual_runs: int = 0
self.N_comparisons: int = 0
self.N_ongoing_runs: int = 0
class Tree:
# To find out how long ago a page was created
current_time = datetime.now()
# Html is wield
white_space: str = "  "
# Display used parameters in html format
run_param_page_icon = f"""<i style="float:right; color:#A9A9A9; font-size:13px" class="fa">  {white_space}</i>"""
used_param_file_name = "used_parameters.html"
# Link and icon for morphology page
morphology_page_icon = f"""<i style="float:right; color:#FF8C00; font-size:18px" class="fa"> {white_space}</i>"""
morphology_file_name = "morphology.html"
# Categories to look for
individual = "individual_runs"
comparison = "comparisons"
def __init__(self, tabs=None, plot_webpage_name: str = "index.html"):
if tabs is None:
tabs = {}
self.tabs: Dict[str, Dict[str, Union[str, List]]] = {
key: {
"data": [],
"directory": value
}
for (key, value) in tabs.items()
}
self.data = Buffer()
self.name: str = plot_webpage_name
self.uploads_days_ago: List[int] = []
self.uploads_z: List[float] = []
self.run_links: List[str] = []
self.N_individual_runs: int = 0
self.N_comparisons: int = 0
self.N_ongoing_runs: int = 0
def find_creating_time(self, path_to_dir: str) -> int:
"""
Compute the time difference between one and when the __plot_webpage_name
file was created
Parameters
-------
path_to_dir: str
Returns
-------
output: int
Number of days ago
"""
time = os.stat(f"{path_to_dir}/{self.name}").st_mtime
dt = datetime.fromtimestamp(time)
days_ago = (self.current_time - dt).days
return days_ago
def days_ago_to_string(self, days_ago: int) -> str:
"""
Formats the number of days ago
Parameters
-------
days_ago: int
Returns
-------
output: str
Number of days ago formatted
"""
if days_ago > 1:
string = Content(f"{days_ago:d} days ago")
elif days_ago == 1:
string = Content(f"{days_ago:d} day ago")
else:
string = Content(f"{days_ago:d} New")
string.wrap_text(block="i")
string.wrap_text(block="span", span_style="float:right; color:#0000FF")
return string.getter()
def walk(
self,
abs_path: str,
depth: int = 0,
visual_line: str = "",
prefix: str = "/cosma/home/www/swift.dur.ac.uk/public_html",
):
"""
This function traverses the directory tree collecting necessary information in
order to create a general html page with the runs' info and statistics
"""
relative_path = abs_path.replace(prefix, "")
# Number of sub-directions in the current directory
list_dir = sorted(os.listdir(abs_path))
N_of_dir = sum(
[1 for d in list_dir if os.path.isdir(os.path.join(abs_path, d))])
# Loop over all sub-directories and files in the current directory
for dir_counter, dir_name in enumerate(list_dir):
# Absolute path to the file/sub-directory
absolute_dir_path = os.path.join(abs_path, dir_name)
# Do not do anything if it is a file
if os.path.isdir(absolute_dir_path):
if absolute_dir_path.find(self.individual) > -1:
self.N_individual_runs += 1
elif absolute_dir_path.find(self.comparison) > -1:
self.N_comparisons += 1
else:
continue
# Number of sub-directories in a given sub-directory of this directory.
# Zero means leaf node
N_of_subdir = sum([
1 for sub_dir in os.scandir(absolute_dir_path)
if os.path.isdir(os.path.join(abs_path, sub_dir))
])
is_leaf = N_of_subdir == 0
# Check if the directory is a leaf node
if is_leaf:
# Time data
days_ago = self.find_creating_time(absolute_dir_path)
modified = self.days_ago_to_string(days_ago=days_ago)
self.uploads_days_ago.append(days_ago)
link_to_plots = f"{relative_path}/{dir_name}/{self.name}"
morphology = ""
settings = ""
if os.path.isfile(
f"{absolute_dir_path:}/{self.used_param_file_name}"
):
settings = f"""<a href="{relative_path}/{dir_name}/{self.used_param_file_name}" target="_blank">
{self.run_param_page_icon}</a>"""
if os.path.isfile(
f"{absolute_dir_path:}/{self.morphology_file_name}"
):
settings = f"""<a href="{relative_path}/{dir_name}/{self.morphology_file_name}" target="_blank">
{self.morphology_page_icon}</a>"""
text = f"{dir_name}{modified}"
link_list = f"<li><a href={link_to_plots}>{text}</a>{settings}{morphology}</li>"
self.run_links.append(link_list)
text = f"{visual_line} |_{dir_name}{modified}"
link_tree = f"<li><a href={link_to_plots}>{text}</a>{settings}{morphology}</li>"
self.data.write_to_buffer(link_tree)
try:
redshift = float(dir_name[1:4])
self.uploads_z.append(redshift)
except ValueError:
pass
# Extra tabs
for (tab_name, tab_content) in self.tabs.items():
if link_to_plots.find(tab_content["directory"]) > -1:
string_split = text.split()
if (string_split[:3] ==
f"{self.white_space} {self.white_space} |".
split()):
arg = text.find("|")
text = f" {self.white_space * 3}{text[arg + 2:]}"
elif string_split[:
3] == f"| {self.white_space} |".split(
):
indent = 8
arg = text[indent:].find("|")
text = f" | {self.white_space * 2}{text[indent + arg + 1:]}"
string = f"<li><a href={link_to_plots}>{text}</a>{settings:s}{morphology:s}</li>"
self.tabs[tab_name]["data"].append(string)
# Not a leaf
else:
text = f"{visual_line} |_<b> {dir_name} </b>"
string = f"<li>{text}</li>"
self.data.write_to_buffer(string)
rel_path_dir = f"{relative_path}/{dir_name}"
# Extra tabs
if dir_name == self.individual or dir_name == self.comparison:
for tab_name in self.tabs.keys():
self.tabs[tab_name]["data"].append(string)
for (tab_name, tab_content) in self.tabs.items():
if rel_path_dir.find(tab_content["directory"]) > -1:
string_split = text.split()
if (string_split[:3] ==
f"{self.white_space} {self.white_space} |".
split()):
arg = text.find("|")
text = f" {self.white_space * 3}{text[arg + 2:]}"
elif string_split[:
3] == f"| {self.white_space} |".split(
):
indent = 8
arg = text[indent:].find("|")
text = f" | {self.white_space * 2}{text[indent + arg + 1:]}"
self.tabs[tab_name]["data"].append(
f"<li>{text}</li>")
# Evolve tree representation
if dir_counter < N_of_dir - 1:
new_visual_line = f"{visual_line} | {self.white_space} "
else:
new_visual_line = f"{visual_line} {self.white_space * 2}"
# Recurse
self.walk(
absolute_dir_path,
depth + 1,
new_visual_line,
)
class HtmlPage:
white_space: str = "  "
def __init__(self, filename: str, debug=False):
self.__html_style = """"""
# Open head and body streams (for reading and writing)
self.__html_head = Buffer()
self.__html_body = Buffer()
self.__debug = debug
# Name of the output file
self.__webpage_filename = filename
def __del__(self):
if self.__debug:
print("Destruction: Closing head and body streams")
# Close the streams
del self.__html_head, self.__html_body
return
def write_head(self, text: str):
"""
Writes to head of webpage
Parameters
-------
text: str
text to write to the head
"""
self.__html_head.write_to_buffer(text=text)
return
def write_body(self, text: str):
"""
Writes to the body of webpage
Parameters
-------
text: str
text to write to the body
"""
self.__html_body.write_to_buffer(text=text)
return
def render_webpage(self) -> str:
"""
Renders and returns the html page
Returns
-------
output: str
"""
webpage = f"""
<!DOCTYPE html>
<html>
<head>
{self.__html_style}
{self.__html_head.get_content()}
</head>
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.7.0/css/font-awesome.min.css">
<body>
{self.__html_body.get_content()}
</body>
</html>
"""
return webpage
def set_style(self, style: str):
"""
Sets up webpage style
Parameters
-------
style: str
style in the html format
"""
self.__html_style = style
return
def load_style(self, path_to_file: str):
"""
Loads page style from the provided file and applies it to the webpage
Parameters
-------
path_to_file: str
path to the file with the style for webpage
"""
if path.exists(path_to_file):
with open(path_to_file) as f:
# Read style
loaded_style = f.read()
# Apply style
self.set_style(style=loaded_style)
else:
raise IOError(f"File '{path_to_file}' does not exist!")
return
def save_webpage(self):
"""
Builds and saves the webpage into a file. If the file
already exists, overwrites it.
"""
with open(self.__webpage_filename, "w+") as out:
if self.__debug:
print(f"Saving webpage into {self.__webpage_filename}...")
webpage = self.render_webpage()
out.write(webpage)
return
def visualise_webpage(self):
"""
Opens the webpage using the default browser
"""
# Create and save webpage if the file does not exist yet
self.save_webpage()
webbrowser.open_new_tab(self.__webpage_filename)
return
def create_tab_panel_open(self, tabs: Optional[Dict[str, str]] = None):
"""
Creates tabs with different content. Start the tab panel block
Parameters
-------
tabs: Optional[Dict[str, str]]
Dictionary containing the tabs
"""
if tabs is None:
tabs = {}
self.__html_body.write_to_buffer("""<div class="tab">""")
for (name, display_name) in tabs.items():
button_attrs = f"""tablinks" onclick="openTab(event, '{name}')"""
button = Content(display_name)
button.wrap_text(block="button", button_class=button_attrs)
self.__html_body.write_to_buffer(button.getter())
self.__html_body.write_to_buffer("""</div>""")
return
def create_tab_panel_close(self, tab_content: str = "tabcontent"):
"""
Finish the tab panel block
Parameters
-------
tab_content: str
Tab content object
"""
self.__html_body.write_to_buffer(f"""<script>
function openTab(evt, TabName)
{{
var i, {tab_content}, tablinks;
tabcontent = document.getElementsByClassName("{tab_content}");
for (i = 0; i < {tab_content}.length; i++) {{
{tab_content}[i].style.display = "none";
}}
tablinks = document.getElementsByClassName("tablinks");
for (i = 0; i < tablinks.length; i++) {{
tablinks[i].className = tablinks[i].className.replace(" active", "");
}}
document.getElementById(TabName).style.display = "block";
evt.currentTarget.className += " active";
}}
</script>
""")
return
def create_table_with_links_to_reports(file_name: str = "reports.json") -> str:
"""
Reads off reports' titles and the links to the reports.
Creates a table with this info on the main html webpage.
Parameters
-------
file_name: str
File with the report names and links
Returns
-------
output: str
"""
try:
with open(file_name, "r") as f:
data = json.load(f)
except FileNotFoundError as err:
error_message = (
f"The file with the reports' links '{file_name:s}' is not found.")
raise RuntimeError(error_message) from err
buffer = Buffer()
buffer.write_to_buffer("<h2>Reports</h2>")
for author, reports in data.items():
if reports:
buffer.write_to_buffer(f"<i>Author: {author:s}</i><ol>")
for name, link in reports.items():
buffer.write_to_buffer(
f"<li><a href={link:s}>{name:s}</a></li>")
buffer.write_to_buffer("</ol>")
return buffer.get_content()
def create_webpage(
filename: str = "index.html",
path_to_style: str = "./html/style.html",
debug: bool = False,
):
"""
Create the webpage step by step
Parameters
-------
filename: str
filename for the webpage
path_to_style: str
Path to file with a style for webpage
debug: bool
Extra output and checks for debugging?
"""
today_pretty_format = datetime.now().strftime("%d/%m/%Y at %H:%M:%S")
obj = HtmlPage(filename=filename, debug=debug)
obj.load_style(path_to_file=path_to_style)
tabs = {
"full_tree": "Full tree",
"date": "Sort by date",
"redshift": "Sort by redshift",
}
extra_tabs = {"xmas2020": "NEWEST", "hawk": "HAWK", "zooms": "ZOOMS"}
tabs.update(extra_tabs)
# Traverse tree
PATH = "/home/evgenii/Desktop/PhD/shell/webpage/explore"
tree = traverse_tree(initial_path=PATH, tabs=tabs)
tree_content = Content(tree.data.get_content())
# Compute additional properties
reports = create_table_with_links_to_reports()
current_runs, redshifts_ongoing = fetch_current_runs_names()
plots = create_plots(tree, ongoing_runs=redshifts_ongoing)
# Flex container
obj.write_body(
"""<div class="container" style="display: flex; max-width: 2500px;">"""
)
obj.write_body("""<div class="content">""")
# Open tabs
obj.create_tab_panel_open(tabs=tabs)
tab_tree_top = f"""<h1>Path to current directory: {PATH}
<span style="float:right; color:blue;"> Uploaded
</span>
</h1>"""
# Tab 1
tab1 = Content(tab_tree_top)
tree_content.wrap_text(block="ul")
tab1.append_text(tree_content.getter())
tab1.wrap_text(
block="div",
div_id="full_tree",
div_class="tabcontent",
div_style="display:none",
)
obj.write_body(tab1.getter())
# Tab 2
tab2 = Content(tab_tree_top)
buffer2 = Buffer()
for item_arg in np.argsort(tree.uploads_days_ago):
buffer2.write_to_buffer(tree.run_links[item_arg])
buffer2_out = Content(buffer2.get_content())
buffer2_out.wrap_text("ol")
tab2.append_text(buffer2_out.getter())
tab2.wrap_text(block="div",
div_id="date",
div_class="tabcontent",
div_style="display:none")
obj.write_body(tab2.getter())
# Tab 3
tab3 = Content(tab_tree_top)
buffer3 = Buffer()
for item_arg in np.argsort(tree.uploads_z):
buffer3.write_to_buffer(tree.run_links[item_arg])
buffer3_out = Content(buffer3.get_content())
buffer3_out.wrap_text("ol")
tab3.append_text(buffer3_out.getter())
tab3.wrap_text(block="div",
div_id="redshift",
div_class="tabcontent",
div_style="display:none")
obj.write_body(tab3.getter())
# Extra tabs
for tab_name in extra_tabs:
tabn = Content(tab_tree_top)
buffern = Buffer()
for string in tree.tabs[tab_name]["data"]:
buffern.write_to_buffer(string)
buffern_out = Content(buffern.get_content())
buffern_out.wrap_text("ul")
tabn.append_text(buffern_out.getter())
tabn.wrap_text(
block="div",
div_id=tab_name,
div_class="tabcontent",
div_style="display:none",
)
obj.write_body(tabn.getter())
# Close left column
obj.write_body("</div>")
# Add script to make tab panel interactive
obj.create_tab_panel_close(tab_content="tabcontent")
# Open middle column
obj.write_body(
"""<div class="content"; style="margin-left: 1em; max-width: 550px;">"""
)
# Block with general information
tab = Content(f"""
<h2>General information</h2><ul>
<li>The page was last updated on
<b>{today_pretty_format:s}</b></li>
<li>Number of ongoing runs: <b>{len(redshifts_ongoing):d}</b></li>
<li>Number of analysed runs [individual]:
<b>{tree.N_individual_runs:d}</b></li>
<li>Number of analysed runs [comparisons]:
<b>{tree.N_comparisons:d}</b></li><p>{tree.white_space}</p>
</ul>""")
tab.wrap_text(block="div", div_class="content1")
obj.write_body(tab.getter())
obj.write_body(reports)
obj.write_body(current_runs)
# Close middle column
obj.write_body("""</div>""")
# Plots to the third column
obj.write_body(
"""<div class="content"; style="margin-left: 1em; max-width: 330px">"""
)
obj.write_body(plots)
obj.write_body("</div>")
obj.visualise_webpage()
def fetch_current_runs_names(
file_with_redshifts: str = "./output_list.txt",
file_pattern: str = "colibre*") -> Tuple[str, List[str]]:
"""
Find currently ongoing runs
Parameters
-------
file_with_redshifts: str
File containing the redshifts
file_pattern: str
File patter to use when looking for matches
Returns
-------
output: str
"""
run_names = []
run_redshifts = []
try:
output_z = open(file_with_redshifts, "r")
run_z_list = []
for line in output_z.readlines():
try:
redshift = "{:.2f}".format(float(line.split(", ")[0]))
run_z_list.append(redshift)
# Skip header if there is one
except ValueError:
pass
# See the ongoing runs
process = subprocess.run(["squeue", "-u", "dc-chai1", "-o", "%Z"],
stdout=subprocess.PIPE)
paths = process.stdout.decode("utf-8").split("\n")
# Loop over paths to ongoing runs
for path in paths:
# Looking for names
file_paths = glob.glob(f"{path}/{file_pattern}.yml")
if file_paths:
with open(file_paths[0], "r") as f:
file_lines = f.readlines()
for l in file_lines:
if l.__contains__("run_name:"):
run_names.append(l.split()[-1])
else:
run_names.append("None")
# Looking for redshifts
file_paths = sorted(glob.glob(f"{path}/{file_pattern}.hdf5"))
if file_paths:
# The path must be of the form "*/colibre_????.hdf5"
snp_nums = [
int(path.split("/")[-1][8:-5]) for path in file_paths
]
# Snapshot with the highest number corresponds to the lowest redshift
N_latest_snp = sorted(snp_nums)[-1]
try:
z = run_z_list[N_latest_snp]
except IndexError:
z = "0.5"
run_redshifts.append(z)
else:
run_redshifts.append("Enqueued")
# return run_names, run_redshifts
run_names = run_names[1:-1]
run_redshifts = run_redshifts[1:-1]
except (IOError, ValueError):
print(f"File {file_with_redshifts:s} cannot be open")
# Block with the info on ongoing runs
buffer = Buffer()
buffer.write_to_buffer("""
<h2>Names of ongoing runs<span style="float:right; color:black">Current z</span></h2>
<ol>""")
for idx in np.argsort(run_redshifts):
if run_names[idx] != "None":
buffer.write_to_buffer(f"""<li>{run_names[idx]:s}
<span style="float:right;">{ run_redshifts[idx]:s}</span></li>""")
buffer.write_to_buffer("</ol>")
return buffer.get_content(), run_redshifts