def jvm_usage(self, sessionID:str, manager_node_id: str, from_date: datetime, to_date: datetime, verify_ssl:bool = False) -> Dict[str, str]:
params = {'sID': sessionID }
if len(manager_node_id) > 0: params['managerNodeID'] = manager_node_id
if from_date is not None: params['from']= from_date.ctime()
if to_date is not None: params['to'] = to_date.ctime()
url = self.api_url + "/jvm"
print(params)
response = requests.get(url, verify=verify_ssl, headers=self.headers, params=params)
return response.json()
async def set_rtc(self, val: datetime):
logger.debug('Set RTC: %s', val.ctime())
data = pack('BBB', (val.weekday() + 2) % 7, val.hour, val.minute)
cmd = CACommandPair(CACommand(self._loop, 0x35, data),
CACommand(self._loop, 0x3c))
await self._transaction(cmd)
def _reformat_to_readable_date(self, d: datetime.datetime) -> str:
"""Convert datetime object into readable date label.
Args:
d (datetime.datetime): the datetime object.
Returns:
str. formatted datetime (e.g. 'Sun Oct 4').
"""
return re.fullmatch("(.*) \d+:.*", d.ctime()).group(1)
def _build_filename(self, timestamp:datetime, custom_templ:str=None,
str_param:dict=None):
"""
This function uses _search_files to find the correct
filename and checks if the search was unambiguous.
Parameters
----------
timestamp: datetime
datetime for given filename
custom_tmpl : string, optional
If given the fname_templ is not used but the custom_templ. This
is convenient for some datasets where not all file names follow
the same convention and where the read_image function can choose
between templates based on some condition.
str_param : dict, optional
If given then this dict will be applied to the fname_templ using
the fname_templ.format(**str_param) notation before the resulting
string is put into datetime.strftime.
"""
filename = self._search_files(timestamp, custom_templ=custom_templ,
str_param=str_param)
if len(filename) == 0:
raise IOError("No file found for {:}".format(timestamp.ctime()))
if len(filename) > 1:
filename = sorted(filename)
if self.solve_ambiguity == 'sort_last':
warnings.warn(f'Ambiguous file for {str(timestamp)} found.'
f' Sort and use last: {filename[-1]}, skipped {filename[:-1]}')
filename = [filename[-1]]
elif self.solve_ambiguity == 'sort_first':
warnings.warn(f'Ambiguous file for {str(timestamp)} found.'
f' Sort and use first: {filename[0]}')
filename = [filename[0]]
else:
raise IOError(
"File search is ambiguous {:}".format(filename))
return filename[0]
def dt_to_str(dt: datetime) -> str:
return dt.ctime()
def render_report(output_path: str, now: datetime) -> str:
"""Write out the HTML index and improve the SVGs."""
for svg_path in [
os.path.join(output_path, "peak-memory.svg"),
os.path.join(output_path, "peak-memory-reversed.svg"),
]:
update_svg(svg_path)
index_path = os.path.join(output_path, "index.html")
with open(index_path, "w") as index:
index.write("""
<html>
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Fil Memory Profile ({now})</title>
<style type="text/css">
body {{
font-family: -apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,Oxygen-Sans,Ubuntu,Cantarell,"Helvetica Neue",sans-serif;
line-height: 1.2;
max-width: 40rem;
margin: 4rem auto;
font-size: 18px;
}}
div {{
text-align: center;
}}
</style>
<script>
function compatRequestFullscreen(elem) {{
if (elem.requestFullscreen) {{
return elem.requestFullscreen();
}} else if (elem.webkitRequestFullscreen) {{
return elem.webkitRequestFullscreen();
}} else if (elem.mozRequestFullScreen) {{
return elem.mozRequestFullScreen();
}} else if (elem.msRequestFullscreen) {{
return elem.msRequestFullscreen();
}}
}}
function fullScreen(id) {{
var elem = document.querySelector(id);
var currentHeight = elem.style.height;
elem.style.height = "100%";
compatRequestFullscreen(elem).finally(
(info) => {{elem.style.height = currentHeight;}}
);
}}
</script>
</head>
<body>
<h1>Fil Memory Profile</h1>
<h2>{now}</h2>
<h2>Command</h2>
<p><code>{argv}</code><p>
<h2>Profiling result</h2>
<div><iframe id="peak" src="peak-memory.svg" width="100%" height="200" scrolling="auto" frameborder="0"></iframe><br>
<p><input type="button" onclick="fullScreen('#peak');" value="Full screen"></p></div>
<br>
<div><iframe id="peak-reversed" src="peak-memory-reversed.svg" width="100%" height="200" scrolling="auto" frameborder="0"></iframe><br>
<p><input type="button" onclick="fullScreen('#peak-reversed');" value="Full screen"></p></div>
<h2>Need help, or does something look wrong? <a href="https://github.com/pythonspeed/filprofiler/issues/new?body={bugreport}">Please file an issue</a> and I'll try to help</h2>
<h2>Understanding the graphs</h2>
<p>The flame graphs shows the callstacks responsible for allocations at peak.</p>
<p>The wider (and the redder) the bar, the more memory was allocated by that function or its callers.
If the bar is 100% of width, that's all the allocated memory.</p>
<p>The first graph shows the normal callgraph: if <tt>main()</tt> calls <tt>g()</tt> calls <tt>f()</tt>, let's say, then <tt>main()</tt> will be at the top.
The second graph shows the reverse callgraph, from <tt>f()</tt> upwards.</p>
<p>Why is the second graph useful? If <tt>f()</tt> is called from multiple places, in the first graph it will show up multiple times, at the bottom.
In the second reversed graph all calls to <tt>f()</tt> will be merged together.</p>
<p>Need help reducing your data processing application's memory use? Check out tips and tricks <a href="https://pythonspeed.com/datascience/">here</a>.</p>
</body>
</html>
""".format(
now=now.ctime(),
argv=" ".join(map(shlex.quote, sys.argv)),
bugreport=DEBUGGING_INFO,
))
return index_path
def check_availability(meetings: List[Meeting], tstamp: datetime):
for meeting in meetings:
if tstamp.ctime() >= meeting.start_time.ctime() and tstamp.ctime() <= meeting.end_time.ctime() :
return False
return True
def render_report(output_path: str, now: datetime) -> str:
"""Write out the HTML index and improve the SVGs."""
index_path = os.path.join(output_path, "index.html")
with open(index_path, "w") as index:
index.write("""
<html>
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Fil Memory Profile ({now})</title>
<style type="text/css">
body {{
font-family: -apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,Oxygen-Sans,Ubuntu,Cantarell,"Helvetica Neue",sans-serif;
line-height: 1.2;
max-width: 40rem;
margin: 4rem auto;
font-size: 18px;
}}
blockquote {{ border-width: 1px; border-color: black; border-style: solid; padding: 1em; }}
div {{
text-align: center;
}}
</style>
<script>
function compatRequestFullscreen(elem) {{
if (elem.requestFullscreen) {{
return elem.requestFullscreen();
}} else if (elem.webkitRequestFullscreen) {{
return elem.webkitRequestFullscreen();
}} else if (elem.mozRequestFullScreen) {{
return elem.mozRequestFullScreen();
}} else if (elem.msRequestFullscreen) {{
return elem.msRequestFullscreen();
}}
}}
function fullScreen(id) {{
var elem = document.querySelector(id);
var currentHeight = elem.style.height;
elem.style.height = "100%";
compatRequestFullscreen(elem).finally(
(info) => {{elem.style.height = currentHeight;}}
);
}}
</script>
</head>
<body>
<h1>Fil Memory Profile</h1>
<h2>{now}</h2>
<h2>Command</h2>
<p><code>{argv}</code><p>
<h2>Profiling result</h2>
<div><p><input type="button" onclick="fullScreen('#peak');" value="Full screen"> · <a href="peak-memory.svg" target="_blank"><button>Open in new window</button></a></p>
<iframe id="peak" src="peak-memory.svg" width="100%" height="400" scrolling="auto" frameborder="0"></iframe><br>
</div>
<br>
<br>
<hr>
<br>
<div><p><input type="button" onclick="fullScreen('#peak-reversed');" value="Full screen"> · <a href="peak-memory-reversed.svg" target="_blank"><button>Open in new window</button></a></p>
<iframe id="peak-reversed" src="peak-memory-reversed.svg" width="100%" height="400" scrolling="auto" frameborder="0"></iframe><br>
</div>
<br>
<blockquote><strong>Need help, or does something look wrong?</strong>
<a href="https://pythonspeed.com/fil/docs/">Read the documentation</a>,
and if that doesn't help please
<a href="https://github.com/pythonspeed/filprofiler/issues/new?body={bugreport}">file an issue</a>
and I'll try to help.</blockquote>
<br>
<blockquote><strong>Want memory and performance profiling for your production batch jobs?</strong>
I've also created an
<a href="https://pythonspeed.com/sciagraph/"
>always-on profiler called Sciagraph</a> that is fast and robust enough to run in production.</blockquote>
<br>
<h2>Learn how to reduce memory usage</h2>
<p>Need help reducing your data processing application's memory use? Check out tips and tricks <a href="https://pythonspeed.com/memory/">here</a>.</p>
<h2>Understanding the graphs</h2>
<p>The flame graphs shows the callstacks responsible for allocations at peak.</p>
<p>The wider (and the redder) the bar, the more memory was allocated by that function or its callers.
If the bar is 100% of width, that's all the allocated memory.</p>
<p>The left-right axis has no meaning!
The order of frames is somewhat arbitrary, for example beause multiple calls to the same function may well have been merged into a single callstack.
So you can't tell from the graph which allocations happened first.
All you're getting is that at peak allocation these time, these stacktraces were responsible for these allocations.
</p>
<p>The first graph shows the normal callgraph: if <tt>main()</tt> calls <tt>g()</tt> calls <tt>f()</tt>, let's say, then <tt>main()</tt> will be at the top.
The second graph shows the reverse callgraph, from <tt>f()</tt> upwards.</p>
<p>Why is the second graph useful? If <tt>f()</tt> is called from multiple places, in the first graph it will show up multiple times, at the bottom.
In the second reversed graph all calls to <tt>f()</tt> will be merged together.</p>
<h2>Understanding what Fil tracks</h2>
<p>Fil measures how much memory has been allocated; this is not the same as how much memory the process is actively using, nor is it the same as memory resident in RAM.</p>
<ul>
<li>If the data gets dumped from RAM to swap, Fil still counts it but it's not counted as resident in RAM.</li>
<li>If the memory is a large chunk of all zeros, on Linux no RAM is used by OS until you actually modify that memory, but Fil will still count it.</li>
<li>If you have memory that only gets freed on garbage collection
(this will happen if you have circular references in your data structures),
memory can be freed at inconsistent times across different runs, especially
if you're using threads.</li>
</ul>
<p>See <a href="https://pythonspeed.com/articles/measuring-memory-python/">this article</a> for more details.</p>
</body>
</html>
""".format(
now=now.ctime(),
argv=" ".join(map(shlex.quote, sys.argv)),
bugreport=DEBUGGING_INFO,
))
return index_path
