def add_job(self, job):
"""
Add a job to self.jobs
"""
if not self.running:
logger.error('Scheduler not running. Cannot add job.')
return False
else:
self.jobs[job.id()] = job
# Need to fix this properly
# if True or flask._app_ctx_stack.top is not None:
from digits.webapp import app, socketio
with app.app_context():
# send message to job_management room that the job is added
socketio.emit('job update',
{
'update': 'added',
'job_id': job.id(),
},
namespace='/jobs',
room='job_management',
)
if 'DIGITS_MODE_TEST' not in os.environ:
# Let the scheduler do a little work before returning
time.sleep(utils.wait_time())
return True
def on_status_update(self):
"""
Called when StatusCls.status.setter is used
"""
from digits.webapp import app, socketio
# Send socketio updates
message = {
'task': self.html_id(),
'update': 'status',
'status': self.status.name,
'css': self.status.css,
'show': (self.status in [Status.RUN, Status.ERROR]),
'running': self.status.is_running(),
}
with app.app_context():
message['html'] = flask.render_template('status_updates.html',
updates = self.status_history,
exception = self.exception,
traceback = self.traceback,
)
socketio.emit('task update',
message,
namespace='/jobs',
room=self.job_id,
)
from digits.webapp import scheduler
job = scheduler.get_job(self.job_id)
if job:
job.on_status_update()
def on_status_update(self):
"""
Called when StatusCls.status.setter is used
"""
from digits.webapp import app, socketio
# Send socketio updates
message = {
'task': self.html_id(),
'update': 'status',
'status': self.status.name,
'css': self.status.css,
'show': (self.status in [Status.RUN, Status.ERROR]),
'running': self.status.is_running(),
}
with app.app_context():
message['html'] = flask.render_template('status_updates.html',
updates=self.status_history,
exception=self.exception,
traceback=self.traceback,
)
socketio.emit('task update',
message,
namespace='/jobs',
room=self.job_id,
)
from digits.webapp import scheduler
job = scheduler.get_job(self.job_id)
if job:
job.on_status_update()
def __call__(self, file=None, **kwargs):
if "for_" in kwargs:
kwargs["for"] = kwargs.pop("for_")
else:
kwargs.setdefault("for", self.field_id)
import flask
from digits.webapp import app
html = ""
# get the text from the html file
with app.app_context():
html = flask.render_template(file if file else self.file)
if len(html) == 0:
return ""
return wtforms.widgets.HTMLString(
(
'<div id="%s_explanation" style="display:none;">\n'
"%s"
"</div>\n"
'<a href=# onClick="bootbox.alert($(\'#%s_explanation\').html()); return false;"><span class="glyphicon glyphicon-question-sign"></span></a>\n'
)
% (self.for_name, html, self.for_name)
)
def on_status_update(self):
"""
Called when StatusCls.status.setter is used
"""
from digits.webapp import app, socketio
message = {
'update': 'status',
'status': self.status_of_tasks().name,
'css': self.status_of_tasks().css,
'running': self.status.is_running(),
'job_id': self.id(),
}
with app.app_context():
message['html'] = flask.render_template('status_updates.html', updates=self.status_history)
socketio.emit('job update',
message,
namespace='/jobs',
room=self.id(),
)
# send message to job_management room as well
socketio.emit('job update',
message,
namespace='/jobs',
room='job_management',
)
if not self.status.is_running():
if hasattr(self, 'event'):
# release threads that are waiting for job to complete
self.event.set()
def add_job(self, job):
"""
Add a job to self.jobs
"""
if not self.running:
logger.error('Scheduler not running. Cannot add job.')
return False
else:
self.jobs[job.id()] = job
# Need to fix this properly
# if True or flask._app_ctx_stack.top is not None:
from digits.webapp import app, socketio
with app.app_context():
# send message to job_management room that the job is added
socketio.emit('job update',
{
'update': 'added',
'job_id': job.id(),
},
namespace='/jobs',
room='job_management',
)
if 'DIGITS_MODE_TEST' not in os.environ:
# Let the scheduler do a little work before returning
time.sleep(utils.wait_time())
return True
def on_status_update(self):
"""
Called when StatusCls.status.setter is used
"""
from digits.webapp import app, socketio
message = {
'update': 'status',
'status': self.status.name,
'css': self.status.css,
'running': self.status.is_running(),
'job_id': self.id(),
}
with app.app_context():
message['html'] = flask.render_template('status_updates.html', updates=self.status_history)
socketio.emit('job update',
message,
namespace='/jobs',
room=self.id(),
)
# send message to job_management room as well
socketio.emit('job update',
message,
namespace='/jobs',
room='job_management',
)
def on_status_update(self):
"""
Called when StatusCls.status.setter is used
"""
from digits.webapp import app, socketio
message = {
'update': 'status',
'status': self.status_of_tasks().name,
'css': self.status_of_tasks().css,
'running': self.status.is_running(),
'job_id': self.id(),
}
with app.app_context():
message['html'] = flask.render_template('status_updates.html', updates=self.status_history)
socketio.emit('job update',
message,
namespace='/jobs',
room=self.id(),
)
# send message to job_management room as well
socketio.emit('job update',
message,
namespace='/jobs',
room='job_management',
)
if not self.status.is_running():
if hasattr(self, 'event'):
# release threads that are waiting for job to complete
self.event.set()
def test_api_md():
with app.app_context():
generator = _.ApiDocGenerator(doc)
check_doc_file(
generator,
os.path.join(os.path.dirname(os.path.dirname(__file__)), 'docs',
'API.md'))
def test_flask_routes_md():
with app.app_context():
generator = _.FlaskRoutesDocGenerator(doc)
check_doc_file(
generator,
os.path.join(os.path.dirname(os.path.dirname(__file__)), 'docs',
'FlaskRoutes.md'))
def on_status_update(self):
"""
Called when StatusCls.status.setter is used
"""
from digits.webapp import app, socketio
message = {
'update': 'status',
'status': self.status.name,
'css': self.status.css,
'running': self.status.is_running(),
'job_id': self.id(),
}
with app.app_context():
message['html'] = flask.render_template('status_updates.html', updates=self.status_history)
socketio.emit('job update',
message,
namespace='/jobs',
room=self.id(),
)
# send message to job_management room as well
socketio.emit('job update',
message,
namespace='/jobs',
room='job_management',
)
def hw_socketio_updater(self, gpus):
"""
This thread sends SocketIO messages about hardware utilization
to connected clients
Arguments:
gpus -- a list of identifiers for the GPUs currently being used
"""
from digits.webapp import app, socketio
devices = []
if gpus is not None:
for index in gpus:
device = device_query.get_device(index)
if device:
devices.append((index, device))
else:
raise RuntimeError('Failed to load gpu information for GPU #"%s"' % index)
# this thread continues until killed in after_run()
while True:
# CPU (Non-GPU) Info
data_cpu = {}
if hasattr(self, "p") and self.p is not None:
data_cpu["pid"] = self.p.pid
try:
ps = psutil.Process(self.p.pid) # 'self.p' is the system call object
if ps.is_running():
if psutil.version_info[0] >= 2:
data_cpu["cpu_pct"] = ps.cpu_percent(interval=1)
data_cpu["mem_pct"] = ps.memory_percent()
data_cpu["mem_used"] = ps.memory_info().rss
else:
data_cpu["cpu_pct"] = ps.get_cpu_percent(interval=1)
data_cpu["mem_pct"] = ps.get_memory_percent()
data_cpu["mem_used"] = ps.get_memory_info().rss
except psutil.NoSuchProcess:
# In rare case of instant process crash or PID went zombie (report nothing)
pass
data_gpu = []
for index, device in devices:
update = {"name": device.name, "index": index}
nvml_info = device_query.get_nvml_info(index)
if nvml_info is not None:
update.update(nvml_info)
data_gpu.append(update)
with app.app_context():
html = flask.render_template("models/gpu_utilization.html", data_gpu=data_gpu, data_cpu=data_cpu)
socketio.emit(
"task update",
{"task": self.html_id(), "update": "gpu_utilization", "html": html},
namespace="/jobs",
room=self.job_id,
)
gevent.sleep(1)
def gpu_socketio_updater(self, gpus):
"""
This thread sends SocketIO messages about GPU utilization
to connected clients
Arguments:
gpus -- a list of identifiers for the GPUs currently being used
"""
from digits.webapp import app, socketio
devices = []
for index in gpus:
device = device_query.get_device(index)
if device:
devices.append((index, device))
if not devices:
raise RuntimeError('Failed to load gpu information for "%s"' %
gpus)
# this thread continues until killed in after_run()
while True:
data = []
for index, device in devices:
update = {'name': device.name, 'index': index}
nvml_info = device_query.get_nvml_info(index)
if nvml_info is not None:
update.update(nvml_info)
data.append(update)
with app.app_context():
html = flask.render_template('models/gpu_utilization.html',
data=data)
socketio.emit(
'task update',
{
'task': self.html_id(),
'update': 'gpu_utilization',
'html': html,
},
namespace='/jobs',
room=self.job_id,
)
gevent.sleep(1)
def gpu_socketio_updater(self, gpus):
"""
This thread sends SocketIO messages about GPU utilization
to connected clients
Arguments:
gpus -- a list of identifiers for the GPUs currently being used
"""
from digits.webapp import app, socketio
devices = []
for index in gpus:
device = device_query.get_device(index)
if device:
devices.append((index, device))
if not devices:
raise RuntimeError('Failed to load gpu information for "%s"' % gpus)
# this thread continues until killed in after_run()
while True:
data = []
for index, device in devices:
update = {'name': device.name, 'index': index}
nvml_info = device_query.get_nvml_info(index)
if nvml_info is not None:
update.update(nvml_info)
data.append(update)
with app.app_context():
html = flask.render_template('models/gpu_utilization.html',
data = data)
socketio.emit('task update',
{
'task': self.html_id(),
'update': 'gpu_utilization',
'html': html,
},
namespace='/jobs',
room=self.job_id,
)
gevent.sleep(1)
def add_job(self, job):
"""
Add a job to self.jobs
"""
if not self.running:
logger.error('Scheduler not running. Cannot add job.')
return False
else:
self.jobs.append(job)
# Need to fix this properly
# if True or flask._app_ctx_stack.top is not None:
from digits.webapp import app
with app.app_context():
# send message to job_management room that the job is added
import flask
html = flask.render_template('job_row.html', job=job)
# Convert the html into a list for the jQuery
# DataTable.row.add() method. This regex removes the <tr>
# and <td> tags, and splits the string into one element
# for each cell.
import re
html = re.sub('<tr[^<]*>[\s\n\r]*<td[^<]*>[\s\n\r]*', '', html)
html = re.sub('[\s\n\r]*</td>[\s\n\r]*</tr>', '', html)
html = re.split('</td>[\s\n\r]*<td[^<]*>', html)
from digits.webapp import socketio
socketio.emit(
'job update',
{
'update': 'added',
'job_id': job.id(),
'html': html
},
namespace='/jobs',
room='job_management',
)
if 'DIGITS_MODE_TEST' not in os.environ:
# Let the scheduler do a little work before returning
time.sleep(utils.wait_time())
return True
def add_job(self, job):
"""
Add a job to self.jobs
"""
if not self.running:
logger.error('Scheduler not running. Cannot add job.')
return False
else:
self.jobs[job.id()] = job
# Need to fix this properly
# if True or flask._app_ctx_stack.top is not None:
from digits.webapp import app
with app.app_context():
# send message to job_management room that the job is added
import flask
html = flask.render_template('job_row.html', job = job)
# Convert the html into a list for the jQuery
# DataTable.row.add() method. This regex removes the <tr>
# and <td> tags, and splits the string into one element
# for each cell.
import re
html = re.sub('<tr[^<]*>[\s\n\r]*<td[^<]*>[\s\n\r]*', '', html)
html = re.sub('[\s\n\r]*</td>[\s\n\r]*</tr>', '', html)
html = re.split('</td>[\s\n\r]*<td[^<]*>', html)
from digits.webapp import socketio
socketio.emit('job update',
{
'update': 'added',
'job_id': job.id(),
'html': html
},
namespace='/jobs',
room='job_management',
)
if 'DIGITS_MODE_TEST' not in os.environ:
# Let the scheduler do a little work before returning
time.sleep(utils.wait_time())
return True
def __call__(self, file=None, **kwargs):
if 'for_' in kwargs:
kwargs['for'] = kwargs.pop('for_')
else:
kwargs.setdefault('for', self.field_id)
import flask
from digits.webapp import app
html = ''
# get the text from the html file
with app.app_context():
html = flask.render_template(file if file else self.file)
if len(html) == 0: return ''
return wtforms.widgets.HTMLString((
'<div id="%s_explanation" style="display:none;">\n'
'%s'
'</div>\n'
'<a href=# onClick="bootbox.alert($(\'#%s_explanation\').html()); return false;"><span class="glyphicon glyphicon-question-sign"></span></a>\n'
) % (self.for_name, html, self.for_name))
def test_api_md():
"""API.md out-of-date"""
with app.app_context():
generator = _.ApiDocGenerator(doc)
check_doc_file(generator,
os.path.join(os.path.dirname(os.path.dirname(__file__)), 'docs', 'API.md'))
def test_flask_routes_md():
"""FlaskRoutes.md out-of-date"""
with app.app_context():
generator = _.FlaskRoutesDocGenerator(doc)
check_doc_file(generator, 'docs/FlaskRoutes.md')
def test_api_md():
"""API.md out-of-date"""
with app.app_context():
generator = _.ApiDocGenerator(doc)
check_doc_file(generator, 'docs/API.md')
def hw_socketio_updater(self, gpus):
"""
This thread sends SocketIO messages about hardware utilization
to connected clients
Arguments:
gpus -- a list of identifiers for the GPUs currently being used
"""
from digits.webapp import app, socketio
devices = []
if gpus is not None:
for index in gpus:
device = device_query.get_device(index)
if device:
devices.append((index, device))
else:
raise RuntimeError(
'Failed to load gpu information for GPU #"%s"' % index)
# this thread continues until killed in after_run()
while True:
# CPU (Non-GPU) Info
data_cpu = {}
if hasattr(self, 'p') and self.p is not None:
data_cpu['pid'] = self.p.pid
try:
ps = psutil.Process(
self.p.pid) # 'self.p' is the system call object
if ps.is_running():
if psutil.version_info[0] >= 2:
data_cpu['cpu_pct'] = ps.cpu_percent(interval=1)
data_cpu['mem_pct'] = ps.memory_percent()
data_cpu['mem_used'] = ps.memory_info().rss
else:
data_cpu['cpu_pct'] = ps.get_cpu_percent(
interval=1)
data_cpu['mem_pct'] = ps.get_memory_percent()
data_cpu['mem_used'] = ps.get_memory_info().rss
except psutil.NoSuchProcess:
# In rare case of instant process crash or PID went zombie (report nothing)
pass
data_gpu = []
for index, device in devices:
update = {'name': device.name, 'index': index}
nvml_info = device_query.get_nvml_info(index)
if nvml_info is not None:
update.update(nvml_info)
data_gpu.append(update)
with app.app_context():
html = flask.render_template('models/gpu_utilization.html',
data_gpu=data_gpu,
data_cpu=data_cpu)
socketio.emit(
'task update',
{
'task': self.html_id(),
'update': 'gpu_utilization',
'html': html,
},
namespace='/jobs',
room=self.job_id,
)
gevent.sleep(1)
Generates FlaskRoutes.md
"""
def __init__(self, *args, **kwargs):
super(FlaskRoutesDocGenerator, self).__init__(exclude_groups=['api'], *args, **kwargs)
def print_header(self):
text = """
# Flask Routes
%s
Documentation on the various routes used internally for the web application.
These are all technically RESTful, but they return HTML pages. To get JSON responses, see [this page](API.md).
""" % self.timestamp()
self.w(text.strip())
self.w()
def get_routes(self, group):
for route in self.autodoc.generate(groups=group):
if '.json' not in route['rule']:
yield route
if __name__ == '__main__':
with app.app_context():
ApiDocGenerator(doc).generate('../docs/API.md')
FlaskRoutesDocGenerator(doc).generate('../docs/FlaskRoutes.md')
Generates FlaskRoutes.md
"""
def __init__(self, *args, **kwargs):
super(FlaskRoutesDocGenerator, self).__init__(exclude_groups=['api'], *args, **kwargs)
def print_header(self):
text = """
# Flask Routes
%s
Documentation on the various routes used internally for the web application.
These are all technically RESTful, but they return HTML pages. To get JSON responses, see [this page](API.md).
""" % self.timestamp()
self.w(text.strip())
self.w()
def get_routes(self, group):
for route in self.autodoc.generate(groups=group):
if '.json' not in route['rule']:
yield route
if __name__ == '__main__':
with app.app_context():
ApiDocGenerator(doc).generate('../docs/API.md')
FlaskRoutesDocGenerator(doc).generate('../docs/FlaskRoutes.md')
def test_flask_routes_md():
"""FlaskRoutes.md out-of-date"""
with app.app_context():
generator = _.FlaskRoutesDocGenerator(doc)
check_doc_file(generator,
os.path.join(os.path.dirname(os.path.dirname(__file__)), 'docs', 'FlaskRoutes.md'))
