def run(self):
self._plugins.init_plugins()
self._dispatcher.start()
self._client.rtm_connect()
_thread.start_new_thread(self._keepactive, tuple())
logger.info('connected to slack RTM api')
self._dispatcher.loop()
def image_create(request, **kwargs):
"""Create image.
:param kwargs:
* copy_from: URL from which Glance server should immediately copy
the data and store it in its configured image store.
* data: Form data posted from client.
* location: URL where the data for this image already resides.
In the case of 'copy_from' and 'location', the Glance server
will give us a immediate response from create and handle the data
asynchronously.
In the case of 'data' the process of uploading the data may take
some time and is handed off to a seperate thread.
"""
data = kwargs.pop("data", None)
image = glanceclient(request).images.create(**kwargs)
if data:
if isinstance(data, TemporaryUploadedFile):
# Hack to fool Django, so we can keep file open in the new thread.
data.file.close_called = True
if isinstance(data, InMemoryUploadedFile):
# Clone a new file for InMemeoryUploadedFile.
# Because the old one will be closed by Django.
data = SimpleUploadedFile(data.name, data.read(), data.content_type)
thread.start_new_thread(image_update, (request, image.id), {"data": data, "purge_props": False})
return image
def run(self):
self._plugins.init_plugins()
self._dispatcher.start()
self._client.rtm_connect()
_thread.start_new_thread(self._keepactive, tuple())
logger.info('connected to slack RTM api')
self._dispatcher.loop()
def log_run_info(self, model_name, dataset_name, run_params, test_id=None):
"""Collect most of the TF runtime information for the local env.
The schema of the run info follows official/benchmark/datastore/schema.
Args:
model_name: string, the name of the model.
dataset_name: string, the name of dataset for training and evaluation.
run_params: dict, the dictionary of parameters for the run, it could
include hyperparameters or other params that are important for the run.
test_id: string, the unique name of the test run by the combination of key
parameters, eg batch size, num of GPU. It is hardware independent.
"""
run_info = _gather_run_info(model_name, dataset_name, run_params, test_id)
# Starting new thread for bigquery upload in case it might take long time
# and impact the benchmark and performance measurement. Starting a new
# thread might have potential performance impact for model that run on CPU.
thread.start_new_thread(
self._bigquery_uploader.upload_benchmark_run_json,
(self._bigquery_data_set,
self._bigquery_run_table,
self._run_id,
run_info))
thread.start_new_thread(
self._bigquery_uploader.insert_run_status,
(self._bigquery_data_set,
self._bigquery_run_status_table,
self._run_id,
RUN_STATUS_RUNNING))
def on_finish(self, status):
thread.start_new_thread(
self._bigquery_uploader.update_run_status,
(self._bigquery_data_set,
self._bigquery_run_status_table,
self._run_id,
status))
def create_listener(request, **kwargs):
"""Create a new listener.
"""
data = request.DATA
listenerSpec = {
'protocol': data['listener']['protocol'],
'protocol_port': data['listener']['port'],
'loadbalancer_id': kwargs['loadbalancer_id']
}
if data['listener'].get('name'):
listenerSpec['name'] = data['listener']['name']
if data['listener'].get('description'):
listenerSpec['description'] = data['listener']['description']
if data.get('certificates'):
listenerSpec['default_tls_container_ref'] = data['certificates'][0]
listenerSpec['sni_container_refs'] = data['certificates']
listener = neutronclient(request).create_listener(
{'listener': listenerSpec}).get('listener')
if data.get('pool'):
args = (request, kwargs['loadbalancer_id'], create_pool)
kwargs = {'callback_kwargs': {'listener_id': listener['id']}}
thread.start_new_thread(poll_loadbalancer_status, args, kwargs)
return listener
def image_create(request, **kwargs):
"""Create image.
:param kwargs:
* copy_from: URL from which Glance server should immediately copy
the data and store it in its configured image store.
* data: Form data posted from client.
* location: URL where the data for this image already resides.
In the case of 'copy_from' and 'location', the Glance server
will give us a immediate response from create and handle the data
asynchronously.
In the case of 'data' the process of uploading the data may take
some time and is handed off to a separate thread.
"""
data = kwargs.pop('data', None)
image = glanceclient(request).images.create(**kwargs)
if data:
if isinstance(data, TemporaryUploadedFile):
# Hack to fool Django, so we can keep file open in the new thread.
data.file.close_called = True
if isinstance(data, InMemoryUploadedFile):
# Clone a new file for InMemeoryUploadedFile.
# Because the old one will be closed by Django.
data = SimpleUploadedFile(data.name, data.read(),
data.content_type)
thread.start_new_thread(image_update, (request, image.id), {
'data': data,
'purge_props': False
})
return image
def setUp(self):
# start the server
self.exit = False
def server():
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
s.bind(LOCALPORT); s.listen(1)
while 1:
c, a = s.accept()
if self.exit: c.close(); break
ending_compat = '\r\n\r\n' if not six.PY3 else b'\r\n\r\n'
while not c.recv(4096).endswith(ending_compat): pass
http_compat = 'HTTP/1.1 %d %s\r\n' % self.reply
c.sendall(http_compat if not six.PY3 else http_compat.encode('utf-8'))
if self.content is not None:
cont_length_compat = 'Content-Length: %d\r\n\r\n' % len(self.content)
c.sendall(cont_length_compat if not six.PY3 else cont_length_compat.encode('utf-8'))
c.sendall(self.content if not six.PY3 else self.content.encode('utf-8'))
c.close()
s.close()
self.exit = False
thread.start_new_thread(server, ())
# create grabber and mirror group objects
def failure(obj):
self.code = getattr(obj.exception, 'code', None)
return {}
self.g = URLGrabber()
self.mg = MirrorGroup(self.g, ['http://%s:%d' % LOCALPORT],
failure_callback = failure)
def run(self):
self._plugins.init_plugins()
self._dispatcher.start()
_thread.start_new_thread(self._keep_active, tuple())
_thread.start_new_thread(self._run_jobs, tuple())
self.run_scheduled_update_jobs()
self._dispatcher.loop()
def create_pool(request, **kwargs):
"""Create a new pool.
"""
data = request.DATA
poolSpec = {
'protocol': data['pool']['protocol'],
'lb_algorithm': data['pool']['method'],
'listener_id': kwargs['listener_id']
}
if data['pool'].get('name'):
poolSpec['name'] = data['pool']['name']
if data['pool'].get('description'):
poolSpec['description'] = data['pool']['description']
pool = neutronclient(request).create_lbaas_pool(
{'pool': poolSpec}).get('pool')
if data.get('members'):
args = (request, kwargs['loadbalancer_id'], add_member)
kwargs = {'callback_kwargs': {'pool_id': pool['id'],
'index': 0}}
thread.start_new_thread(poll_loadbalancer_status, args, kwargs)
elif data.get('monitor'):
args = (request, kwargs['loadbalancer_id'], create_health_monitor)
kwargs = {'callback_kwargs': {'pool_id': pool['id']}}
thread.start_new_thread(poll_loadbalancer_status, args, kwargs)
return pool
def run(self):
self._plugins.init_plugins()
self._plugins.trigger_at_start(self._client)
self._dispatcher.start()
_thread.start_new_thread(self._keep_active, tuple())
_thread.start_new_thread(self._run_jobs, tuple())
self._dispatcher.loop()
def update_member_list(request, **kwargs):
"""Update the list of members by adding or removing the necessary members.
"""
data = request.DATA
loadbalancer_id = data.get('loadbalancer_id')
pool_id = kwargs.get('pool_id')
existing_members = kwargs.get('existing_members')
members_to_add = kwargs.get('members_to_add')
members_to_delete = kwargs.get('members_to_delete')
if members_to_delete:
kwargs = {'existing_members': existing_members,
'members_to_add': members_to_add,
'members_to_delete': members_to_delete,
'pool_id': pool_id}
remove_member(request, **kwargs)
elif members_to_add:
kwargs = {'existing_members': existing_members,
'members_to_add': members_to_add,
'members_to_delete': members_to_delete,
'pool_id': pool_id}
add_member(request, **kwargs)
elif data.get('monitor'):
args = (request, loadbalancer_id, update_monitor)
thread.start_new_thread(poll_loadbalancer_status, args)
def log_run_info(self, model_name, dataset_name, run_params, test_id=None):
"""Collect most of the TF runtime information for the local env.
The schema of the run info follows official/benchmark/datastore/schema.
Args:
model_name: string, the name of the model.
dataset_name: string, the name of dataset for training and evaluation.
run_params: dict, the dictionary of parameters for the run, it could
include hyperparameters or other params that are important for the run.
test_id: string, the unique name of the test run by the combination of key
parameters, eg batch size, num of GPU. It is hardware independent.
"""
run_info = _gather_run_info(model_name, dataset_name, run_params,
test_id)
# Starting new thread for bigquery upload in case it might take long time
# and impact the benchmark and performance measurement. Starting a new
# thread might have potential performance impact for model that run on CPU.
thread.start_new_thread(
self._bigquery_uploader.upload_benchmark_run_json,
(self._bigquery_data_set, self._bigquery_run_table, self._run_id,
run_info))
thread.start_new_thread(
self._bigquery_uploader.insert_run_status,
(self._bigquery_data_set, self._bigquery_run_status_table,
self._run_id, RUN_STATUS_RUNNING))
def log_metric(self,
name,
value,
unit=None,
global_step=None,
extras=None):
"""Log the benchmark metric information to bigquery.
Args:
name: string, the name of the metric to log.
value: number, the value of the metric. The value will not be logged if it
is not a number type.
unit: string, the unit of the metric, E.g "image per second".
global_step: int, the global_step when the metric is logged.
extras: map of string:string, the extra information about the metric.
"""
metric = _process_metric_to_json(name, value, unit, global_step,
extras)
if metric:
# Starting new thread for bigquery upload in case it might take long time
# and impact the benchmark and performance measurement. Starting a new
# thread might have potential performance impact for model that run on
# CPU.
thread.start_new_thread(
self._bigquery_uploader.upload_benchmark_metric_json,
(self._bigquery_data_set, self._bigquery_metric_table,
self._run_id, [metric]))
def testConNeg():
_thread.start_new_thread(runHttpServer, tuple())
# hang on a second while server starts
time.sleep(1)
graph = Graph()
graph.parse("http://localhost:12345/foo", format="xml")
graph.parse("http://localhost:12345/foo", format="n3")
graph.parse("http://localhost:12345/foo", format="nt")
def startUpgrade(self, datafile, device, firmware): if firmware == 'fpga': self.fu = FPGAUpgradeCore(firmwarefile=datafile, devicefile=device) elif firmware == 'fp': self.fu = FPUpgradeCore(firmwarefile=datafile, devicefile=device) elif firmware == 'vfd': self.fu = VFDCtrlUpgradeCore(firmwarefile=datafile, devicefile=device) _thread.start_new_thread(self.fu.upgradeMain, ())
def _parallel_split(obj, eng, calls):
lock = thread.allocate_lock()
i = 0
eng.setVar('lock', lock)
for func in calls:
new_eng = duplicate_engine_instance(eng)
new_eng.setWorkflow([lambda o, e: e.setVar('lock', lock), func])
thread.start_new_thread(new_eng.process, ([obj], ))
def _parallel_split(obj, eng, calls):
lock = thread.allocate_lock()
i = 0
eng.setVar('lock', lock)
for func in calls:
new_eng = duplicate_engine_instance(eng)
new_eng.setWorkflow([lambda o, e: e.setVar('lock', lock), func])
thread.start_new_thread(new_eng.process, ([obj], ))
def _parallel_split(obj, eng, calls):
lock = thread.allocate_lock()
eng.store['lock'] = lock
for func in calls:
new_eng = eng.duplicate()
new_eng.setWorkflow(
[lambda o, e: e.store.update({'lock': lock}), func])
thread.start_new_thread(new_eng.process, ([obj], ))
def testConNeg():
_thread.start_new_thread(runHttpServer, tuple())
# hang on a second while server starts
time.sleep(1)
graph=Graph()
graph.parse("http://localhost:12345/foo", format="xml")
graph.parse("http://localhost:12345/foo", format="n3")
graph.parse("http://localhost:12345/foo", format="nt")
def image_create(request, **kwargs):
"""Create image.
:param kwargs:
* copy_from: URL from which Glance server should immediately copy
the data and store it in its configured image store.
* data: Form data posted from client.
* location: URL where the data for this image already resides.
In the case of 'copy_from' and 'location', the Glance server
will give us a immediate response from create and handle the data
asynchronously.
In the case of 'data' the process of uploading the data may take
some time and is handed off to a separate thread.
"""
data = kwargs.pop('data', None)
location = None
if VERSIONS.active >= 2:
location = kwargs.pop('location', None)
image = glanceclient(request).images.create(**kwargs)
if location is not None:
glanceclient(request).images.add_location(image.id, location, {})
if data:
if isinstance(data, six.string_types):
# The image data is meant to be uploaded externally, return a
# special wrapper to bypass the web server in a subsequent upload
return ExternallyUploadedImage(image, request)
elif isinstance(data, TemporaryUploadedFile):
# Hack to fool Django, so we can keep file open in the new thread.
data.file.close_called = True
elif isinstance(data, InMemoryUploadedFile):
# Clone a new file for InMemeoryUploadedFile.
# Because the old one will be closed by Django.
data = SimpleUploadedFile(data.name, data.read(), data.content_type)
if VERSIONS.active < 2:
thread.start_new_thread(image_update, (request, image.id), {'data': data})
else:
def upload():
try:
return glanceclient(request).images.upload(image.id, data)
finally:
filename = str(data.file.name)
try:
os.remove(filename)
except OSError as e:
LOG.warning('Failed to remove temporary image file '
'%(file)s (%(e)s)', {
'file': filename,
'e': e
})
thread.start_new_thread(upload, ())
return Image(image)
def image_create(request, **kwargs):
"""Create image.
:param kwargs:
* copy_from: URL from which Glance server should immediately copy
the data and store it in its configured image store.
* data: Form data posted from client.
* location: URL where the data for this image already resides.
In the case of 'copy_from' and 'location', the Glance server
will give us a immediate response from create and handle the data
asynchronously.
In the case of 'data' the process of uploading the data may take
some time and is handed off to a separate thread.
"""
data = kwargs.pop('data', None)
location = None
if VERSIONS.active >= 2:
location = kwargs.pop('location', None)
image = glanceclient(request).images.create(**kwargs)
if location is not None:
glanceclient(request).images.add_location(image.id, location, {})
if data:
if isinstance(data, six.string_types):
# The image data is meant to be uploaded externally, return a
# special wrapper to bypass the web server in a subsequent upload
return ExternallyUploadedImage(image, request)
elif isinstance(data, TemporaryUploadedFile):
# Hack to fool Django, so we can keep file open in the new thread.
data.file.close_called = True
elif isinstance(data, InMemoryUploadedFile):
# Clone a new file for InMemeoryUploadedFile.
# Because the old one will be closed by Django.
data = SimpleUploadedFile(data.name,
data.read(),
data.content_type)
if VERSIONS.active < 2:
thread.start_new_thread(image_update,
(request, image.id),
{'data': data})
else:
def upload():
try:
return glanceclient(request).images.upload(image.id, data)
finally:
filename = str(data.file.name)
try:
os.remove(filename)
except OSError as e:
LOG.warning('Failed to remove temporary image file '
'%(file)s (%(e)s)',
{'file': filename, 'e': e})
thread.start_new_thread(upload, ())
return Image(image)
def _parallel_split(obj, eng, calls):
lock = thread.allocate_lock()
eng.store['lock'] = lock
for func in calls:
new_eng = eng.duplicate()
new_eng.setWorkflow(
[lambda o, e: e.store.update({'lock': lock}), func]
)
thread.start_new_thread(new_eng.process, ([obj], ))
def _reloader(app_run_func, args, kwargs):
if os.environ.get('RUN_MAIN_APP') != 'true':
exit_code = _restart_application_with_autoreload()
if exit_code < 0:
os.kill(os.getpid(), -exit_code)
else:
sys.exit(exit_code)
else:
thread.start_new_thread(app_run_func, args, kwargs)
_monitor_needs_reloading()
def start_xmlrpc_server(addr='localhost', port=8000):
server = SimpleXMLRPCServer((addr, port))
functions = [
getRootNodePath, getitem, getChildNodes, getMethods, getMethodDoc,
getRepr, callMethod
]
for function in functions:
server.register_function(function, function.__name__)
thread.start_new_thread(server.serve_forever, tuple())
return server
def parameter_value_changed(frame, event):
""" Event handler for when a value of a parameter in the grid has been updated.
:param frame:
:param event:
:return:
"""
frame.simulation_queue_counter += 1
if frame.mb_fit_autosim.IsChecked() and not frame.flag_simulating:
_thread.start_new_thread(simulation_loop, (frame, ))
def _rtm_connect(self):
r = self.slacker.rtm.start().body
self.driver_username = r['self']['name']
self.driver_userid = r['self']['id']
self.users = {u['name']: u['id'] for u in r['users']}
self.testbot_userid = self.users[self.testbot_username]
self._websocket = websocket.create_connection(r['url'])
self._websocket.sock.setblocking(0)
_thread.start_new_thread(self._rtm_read_forever, tuple())
def start_monitor(unused):
"""start the request monitor if configured."""
from App.config import getConfiguration
config = getConfiguration().product_config.get('requestmonitor')
if config is None:
return # not configured
# register publication observers
provideHandler(handle_request_start)
provideHandler(handle_request_end)
monitor = _Monitor(config)
start_new_thread(monitor.run, ())
def _rtm_connect(self):
r = self.slacker.rtm.start().body
self.driver_username = r['self']['name']
self.driver_userid = r['self']['id']
self.users = {u['name']: u['id'] for u in r['users']}
self.testbot_userid = self.users[self.testbot_username]
self._websocket = websocket.create_connection(r['url'])
self._websocket.sock.setblocking(0)
_thread.start_new_thread(self._rtm_read_forever, tuple())
def __init__(self, bin=None, options=''):
self.execStart = time.time()
self.maxProcCycles = 1.0
# start CQP as a child process of this wrapper
if bin == None:
print("ERROR: Path to CQP binaries undefined", file=sys.stderr)
sys.exit(1)
self.CQP_process = subprocess.Popen(bin + ' ' + options,
shell=True,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
close_fds=True,
encoding='utf8')
self.CQPrunning = True
thread.start_new_thread(self._progressController, ())
# "cqp -c" should print version on startup:
version_string = str(self.CQP_process.stdout.readline())
version_string = version_string.rstrip() # Equivalent to Perl's chomp
# test modifs
if (re.match(r'^b\'.+?\'', version_string)):
version_string = re.sub(r'^b\'(.+?)\\n\'', r'\1', version_string)
version_regexp = re.compile(
r'^CQP\s+(?:\w+\s+)*([0-9]+)\.([0-9]+)(?:\.b?([0-9]+))?(?:\s+(.*))?$'
)
match = version_regexp.match(version_string)
if not match:
print("ERROR: CQP backend startup failed", file=sys.stderr)
sys.exit(1)
self.major_version = int(match.group(1))
self.minor_version = int(match.group(2))
self.beta_version = int(match.group(3))
self.compile_date = match.group(4)
# We need cqp-2.2.b41 or newer (for query lock):
if not (self.major_version >= 3 or
(self.major_version == 2 and self.minor_version == 2
and self.beta_version >= 41)):
print("ERROR: CQP version too old: " + version_string,
file=sys.stderr)
sys.exit(1)
# Error handling:
self.error_handler = None
self.status = 'ok'
self.error_message = '' # we store compound error messages as a STRING
self.errpipe = self.CQP_process.stderr.fileno()
# Debugging (prints more or less everything on stdout)
self.debug = False
# CQP defaults:
self.Exec('set PrettyPrint off')
self.execStart = None
def _process(self, event):
""" 基于多线程的处理事件 """
if event.route not in self._routes:
log.warning("事件%s 没有被处理" % event.route)
return
for handler in self._routes[event.route]:
try:
log.debug("处理事件%s" % event.route)
_thread.start_new_thread(handler, (event,))
#handler(event)
except Exception as e:
log.error(e)
def image_create(request, **kwargs):
copy_from = kwargs.pop("copy_from", None)
data = kwargs.pop("data", None)
image = glanceclient(request).images.create(**kwargs)
if data:
thread.start_new_thread(image_update, (request, image.id), {"data": data, "purge_props": False})
elif copy_from:
thread.start_new_thread(image_update, (request, image.id), {"copy_from": copy_from, "purge_props": False})
return image
def subscribe(self, notification, cb, data=None):
np_data = {
"running": True,
"notification": notification,
"callback": cb,
"userdata": data,
}
thread.start_new_thread( self.notifier, ("NotificationProxyNotifier_"+notification, np_data, ) )
while(1):
time.sleep(1)
def _process(self, event):
""" 基于多线程的处理事件 """
if event.route not in self._routes:
log.warning("事件%s 没有被处理" % event.route)
return
for handler in self._routes[event.route]:
try:
log.debug("处理事件%s" % event.route)
_thread.start_new_thread(handler, (event, ))
#handler(event)
except Exception as e:
log.error(e)
def server(*settings):
try:
dock_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
dock_socket.bind(("127.0.0.1", settings[2]))
dock_socket.listen(5)
while True:
client_socket = dock_socket.accept()[0]
server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server_socket.connect((settings[0], settings[1]))
thread.start_new_thread(forward, (client_socket, server_socket))
thread.start_new_thread(forward, (server_socket, client_socket))
except Exception:
print_exc()
def __init__(self, bin=None, options=''):
"""Class constructor."""
self.__logger = logging.getLogger(self.__class__.__name__)
self.execStart = time.time()
self.maxProcCycles = 1.0
# start CQP as a child process of this wrapper
if bin is None:
self.__logger.error("Path to CQP binaries undefined")
sys.exit(1)
self.CQP_process = subprocess.Popen(bin + ' ' + options,
shell=True,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
universal_newlines=True,
close_fds=True)
self.CQPrunning = True
thread.start_new_thread(self._progressController, ())
# "cqp -c" should print version on startup:
version_string = self.CQP_process.stdout.readline()
version_string = version_string.rstrip() # Equivalent to Perl's chomp
self.CQP_process.stdout.flush()
self.__logger.info("Test " + version_string)
version_regexp = re.compile(
r'^CQP\s+(?:\w+\s+)*([0-9]+)\.([0-9]+)(?:\.b?([0-9]+))?(?:\s+(.*))?$'
)
match = version_regexp.match(version_string)
if not match:
self.__logger.error("CQP backend startup failed")
sys.exit(1)
self.major_version = int(match.group(1))
self.minor_version = int(match.group(2))
self.beta_version = int(match.group(3))
self.compile_date = match.group(4)
# We need cqp-2.2.b41 or newer (for query lock):
if not (self.major_version >= 3 or
(self.major_version == 2 and self.minor_version == 2
and self.beta_version >= 41)):
self.__logger.error("CQP version too old: %s", version_string)
sys.exit(1)
# Error handling:
self.error_handler = None
self.status = 'ok'
self.error_message = '' # we store compound error messages as a STRING
self.errpipe = self.CQP_process.stderr.fileno()
# CQP defaults:
self.Exec('set PrettyPrint off')
self.execStart = None
def run(self):
"""Start a new thread for the player.
Call this function before trying to play any music with
play_file() or play().
"""
# If we don't use the MainLoop, messages are never sent.
def start():
loop = GLib.MainLoop()
loop.run()
_thread.start_new_thread(start, ())
def run(self):
"""Start a new thread for the player.
Call this function before trying to play any music with
play_file() or play().
"""
# If we don't use the MainLoop, messages are never sent.
def start():
loop = GLib.MainLoop()
loop.run()
_thread.start_new_thread(start, ())
def subscribe(self, notification, cb, data=None):
np_data = {
"running": True,
"notification": notification,
"callback": cb,
"userdata": data,
}
thread.start_new_thread(self.notifier, (
"NotificationProxyNotifier_" + notification,
np_data,
))
while (1):
time.sleep(1)
def notifier(self, name, args=None):
if args == None:
return None
self.observe_notification(args.get("notification"))
while args.get("running") == True:
np_name = self.get_notification(args.get("notification"))
if np_name:
userdata = args.get("userdata")
try:
thread.start_new_thread( args.get("callback") , (np_name, userdata, ) )
except:
self.logger.error("Error: unable to start thread")
def explore(self):
"""
Explore model evaluation qualitatively through a GUI assisted application.
"""
if self._data["test_data"][self._data["feature"]].dtype == _tc.Image:
print("Resizing image data... ", end='')
self._data["test_data"][self._data["feature"]] = self._data["test_data"][self._data["feature"]].apply(_image_conversion)
self._data["test_data"].materialize()
print("Done.")
params = (self._get_eval_json()+"\n", self._data["test_data"], self, )
# Suppress visualization output if 'none' target is set
from ...visualization._plot import _target
if _target == 'none':
return
_thread.start_new_thread(_start_process, params)
def python_reloader(main_func, args, kwargs):
if os.environ.get("RUN_MAIN") == "true":
thread.start_new_thread(main_func, args, kwargs)
try:
reloader_thread()
except KeyboardInterrupt:
pass
else:
try:
exit_code = restart_with_reloader()
if exit_code < 0:
os.kill(os.getpid(), -exit_code)
else:
sys.exit(exit_code)
except KeyboardInterrupt:
pass
def start_xmlrpc_server(addr='localhost', port=8000):
server = SimpleXMLRPCServer((addr, port))
functions = [
getRootNodePath,
getitem,
getChildNodes,
getMethods,
getMethodDoc,
getRepr,
callMethod
]
for function in functions:
server.register_function(function, function.__name__)
thread.start_new_thread(server.serve_forever, tuple())
return server
def readdir(self, path, file_id):
"""Send a list of entries in this directory"""
logger.debug("Getting directory list for {0}".format(path))
# reading from VOSpace can be slow, we'll do this in a thread
from six.moves import _thread
with self.condition:
if not self.loading_dir.get(path, False):
self.loading_dir[path] = True
_thread.start_new_thread(self.load_dir, (path, ))
while self.loading_dir.get(path, False):
logger.debug("Waiting ... ")
self.condition.wait()
return ['.', '..'] + [
e.name
for e in self.getNode(path, force=False, limit=None).node_list
]
def execute(cmd, memprof_file, sample_interval):
"""https://stackoverflow.com/questions/4417546/constantly-print-subprocess-output-while-process-is-running"""
exe = which(cmd[0])
if exe is None:
logging.error('Could not find executable "%s"', cmd[0])
raise OSError(errno.ENOENT, os.strerror(errno.ENOENT), cmd[0])
popen = subprocess.Popen(
cmd, stdout=subprocess.PIPE, universal_newlines=True)
thread.start_new_thread(memory_profile, (' '.join(cmd), popen, memprof_file, sample_interval))
for stdout_line in iter(popen.stdout.readline, ""):
yield stdout_line
popen.stdout.close()
return_code = popen.wait()
if return_code:
if popen.stderr:
logging.error(popen.stderr.read())
raise subprocess.CalledProcessError(return_code, cmd)
def image_create(request, **kwargs):
copy_from = kwargs.pop('copy_from', None)
data = kwargs.pop('data', None)
image = glanceclient(request).images.create(**kwargs)
if data:
thread.start_new_thread(image_update,
(request, image.id),
{'data': data,
'purge_props': False})
elif copy_from:
thread.start_new_thread(image_update,
(request, image.id),
{'copy_from': copy_from,
'purge_props': False})
return image
def notifier(self, name, args=None):
if args == None:
return None
self.observe_notification(args.get("notification"))
while args.get("running") == True:
np_name = self.get_notification(args.get("notification"))
if np_name:
userdata = args.get("userdata")
try:
thread.start_new_thread(args.get("callback"), (
np_name,
userdata,
))
except:
self.logger.error("Error: unable to start thread")
def slotConnect(self, data):
"""
connect to a core
if connection is local, parse the core config file for data
if internal, start pyLoadCore
set up connector, show main window
"""
self.connWindow.hide()
if data["type"] not in ("remote", "internal"):
coreparser = ConfigParser(self.configdir)
if not coreparser.config:
self.connector.setConnectionData("127.0.0.1", 7227,
"anonymous", "anonymous",
False)
else:
self.connector.setConnectionData(
"127.0.0.1", coreparser.get("remote", "port"), "anonymous",
"anonymous")
elif data["type"] == "remote":
self.connector.setConnectionData(data["host"], data["port"],
data["user"], data["password"])
elif data["type"] == "internal":
from pyLoadCore import Core
from module.ConfigParser import ConfigParser as CoreConfig
if not self.core:
config = CoreConfig(
) #create so at least default config exists
self.core = Core()
self.core.startedInGui = True
thread.start_new_thread(self.core.start, (False, False))
while not self.core.running:
sleep(0.5)
self.connector.proxy = self.core.api
self.connector.internal = True
#self.connector.setConnectionData("127.0.0.1", config.get("remote","port"), "anonymous", "anonymous")
self.startMain()
def log_run_info(self, model_name, dataset_name, run_params):
"""Collect most of the TF runtime information for the local env.
The schema of the run info follows official/benchmark/datastore/schema.
Args:
model_name: string, the name of the model.
dataset_name: string, the name of dataset for training and evaluation.
run_params: dict, the dictionary of parameters for the run, it could
include hyperparameters or other params that are important for the run.
"""
run_info = _gather_run_info(model_name, dataset_name, run_params)
# Starting new thread for bigquery upload in case it might take long time
# and impact the benchmark and performance measurement. Starting a new
# thread might have potential performance impact for model that run on CPU.
thread.start_new_thread(
self._bigquery_uploader.upload_benchmark_run_json,
(self._bigquery_data_set, self._bigquery_run_table, self._run_id,
run_info))
def test_initialization_race_condition(self):
# Tries to exercise a former race condition where
# FSObject._updateFromFS() set self._parsed before the
# object was really parsed.
for _n in range(10):
f = Folder()
script = self._makeOne('test1', 'test1.py').__of__(f)
res = []
def call_script(script=script, res=res):
try:
res.append(script())
except Exception:
res.append('%s: %s' % exc_info()[:2])
start_new_thread(call_script, ())
call_script()
while len(res) < 2:
sleep(0.05)
self.assertEqual(res, ['test1', 'test1'], res)
def remove_member(request, **kwargs):
"""Remove a member from the pool.
"""
data = request.DATA
loadbalancer_id = data.get('loadbalancer_id')
pool_id = kwargs.get('pool_id')
if kwargs.get('members_to_delete'):
members_to_delete = kwargs['members_to_delete']
member_id = members_to_delete.pop(0)
neutronclient(request).delete_lbaas_member(member_id, pool_id)
args = (request, loadbalancer_id, update_member_list)
kwargs = {'callback_kwargs': {
'existing_members': kwargs.get('existing_members'),
'members_to_add': kwargs.get('members_to_add'),
'members_to_delete': members_to_delete}}
thread.start_new_thread(poll_loadbalancer_status, args, kwargs)
def nova_live_migrate(node):
loader = loading.get_plugin_loader('password')
auth = loader.load_from_options(
auth_url=os.environ["OS_AUTH_URL"],
username=os.environ["OS_USERNAME"],
password=os.environ["OS_PASSWORD"],
user_domain_name=os.environ["OS_USER_DOMAIN_NAME"],
project_domain_name=os.environ["OS_PROJECT_DOMAIN_NAME"],
project_name=os.environ["OS_PROJECT_NAME"])
OS_COMPUTE_API_VERSION = "2"
sess = session.Session(auth=auth)
nova = client.Client(OS_COMPUTE_API_VERSION, session=sess)
LOG.info("Disabling nova-compute service on: %s", node)
nova.services.disable(node, "nova-compute")
for server in nova.servers.list(search_opts={'host': node}):
LOG.info("Live-migrating instance: %s from node: %s", server.name,
node)
server.live_migrate(block_migration=True)
thread.start_new_thread(live_migration_watcher_thread, (nova, node))
def log_run_info(self, model_name, dataset_name, run_params):
"""Collect most of the TF runtime information for the local env.
The schema of the run info follows official/benchmark/datastore/schema.
Args:
model_name: string, the name of the model.
dataset_name: string, the name of dataset for training and evaluation.
run_params: dict, the dictionary of parameters for the run, it could
include hyperparameters or other params that are important for the run.
"""
run_info = _gather_run_info(model_name, dataset_name, run_params)
# Starting new thread for bigquery upload in case it might take long time
# and impact the benchmark and performance measurement. Starting a new
# thread might have potential performance impact for model that run on CPU.
thread.start_new_thread(
self._bigquery_uploader.upload_benchmark_run_json,
(self._bigquery_data_set,
self._bigquery_run_table,
self._run_id,
run_info))
def RepeatTimer(interval, function, iterations=0, *args, **kwargs):
"""Repeating timer. Returns a thread id."""
def __repeat_timer(interval, function, iterations, args, kwargs):
"""Inner function, run in background thread."""
count = 0
while iterations <= 0 or count < iterations:
sleep(interval)
function(*args, **kwargs)
count += 1
return start_new_thread(__repeat_timer, (interval, function, iterations, args, kwargs))
def performance_test( mod, hash_alg, ng_type, niter=10, nthreads=1 ):
global NLEFT
_s, _v = srp.create_salted_verification_key( username, password, hash_alg, ng_type )
NLEFT = niter
def test_thread():
global NLEFT
while NLEFT > 0:
do_auth( mod, hash_alg, ng_type, _s, _v )
NLEFT -= 1
start = time.time()
while nthreads > 1:
_thread.start_new_thread( test_thread, () )
nthreads -= 1
test_thread()
duration = time.time() - start
return duration
def image_create(request, **kwargs):
copy_from = kwargs.pop('copy_from', None)
data = kwargs.pop('data', None)
location = kwargs.pop('location', None)
image = glanceclient(request).images.create(**kwargs)
if data:
file_path = os.path.join("/tmp", uuid.uuid1().hex)
if isinstance(data, uploadedfile.TemporaryUploadedFile):
shutil.copyfile(data.temporary_file_path(), file_path)
elif isinstance(data, uploadedfile.InMemoryUploadedFile):
open(file_path, "wb").write(data.file.read())
thread.start_new_thread(image_update,
(request, image.id),
{'file_path': file_path,
'data': data,
'purge_props': False})
elif copy_from:
thread.start_new_thread(image_update,
(request, image.id),
{'copy_from': copy_from,
'purge_props': False})
elif location:
thread.start_new_thread(image_update,
(request, image.id),
{'location': location,
'purge_props': False})
return image
def image_create(request, **kwargs):
copy_from = kwargs.pop('copy_from', None)
data = kwargs.pop('data', None)
location = kwargs.pop('location', None)
image = glanceclient(request).images.create(**kwargs)
if data:
if isinstance(data, TemporaryUploadedFile):
# Hack to fool Django, so we can keep file open in the new thread.
data.file.close_called = True
if isinstance(data, InMemoryUploadedFile):
# Clone a new file for InMemeoryUploadedFile.
# Because the old one will be closed by Django.
data = SimpleUploadedFile(data.name,
data.read(),
data.content_type)
thread.start_new_thread(image_update,
(request, image.id),
{'data': data,
'purge_props': False})
elif copy_from:
thread.start_new_thread(image_update,
(request, image.id),
{'copy_from': copy_from,
'purge_props': False})
elif location:
thread.start_new_thread(image_update,
(request, image.id),
{'location': location,
'purge_props': False})
return image
def update_listener(request, **kwargs):
"""Update a listener.
"""
data = request.DATA
listener_spec = {}
listener_id = data['listener'].get('id')
loadbalancer_id = data.get('loadbalancer_id')
if data['listener'].get('name'):
listener_spec['name'] = data['listener']['name']
if data['listener'].get('description'):
listener_spec['description'] = data['listener']['description']
listener = neutronclient(request).update_listener(
listener_id, {'listener': listener_spec}).get('listener')
if data.get('pool'):
args = (request, loadbalancer_id, update_pool)
thread.start_new_thread(poll_loadbalancer_status, args)
return listener
def log_metric(self, name, value, unit=None, global_step=None, extras=None):
"""Log the benchmark metric information to bigquery.
Args:
name: string, the name of the metric to log.
value: number, the value of the metric. The value will not be logged if it
is not a number type.
unit: string, the unit of the metric, E.g "image per second".
global_step: int, the global_step when the metric is logged.
extras: map of string:string, the extra information about the metric.
"""
metric = _process_metric_to_json(name, value, unit, global_step, extras)
if metric:
# Starting new thread for bigquery upload in case it might take long time
# and impact the benchmark and performance measurement. Starting a new
# thread might have potential performance impact for model that run on
# CPU.
thread.start_new_thread(
self._bigquery_uploader.upload_benchmark_metric_json,
(self._bigquery_data_set,
self._bigquery_metric_table,
self._run_id,
[metric]))
