def test__read_text(c, s, a, b):
with make_hdfs() as (hdfs, basedir):
with hdfs.open('%s/text.1.txt' % basedir, 'wb') as f:
f.write('Alice 100\nBob 200\nCharlie 300'.encode())
with hdfs.open('%s/text.2.txt' % basedir, 'wb') as f:
f.write('Dan 400\nEdith 500\nFrank 600'.encode())
with hdfs.open('%s/other.txt' % basedir, 'wb') as f:
f.write('a b\nc d'.encode())
b = db.read_text('hdfs://%s/text.*.txt' % basedir)
yield gen.sleep(0.5)
assert not s.tasks
import dask
b.compute(get=dask.get)
coll = b.str.strip().str.split().map(len)
future = c.compute(coll)
yield gen.sleep(0.5)
result = yield future._result()
assert result == [2, 2, 2, 2, 2, 2]
b = db.read_text('hdfs://%s/other.txt' % basedir)
b = c.persist(b)
future = c.compute(b.str.split().concat())
result = yield future._result()
assert result == ['a', 'b', 'c', 'd']
def test_scale_retires_workers():
class MyCluster(LocalCluster):
def scale_down(self, *args, **kwargs):
pass
loop = IOLoop.current()
cluster = yield MyCluster(0, scheduler_port=0, processes=False,
silence_logs=False, diagnostics_port=None,
loop=loop, asynchronous=True)
c = yield Client(cluster, loop=loop, asynchronous=True)
assert not cluster.workers
yield cluster.scale(2)
start = time()
while len(cluster.scheduler.workers) != 2:
yield gen.sleep(0.01)
assert time() < start + 3
yield cluster.scale(1)
start = time()
while len(cluster.scheduler.workers) != 1:
yield gen.sleep(0.01)
assert time() < start + 3
yield c.close()
yield cluster.close()
def test_failed_worker_without_warning(c, s, a, b):
L = c.map(inc, range(10))
yield wait(L)
original_pid = a.pid
with ignoring(CommClosedError):
yield c._run(os._exit, 1, workers=[a.worker_address])
start = time()
while a.pid == original_pid:
yield gen.sleep(0.01)
assert time() - start < 10
yield gen.sleep(0.5)
start = time()
while len(s.ncores) < 2:
yield gen.sleep(0.01)
assert time() - start < 10
yield wait(L)
L2 = c.map(inc, range(10, 20))
yield wait(L2)
assert all(len(keys) > 0 for keys in s.has_what.values())
ncores2 = dict(s.ncores)
yield c._restart()
L = c.map(inc, range(10))
yield wait(L)
assert all(len(keys) > 0 for keys in s.has_what.values())
assert not (set(ncores2) & set(s.ncores)) # no overlap
def run_worker(context):
poller = Poller()
liveness = HEARTBEAT_LIVENESS
interval = INTERVAL_INIT
heartbeat_at = time.time() + HEARTBEAT_INTERVAL
worker = yield worker_socket(context, poller)
cycles = 0
while True:
socks = yield poller.poll(HEARTBEAT_INTERVAL * 1000)
socks = dict(socks)
# Handle worker activity on backend
if socks.get(worker) == zmq.POLLIN:
# Get message
# - 3-part envelope + content -> request
# - 1-part HEARTBEAT -> heartbeat
frames = yield worker.recv_multipart()
if not frames:
break # Interrupted
if len(frames) == 3:
# Simulate various problems, after a few cycles
cycles += 1
if cycles > 3 and randint(0, 5) == 0:
print("I: Simulating a crash")
break
if cycles > 3 and randint(0, 5) == 0:
print("I: Simulating CPU overload")
yield gen.sleep(3)
print("I: Normal reply")
yield worker.send_multipart(frames)
liveness = HEARTBEAT_LIVENESS
yield gen.sleep(1) # Do some heavy work
elif len(frames) == 1 and frames[0] == PPP_HEARTBEAT:
print("I: Queue heartbeat")
liveness = HEARTBEAT_LIVENESS
else:
print("E: Invalid message: %s" % frames)
interval = INTERVAL_INIT
else:
liveness -= 1
if liveness == 0:
print("W: Heartbeat failure, can't reach queue")
print("W: Reconnecting in %0.2fs..." % interval)
yield gen.sleep(interval)
if interval < INTERVAL_MAX:
interval *= 2
poller.unregister(worker)
worker.setsockopt(zmq.LINGER, 0)
worker.close()
worker = yield worker_socket(context, poller)
liveness = HEARTBEAT_LIVENESS
if time.time() > heartbeat_at:
heartbeat_at = time.time() + HEARTBEAT_INTERVAL
print("I: Worker heartbeat")
yield worker.send(PPP_HEARTBEAT)
def test_upload_large_file(c, s, a, b):
pytest.importorskip('crick')
yield gen.sleep(0.05)
with rpc(a.address) as aa:
yield aa.upload_file(filename='myfile.dat', data=b'0' * 100000000)
yield gen.sleep(0.05)
assert a.digests['tick-duration'].components[0].max() < 0.050
def test_worker_who_has_clears_after_failed_connection(c, s, a, b):
n = Nanny(s.ip, s.port, ncores=2, loop=s.loop)
n.start(0)
start = time()
while len(s.ncores) < 3:
yield gen.sleep(0.01)
assert time() < start + 5
futures = c.map(slowinc, range(20), delay=0.01,
key=['f%d' % i for i in range(20)])
yield wait(futures)
result = yield c.submit(sum, futures, workers=a.address)
for dep in set(a.dep_state) - set(a.task_state):
a.release_dep(dep, report=True)
n_worker_address = n.worker_address
with ignoring(CommClosedError):
yield c._run(os._exit, 1, workers=[n_worker_address])
while len(s.workers) > 2:
yield gen.sleep(0.01)
total = c.submit(sum, futures, workers=a.address)
yield total
assert not a.has_what.get(n_worker_address)
assert not any(n_worker_address in s for s in a.who_has.values())
yield n._close()
def __wait(self):
log.debug("Waiting for events")
while not (yield sleep(0.001)):
try:
state = self.__connection.poll()
except QueryCanceledError:
yield sleep(0.1)
continue
f = Future()
def resolve(fileno, io_op):
if f.running():
f.set_result(True)
self.__io_loop.remove_handler(fileno)
if state == psycopg2.extensions.POLL_OK:
raise Return(True)
elif state == psycopg2.extensions.POLL_READ:
self.__io_loop.add_handler(self.__connection.fileno(), resolve, IOLoop.READ)
yield f
elif state == psycopg2.extensions.POLL_WRITE:
self.__io_loop.add_handler(self.__connection.fileno(), resolve, IOLoop.WRITE)
yield f
def test_Royal_Seal(self):
tu.print_test_header("test Royal Seal")
royal_seal = prosperity.Royal_Seal(self.game, self.player1)
workers_village = prosperity.Workers_Village(self.game, self.player1)
copper = supply_cards.Copper(self.game, self.player1)
self.player1.hand.add(royal_seal)
self.player1.hand.add(copper)
tu.add_many_to_hand(self.player1, workers_village, 3)
workers_village.play()
workers_village.play()
workers_village.play()
royal_seal.play()
yield tu.send_input(self.player1, "buyCard", "Curse")
self.assertTrue(self.player1.last_mode["mode"] == "select")
yield tu.send_input(self.player1, "post_selection", ["Yes"])
self.assertTrue(self.player1.deck[-1].title == "Curse")
yield gen.sleep(.2)
self.assertTrue(self.player1.last_mode["mode"] == "buy")
yield tu.send_input(self.player1, "buyCard", "Silver")
self.assertTrue(self.player1.last_mode["mode"] == "select")
yield tu.send_input(self.player1, "post_selection", ["No"])
self.assertTrue(self.player1.discard_pile[-1].title == "Silver")
yield gen.sleep(.2)
self.assertTrue(self.player1.last_mode["mode"] == "buy")
yield tu.send_input(self.player1, "buyCard", "Mint")
self.assertTrue(self.player1.last_mode["mode"] == "buy")
def send():
c = BufferedClient()
c.send({'host': 'localhost', 'service': 'buffer test'})
yield sleep(0.5)
c.send({'host': 'localhost', 'service': 'buffer test'})
c.send({'host': 'localhost', 'service': 'buffer test'})
yield sleep(1.5)
def test_serializers():
with echo_server() as e:
comm = yield connect(e.address)
b = BatchedSend(interval='10ms', serializers=['msgpack'])
b.start(comm)
b.send({'x': to_serialize(123)})
b.send({'x': to_serialize('hello')})
yield gen.sleep(0.100)
b.send({'x': to_serialize(lambda x: x + 1)})
with captured_logger('distributed.protocol') as sio:
yield gen.sleep(0.100)
value = sio.getvalue()
assert 'serialize' in value
assert 'type' in value
assert 'function' in value
msg = yield comm.read()
assert list(msg) == [{'x': 123}, {'x': 'hello'}]
with pytest.raises(gen.TimeoutError):
msg = yield gen.with_timeout(timedelta(milliseconds=100), comm.read())
def _call(self, api_function, *args, **kwargs):
"""Calls the provided api_function in a background thread.
If the api function returns a response cleanly, this will return it.
If the api function raises an exception, this raises it up.
For as long as the api function returns a boto2 or boto3
rate limiting exception, this will backoff and try again.
"""
while True:
try:
result = yield self._thread(api_function, *args, **kwargs)
self._decrease_delay()
raise gen.Return(result)
except boto_exception.BotoServerError as e:
# Boto2 exception.
if e.error_code in self.boto2_throttle_strings:
self._increase_delay()
yield gen.sleep(self.delay)
else:
self._decrease_delay()
raise e
except botocore_exceptions.ClientError as e:
# Boto3 exception.
if e.response['Error']['Code'] == 'Throttling':
self._increase_delay()
yield gen.sleep(self.delay)
else:
self._decrease_delay()
raise e
def test_advertiser_intermediate_failure():
@gen.coroutine
def handle(request, response):
body = yield request.get_body()
if hb.count == 2:
# fail the second request only
raise Exception('great sadness')
response.write_body(body)
hb = Fakebahn(handle)
try:
hb.start()
adv = hyperbahn.Advertiser(
'foo', TChannel('foo', known_peers=[hb.hostport]),
interval_secs=0.2,
interval_max_jitter_secs=0.0,
)
yield adv.start()
assert 1 == hb.count
yield gen.sleep(0.25)
assert 2 == hb.count
yield gen.sleep(0.25)
assert 3 == hb.count
finally:
hb.stop()
def get(self, path):
log.debug("Slow Sending file: %s (This may take several minutes)" % self.request.path)
seenfiles.append(os.path.basename(str(self.request.path)))
f = open(os.path.join('static', path), 'rb')
f.seek(0, 2)
length = f.tell()
f.seek(0, 0)
self.set_header('Content-Length', str(length))
self.set_header('X-Powered-By', 'Express')
self.set_header('Transfer-Encoding', '')
self.set_header('Content-Disposition',
'attachment; filename="{}"'.format(path))
self.set_header('Accept-Ranges', 'bytes')
self.set_header('Cache-Control', 'public, max-age=0')
self.set_header('Content-Type', 'application/octet-stream')
chunk = f.read(10)
self.write(chunk)
yield self.flush()
yield gen.sleep(0.9)
while True:
chunk = f.read(1400)
if not chunk:
break
self.write(chunk)
yield self.flush()
print(" {}% ".format(int(f.tell()*100/length)), end="\r", flush=True)
yield gen.sleep(0.3)
def traktToken(self, trakt_pin=None, refresh=False, count=0):
if count > 3:
sickrage.TRAKT_ACCESS_TOKEN = ''
return False
elif count > 0:
gen.sleep(2)
data = {
'client_id': sickrage.TRAKT_API_KEY,
'client_secret': sickrage.TRAKT_API_SECRET,
'redirect_uri': 'urn:ietf:wg:oauth:2.0:oob'
}
if refresh:
data['grant_type'] = 'refresh_token'
data['refresh_token'] = sickrage.TRAKT_REFRESH_TOKEN
else:
data['grant_type'] = 'authorization_code'
if not None == trakt_pin:
data['code'] = trakt_pin
headers = {
'Content-Type': 'application/json'
}
resp = self.traktRequest('oauth/token', data=data, headers=headers, url=self.auth_url, method='POST',
count=count)
if 'access_token' in resp:
sickrage.TRAKT_ACCESS_TOKEN = resp['access_token']
if 'refresh_token' in resp:
sickrage.TRAKT_REFRESH_TOKEN = resp['refresh_token']
return True
return False
def f():
nn = rpc(ip=n.ip, port=n.port)
yield n._start()
ww = rpc(ip=n.ip, port=n.worker_port)
yield ww.update_data(data={'x': 1, 'y': 2})
with ignoring(StreamClosedError):
yield ww.compute(function=sys.exit, args=(0,), key='z')
start = time()
while n.process.is_alive(): # wait while process dies
yield gen.sleep(0.01)
assert time() - start < 2
start = time()
while not n.process.is_alive(): # wait while process comes back
yield gen.sleep(0.01)
assert time() - start < 2
start = time()
while n.worker_address not in c.ncores:
yield gen.sleep(0.01)
assert time() - start < 2
yield n._close()
c.stop()
def f():
yield a._start()
yield b._start()
yield s._sync_center()
done = s.start()
try:
assert s.ncores == {('127.0.0.1', a.worker_port): 2,
('127.0.0.1', b.worker_port): 2}
assert s.nannies == {(n.ip, n.worker_port): n.port
for n in [a, b]}
while any(len(v) < 3 for v in s.resource_logs.values()):
yield gen.sleep(0.01)
yield gen.sleep(0.1)
assert set(s.resource_logs) == {(a.ip, a.port), (b.ip, b.port)}
assert all(len(v) == 3 for v in s.resource_logs.values())
s.put({'op': 'close'})
yield done
finally:
yield a._close()
yield b._close()
c.stop()
def test_broken_worker_during_computation(c, s, a, b):
n = Nanny(s.ip, s.port, ncores=2, loop=s.loop)
n.start(0)
start = time()
while len(s.ncores) < 3:
yield gen.sleep(0.01)
assert time() < start + 5
L = c.map(inc, range(256))
for i in range(8):
L = c.map(add, *zip(*partition_all(2, L)))
from random import random
yield gen.sleep(random() / 2)
with ignoring(OSError):
n.process.terminate()
yield gen.sleep(random() / 2)
with ignoring(OSError):
n.process.terminate()
result = yield c._gather(L)
assert isinstance(result[0], int)
yield n._close()
def wait_for_http_server(url, timeout=10):
"""Wait for an HTTP Server to respond at url
Any non-5XX response code will do, even 404.
"""
loop = ioloop.IOLoop.current()
tic = loop.time()
client = AsyncHTTPClient()
while loop.time() - tic < timeout:
try:
r = yield client.fetch(url, follow_redirects=False)
except HTTPError as e:
if e.code >= 500:
# failed to respond properly, wait and try again
if e.code != 599:
# we expect 599 for no connection,
# but 502 or other proxy error is conceivable
app_log.warn("Server at %s responded with error: %s", url, e.code)
yield gen.sleep(0.1)
else:
app_log.debug("Server at %s responded with %s", url, e.code)
return
except (OSError, socket.error) as e:
if e.errno not in {errno.ECONNABORTED, errno.ECONNREFUSED, errno.ECONNRESET}:
app_log.warn("Failed to connect to %s (%s)", url, e)
yield gen.sleep(0.1)
else:
return
raise TimeoutError("Server at {url} didn't respond in {timeout} seconds".format(
**locals()
))
def test_failed_worker_without_warning(c, s, a, b):
L = c.map(inc, range(10))
yield _wait(L)
original_process = a.process
a.process.terminate()
start = time()
while a.process is original_process and not isalive(a.process):
yield gen.sleep(0.01)
assert time() - start < 10
yield gen.sleep(0.5)
start = time()
while len(s.ncores) < 2:
yield gen.sleep(0.01)
assert time() - start < 10
yield _wait(L)
L2 = c.map(inc, range(10, 20))
yield _wait(L2)
assert all(len(keys) > 0 for keys in s.has_what.values())
ncores2 = s.ncores.copy()
yield c._restart()
L = c.map(inc, range(10))
yield _wait(L)
assert all(len(keys) > 0 for keys in s.has_what.values())
assert not (set(ncores2) & set(s.ncores)) # no overlap
def do_connect(self, reconnect = False):
# Create the socket and connect to the server
if reconnect == True:
logger.warning('Connection failed, retrying in '+str(self._retrydelay)+ ' seconds')
yield gen.sleep(self._retrydelay)
while self._connection == None:
logger.debug('Connecting to {}:{}'.format(config.ENVISALINKHOST, config.ENVISALINKPORT))
try:
self._connection = yield self.tcpclient.connect(config.ENVISALINKHOST, config.ENVISALINKPORT)
self._connection.set_close_callback(self.handle_close)
except StreamClosedError:
#failed to connect, but got no connection object so we will loop here
logger.warning('Connection failed, retrying in '+str(self._retrydelay)+ ' seconds')
yield gen.sleep(self._retrydelay)
continue
try:
line = yield self._connection.read_until(self._terminator)
except StreamClosedError:
#in this state, since the connection object isnt none, its going to throw the callback for handle_close so we just bomb out.
#and let handle_close deal with this
return
logger.debug("Connected to %s:%i" % (config.ENVISALINKHOST, config.ENVISALINKPORT))
self.handle_line(line)
def test_pool_shrinking_with_shrink_delay(self):
db = yield self.build_pool(auto_shrink=True, shrink_delay=datetime.timedelta(seconds=1),
shrink_period=datetime.timedelta(milliseconds=500))
f1 = db.execute("SELECT 1")
f2 = db.execute("SELECT 2")
f3 = db.execute("SELECT 3")
f4 = db.execute("SELECT 4")
f5 = db.execute("SELECT 5")
cursors = yield [f1, f2, f3, f4, f5]
yield gen.sleep(.7)
self.assertEqual(db.conns.total, 5)
self.assertEqual(cursors[0].fetchone()[0], 1)
self.assertEqual(cursors[1].fetchone()[0], 2)
self.assertEqual(cursors[2].fetchone()[0], 3)
self.assertEqual(cursors[3].fetchone()[0], 4)
self.assertEqual(cursors[4].fetchone()[0], 5)
f1 = db.execute("SELECT 1")
f2 = db.execute("SELECT 2")
f3 = db.execute("SELECT 3")
cursors = yield [f1, f2, f3]
self.assertEqual(cursors[0].fetchone()[0], 1)
self.assertEqual(cursors[1].fetchone()[0], 2)
self.assertEqual(cursors[2].fetchone()[0], 3)
yield gen.sleep(1)
self.assertEqual(db.conns.total, 3)
def test_spawner_poll(db):
first_spawner = new_spawner(db)
user = first_spawner.user
yield first_spawner.start()
proc = first_spawner.proc
status = yield first_spawner.poll()
assert status is None
if user.state is None:
user.state = {}
first_spawner.orm_spawner.state = first_spawner.get_state()
assert 'pid' in first_spawner.orm_spawner.state
# create a new Spawner, loading from state of previous
spawner = new_spawner(db, user=first_spawner.user)
spawner.start_polling()
# wait for the process to get to the while True: loop
yield gen.sleep(1)
status = yield spawner.poll()
assert status is None
# kill the process
proc.terminate()
for i in range(10):
if proc.poll() is None:
yield gen.sleep(1)
else:
break
assert proc.poll() is not None
yield gen.sleep(2)
status = yield spawner.poll()
assert status is not None
def test_spawner_poll(db, io_loop):
first_spawner = new_spawner(db)
user = first_spawner.user
io_loop.run_sync(first_spawner.start)
proc = first_spawner.proc
status = io_loop.run_sync(first_spawner.poll)
assert status is None
user.state = first_spawner.get_state()
assert "pid" in user.state
# create a new Spawner, loading from state of previous
spawner = new_spawner(db, user=first_spawner.user)
spawner.start_polling()
# wait for the process to get to the while True: loop
io_loop.run_sync(lambda: gen.sleep(1))
status = io_loop.run_sync(spawner.poll)
assert status is None
# kill the process
proc.terminate()
for i in range(10):
if proc.poll() is None:
time.sleep(1)
else:
break
assert proc.poll() is not None
io_loop.run_sync(lambda: gen.sleep(2))
status = io_loop.run_sync(spawner.poll)
assert status is not None
def run(self):
QueueItem.run(self)
try:
sickrage.srLogger.info("Beginning daily search for new episodes")
foundResults = searchForNeededEpisodes()
if not len(foundResults):
sickrage.srLogger.info("No needed episodes found")
else:
for result in foundResults:
# just use the first result for now
sickrage.srLogger.info("Downloading " + result.name + " from " + result.provider.name)
self.success = snatchEpisode(result)
# give the CPU a break
gen.sleep(cpu_presets[sickrage.srConfig.CPU_PRESET])
QueueItem.finish(self)
except Exception:
sickrage.srLogger.debug(traceback.format_exc())
if self.success is None:
self.success = False
self.finish()
def test_here_now_multiple_channels(self):
ch1 = 'test-here-now-channel1'
ch2 = 'test-here-now-channel2'
self.pubnub.config.uuid = 'test-here-now-uuid'
# print("connecting to the first...")
yield connect_to_channel(self.pubnub, ch1)
# print("...connected to the first")
yield gen.sleep(1)
# print("connecting to the second...")
self.pubnub.subscribe().channels(ch2).execute()
# print("...connected to the second")
yield gen.sleep(5)
env = yield self.pubnub.here_now() \
.channels([ch1, ch2]) \
.future()
assert env.result.total_channels == 2
assert env.result.total_occupancy >= 1
channels = env.result.channels
assert len(channels) == 2
assert channels[0].occupancy >= 1
assert channels[0].occupants[0].uuid == self.pubnub.uuid
assert channels[1].occupancy >= 1
assert channels[1].occupants[0].uuid == self.pubnub.uuid
yield disconnect_from_channel(self.pubnub, [ch1, ch2])
self.pubnub.stop()
self.stop()
def run(self):
QueueItem.run(self)
try:
sickrage.srLogger.info("Beginning manual search for: [" + self.segment.prettyName() + "]")
self.started = True
searchResult = searchProviders(self.show, [self.segment], True, self.downCurQuality)
if searchResult:
# just use the first result for now
sickrage.srLogger.info("Downloading " + searchResult[0].name + " from " + searchResult[0].provider.name)
self.success = snatchEpisode(searchResult[0])
# give the CPU a break
gen.sleep(cpu_presets[sickrage.srConfig.CPU_PRESET])
else:
notifications.message('No downloads were found',
"Couldn't find a download for <i>%s</i>" % self.segment.prettyName())
sickrage.srLogger.info("Unable to find a download for: [" + self.segment.prettyName() + "]")
except Exception:
sickrage.srLogger.debug(traceback.format_exc())
### Keep a list with the 100 last executed searches
fifo(MANUAL_SEARCH_HISTORY, self, MANUAL_SEARCH_HISTORY_SIZE)
if self.success is None:
self.success = False
self.finish()
def do_connect(self, reconnect = False):
# Create the socket and connect to the server
if reconnect == True:
logger.warning('Connection failed, retrying in %s seconds' % str(self._retrydelay))
yield gen.sleep(self._retrydelay)
while self._connection == None:
logger.debug('Connecting to {}:{}'.format(config.ENVISALINKHOST, config.ENVISALINKPORT))
try:
self._connection = yield self.tcpclient.connect(config.ENVISALINKHOST, config.ENVISALINKPORT)
self._connection.set_close_callback(self.handle_close)
except StreamClosedError:
#failed to connect, but got no connection object so we will loop here
logger.warning('Connection failed, retrying in %s seconds' % str(self._retrydelay))
yield gen.sleep(self._retrydelay)
continue
except gaierror:
#could not resolve host provided, if this is a reconnect, will retry, if not, will fail
if reconnect == True:
logger.warning('Connection failed, unable to resolve hostname %s, retrying in %s seconds' % (config.ENVISALINKHOST, str(self._retrydelay)))
yield gen.sleep(self._retrydelay)
continue
else:
logger.warning('Connection failed, unable to resolve hostname %s. Exiting due to incorrect hostname.' % config.ENVISALINKHOST)
sys.exit(0)
try:
line = yield self._connection.read_until(self._terminator)
except StreamClosedError:
#in this state, since the connection object isnt none, its going to throw the callback for handle_close so we just bomb out.
#and let handle_close deal with this
return
logger.debug("Connected to %s:%i" % (config.ENVISALINKHOST, config.ENVISALINKPORT))
self.handle_line(line)
def instantiate(self, stream=None, environment=None):
""" Start a local worker process
Blocks until the process is up and the scheduler is properly informed
"""
if environment:
if not os.path.isabs(environment):
environment = os.path.join(self.local_dir, environment)
self.environment = environment
with log_errors():
if self.process and isalive(self.process):
raise ValueError("Existing process still alive. Please kill first")
if self.environment != nanny_environment:
with tmpfile() as fn:
self.process = run_worker_subprocess(self.environment, self.ip,
self.scheduler.ip, self.scheduler.port, self.ncores,
self.port, self._given_worker_port, self.name,
self.memory_limit, self.loop, fn, self.quiet)
while not os.path.exists(fn):
yield gen.sleep(0.01)
while True:
try:
with open(fn) as f:
msg = json.load(f)
self.worker_port = msg['port']
self.worker_dir = msg['local_directory']
break
except JSONDecodeError:
yield gen.sleep(0.01)
else:
q = Queue()
self.process = Process(target=run_worker_fork,
args=(q, self.ip, self.scheduler.ip,
self.scheduler.port, self.ncores,
self.port, self._given_worker_port,
self.local_dir, self.services, self.name,
self.memory_limit))
self.process.daemon = True
self.process.start()
while True:
try:
msg = q.get_nowait()
if isinstance(msg, Exception):
raise msg
self.worker_port = msg['port']
self.worker_dir = msg['dir']
assert self.worker_port
break
except queues.Empty:
yield gen.sleep(0.1)
logger.info("Nanny %s:%d starts worker process %s:%d",
self.ip, self.port, self.ip, self.worker_port)
raise gen.Return('OK')
def f():
nn = rpc(ip=n.ip, port=n.port)
yield n._start()
first_dir = n.worker_dir
assert os.path.exists(first_dir)
ww = rpc(ip=n.ip, port=n.worker_port)
yield ww.update_data(data={'x': 1, 'y': 2})
with ignoring(StreamClosedError):
yield ww.compute(function=sys.exit, args=(0,), key='z')
start = time()
while n.process.is_alive(): # wait while process dies
yield gen.sleep(0.01)
assert time() - start < 2
start = time()
while not n.process.is_alive(): # wait while process comes back
yield gen.sleep(0.01)
assert time() - start < 2
start = time()
while n.worker_address not in c.ncores or n.worker_dir is None:
yield gen.sleep(0.01)
assert time() - start < 2
second_dir = n.worker_dir
yield n._close()
assert not os.path.exists(second_dir)
assert not os.path.exists(first_dir)
assert first_dir != n.worker_dir
c.stop()
def test__read_text(e, s, a, b):
with make_hdfs() as hdfs:
with hdfs.open('/tmp/test/text.1.txt', 'wb') as f:
f.write('Alice 100\nBob 200\nCharlie 300'.encode())
with hdfs.open('/tmp/test/text.2.txt', 'wb') as f:
f.write('Dan 400\nEdith 500\nFrank 600'.encode())
with hdfs.open('/tmp/test/other.txt', 'wb') as f:
f.write('a b\nc d'.encode())
b = db.read_text('hdfs:///tmp/test/text.*.txt',
collection=True)
yield gen.sleep(0.5)
assert not s.tasks
import dask
b.compute(get=dask.get)
future = e.compute(b.str.strip().str.split().map(len))
yield gen.sleep(0.5)
result = yield future._result()
assert result == [2, 2, 2, 2, 2, 2]
b = db.read_text('hdfs:///tmp/test/other.txt', collection=True)
b = e.persist(b)
future = e.compute(b.str.split().concat())
result = yield future._result()
assert result == ['a', 'b', 'c', 'd']
L = db.read_text('hdfs:///tmp/test/text.*.txt', collection=False)
assert all(isinstance(x, Delayed) for x in L)
def test_learn_occupancy_2(c, s, a, b):
future = c.map(slowinc, range(1000), delay=0.2)
while not any(ts.who_has for ts in s.tasks.values()):
yield gen.sleep(0.01)
assert 100 < s.total_occupancy < 1000
def test_retire_workers_close(c, s, a, b):
yield s.retire_workers(close_workers=True)
assert not s.workers
while a.status != "closed" and b.status != "closed":
yield gen.sleep(0.01)
def _stop_worker_gently(self, worker):
yield self.scheduler.close_worker(worker=worker)
yield gen.sleep(0.9)
yield self.stop_worker(worker)
def do_test(self):
yield gen.sleep(3)
raise gen.Return('test')
def printresults():
logger.log(
STATS, '{} {} {} {} {}'.format(start.strftime("%Y-%m-%d %H:%M:%S"),
end.strftime("%Y-%m-%d %H:%M:%S"),
runtime, queryPerSec, queryratio))
querypermin = queryPerSec * 60
endtime = datetime.now(timezone('UTC')) + timedelta(minutes=runtime)
line = list()
popularitylist = list()
newquerylist = list()
if filename != "":
newquerylist = QueryGenerator.generateQueriesFromFile(
start, end, querypermin * runtime, timeAccessGenerator,
periodAccessGenerator, querytype, queryratio, filename)
elif isbatch == True:
newquerylist = QueryGenerator.generateQueries(
start, end, querypermin * runtime, timeAccessGenerator,
periodAccessGenerator, popularitylist, querytype, queryratio,
logger)
else:
#logger.info("Run.py start queryendtime "+str(start)+", "+str(endtime))
queryStartInterval = start
queryEndInterval = start + timedelta(minutes=1)
for i in range(0, runtime):
logger.info("Start generating queries for interval " +
str(queryStartInterval) + " - " +
str(queryEndInterval))
newquerylist.extend(
QueryGenerator.generateQueries(
queryStartInterval, queryEndInterval, querypermin,
timeAccessGenerator, periodAccessGenerator,
popularitylist, querytype, queryratio, logger))
queryEndInterval = queryEndInterval + timedelta(minutes=1)
logger.info("Finished generating queries. num queries generated " +
str(len(newquerylist)))
if filename != "" or isbatch == True:
count = 0
time = datetime.now(timezone('UTC'))
logger.info("Time: {}".format(time.strftime("%Y-%m-%d %H:%M:%S")))
nextminute = time + timedelta(minutes=1)
for query in newquerylist:
try:
line.append(
applyOperation(query, config, brokernameurl, logger))
except Exception as inst:
logger.error(type(inst)) # the exception instance
logger.error(inst.args) # arguments stored in .args
logger.error(
inst) # __str__ allows args to be printed directly
x, y = inst.args
logger.error('x =', x)
logger.error('y =', y)
count = count + 1
if count >= querypermin:
timediff = (nextminute -
datetime.now(timezone('UTC'))).total_seconds()
if timediff > 0:
yield gen.sleep(timediff)
count = 0
time = datetime.now(timezone('UTC'))
logger.info("Time: {}".format(
time.strftime("%Y-%m-%d %H:%M:%S")))
nextminute = time + timedelta(minutes=1)
else:
# frequency of queries per millisecond
queryPerMilliSecond = float(queryPerSec) / 1000
# number of samples spaced by 1 millisecond
numSamples = runtime * 60 * 1000
numQueries, querySchedule = genPoissonQuerySchedule(
queryPerMilliSecond, numSamples)
logger.info("Poisson numQueries = " + str(numQueries))
queryScheduleIdx = 0
count = 0
while count < len(newquerylist):
sample = querySchedule[queryScheduleIdx]
#logger.info("Poisson sample is "+str(sample[0])+", "+str(sample[1]))
if (sample[0] == 0):
#logger.info("Sleeping for "+str(sample[1]))
yield gen.sleep(
float(sample[1]) /
1000) # divide by 1000 to convert it into seconds
else:
for i in range(0, sample[0]):
try:
line.append(
applyOperation(newquerylist[count], config,
brokernameurl, logger))
#applyOperation(newquerylist[count], config, brokernameurl, logger)
newquerylist[count].setTxTime(datetime.now())
#logger.info("Running query "+str(sample[0]))
except Exception as inst:
logger.error(type(inst)) # the exception instance
logger.error(
inst.args) # arguments stored in .args
logger.error(
inst
) # __str__ allows args to be printed directly
count = count + 1
if count >= len(newquerylist):
break
queryScheduleIdx = queryScheduleIdx + 1
wait_iterator = gen.WaitIterator(*line)
while not wait_iterator.done():
try:
result = yield wait_iterator.next()
except Exception as e:
logger.error("Error {} from {}".format(
e, wait_iterator.current_future))
def callback(self, back_result):
master = self.master
if back_result == '9':
self.refund()
self.update_upstream(refund=True)
try:
self.back_result = str(self.back_result or self.up_back_result
or self.up_result or back_result)
master.hset('order:%s' % self.order_id, 'back_result',
self.back_result)
except Exception as e:
root_log.error("SET", e)
downstream = self.application.config['downstream'][self.user_id]
url = self.back_url
back_time = time.localtime()
if self.product == 'data':
body = json.dumps({
'order_id':
self.sp_order_id,
'transactionid':
self.order_id,
'orderstatus': ((back_result == '1') and 'finish') or 'fail',
'result_code':
self.back_result,
'plat_offer_id':
self.plat_offer_id,
'facevalue':
self.price,
'phone_id':
self.mobile,
'ordertime':
time.strftime("%Y-%m-%d %H:%M:%S", back_time)
})
else:
body = 'userid=%s&orderid=%s&sporderid=%s&merchantsubmittime=%s&resultno=%s' % (
self.user_id, self.order_id, self.sp_order_id,
time.strftime("%Y%m%d%H%M%S", back_time), back_result)
sign = signature(body + '&key=' + downstream['key'])
body += "&sign=" + sign
request_log.info('CALLBACK %s - %s' % (url, body),
extra={'orderid': self.order_id})
h = None
if downstream.get('content'):
h = {'Content-Type': 'application/json;charset=UTF-8'}
for i in range(3):
http_client = AsyncHTTPClient()
try:
response = yield http_client.fetch(url,
method='POST',
headers=h,
body=body)
if response and response.code == 200:
break
except Exception as e:
request_log.warn('CALLBACK FAIL - %s',
e,
extra={'orderid': self.order_id})
finally:
http_client.close()
# wait for 5*i secs
yield gen.sleep(60 * (i + 1))
# finish order
back_time = time.localtime()
try:
master.hset('order:%s' % self.order_id, 'back_time',
time.mktime(back_time))
master.sadd('list:finish', self.order_id)
master.srem('list:create', self.order_id)
except Exception as e:
root_log.error("MOVE ORDER %s", e)
def get(self):
self.write("asdf")
self.flush()
# Wait a bit to ensure the chunks are sent and received separately.
yield gen.sleep(0.01)
self.write("qwer")
def ping():
"""print dots to indicate idleness"""
while True:
yield gen.sleep(0.25)
print('.')
def f(dask_worker=None):
yield gen.sleep(0.001)
raise gen.Return(dask_worker.id)
def test_heartbeats(c, s, a, b):
pytest.importorskip('psutil')
start = time()
while not all(s.worker_info[w].get('memory-rss') for w in s.workers):
yield gen.sleep(0.01)
assert time() < start + 2
def test_wait_timeout(self):
c = locks.Condition()
wait = c.wait(timedelta(seconds=0.01))
self.io_loop.call_later(0.02, c.notify) # Too late.
yield gen.sleep(0.03)
self.assertFalse((yield wait))
def f(index):
with (yield sem.acquire()):
history.append('acquired %d' % index)
yield gen.sleep(0.01)
history.append('release %d' % index)
async def start(self, rate: int = 32): # Match client rate
self.prev_time = time.time()
while True:
timer = gen.sleep(1 / rate)
await self.tick()
await timer
def f(x, y):
yield gen.sleep(x)
raise gen.Return(y + 1)
def geninc(x, delay=0.02):
yield gen.sleep(delay)
raise gen.Return(x + 1)
def _start_instance(self, version, port):
""" Starts a Google App Engine application on this machine. It
will start it up and then proceed to fetch the main page.
Args:
version: A Version object.
port: An integer specifying a port to use.
"""
version_details = version.version_details
runtime = version_details['runtime']
env_vars = version_details.get('envVariables', {})
runtime_params = self._deployment_config.get_config('runtime_parameters')
max_memory = runtime_params.get('default_max_appserver_memory',
DEFAULT_MAX_APPSERVER_MEMORY)
if 'instanceClass' in version_details:
max_memory = INSTANCE_CLASSES.get(version_details['instanceClass'],
max_memory)
source_archive = version_details['deployment']['zip']['sourceUrl']
api_server_port = yield self._ensure_api_server(version.project_id)
yield self._source_manager.ensure_source(
version.revision_key, source_archive, runtime)
logger.info('Starting {}:{}'.format(version, port))
pidfile = PIDFILE_TEMPLATE.format(revision=version.revision_key, port=port)
if runtime == GO:
env_vars['GOPATH'] = os.path.join(UNPACK_ROOT, version.revision_key,
'gopath')
env_vars['GOROOT'] = os.path.join(GO_SDK, 'goroot')
watch = ''.join([MONIT_INSTANCE_PREFIX, version.revision_key])
if runtime in (PYTHON27, GO, PHP):
start_cmd = create_python27_start_cmd(
version.project_id,
self._login_server,
port,
pidfile,
version.revision_key,
api_server_port)
env_vars.update(create_python_app_env(self._login_server,
version.project_id))
elif runtime == JAVA:
# Account for MaxPermSize (~170MB), the parent process (~50MB), and thread
# stacks (~20MB).
max_heap = max_memory - 250
if max_heap <= 0:
raise BadConfigurationException(
'Memory for Java applications must be greater than 250MB')
start_cmd = create_java_start_cmd(
version.project_id,
port,
self._login_server,
max_heap,
pidfile,
version.revision_key,
api_server_port
)
env_vars.update(create_java_app_env(self._deployment_config))
else:
raise BadConfigurationException(
'Unknown runtime {} for {}'.format(runtime, version.project_id))
logging.info("Start command: " + str(start_cmd))
logging.info("Environment variables: " + str(env_vars))
monit_app_configuration.create_config_file(
watch,
start_cmd,
pidfile,
port,
env_vars,
max_memory,
self._syslog_server,
check_port=True,
kill_exceeded_memory=True)
full_watch = '{}-{}'.format(watch, port)
yield self._monit_operator.reload(self._thread_pool)
# The reload command does not block, and we don't have a good way to check
# if Monit is ready with its new configuration yet. If the daemon begins
# reloading while it is handling the 'start', it can end up in a state
# where it never starts the process. As a temporary workaround, this
# small period allows it to finish reloading. This can be removed if
# instances are started inside a cgroup.
yield gen.sleep(0.5)
yield self._monit_operator.send_command_retry_process(full_watch, 'start')
# Make sure the version registration node exists.
self._zk_client.ensure_path(
'/'.join([VERSION_REGISTRATION_NODE, version.version_key]))
instance = Instance(version.revision_key, port)
yield self._add_routing(instance)
if version.project_id == DASHBOARD_PROJECT_ID:
log_size = DASHBOARD_LOG_SIZE
else:
log_size = APP_LOG_SIZE
if not setup_logrotate(version.project_id, log_size):
logging.error("Error while setting up log rotation for application: {}".
format(version.project_id))
def f():
yield gen.sleep(1)
def test_non_existent_worker(c, s):
s.add_worker(address='127.0.0.1:5738', ncores=2, nbytes={}, host_info={})
futures = c.map(inc, range(10))
yield gen.sleep(4)
assert not s.workers
assert all(ts.state == 'no-worker' for ts in s.tasks.values())
def test_basic(c, s, a, b):
X, y = make_classification(n_samples=1000, n_features=5, chunks=100)
model = SGDClassifier(tol=1e-3, penalty="elasticnet")
params = {"alpha": np.logspace(-2, 1, num=50), "l1_ratio": [0.01, 1.0]}
X_test, y_test = X[:100], y[:100]
X_train = X[100:]
y_train = y[100:]
n_parameters = 5
param_list = list(ParameterSampler(params, n_parameters))
def additional_calls(info):
pf_calls = {k: v[-1]["partial_fit_calls"] for k, v in info.items()}
ret = {k: int(calls < 10) for k, calls in pf_calls.items()}
if len(ret) == 1:
return {list(ret)[0]: 0}
# Don't train one model
some_keys = set(ret.keys()) - {0}
del ret[random.choice(list(some_keys))]
return ret
info, models, history, best = yield fit(
model,
param_list,
X_train,
y_train,
X_test,
y_test,
additional_calls,
fit_params={"classes": [0, 1]},
)
# Ensure that we touched all data
keys = {t[0] for t in s.transition_log}
L = [str(k) in keys for kk in X_train.__dask_keys__() for k in kk]
assert all(L)
for model in models.values():
assert isinstance(model, Future)
model2 = yield model
assert isinstance(model2, SGDClassifier)
XX_test, yy_test = yield c.compute([X_test, y_test])
model = yield models[0]
assert model.score(XX_test, yy_test) == info[0][-1]["score"]
# `<` not `==` because we randomly dropped one model
assert len(history) < n_parameters * 10
for h in history:
assert {
"partial_fit_time",
"score_time",
"score",
"model_id",
"params",
"partial_fit_calls",
}.issubset(set(h.keys()))
groups = toolz.groupby("partial_fit_calls", history)
assert len(groups[1]) > len(groups[2]) > len(groups[3]) > len(
groups[max(groups)])
assert max(groups) == n_parameters
keys = list(models.keys())
for key in keys:
del models[key]
while c.futures or s.tasks: # Cleans up cleanly after running
yield gen.sleep(0.01)
# smoke test for ndarray X_test and y_test
X_test, y_test = yield c.compute([X_test, y_test])
info, models, history, best = yield fit(
model,
param_list,
X_train,
y_train,
X_test,
y_test,
additional_calls,
fit_params={"classes": [0, 1]},
)
def get(self):
yield gen.sleep(10)
print 'NoBlockingHnadler', datetime.datetime.now().strftime(
"%Y-%m-%d %H:%M:%S")
self.write('NoBlockingHnadler Request')
def minute_loop(logic):
while True:
print("logged players:", logic.players)
yield gen.sleep(20)
def sleep_some():
self.io_loop.run_sync(lambda: gen.sleep(0.05))
def get(self):
try:
yield self.git_lock.acquire(1)
except gen.TimeoutError:
self.emit({
'phase':
'error',
'message':
'Another git operations is currently running, try again in a few minutes'
})
return
try:
repo = self.get_argument('repo')
branch = self.get_argument('branch')
repo_dir = repo.split('/')[-1]
# We gonna send out event streams!
self.set_header('content-type', 'text/event-stream')
self.set_header('cache-control', 'no-cache')
gp = GitPuller(repo, branch, repo_dir)
q = Queue()
def pull():
try:
for line in gp.pull():
q.put_nowait(line)
# Sentinel when we're done
q.put_nowait(None)
except Exception as e:
q.put_nowait(e)
raise e
self.gp_thread = threading.Thread(target=pull)
self.gp_thread.start()
while True:
try:
progress = q.get_nowait()
except Empty:
yield gen.sleep(0.5)
continue
if progress is None:
break
if isinstance(progress, Exception):
self.emit({
'phase':
'error',
'message':
str(progress),
'output':
'\n'.join([
l.strip() for l in traceback.format_exception(
type(progress), progress,
progress.__traceback__)
])
})
return
self.emit({'output': progress, 'phase': 'syncing'})
self.emit({'phase': 'finished'})
except Exception as e:
self.emit({
'phase':
'error',
'message':
str(e),
'output':
'\n'.join([
l.strip() for l in traceback.format_exception(
type(e), e, e.__traceback__)
])
})
finally:
self.git_lock.release()
def test_explicit(c, s, a, b):
X, y = make_classification(n_samples=1000, n_features=10, chunks=(200, 10))
model = SGDClassifier(tol=1e-3, penalty="elasticnet")
params = [{"alpha": 0.1}, {"alpha": 0.2}]
def additional_calls(scores):
""" Progress through predefined updates, checking along the way """
ts = scores[0][-1]["partial_fit_calls"]
ts -= 1 # partial_fit_calls = time step + 1
if ts == 0:
assert len(scores) == len(params)
assert len(scores[0]) == 1
assert len(scores[1]) == 1
return {k: 2 for k in scores}
if ts == 2:
assert len(scores) == len(params)
assert len(scores[0]) == 2
assert len(scores[1]) == 2
return {0: 1, 1: 0}
elif ts == 3:
assert len(scores) == len(params)
assert len(scores[0]) == 3
assert len(scores[1]) == 2
return {0: 3}
elif ts == 6:
assert len(scores) == 1
assert len(scores[0]) == 4
return {0: 0}
else:
raise Exception()
info, models, history, best = yield fit(
model,
params,
X,
y,
X.blocks[-1],
y.blocks[-1],
additional_calls,
scorer=None,
fit_params={"classes": [0, 1]},
)
assert all(model.done() for model in models.values())
models = yield models
model = models[0]
meta = info[0][-1]
assert meta["params"] == {"alpha": 0.1}
assert meta["partial_fit_calls"] == 6 + 1
assert len(info) > len(models) == 1
assert set(models.keys()).issubset(set(info.keys()))
assert meta["partial_fit_calls"] == history[-1]["partial_fit_calls"]
calls = {
k: [h["partial_fit_calls"] for h in hist]
for k, hist in info.items()
}
for k, call in calls.items():
assert (np.diff(call) >= 1).all()
assert set(models.keys()) == {0}
del models[0]
while s.tasks or c.futures: # all data clears out
yield gen.sleep(0.01)
class ProcessManager(object):
'''
A class which will manage processes that should be running
'''
def __init__(self, name=None, wait_for_kill=1):
# pid -> {tgt: foo, Process: object, args: args, kwargs: kwargs}
self._process_map = {}
self.name = name
if self.name is None:
self.name = self.__class__.__name__
self.wait_for_kill = wait_for_kill
# store some pointers for the SIGTERM handler
self._pid = os.getpid()
self._sigterm_handler = signal.getsignal(signal.SIGTERM)
self._restart_processes = True
def add_process(self, tgt, args=None, kwargs=None, name=None):
'''
Create a processes and args + kwargs
This will deterimine if it is a Process class, otherwise it assumes
it is a function
'''
if args is None:
args = []
if kwargs is None:
kwargs = {}
if salt.utils.platform.is_windows():
# Need to ensure that 'log_queue' is correctly transferred to
# processes that inherit from 'MultiprocessingProcess'.
if type(MultiprocessingProcess) is type(tgt) and (issubclass(
tgt, MultiprocessingProcess)):
need_log_queue = True
else:
need_log_queue = False
if need_log_queue and 'log_queue' not in kwargs:
if hasattr(self, 'log_queue'):
kwargs['log_queue'] = self.log_queue
else:
kwargs['log_queue'] = (
salt.log.setup.get_multiprocessing_logging_queue())
# create a nicer name for the debug log
if name is None:
if isinstance(tgt, types.FunctionType):
name = '{0}.{1}'.format(
tgt.__module__,
tgt.__name__,
)
else:
name = '{0}{1}.{2}'.format(
tgt.__module__,
'.{0}'.format(tgt.__class__)
if str(tgt.__class__) != "<type 'type'>" else '',
tgt.__name__,
)
if type(multiprocessing.Process) is type(tgt) and issubclass(
tgt, multiprocessing.Process):
process = tgt(*args, **kwargs)
else:
process = multiprocessing.Process(target=tgt,
args=args,
kwargs=kwargs,
name=name)
if isinstance(process, SignalHandlingMultiprocessingProcess):
with default_signals(signal.SIGINT, signal.SIGTERM):
process.start()
else:
process.start()
log.debug("Started '{0}' with pid {1}".format(name, process.pid))
self._process_map[process.pid] = {
'tgt': tgt,
'args': args,
'kwargs': kwargs,
'Process': process
}
return process
def restart_process(self, pid):
'''
Create new process (assuming this one is dead), then remove the old one
'''
if self._restart_processes is False:
return
log.info('Process {0} ({1}) died with exit status {2},'
' restarting...'.format(
self._process_map[pid]['tgt'], pid,
self._process_map[pid]['Process'].exitcode))
# don't block, the process is already dead
self._process_map[pid]['Process'].join(1)
self.add_process(self._process_map[pid]['tgt'],
self._process_map[pid]['args'],
self._process_map[pid]['kwargs'])
del self._process_map[pid]
def stop_restarting(self):
self._restart_processes = False
def send_signal_to_processes(self, signal_):
if (salt.utils.platform.is_windows()
and signal_ in (signal.SIGTERM, signal.SIGINT)):
# On Windows, the subprocesses automatically have their signal
# handlers invoked. If you send one of these signals while the
# signal handler is running, it will kill the process where it
# is currently running and the signal handler will not finish.
# This will also break the process tree: children of killed
# children will become parentless and not findable when trying
# to kill the process tree (they don't inherit their parent's
# parent). Hence the 'MWorker' processes would be left over if
# the 'ReqServer' process is killed this way since 'taskkill'
# with the tree option will not be able to find them.
return
for pid in six.iterkeys(self._process_map.copy()):
try:
os.kill(pid, signal_)
except OSError as exc:
if exc.errno not in (errno.ESRCH, errno.EACCES):
# If it's not a "No such process" error, raise it
raise
# Otherwise, it's a dead process, remove it from the process map
del self._process_map[pid]
@gen.coroutine
def run(self, async=False):
'''
Load and start all available api modules
'''
log.debug('Process Manager starting!')
salt.utils.appendproctitle(self.name)
# make sure to kill the subprocesses if the parent is killed
if signal.getsignal(signal.SIGTERM) is signal.SIG_DFL:
# There are no SIGTERM handlers installed, install ours
signal.signal(signal.SIGTERM, self.kill_children)
if signal.getsignal(signal.SIGINT) is signal.SIG_DFL:
# There are no SIGINT handlers installed, install ours
signal.signal(signal.SIGINT, self.kill_children)
while True:
log.trace('Process manager iteration')
try:
# in case someone died while we were waiting...
self.check_children()
# The event-based subprocesses management code was removed from here
# because os.wait() conflicts with the subprocesses management logic
# implemented in `multiprocessing` package. See #35480 for details.
if async:
yield gen.sleep(10)
else:
time.sleep(10)
if len(self._process_map) == 0:
break
# OSError is raised if a signal handler is called (SIGTERM) during os.wait
except OSError:
break
except IOError as exc:
# IOError with errno of EINTR (4) may be raised
# when using time.sleep() on Windows.
if exc.errno != errno.EINTR:
raise
break
def generate_sequence():
for number in fibonacci():
store.submit(number)
yield gen.sleep(1)
if finish.is_set():
break
def cb(self):
while True:
L, self.buffer = self.buffer, []
self.last = self.emit(L)
yield self.last
yield gen.sleep(self.interval)
def get(self):
try:
id = self.get_argument('id')
# We gonna send out event streams!
self.set_header('content-type', 'text/event-stream')
self.set_header('cache-control', 'no-cache')
hs = HSPuller(id, self.settings['hydroshare'])
q = Queue()
def pull():
try:
for line in hs.pull():
q.put_nowait(line)
# Sentinel when we're done
q.put_nowait(None)
except Exception as e:
q.put_nowait(e)
raise e
self.hs_thread = threading.Thread(target=pull)
self.hs_thread.start()
while True:
try:
progress = q.get_nowait()
except Empty:
yield gen.sleep(0.5)
continue
if progress is None:
break
if isinstance(progress, Exception):
self.emit({
'phase':
'error',
'message':
str(progress),
'output':
'\n'.join([
l.strip() for l in traceback.format_exception(
type(progress), progress,
progress.__traceback__)
])
})
return
self.emit({'output': progress, 'phase': 'syncing'})
self.emit({'phase': 'finished'})
except Exception as e:
self.emit({
'phase':
'error',
'message':
str(e),
'output':
'\n'.join([
l.strip() for l in traceback.format_exception(
type(e), e, e.__traceback__)
])
})
def run():
yield [n._start(addr) for n in nannies]
while all(n.status != 'closed' for n in nannies):
yield gen.sleep(0.2)
def worker():
while True:
item = yield q.get()
self.accumulator += item
q.task_done()
yield gen.sleep(random() * 0.01)
