def run(self):
"""This is a dummy implementation. It will return immediately."""
self.progress_queue.put((ProgressQueueMsg.NETWORK_DONE, {
"ret": True,
"processed": self.n_requests,
"retried": self.n_retried,
"consumed": self.n_consumed,
"rusage": get_rusage(),
}))
def run(self):
"""This is a dummy implementation. It will return immediately."""
self.progress_queue.put((ProgressQueueMsg.NETWORK_DONE, {
"ret": True,
"processed": self.n_requests,
"retried": self.n_retried,
"consumed": self.n_consumed,
"rusage": get_rusage(),
}))
def run(self, dry_run=False):
if dry_run:
i = 0
for _ in self.perform_requests(True):
i += 1
return i
self._executor = ThreadPoolExecutor(self.concurrency)
self.session = requests.Session()
adapter = requests.adapters.HTTPAdapter(
pool_connections=self.concurrency, pool_maxsize=self.concurrency)
self.session.mount('http://', adapter)
self.session.mount('https://', adapter)
t0 = time()
last_report = time()
i = 0
r = None
for r in self.perform_requests():
if r is not True:
i += 1
self.ui.info('{} responses sent | time elapsed {}s'.format(
i,
time() - t0))
if time() - last_report > REPORT_INTERVAL:
self.progress_queue.put(
(ProgressQueueMsg.NETWORK_PROGRESS, {
"processed": self.n_requests,
"retried": self.n_retried,
"consumed": self.n_consumed,
"rusage": get_rusage(),
}))
last_report = time()
self.progress_queue.put((ProgressQueueMsg.NETWORK_DONE, {
"ret": r,
"processed": self.n_requests,
"retried": self.n_retried,
"consumed": self.n_consumed,
"rusage": get_rusage(),
}))
def run(self):
self._executor = ThreadPoolExecutor(self.concurrency)
self.session = requests.Session()
adapter = requests.adapters.HTTPAdapter(
pool_connections=self.concurrency, pool_maxsize=self.concurrency)
self.session.mount('http://', adapter)
self.session.mount('https://', adapter)
self.session.verify = self.verify_ssl
t0 = time()
last_report = time()
i = 0
r = None
for r in self.perform_requests():
if r is not True:
i += 1
self.ui.info('{} responses sent | time elapsed {}s'
.format(i, time() - t0))
if time() - last_report > REPORT_INTERVAL:
self.progress_queue.put((
ProgressQueueMsg.NETWORK_PROGRESS, {
"processed": self.n_requests,
"retried": self.n_retried,
"consumed": self.n_consumed,
"rusage": get_rusage(),
}))
last_report = time()
self.progress_queue.put((ProgressQueueMsg.NETWORK_DONE, {
"ret": r,
"processed": self.n_requests,
"retried": self.n_retried,
"consumed": self.n_consumed,
"rusage": get_rusage(),
}))
def _shove(self, args, serialized_dialect, queue):
dialect = serialized_dialect.to_dialect()
signal.signal(signal.SIGINT, self.exit_fast)
signal.signal(signal.SIGTERM, self.exit_fast)
t2 = time()
last_report = time()
_ui = args[4]
_ui.info('Shovel process started')
csv.register_dialect('dataset_dialect', dialect)
batch_generator = BatchGenerator(*args)
batch = None
try:
n = 0
self.shovel_status.value = b"R"
for batch in batch_generator:
if batch:
_ui.debug('queueing batch {}'.format(batch.id))
self.shovel_status.value = b"P"
while True:
try:
queue.put(batch, timeout=1)
break
except Full:
_ui.debug('put timed out')
if self.abort_flag.value:
_ui.info('shoveling abort requested')
self.exit_fast(None, None)
break
continue
n += 1
if self.abort_flag.value:
_ui.info('shoveling abort requested')
self.exit_fast(None, None)
break
if time() - last_report > REPORT_INTERVAL:
self.progress_queue.put((
ProgressQueueMsg.SHOVEL_PROGRESS, {
"produced": n,
"read": batch_generator.n_read,
"skipped": batch_generator.n_skipped,
"rusage": get_rusage()
}))
last_report = time()
self.shovel_status.value = b"R"
self.shovel_status.value = b"D"
_ui.info('shoveling complete | total time elapsed {}s'
''.format(time() - t2))
self.progress_queue.put((ProgressQueueMsg.SHOVEL_DONE,
{
"produced": n,
"read": batch_generator.n_read,
"skipped": batch_generator.n_skipped,
"rusage": get_rusage()
}))
except csv.Error as e:
self.shovel_status.value = b"C"
self.progress_queue.put((
ProgressQueueMsg.SHOVEL_CSV_ERROR,
{
"batch": batch and batch._replace(data=[]) or [],
"error": str(e),
"produced": n,
"read": batch_generator.n_read,
"skipped": batch_generator.n_skipped,
"rusage": get_rusage(),
},
))
raise
except Exception as e:
self.shovel_status.value = b"E"
self.progress_queue.put((
ProgressQueueMsg.SHOVEL_ERROR,
{
"batch": batch and batch._replace("data", []) or [],
"error": str(e),
"produced": n,
"read": batch_generator.n_read,
"skipped": batch_generator.n_skipped,
"rusage": get_rusage(),
},
))
raise
finally:
if os.name is 'nt':
_ui.close()
def process_response(self):
signal.signal(signal.SIGINT, self.exit_fast)
signal.signal(signal.SIGTERM, self.exit_fast)
"""Process a successful request. """
self._ui.debug('Writer Process started - {}'
''.format(multiprocessing.current_process().name))
rows_done = sum(rows for _, rows in self.ctx.db['checkpoints'])
success = False
processed = 0
written = 0
idle_cycles = 0
last_report = time()
try:
while True:
if self.abort_flag.value or self.local_abort_flag:
self._ui.debug('abort requested')
break
try:
if idle_cycles > 2:
self.writer_status.value = b"I"
else:
self.writer_status.value = b"G"
msg, args = self.writer_queue.get(timeout=1)
except queue.Empty:
idle_cycles += 1
continue
idle_cycles = 0
self.writer_status.value = b"W"
if msg == WriterQueueMsg.CTX_ERROR:
# pred_name is a message if ERROR or WARNING
self._ui.debug('Writer ERROR')
self.ctx.save_error(args["batch"], error=args["error"])
continue
elif msg == WriterQueueMsg.CTX_WARNING:
self._ui.debug('Writer WARNING')
self.ctx.save_warning(args["batch"], error=args["error"])
continue
elif msg == WriterQueueMsg.SENTINEL:
self._ui.debug('Writer received SENTINEL')
break
elif msg == WriterQueueMsg.RESPONSE:
processed += 1
batch = args["batch"]
unpack_data, format_data = self.response_handlers
try:
data, exec_time, elapsed_seconds = \
unpack_data(args['request'])
debug_msg = ('successful response {}-{}: exec time '
'{:.0f}msec | round-trip: {:.0f}msec')
self._ui.debug(
debug_msg.format(batch.id, batch.rows, exec_time,
elapsed_seconds * 1000))
except Exception as e:
self._ui.warning('{} response parse error: {} -- retry'
''.format(batch.id, e))
self.deque_failed_batch(batch)
continue
try:
written_fields, comb = format_data(
data, batch,
pred_name=self.ctx.pred_name,
pred_threshold_name=self.ctx.pred_threshold_name,
pred_decision_name=self.ctx.pred_decision_name,
keep_cols=self.ctx.keep_cols,
skip_row_id=self.ctx.skip_row_id,
fast_mode=self.ctx.fast_mode,
delimiter=self.ctx.dialect.delimiter,
max_prediction_explanations=(
self.ctx.max_prediction_explanations
)
)
except UnexpectedKeptColumnCount:
self._ui.fatal('Unexpected number of kept columns ' +
'retrieved. This can happen in ' +
'--fast mode with --keep_cols where ' +
'some cells contain quoted delimiters')
except NoPredictionThresholdInResult:
self._ui.fatal('No predictionThreshold returned from '
'API. --pred_threshold should be used '
'only for binary classification '
'predictions')
except Exception as e:
self._ui.fatal(e)
self.ctx.checkpoint_batch(batch, written_fields, comb)
written += 1
rows_done += batch.rows
if time() - last_report > REPORT_INTERVAL:
self.progress_queue.put((
ProgressQueueMsg.WRITER_PROGRESS, {
"processed": processed,
"written": written,
"rows": rows_done,
"rusage": get_rusage()
}))
last_report = time()
else:
self._ui.error('Unknown Writer Queue msg: "{}", args={}'
''.format(msg, args))
self._ui.debug('---Writer Exiting---')
success = True
if self.local_abort_flag:
success = False
except Exception as e:
# Note this won't catch SystemExit which is raised by
# sigterm_handler because it's based on BaseException
self._ui.error('Writer Process error: batch={}, error={}'
''.format(batch.id, e))
finally:
self.writer_status.value = b"D"
self.progress_queue.put((ProgressQueueMsg.WRITER_DONE, {
"ret": success,
"processed": processed,
"written": written,
"rows": rows_done,
"rusage": get_rusage()
}))
for o in [self.ctx, self.queue, self.writer_queue, self.deque,
self._ui]:
if hasattr(o, 'close'):
# On Windows the Queue doesn't have a close attr
o.close()
if success:
sys.exit(0)
else:
sys.exit(1)
def process_response(self):
signal.signal(signal.SIGINT, self.exit_fast)
signal.signal(signal.SIGTERM, self.exit_fast)
"""Process a successful request. """
self._ui.debug('Writer Process started - {}'
''.format(multiprocessing.current_process().name))
rows_done = 0
for _, rows in self.ctx.db['checkpoints']:
rows_done += rows
success = False
processed = 0
written = 0
idle_cycles = 0
last_report = time()
try:
while True:
if self.abort_flag.value or self.local_abort_flag:
self._ui.debug('abort requested')
break
try:
if idle_cycles > 2:
self.writer_status.value = b"I"
else:
self.writer_status.value = b"G"
msg, args = self.writer_queue.get(timeout=1)
except queue.Empty:
idle_cycles += 1
continue
idle_cycles = 0
self.writer_status.value = b"W"
if msg == WriterQueueMsg.CTX_ERROR:
# pred_name is a message if ERROR or WARNING
self._ui.debug('Writer ERROR')
self.ctx.save_error(args["batch"], error=args["error"])
continue
elif msg == WriterQueueMsg.CTX_WARNING:
self._ui.debug('Writer WARNING')
self.ctx.save_warning(args["batch"], error=args["error"])
continue
elif msg == WriterQueueMsg.SENTINEL:
self._ui.debug('Writer received SENTINEL')
break
elif msg == WriterQueueMsg.RESPONSE:
processed += 1
batch = args["batch"]
result = self.unpack_request_object(args["request"], batch)
if result is False: # unpack_request_object failed
continue
(written_fields,
comb) = self.format_result_data(result, batch)
self.ctx.checkpoint_batch(batch, written_fields, comb)
written += 1
rows_done += batch.rows
if time() - last_report > REPORT_INTERVAL:
self.progress_queue.put(
(ProgressQueueMsg.WRITER_PROGRESS, {
"processed": processed,
"written": written,
"rows": rows_done,
"rusage": get_rusage()
}))
last_report = time()
else:
self._ui.error('Unknown Writer Queue msg: "{}", args={}'
''.format(msg, args))
self._ui.debug('---Writer Exiting---')
success = True
if self.local_abort_flag:
success = False
except Exception as e:
# Note this won't catch SystemExit which is raised by
# sigterm_handler because it's based on BaseException
self._ui.error('Writer Process error: batch={}, error={}'
''.format(batch.id, e))
finally:
self.writer_status.value = b"D"
self.progress_queue.put((ProgressQueueMsg.WRITER_DONE, {
"ret": success,
"processed": processed,
"written": written,
"rows": rows_done,
"rusage": get_rusage()
}))
for o in [
self.ctx, self.queue, self.writer_queue, self.deque,
self._ui
]:
if hasattr(o, 'close'):
# On Windows the Queue doesn't have a close attr
o.close()
if success:
sys.exit(0)
else:
sys.exit(1)
def _shove(self, args, serialized_dialect, queue):
dialect = serialized_dialect.to_dialect()
signal.signal(signal.SIGINT, self.exit_fast)
signal.signal(signal.SIGTERM, self.exit_fast)
t2 = time()
last_report = time()
_ui = args[4]
_ui.info('Shovel process started')
csv.register_dialect('dataset_dialect', dialect)
batch_generator = BatchGenerator(*args)
try:
n = 0
self.shovel_status.value = b"R"
for batch in batch_generator:
if batch:
_ui.debug('queueing batch {}'.format(batch.id))
self.shovel_status.value = b"P"
while True:
try:
queue.put(batch, timeout=1)
break
except Full:
_ui.debug('put timed out')
if self.abort_flag.value:
_ui.info('shoveling abort requested')
self.exit_fast(None, None)
break
continue
n += 1
if self.abort_flag.value:
_ui.info('shoveling abort requested')
self.exit_fast(None, None)
break
if time() - last_report > REPORT_INTERVAL:
self.progress_queue.put((
ProgressQueueMsg.SHOVEL_PROGRESS, {
"produced": n,
"read": batch_generator.n_read,
"skipped": batch_generator.n_skipped,
"rusage": get_rusage()
}))
last_report = time()
self.shovel_status.value = b"R"
self.shovel_status.value = b"D"
_ui.info('shoveling complete | total time elapsed {}s'
''.format(time() - t2))
self.progress_queue.put((ProgressQueueMsg.SHOVEL_DONE,
{
"produced": n,
"read": batch_generator.n_read,
"skipped": batch_generator.n_skipped,
"rusage": get_rusage()
}))
except csv.Error as e:
self.shovel_status.value = b"C"
self.progress_queue.put((ProgressQueueMsg.SHOVEL_CSV_ERROR,
{
"batch": batch._replace(data=[]),
"error": str(e),
"produced": n,
"read": batch_generator.n_read,
"skipped": batch_generator.n_skipped,
"rusage": get_rusage()
}))
raise
except Exception as e:
self.shovel_status.value = b"E"
self.progress_queue.put((ProgressQueueMsg.SHOVEL_ERROR,
{
"batch": batch._replace("data", []),
"error": str(e),
"produced": n,
"read": batch_generator.n_read,
"skipped": batch_generator.n_skipped,
"rusage": get_rusage()
}))
raise
finally:
if os.name is 'nt':
_ui.close()
def process_response(self):
signal.signal(signal.SIGINT, self.exit_fast)
signal.signal(signal.SIGTERM, self.exit_fast)
"""Process a successful request. """
self._ui.debug('Writer Process started - {}'
''.format(multiprocessing.current_process().name))
rows_done = sum(rows for _, rows in self.ctx.db['checkpoints'])
success = False
processed = 0
written = 0
idle_cycles = 0
last_report = time()
try:
while True:
if self.abort_flag.value or self.local_abort_flag:
self._ui.debug('abort requested')
break
try:
if idle_cycles > 2:
self.writer_status.value = b"I"
else:
self.writer_status.value = b"G"
msg, args = self.writer_queue.get(timeout=1)
except queue.Empty:
idle_cycles += 1
continue
idle_cycles = 0
self.writer_status.value = b"W"
if msg == WriterQueueMsg.CTX_ERROR:
# pred_name is a message if ERROR or WARNING
self._ui.debug('Writer ERROR')
self.ctx.save_error(args["batch"], error=args["error"])
continue
elif msg == WriterQueueMsg.CTX_WARNING:
self._ui.debug('Writer WARNING')
self.ctx.save_warning(args["batch"], error=args["error"])
continue
elif msg == WriterQueueMsg.SENTINEL:
self._ui.debug('Writer received SENTINEL')
break
elif msg == WriterQueueMsg.RESPONSE:
processed += 1
batch = args["batch"]
unpack_data, format_data = self.response_handlers
try:
data, exec_time, elapsed_seconds = \
unpack_data(args['request'])
debug_msg = ('successful response {}-{}: exec time '
'{:.0f}msec | round-trip: {:.0f}msec')
self._ui.debug(
debug_msg.format(batch.id, batch.rows, exec_time,
elapsed_seconds * 1000))
except Exception as e:
self._ui.warning('{} response parse error: {} -- retry'
''.format(batch.id, e))
self.deque_failed_batch(batch)
continue
try:
written_fields, comb = format_data(
data, batch,
pred_name=self.ctx.pred_name,
keep_cols=self.ctx.keep_cols,
skip_row_id=self.ctx.skip_row_id,
fast_mode=self.ctx.fast_mode,
delimiter=self.ctx.dialect.delimiter)
except Exception as e:
self._ui.fatal(e)
self.ctx.checkpoint_batch(batch, written_fields, comb)
written += 1
rows_done += batch.rows
if time() - last_report > REPORT_INTERVAL:
self.progress_queue.put((
ProgressQueueMsg.WRITER_PROGRESS, {
"processed": processed,
"written": written,
"rows": rows_done,
"rusage": get_rusage()
}))
last_report = time()
else:
self._ui.error('Unknown Writer Queue msg: "{}", args={}'
''.format(msg, args))
self._ui.debug('---Writer Exiting---')
success = True
if self.local_abort_flag:
success = False
except Exception as e:
# Note this won't catch SystemExit which is raised by
# sigterm_handler because it's based on BaseException
self._ui.error('Writer Process error: batch={}, error={}'
''.format(batch.id, e))
finally:
self.writer_status.value = b"D"
self.progress_queue.put((ProgressQueueMsg.WRITER_DONE, {
"ret": success,
"processed": processed,
"written": written,
"rows": rows_done,
"rusage": get_rusage()
}))
for o in [self.ctx, self.queue, self.writer_queue, self.deque,
self._ui]:
if hasattr(o, 'close'):
# On Windows the Queue doesn't have a close attr
o.close()
if success:
sys.exit(0)
else:
sys.exit(1)
