def logview():
term = blessings.Terminal()
if not term.is_a_tty:
print("logview output cannot be piped, it is for display only")
elif term.number_of_colors < 16:
print(
"logview requires a terminal that supports a minimum of 16 colours"
)
else:
global config
config = parse_args()
print(f"\x1b]2;{TERM_TITLE}\x07", end="")
colors.extend(COLORS[term.number_of_colors])
for filename in config.file:
filepath = os.path.abspath(os.path.join(config.dir, filename))
if os.access(filepath, os.R_OK, effective_ids=True):
windows.append(Window(filepath, config, Logger))
elif os.access(filepath, os.F_OK, effective_ids=True):
print(f"File '{filepath}' is not readable")
else:
print(f"File '{filepath}' does not exist")
if windows and not windows[-1].reader.isOpen:
windows.pop(len(windows) - 1)
if windows:
screen = None
try:
screen = curses.initscr()
curses.noecho()
curses.cbreak()
screen.keypad(True)
screen.nodelay(True)
screen.getch()
print(term.enter_fullscreen + term.hide_cursor +
term.color(colors[APP_FG]) +
term.on_color(colors[APP_BG]) + term.clear)
draw_app(term)
calculate_windows(term, False)
for window in windows:
window.start(term)
# window.load()
# window.refresh(True)
with futures.ThreadPoolExecutor() as pool:
threads = {
pool.submit(window.load): window
for window in windows
}
for future in futures.as_completed(threads):
threads[future].refresh(True)
main_loop(term, screen)
except Exception as err:
raise err
finally:
tprint(term, term.exit_fullscreen + term.normal_cursor)
curses.nocbreak()
if screen:
screen.keypad(False)
curses.echo()
curses.endwin()
print(*log, sep="\n")
exit(0)
exit(1)
def gather_stage(self, harvest_job):
self.job = harvest_job
self._set_source_config(self.job.source.config)
log.debug('Landsat-8 Harvester gather_stage for job: %r', harvest_job)
self.provider = 'usgs'
if not hasattr(self, 'provider_logger'):
self.provider_logger = self.make_provider_logger()
last_path_row = self._get_last_harvesting_tile(harvest_job.source_id)
if last_path_row is not None:
path = int(last_path_row[0])
row = int(last_path_row[1])
if path + 1 > PATH_MAX:
path = 1
row = 1 if row + 1 > ROW_MAX else row + 1
else:
path = path + 1
else:
path = self.source_config.get('path', 1)
row = self.source_config.get('row', 1)
update_all = self.source_config.get('update_all', False)
bucket = self.source_config['bucket']
s3 = self._set_s3_session()
_ls_worker = partial(aws.list_directory, bucket, s3=s3)
ids = []
log.info('Harvesting Path: {} and Row: {}'.format(path, row))
path = self._zeropad(path, 3)
row = self._zeropad(row, 3)
prefixes = ['c1/L8/{}/{}/'.format(path, row)]
with futures.ThreadPoolExecutor(max_workers=2) as executor:
results = executor.map(_ls_worker, prefixes)
results = itertools.chain.from_iterable(results)
scene_ids = [os.path.basename(key.strip('/')) for key in results]
while not scene_ids:
path = int(path)
row = int(row)
if path + 1 > PATH_MAX:
path = 1
row = 1 if row + 1 > ROW_MAX else row + 1
else:
path = path + 1
log.info('Harvesting Path: {} and Row: {}'.format(path, row))
path = self._zeropad(path, 3)
row = self._zeropad(row, 3)
prefixes = ['c1/L8/{}/{}/'.format(path, row)]
with futures.ThreadPoolExecutor(max_workers=2) as executor:
results = executor.map(_ls_worker, prefixes)
results = itertools.chain.from_iterable(results)
scene_ids = [os.path.basename(key.strip('/')) for key in results]
for scene in scene_ids:
_id = self._gather_entry(scene, int(path), int(row), update_all)
if _id:
ids.append(_id)
return ids
def main():
server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
helloworld_pb2_grpc.add_GreeterServicer_to_server(Greeter(), server)
server.add_insecure_port('[::]:50051')
server.start()
server.wait_for_termination()
proxies_list = ['http://' + proxy for proxy in args.proxies]
else:
proxies_list = []
num_threads = args.threads
cnpj_list_to_import, filesFound, filesNotFound = read_cnpj_source_files(
args.args)
temp_dataset = load_temp_dataset()
cnpj_list = remaining_cnpjs(cnpj_list_to_import, temp_dataset)
print('%i CNPJ\'s to be fetched' % len(cnpj_list))
print('Starting fetch. {0} worker threads and {1} http proxies'.format(
num_threads, len(proxies_list)))
# Try again in case of error during fetch_cnpj_info
while len(cnpj_list) > 0:
with futures.ThreadPoolExecutor(max_workers=num_threads) as executor:
future_to_cnpj_info = dict((executor.submit(fetch_cnpj_info, cnpj), cnpj)
for cnpj in cnpj_list)
last_saving_point = 0
for future in futures.as_completed(future_to_cnpj_info):
cnpj = future_to_cnpj_info[future]
if future.exception() is None and future.result() is not None and future.result()['status'] == 'OK':
result_translated = transform_and_translate_data(
future.result())
temp_dataset = pd.concat([temp_dataset, result_translated])
if last_saving_point < divmod(len(temp_dataset.index), 100)[0]:
last_saving_point = divmod(len(temp_dataset.index), 100)[0]
print('###################################')
print('Saving information already fetched. {0} records'.format(
len(temp_dataset.index)))
temp_dataset.to_csv(TEMP_DATASET_PATH,
# create a class to define the server functions, derived from
# calculator_pb2_grpc.CalculatorServicer
class CalculatorServicer(calculator_pb2_grpc.CalculatorServicer):
# calculator.square_root is exposed here
# the request and response are of the data type
# calculator_pb2.Number
def SquareRoot(self, request, context):
response = calculator_pb2.Number()
response.value = calculator.square_root(request.value)
return response
# create a gRPC server
server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
# use the generated function `add_CalculatorServicer_to_server`
# to add the defined class to the server
calculator_pb2_grpc.add_CalculatorServicer_to_server(CalculatorServicer(),
server)
# listen on port 50051
print('Starting server. Listening on port 50051.')
server.add_insecure_port('[::]:50051')
server.start()
# since server.start() will not block,
# a sleep-loop is added to keep alive
try:
while True:
def test_abort(self):
"""Check that we can catch an abort properly"""
# Intercept gRPC calls...
interceptor = server_interceptor()
# our detailed failure message
failure_message = "This is a test failure"
# aborting RPC handler
def handler(request, context):
context.abort(grpc.StatusCode.FAILED_PRECONDITION, failure_message)
with futures.ThreadPoolExecutor(max_workers=1) as executor:
server = grpc.server(
executor,
options=(("grpc.so_reuseport", 0), ),
interceptors=[interceptor],
)
server.add_generic_rpc_handlers((UnaryUnaryRpcHandler(handler), ))
port = server.add_insecure_port("[::]:0")
channel = grpc.insecure_channel(f"localhost:{port:d}")
rpc_call = "TestServicer/handler"
server.start()
# unfortunately, these are just bare exceptions in grpc...
with self.assertRaises(Exception):
channel.unary_unary(rpc_call)(b"")
server.stop(None)
spans_list = self.memory_exporter.get_finished_spans()
self.assertEqual(len(spans_list), 1)
span = spans_list[0]
self.assertEqual(span.name, rpc_call)
self.assertIs(span.kind, trace.SpanKind.SERVER)
# Check version and name in span's instrumentation info
self.assertEqualSpanInstrumentationInfo(
span, opentelemetry.instrumentation.grpc)
# make sure this span errored, with the right status and detail
self.assertEqual(span.status.status_code, StatusCode.ERROR)
self.assertEqual(
span.status.description,
f"{grpc.StatusCode.FAILED_PRECONDITION}:{failure_message}",
)
# Check attributes
self.assertSpanHasAttributes(
span,
{
SpanAttributes.NET_PEER_IP:
"[::1]",
SpanAttributes.NET_PEER_NAME:
"localhost",
SpanAttributes.RPC_METHOD:
"handler",
SpanAttributes.RPC_SERVICE:
"TestServicer",
SpanAttributes.RPC_SYSTEM:
"grpc",
SpanAttributes.RPC_GRPC_STATUS_CODE:
grpc.StatusCode.FAILED_PRECONDITION.value[0],
},
)
class Crawler:
'''Blueprint for creating new spiders'''
home_url = ''
novel_url = ''
last_visited_url = None
scraper = cfscrape.create_scraper()
executor = futures.ThreadPoolExecutor(max_workers=5)
'''Must resolve these fields inside `read_novel_info`'''
novel_title = 'N/A'
novel_author = 'N/A'
novel_cover = None
novel_summary = None
'''
Each item must contain these keys:
`title` - the title of the volume
'''
volumes = []
'''
Each item must contain these keys:
`id` - the index of the chapter
`title` - the title name
`volume` - the volume id of this chapter
`url` - the link where to download the chapter
`name` - the chapter name, e.g: 'Chapter 3' or 'Chapter 12 (Special)'
'''
chapters = []
def __init__(self):
self.scraper.verify = False
# ------------------------------------------------------------------------- #
# Implement these methods
# ------------------------------------------------------------------------- #
def initialize(self):
pass
def dispose(self):
pass
@property
def supports_login(self):
'''Whether the crawler supports login() and logout method'''
return False
def login(self, email, password):
pass
def logout(self):
pass
def read_novel_info(self, url):
'''Get novel title, autor, cover etc'''
pass
def download_chapter_list(self):
'''Download list of chapters and volumes.'''
pass
def download_chapter_body(self, chapter):
'''Download body of a single chapter and return as clean html format.'''
pass
def get_chapter_index_of(self, url):
'''Return the index of chapter by given url or 0'''
url = (url or '').strip().strip('/')
for chapter in self.chapters:
if chapter['url'] == url:
return chapter['id']
return 0
# ------------------------------------------------------------------------- #
# Helper methods to be used
# ------------------------------------------------------------------------- #
@property
def headers(self):
return self.scraper.headers.copy()
@property
def cookies(self):
return {x.name: x.value for x in self.scraper.cookies}
def absolute_url(self, url):
if not url or len(url) == 0:
return None
elif url.startswith('//'):
return 'http:' + url
elif url.find('//') >= 0:
return url
elif url.startswith('/'):
return self.home_url + url
else:
return (self.last_visited_url or self.home_url) + '/' + url
def get_response(self, url, incognito=False):
self.last_visited_url = url.strip('/')
response = self.scraper.get(url)
response.encoding = 'utf-8'
self.cookies.update({x.name: x.value for x in response.cookies})
return response
def submit_form(self, url, multipart=False, headers={}, **data):
'''Submit a form using post request'''
headers = {
'content-type':
'multipart/form-data'
if multipart else 'application/x-www-form-urlencoded'
}
response = self.scraper.post(url, data=data, headers=headers)
self.cookies.update({x.name: x.value for x in response.cookies})
return response
def download_cover(self, output_file):
response = self.get_response(self.novel_cover)
with open(output_file, 'wb') as f:
f.write(response.content)
# end with
blacklist_patterns = [
r'^(volume|chapter) .?\d+$',
]
def not_blacklisted(self, text):
for pattern in self.blacklist_patterns:
if re.search(pattern, text, re.IGNORECASE):
return False
return True
def extract_contents(self, contents, level=0):
body = []
for elem in contents:
if ['script', 'iframe', 'form', 'a', 'br', 'img'].count(elem.name):
continue
elif ['h3', 'div', 'p'].count(elem.name):
body += self.extract_contents(elem.contents, level + 1)
continue
if not elem.name:
text = str(elem).strip()
else:
text = '<%s>%s</%s>' % (elem.name, elem.text.strip(),
elem.name)
patterns = [
re.compile(r'<!--(.|\n)*-->', re.MULTILINE),
re.compile(r'\[if (.|\n)*!\[endif\]', re.MULTILINE),
]
for x in patterns:
text = x.sub('', text).strip()
if text:
body.append(text)
if level == 0:
return [x for x in body if len(x) and self.not_blacklisted(x)]
else:
return body
class Hexitec():
"""Hexitec: Class that extracts and stores information about system-level parameters."""
# Thread executor used for background tasks
thread_executor = futures.ThreadPoolExecutor(max_workers=3)
def __init__(self, options):
"""Initialise the Hexitec object.
This constructor initialises the Hexitec object, building a
parameter tree and launching a background task if enabled
"""
defaults = HexitecDetectorDefaults()
self.file_dir = options.get("save_dir", defaults.save_dir)
self.file_name = options.get("save_file", defaults.save_file)
self.number_frames = options.get("acquisition_num_frames",
defaults.number_frames)
self.number_frames_to_request = self.number_frames
self.total_delay = 0.0
# Backup number_frames as first initialisation temporary sets number_frames = 2
self.backed_up_number_frames = self.number_frames
self.duration = 1
self.duration_enable = False
self.daq = HexitecDAQ(self, self.file_dir, self.file_name)
self.adapters = {}
self.fem = None
for key, value in options.items():
if "fem" in key:
fem_info = value.split(',')
fem_info = [(i.split('=')[0], i.split('=')[1])
for i in fem_info]
fem_dict = {
fem_key.strip(): fem_value.strip()
for (fem_key, fem_value) in fem_info
}
logging.debug(fem_dict)
self.fem = HexitecFem(
self,
fem_dict.get("server_ctrl_ip_addr",
defaults.fem["server_ctrl_ip"]),
fem_dict.get("camera_ctrl_ip_addr",
defaults.fem["camera_ctrl_ip"]),
fem_dict.get("server_data_ip_addr",
defaults.fem["server_data_ip"]),
fem_dict.get("camera_data_ip_addr",
defaults.fem["camera_data_ip"]))
if not self.fem:
self.fem = HexitecFem(
parent=self,
server_ctrl_ip_addr=defaults.fem["server_ctrl_ip"],
camera_ctrl_ip_addr=defaults.fem["camera_ctrl_ip"],
server_data_ip_addr=defaults.fem["server_data_ip"],
camera_data_ip_addr=defaults.fem["camera_data_ip"])
self.fem_health = True
# Bias (clock) tracking variables #
self.bias_clock_running = False
self.bias_init_time = 0 # Placeholder
self.bias_blocking_acquisition = False
self.extended_acquisition = False # Track acquisition spanning bias window(s)
self.frames_already_acquired = 0 # Track frames acquired across collection windows
self.collect_and_bias_time = self.fem.bias_refresh_interval + \
self.fem.bias_voltage_settle_time + self.fem.time_refresh_voltage_held
# Tracks whether first acquisition of multiple, bias-window(s), collection
self.initial_acquisition = True
# Tracks whether 2 frame fudge collection: (during cold initialisation)
self.first_initialisation = True
self.acquisition_in_progress = False
# Watchdog variables
self.error_margin = 400 # TODO: Revisit timeouts
self.fem_tx_timeout = 5000
self.daq_rx_timeout = self.collect_and_bias_time + self.error_margin
self.fem_start_timestamp = 0
self.time_waiting_for_data_arrival = 0
# Store initialisation time
self.init_time = time.time()
self.system_health = True
self.status_message = ""
self.status_error = ""
self.elog = ""
self.number_nodes = 1
# Software states:
# Cold, Disconnected, Idle, Acquiring
self.software_state = "Cold"
self.cold_initialisation = True
detector = ParameterTree({
"fem":
self.fem.param_tree,
"daq":
self.daq.param_tree,
"connect_hardware": (None, self.connect_hardware),
"initialise_hardware": (None, self.initialise_hardware),
"disconnect_hardware": (None, self.disconnect_hardware),
"collect_offsets": (None, self._collect_offsets),
"commit_configuration": (None, self.commit_configuration),
"software_state": (lambda: self.software_state, None),
"cold_initialisation": (lambda: self.cold_initialisation, None),
"hv_on": (None, self.hv_on),
"hv_off": (None, self.hv_off),
"acquisition": {
"number_frames":
(lambda: self.number_frames, self.set_number_frames),
"duration": (lambda: self.duration, self.set_duration),
"duration_enable":
(lambda: self.duration_enable, self.set_duration_enable),
"start_acq": (None, self.acquisition),
"stop_acq": (None, self.cancel_acquisition)
},
"status": {
"system_health": (lambda: self.system_health, None),
"status_message": (lambda: self.status_message, None),
"status_error": (lambda: self.status_error, None),
"elog": (lambda: self.elog, self.set_elog),
"fem_health": (lambda: self.fem_health, None),
"number_nodes":
(lambda: self.number_nodes, self.set_number_nodes)
}
})
self.system_info = SystemInfo()
# Store all information in a parameter tree
self.param_tree = ParameterTree({
"system_info": self.system_info.param_tree,
"detector": detector
})
self._start_polling()
def _start_polling(self):
IOLoop.instance().add_callback(self.polling)
def polling(self): # pragma: no cover
"""Poll FEM for status.
Check if acquisition completed (if initiated), for error(s) and
whether DAQ/FEM watchdogs timed out.
"""
# Poll FEM acquisition & health status
self.poll_fem()
# Watchdog: Watch FEM in case no data from hardware triggered by fem.acquire_data()
self.check_fem_watchdog()
# TODO: WATCHDOG, monitor HexitecDAQ rate of frames_processed updated.. (Break if stalled)
self.check_daq_watchdog()
IOLoop.instance().call_later(1.0, self.polling)
def get_frames_processed(self):
"""Get number of frames processed across node(s)."""
status = self._get_od_status("fp")
frames_processed = 0
for index in status:
# rank = index.get('hdf', None).get('rank')
# frames = index.get('histogram').get('frames_processed')
# print(" g_f_p(), rank: {} frames_processed: {}".format(rank, frames))
frames_processed = frames_processed + index.get('histogram').get(
'frames_processed')
return frames_processed
def poll_fem(self):
"""Poll FEM for acquisition and health status."""
if self.fem.acquisition_completed:
frames_processed = self.get_frames_processed()
# Either cold initialisation (first_initialisation is True, therefore only 2 frames
# expected) or, ordinary collection (self.number_frames frames expected)
if ((self.first_initialisation and (frames_processed == 2))
or (frames_processed == self.number_frames)): # noqa: W503
if self.first_initialisation:
self.first_initialisation = False
self.number_frames = self.backed_up_number_frames # TODO: redundant
# Reset FEM's acquisiton status ahead of future acquisitions
self.fem.acquisition_completed = False
# TODO: Also check sensor values?
# ..
fem_health = self.fem.get_health()
self.fem_health = fem_health
if self.system_health:
self.status_error = self.fem._get_status_error()
self.status_message = self.fem._get_status_message()
self.system_health = self.system_health and self.fem_health
def check_fem_watchdog(self):
"""Check data sent when FEM acquiring data."""
if self.acquisition_in_progress:
# TODO: Monitor FEM in case no data following fem.acquire_data() call
if (self.fem.hardware_busy):
fem_begun = self.fem.acquire_timestamp
delta_time = time.time() - fem_begun
logging.debug(" FEM w-dog: {0:.2f} < {1:.2f}".format(
delta_time, self.fem_tx_timeout))
if (delta_time > self.fem_tx_timeout):
self.fem.stop_acquisition = True
self.shutdown_processing()
logging.error("FEM data transmission timed out")
error = "Timed out waiting ({0:.2f} seconds) for FEM data".format(
delta_time)
self.fem._set_status_message(error)
def check_daq_watchdog(self):
"""Monitor DAQ's frames_processed while data processed.
Ensure frames_processed increments, completes within reasonable time of acquisition.
Failure to do so indicate missing/dropped packet(s), stop processing if stalled.
"""
if self.daq.in_progress:
processed_timestamp = self.daq.processed_timestamp
delta_time = time.time() - processed_timestamp
if (delta_time > self.daq_rx_timeout):
logging.error(" DAQ -- PROCESSING TIMED OUT")
# DAQ: Timed out waiting for next frame to process
self.shutdown_processing()
logging.error(
"DAQ processing timed out; Saw %s expected %s frames" %
(self.daq.frames_processed,
self.daq.frame_end_acquisition))
self.fem._set_status_error(
"Processing timed out: {0:.2f} seconds \
(exceeded {1:.2f}); Expected {2} got {3} frames\
".format(delta_time, self.daq_rx_timeout,
self.daq.frame_end_acquisition,
self.daq.frames_processed))
self.fem._set_status_message("Processing abandoned")
def shutdown_processing(self):
"""Stop processing in DAQ."""
self.daq.shutdown_processing = True
self.acquisition_in_progress = False
def _get_od_status(self, adapter):
"""Get status from adapter."""
try:
request = ApiAdapterRequest(None, content_type="application/json")
response = self.adapters[adapter].get("status", request)
response = response.data["value"]
except KeyError:
logging.warning("%s Adapter Not Found" % adapter)
response = [{"Error": "Adapter {} not found".format(adapter)}]
finally:
return response
def connect_hardware(self, msg):
"""Set up watchdog timeout, start bias clock and connect with hardware."""
# TODO: Must recalculate collect and bias time both here and in initialise()
# Logically, commit_configuration() is the best place but it updates variables before
# reading .ini file
self.collect_and_bias_time = self.fem.bias_refresh_interval + \
self.fem.bias_voltage_settle_time + self.fem.time_refresh_voltage_held
self.daq_rx_timeout = self.collect_and_bias_time + self.error_margin
# Start bias clock if not running
if not self.bias_clock_running:
IOLoop.instance().add_callback(self.start_bias_clock)
self.fem.connect_hardware(msg)
self.software_state = "Idle"
def start_bias_clock(self):
"""Set up bias 'clock'."""
if not self.bias_clock_running:
self.bias_init_time = time.time()
self.bias_clock_running = True
self.poll_bias_clock()
def poll_bias_clock(self):
"""Call periodically (0.1 seconds often enough??) to bias window status.
Are we in bias refresh intv / refresh volt held / Settle time ?
Example: 60000 / 3000 / 2000: Collect for 60s, pause for 3+2 secs
"""
current_time = time.time()
time_elapsed = current_time - self.bias_init_time
if (time_elapsed < self.fem.bias_refresh_interval):
# Still within collection window - acquiring data is allowed
pass
else:
if (time_elapsed < self.collect_and_bias_time):
# Blackout period - Wait for electrons to replenish/voltage to stabilise
self.bias_blocking_acquisition = True
else:
# Beyond blackout period - Back within bias
# Reset bias clock
self.bias_init_time = current_time
self.bias_blocking_acquisition = False
IOLoop.instance().call_later(0.1, self.poll_bias_clock)
def initialise_hardware(self, msg):
"""Initialise hardware.
Recalculate collect and bias timing, update watchdog timeout.
"""
# TODO: Must recalculate collect and bias time both here and in initialise();
# Logically, commit_configuration() is the best place but it updates variables before
# values read from .ini file
self.collect_and_bias_time = self.fem.bias_refresh_interval + \
self.fem.bias_voltage_settle_time + self.fem.time_refresh_voltage_held
self.daq_rx_timeout = self.collect_and_bias_time + self.error_margin
# If first initialisation, ie fudge, temporarily change number_frames to 2
# Adapter also controls this change in FEM
if self.first_initialisation:
self.backed_up_number_frames = self.number_frames
self.number_frames = 2
# TODO: Fix this fudge?
self.fem.acquire_timestamp = time.time()
self.acquisition_in_progress = True
self.fem.initialise_hardware(msg)
# Wait for fem initialisation/fudge frames
IOLoop.instance().call_later(0.5, self.monitor_fem_progress)
def disconnect_hardware(self, msg):
"""Disconnect FEM's hardware connection."""
if self.daq.in_progress:
# Stop hardware if still in acquisition
if self.fem.hardware_busy:
self.cancel_acquisition()
# Reset daq
self.shutdown_processing()
# Allow processing to shutdown before disconnecting hardware
IOLoop.instance().call_later(0.2, self.fem.disconnect_hardware)
else:
# Nothing in progress, disconnect hardware
self.fem.disconnect_hardware(msg)
self.software_state = "Disconnected"
# Reset system status
self.status_error = ""
self.status_message = ""
self.system_health = True
# Stop bias clock
if self.bias_clock_running:
self.bias_clock_running = False
def set_duration_enable(self, duration_enable):
"""Set duration enable, calculating number of frames accordingly."""
self.duration_enable = duration_enable
self.fem.set_duration_enable(duration_enable)
# Ensure DAQ, FEM have correct duration/number of frames configured
if duration_enable:
self.set_duration(self.duration)
else:
# print("\n\tadp.set_duration_enable({}) number_frames: {}\n".format(duration_enable, self.number_frames))
self.set_number_frames(self.number_frames)
def set_number_frames(self, frames):
"""Set number of frames in DAQ, FEM."""
# print("\n\tadp.set_number_frames({}) -> number_frames: {}\n".format(frames, self.number_frames))
self.number_frames = frames
# Update number of frames in Hardware, and (via DAQ) in histogram and hdf plugins
self.fem.set_number_frames(self.number_frames)
self.daq.set_number_frames(self.number_frames)
def set_duration(self, duration):
"""Set duration, calculate frames from frame rate and update DAQ, FEM."""
self.duration = duration
self.fem.set_duration(self.duration)
# print("\n\tadp.set_duration({}) number_frames {} -> {}\n".format(duration, self.fem.get_number_frames(), self.number_frames))
self.number_frames = self.fem.get_number_frames()
self.daq.set_number_frames(self.number_frames)
def set_elog(self, entry):
"""Set the elog entry provided by the user through the UI."""
self.elog = entry
def set_number_nodes(self, number_nodes):
"""Set number of nodes."""
self.number_nodes = number_nodes
self.daq.set_number_nodes(self.number_nodes)
def initialize(self, adapters):
"""Get references to adapters, and pass these to the classes that need to use them."""
self.adapters = dict(
(k, v) for k, v in adapters.items() if v is not self)
self.daq.initialize(self.adapters)
def acquisition(self, put_data=None):
"""Instruct DAQ and FEM to acquire data."""
# Synchronise first_initialisation status (i.e. collect 2 fudge frames) with FEM
if self.first_initialisation:
self.first_initialisation = self.fem.first_initialisation
else:
# Clear (any previous) daq error
self.daq.in_error = False
if self.extended_acquisition is False:
if self.daq.in_progress:
logging.warning("Cannot Start Acquistion: Already in progress")
self.fem._set_status_error(
"Cannot Start Acquistion: Already in progress")
return
self.total_delay = 0
self.number_frames_to_request = self.number_frames
if self.fem.bias_voltage_refresh:
# Did the acquisition coincide with bias dead time?
if self.bias_blocking_acquisition:
IOLoop.instance().call_later(0.1, self.acquisition)
return
# Work out how many frames can be acquired before next bias refresh
time_into_window = time.time() - self.bias_init_time
time_available = self.fem.bias_refresh_interval - time_into_window
if time_available < 0:
IOLoop.instance().call_later(0.09, self.acquisition)
return
frames_before_bias = self.fem.frame_rate * time_available
number_frames_before_bias = int(round(frames_before_bias))
self.number_frames_to_request = self.number_frames - self.frames_already_acquired
# Can we obtain all required frames within current bias window?
if (number_frames_before_bias < self.number_frames_to_request):
# Need >1 bias window to fulfil acquisition
self.extended_acquisition = True
self.number_frames_to_request = number_frames_before_bias
self.total_delay = time_available + self.fem.bias_voltage_settle_time + \
self.fem.time_refresh_voltage_held
# # TODO: Remove once Firmware made to reset on each new acquisition
# # TODO: WILL BE NON 0 VALUE IN THE FUTURE - TO SUPPORT BIAS REFRESH INTV
# # BUT, if nonzero then won't FP's Acquisition time out before processing done?????
# #
# Reset Reorder plugin's frame_number (to current frame number, for multi-window acquire)
command = "config/reorder/frame_number"
request = ApiAdapterRequest(self.file_dir,
content_type="application/json")
request.body = "{}".format(self.frames_already_acquired)
self.adapters["fp"].put(command, request)
# TODO: To be removed once firmware updated? FP may be slow to process frame_number reset
time.sleep(0.5)
# Reset histograms, call DAQ's prepare_daq() once per acquisition
if self.initial_acquisition:
# Issue reset to histogram
command = "config/histogram/reset_histograms"
request = ApiAdapterRequest(self.file_dir,
content_type="application/json")
request.body = "{}".format(1)
self.adapters["fp"].put(command, request)
self.daq_target = time.time()
self.daq.prepare_daq(self.number_frames)
self.initial_acquisition = False
# Acquisition (whether single/multi-run) starts here
self.acquisition_in_progress = True
# Wait for DAQ (i.e. file writer) to be enabled before FEM told to collect data
# IOLoop.instance().call_later(0.1, self.await_daq_ready)
IOLoop.instance().add_callback(self.await_daq_ready)
def await_daq_ready(self):
"""Wait until DAQ has configured, enabled file writer."""
if (self.daq.in_error):
# Reset state variables
self.reset_state_variables()
elif (self.daq.file_writing is False):
IOLoop.instance().call_later(0.05, self.await_daq_ready)
else:
self.software_state = "Acquiring"
# Add additional 8 ms delay to ensure file writer's file open before first frame arrives
IOLoop.instance().call_later(0.08, self.trigger_fem_acquisition)
def trigger_fem_acquisition(self):
"""Trigger data acquisition in fem."""
# TODO: Temp hack: Prevent frames being 1 (continuous readout) by setting to 2 if it is
self.number_frames_to_request = 2 if (self.number_frames_to_request == 1) else \
self.number_frames_to_request
self.fem.set_number_frames(self.number_frames_to_request)
self.fem.collect_data()
self.frames_already_acquired += self.number_frames_to_request
# Note when FEM told to begin collecting data
self.fem_start_timestamp = time.time()
IOLoop.instance().call_later(self.total_delay,
self.monitor_fem_progress)
def monitor_fem_progress(self):
"""Check fem hardware progress.
Busy either:
-Initialising from cold (2 fudge frames)
-Normal initialisation
-Waiting for data collection to complete, either single/multi run
"""
if (self.fem.hardware_busy):
# Still sending data
IOLoop.instance().call_later(0.5, self.monitor_fem_progress)
return
else:
# Current collection completed; Do we have all the requested frames?
if self.extended_acquisition:
if (self.frames_already_acquired < self.number_frames):
# Need further bias window(s)
IOLoop.instance().add_callback(self.acquisition)
return
# Issue reset to summed_image
command = "config/summed_image/reset_image"
request = ApiAdapterRequest(self.file_dir,
content_type="application/json")
request.body = "{}".format(1)
self.adapters["fp"].put(command, request)
rc = self.daq.prepare_odin()
if not rc:
message = "Prepare Odin failed!"
self.fem._set_status_error(message)
self.status_error = message
self.reset_state_variables()
def reset_state_variables(self):
"""Reset state variables.
Utilised by await_daq_ready(), monitor_fem_progress()
"""
self.initial_acquisition = True
self.extended_acquisition = False
self.acquisition_in_progress = False
self.frames_already_acquired = 0
self.software_state = "Idle"
def cancel_acquisition(self, put_data=None):
"""Cancel ongoing acquisition in Software.
Not yet possible to stop FEM, mid-acquisition
"""
self.fem.stop_acquisition = True
# Inject End of Acquisition Frame
command = "config/inject_eoa"
request = ApiAdapterRequest("", content_type="application/json")
self.adapters["fp"].put(command, request)
self.shutdown_processing()
self.software_state = "Idle"
def _collect_offsets(self, msg):
"""Instruct FEM to collect offsets."""
self.fem.collect_offsets()
def commit_configuration(self, msg):
"""Push HexitecDAQ's 'config/' ParameterTree settings into FP's plugins."""
self.daq.commit_configuration()
# Clear cold initialisation if first config commit
if self.cold_initialisation:
self.cold_initialisation = False
def hv_on(self, msg):
"""Switch HV on."""
# TODO: Complete placeholder
self.fem.hv_bias_enabled = True
def hv_off(self, msg):
"""Switch HV off."""
# TODO: Complete placeholder
self.fem.hv_bias_enabled = False
def get(self, path):
"""
Get the parameter tree.
This method returns the parameter tree for use by clients via the Hexitec adapter.
:param path: path to retrieve from tree
"""
return self.param_tree.get(path)
def set(self, path, data):
"""
Set parameters in the parameter tree.
This method simply wraps underlying ParameterTree method so that an exception can be
re-raised with an appropriate HexitecError.
:param path: path of parameter tree to set values for
:param data: dictionary of new data values to set in the parameter tree
"""
try:
self.param_tree.set(path, data)
except ParameterTreeError as e:
raise HexitecError(e)
def copy_dataset(input_path,
input_key,
output_path,
output_key,
is_h5,
convert_dtype=False,
chunks=None,
n_threads=1,
overwrite=False):
with open_file(input_path, 'r') as f_in, open_file(output_path,
'a') as f_out:
ds_in = f_in[input_key]
shape = ds_in.shape
have_data = output_key in f_out
if have_data and not overwrite:
return True
# validate chunks
if chunks is None:
chunks_ = True if is_h5 else None
else:
chunks_ = tuple(min(ch, sh) for sh, ch in zip(shape, chunks))
if convert_dtype:
out_dtype = get_new_dtype(ds_in.dtype)
else:
out_dtype = ds_in.dtype
if overwrite:
del f_out[output_key]
# create the output dataset and get the effective chunks
ds_out = f_out.create_dataset(output_key,
shape=shape,
chunks=chunks_,
compression='gzip',
dtype=out_dtype)
ds_chunks = ds_out.chunks
def copy_chunk(bb):
data = ds_in[bb]
# skip empty chunks
if data.sum() == 0:
return
if convert_dtype:
data = convert_to_bdv_dtype(data)
ds_out[bb] = data
print("Copy initial dataset from: %s:%s to %s:%s" %
(input_path, input_key, output_path, output_key))
blocks = list(blocking(shape, ds_chunks))
if n_threads > 1:
with futures.ThreadPoolExecutor(n_threads) as tp:
list(tqdm(tp.map(copy_chunk, blocks), total=len(blocks)))
else:
for bb in tqdm(blocks, total=len(blocks)):
copy_chunk(bb)
return False
def home2():
"""Doesn't work and should return the value from env."""
with rasterio.Env(GDAL_DISABLE_READDIR_ON_OPEN="FALSE"):
with futures.ThreadPoolExecutor() as executor:
res = list(executor.map(f, range(1)))[0]
return {"env": res}
async def home3():
"""Works and should return FALSE."""
with rasterio.Env(GDAL_DISABLE_READDIR_ON_OPEN="FALSE"):
with futures.ThreadPoolExecutor() as executor:
res = list(executor.map(f, range(1)))[0]
return {"env": res}
async def home3():
"""Works and should return FALSE."""
with futures.ThreadPoolExecutor() as executor:
res = list(executor.map(f, range(1)))[0]
return {"env": res}
def home2():
"""Doesn't work and should return the value from env."""
with futures.ThreadPoolExecutor() as executor:
res = list(executor.map(f, range(1)))[0]
return {"env": res}
class IBMQJob(BaseJob):
"""IBM Q Job class
Attributes:
_executor (futures.Executor): executor to handle asynchronous jobs
_final_states (list(JobStatus)): terminal states of async jobs
"""
_executor = futures.ThreadPoolExecutor()
_final_states = [
JobStatus.DONE,
JobStatus.CANCELLED,
JobStatus.ERROR
]
def __init__(self, qobj, api, is_device):
"""IBMQJob init function.
Args:
qobj (dict): job description
api (IBMQuantumExperience): IBM Q API
is_device (bool): whether backend is a real device # TODO: remove this after Qobj
"""
super().__init__()
self._qobj = qobj
self._api = api
self._id = None # this must be before creating the future
self._backend_name = self._qobj.get('config').get('backend_name')
self._status = JobStatus.INITIALIZING
self._future_submit = self._executor.submit(self._submit)
self._status_msg = 'Job is initializing. Please, wait a moment.'
self._queue_position = None
self._cancelled = False
self._exception = None
self._is_device = is_device
self.creation_date = datetime.datetime.utcnow().replace(
tzinfo=datetime.timezone.utc).isoformat()
@classmethod
def from_api(cls, job_info, api, is_device):
"""Instantiates job using information returned from
IBMQuantumExperience about a particular job.
Args:
job_info (dict): This is the information about a job returned from
the API. It has the simplified structure::
{'backend': {'id', 'backend id string',
'name', 'ibmqx4'},
'id': 'job id string',
'qasms': [{'executionId': 'id string',
'qasm': 'qasm string'},
]
'status': 'status string',
'seed': '1',
'shots': 1024,
'status': 'status string',
'usedCredits': 3,
'creationDate': '2018-06-13T04:31:13.175Z'
'userId': 'user id'}
api (IBMQuantumExperience): IBM Q API
is_device (bool): whether backend is a real device # TODO: remove this after Qobj
Returns:
IBMQJob: an instance of this class
"""
job_instance = cls.__new__(cls)
job_instance._status = JobStatus.QUEUED
job_instance._backend_name = job_info.get('backend').get('name')
job_instance._api = api
job_instance._id = job_info.get('id')
job_instance._exception = None # needs to be before status call below
job_instance._status_msg = None
job_instance._queue_position = None
job_instance._cancelled = False
job_instance._is_device = is_device
job_instance.creation_date = job_info.get('creationDate')
return job_instance
def result(self, timeout=None, wait=5):
"""Return the result from the job.
Args:
timeout (int): number of seconds to wait for job
wait (int): time between queries to IBM Q server
Returns:
Result: Result object
Raises:
IBMQJobError: exception raised during job initialization
"""
# pylint: disable=arguments-differ
while self._status == JobStatus.INITIALIZING:
if self._future_submit.exception():
raise IBMQJobError('error submitting job: {}'.format(
repr(self._future_submit.exception())))
time.sleep(0.1)
try:
this_result = self._wait_for_job(timeout=timeout, wait=wait)
except TimeoutError as err:
# A timeout error retrieving the results does not imply the job
# is failing. The job can be still running.
return Result({'id': self._id, 'status': 'ERROR',
'result': str(err)})
if self._is_device and self.done:
_reorder_bits(this_result)
if self._status not in self._final_states:
if this_result.get_status() == 'ERROR':
self._status = JobStatus.ERROR
else:
self._status = JobStatus.DONE
return this_result
def cancel(self):
"""Attempt to cancel job. Currently this is only possible on
commercial systems.
Returns:
bool: True if job can be cancelled, else False.
Raises:
IBMQJobError: if server returned error
"""
if self._is_commercial:
hub = self._api.config['hub']
group = self._api.config['group']
project = self._api.config['project']
response = self._api.cancel_job(self._id, hub, group, project)
if 'error' in response:
err_msg = response.get('error', '')
error = IBMQJobError('Error cancelling job: %s' % err_msg)
self._exception = error
raise error
else:
self._cancelled = True
return True
else:
self._cancelled = False
return False
@property
def status(self):
self._update_status()
stats = {
'job_id': self._id,
'status': self._status,
'status_msg': self._status_msg
}
if self._queue_position:
stats['queue_position'] = self._queue_position
# Reset once consumed to allow _update_status to regenerate the
# value if needed.
self._queue_position = None
return stats
def _update_status(self):
"""Query the API to update the status."""
if (self._status in self._final_states or
self._status == JobStatus.INITIALIZING):
return None
try:
api_job = self._api.get_job(self.id)
if 'status' not in api_job:
raise QISKitError('get_job didn\'t return status: %s' %
pprint.pformat(api_job))
# pylint: disable=broad-except
except Exception as err:
self._status = JobStatus.ERROR
self._exception = err
self._status_msg = '{}'.format(err)
return None
if api_job['status'] == 'RUNNING':
self._status = JobStatus.RUNNING
self._status_msg = self._status.value
queued, queue_position = self._is_job_queued(api_job)
if queued:
self._status = JobStatus.QUEUED
self._status_msg = self._status.value
if queue_position:
self._queue_position = queue_position
elif api_job['status'] == 'COMPLETED':
self._status = JobStatus.DONE
self._status_msg = self._status.value
elif api_job['status'] == 'CANCELLED':
self._status = JobStatus.CANCELLED
self._status_msg = self._status.value
self._cancelled = True
elif 'ERROR' in api_job['status']:
# ERROR_CREATING_JOB or ERROR_RUNNING_JOB
self._status = JobStatus.ERROR
self._status_msg = api_job['status']
elif self.exception or self._future_submit.exception():
self._status = JobStatus.ERROR
if self._future_submit.exception():
self._exception = self._future_submit.exception()
self._status_msg = str(self.exception)
else:
self._status = JobStatus.ERROR
self._exception = IBMQJobError(
'Unrecognized result: \n{}'.format(pprint.pformat(api_job)))
self._status_msg = '{}'.format(self._exception)
return api_job
def _is_job_queued(self, api_job):
is_queued, position = False, None
if 'infoQueue' in api_job:
if 'status' in api_job['infoQueue']:
queue_status = api_job['infoQueue']['status']
is_queued = queue_status == 'PENDING_IN_QUEUE'
if 'position' in api_job['infoQueue']:
position = api_job['infoQueue']['position']
return is_queued, position
@property
def queued(self):
"""
Returns whether job is queued.
Returns:
bool: True if job is queued, else False.
Raises:
QISKitError: couldn't get job status from server
"""
return self.status['status'] == JobStatus.QUEUED
@property
def running(self):
"""
Returns whether job is actively running
Returns:
bool: True if job is running, else False.
Raises:
QISKitError: couldn't get job status from server
"""
return self.status['status'] == JobStatus.RUNNING
@property
def done(self):
"""
Returns True if job successfully finished running.
Note behavior is slightly different than Future objects which would
also return true if successfully cancelled.
"""
return self.status['status'] == JobStatus.DONE
@property
def cancelled(self):
return self._cancelled
@property
def exception(self):
"""
Return Exception object previously raised by job else None
Returns:
Exception: exception raised by job
"""
if isinstance(self._exception, Exception):
self._status_msg = str(self._exception)
return self._exception
@property
def _is_commercial(self):
config = self._api.config
# this check may give false positives so should probably be improved
return config.get('hub') and config.get('group') and config.get('project')
@property
def id(self):
"""
Return backend determined id (also available in status method).
"""
# pylint: disable=invalid-name
while self._id is None and self._status not in self._final_states:
if self._future_submit.exception():
self._status = JobStatus.ERROR
self._exception = self._future_submit.exception()
# job is initializing and hasn't gotten a id yet.
time.sleep(0.1)
return self._id
@property
def backend_name(self):
"""
Return backend name used for this job
"""
return self._backend_name
def _submit(self):
"""Submit job to IBM Q.
Returns:
dict: submission info including job id from server
Raises:
QISKitError: The backend name in the job doesn't match this backend.
ResultError: If the API reported an error with the submitted job.
RegisterSizeError: If the requested register size exceeded device
capability.
"""
qobj = self._qobj
api_jobs = []
for circuit in qobj['circuits']:
job = {}
if (('compiled_circuit_qasm' not in circuit) or
(circuit['compiled_circuit_qasm'] is None)):
compiled_circuit = transpile(circuit['circuit'])
circuit['compiled_circuit_qasm'] = compiled_circuit.qasm(qeflag=True)
if isinstance(circuit['compiled_circuit_qasm'], bytes):
job['qasm'] = circuit['compiled_circuit_qasm'].decode()
else:
job['qasm'] = circuit['compiled_circuit_qasm']
if 'name' in circuit:
job['name'] = circuit['name']
# convert numpy types for json serialization
compiled_circuit = json.loads(
json.dumps(circuit['compiled_circuit'],
default=_numpy_type_converter))
job['metadata'] = {'compiled_circuit': compiled_circuit}
api_jobs.append(job)
seed0 = qobj['circuits'][0]['config']['seed']
hpc = None
if 'hpc' in qobj['config']:
try:
# Use CamelCase when passing the hpc parameters to the API.
hpc = {
'multiShotOptimization':
qobj['config']['hpc']['multi_shot_optimization'],
'ompNumThreads':
qobj['config']['hpc']['omp_num_threads']
}
except (KeyError, TypeError):
hpc = None
backend_name = qobj['config']['backend_name']
if backend_name != self._backend_name:
raise QISKitError("inconsistent qobj backend "
"name ({0} != {1})".format(backend_name,
self._backend_name))
submit_info = {}
try:
submit_info = self._api.run_job(api_jobs, backend=backend_name,
shots=qobj['config']['shots'],
max_credits=qobj['config']['max_credits'],
seed=seed0,
hpc=hpc)
# pylint: disable=broad-except
except Exception as err:
self._status = JobStatus.ERROR
self._status_msg = str(err)
self._exception = err
return None
if 'error' in submit_info:
self._status = JobStatus.ERROR
self._status_msg = str(submit_info['error'])
self._exception = IBMQJobError(self._status_msg)
return submit_info
self._id = submit_info.get('id')
self.creation_date = submit_info.get('creationDate')
self._status = JobStatus.QUEUED
return submit_info
def _wait_for_job(self, timeout=60, wait=5):
"""Wait until all online ran circuits of a qobj are 'COMPLETED'.
Args:
timeout (float or None): seconds to wait for job. If None, wait
indefinitely.
wait (float): seconds between queries
Returns:
Result: A result object.
Raises:
QISKitError: job didn't return status or reported error in status
TimeoutError: if the job does not return results before an
specified timeout.
"""
start_time = time.time()
api_result = self._update_status()
while self._status not in self._final_states:
elapsed_time = time.time() - start_time
if timeout is not None and elapsed_time >= timeout:
raise TimeoutError('QISKit timed out')
logger.info('status = %s (%d seconds)', api_result['status'],
elapsed_time)
if 'status' not in api_result:
self._exception = QISKitError("get_job didn't return status: %s" %
(pprint.pformat(api_result)))
raise QISKitError("get_job didn't return status: %s" %
(pprint.pformat(api_result)))
if (api_result['status'] == 'ERROR_CREATING_JOB' or
api_result['status'] == 'ERROR_RUNNING_JOB'):
job_result = {'id': self._id, 'status': 'ERROR',
'result': api_result['status']}
return Result(job_result)
time.sleep(wait)
api_result = self._update_status()
if self.cancelled:
job_result = {'id': self._id, 'status': 'CANCELLED',
'result': 'job cancelled'}
return Result(job_result)
elif self.exception:
job_result = {'id': self._id, 'status': 'ERROR',
'result': str(self.exception)}
return Result(job_result)
if api_result is None:
api_result = self._api.get_job(self._id)
job_result_list = []
for circuit_result in api_result['qasms']:
this_result = {'data': circuit_result['data'],
'name': circuit_result.get('name'),
'compiled_circuit_qasm': circuit_result.get('qasm'),
'status': circuit_result['status']}
if 'metadata' in circuit_result:
this_result['metadata'] = circuit_result['metadata']
job_result_list.append(this_result)
job_result = {'id': self._id,
'status': api_result['status'],
'used_credits': api_result.get('usedCredits'),
'result': job_result_list}
job_result['backend_name'] = self.backend_name
return Result(job_result)
def test_create_two_spans_streaming(self):
"""Verify that the interceptor captures sub spans in a
streaming call, within the given trace"""
class TwoSpanServicer(GRPCTestServerServicer):
# pylint:disable=C0103
def ServerStreamingMethod(self, request, context):
# create another span
tracer = trace.get_tracer(__name__)
with tracer.start_as_current_span("child") as child:
child.add_event("child event")
for data in ("one", "two", "three"):
yield Response(
server_id=request.client_id,
response_data=data,
)
# Intercept gRPC calls...
interceptor = server_interceptor()
with futures.ThreadPoolExecutor(max_workers=1) as executor:
server = grpc.server(
executor,
options=(("grpc.so_reuseport", 0), ),
interceptors=[interceptor],
)
add_GRPCTestServerServicer_to_server(TwoSpanServicer(), server)
port = server.add_insecure_port("[::]:0")
channel = grpc.insecure_channel(f"localhost:{port:d}")
# setup the RPC
rpc_call = "/GRPCTestServer/ServerStreamingMethod"
request = Request(client_id=1, request_data="test")
msg = request.SerializeToString()
try:
server.start()
list(channel.unary_stream(rpc_call)(msg))
finally:
server.stop(None)
spans_list = self.memory_exporter.get_finished_spans()
self.assertEqual(len(spans_list), 2)
child_span = spans_list[0]
parent_span = spans_list[1]
self.assertEqual(parent_span.name, rpc_call)
self.assertIs(parent_span.kind, trace.SpanKind.SERVER)
# Check version and name in span's instrumentation info
self.assertEqualSpanInstrumentationInfo(
parent_span, opentelemetry.instrumentation.grpc)
# Check attributes
self.assertSpanHasAttributes(
parent_span,
{
SpanAttributes.NET_PEER_IP: "[::1]",
SpanAttributes.NET_PEER_NAME: "localhost",
SpanAttributes.RPC_METHOD: "ServerStreamingMethod",
SpanAttributes.RPC_SERVICE: "GRPCTestServer",
SpanAttributes.RPC_SYSTEM: "grpc",
SpanAttributes.RPC_GRPC_STATUS_CODE:
grpc.StatusCode.OK.value[0],
},
)
# Check the child span
self.assertEqual(child_span.name, "child")
self.assertEqual(parent_span.context.trace_id,
child_span.context.trace_id)
def serve(max_workers=10, port=7777):
server = grpc.server(futures.ThreadPoolExecutor(max_workers=max_workers))
grpc_bt_grpc.add_DeepFakesFaceSwapServicer_to_server(
DeepFakesFaceSwapServicer(), server)
server.add_insecure_port("[::]:{}".format(port))
return server
def test_concurrent_server_spans(self):
"""Check that concurrent RPC calls don't interfere with each other.
This is the same check as test_sequential_server_spans except that the
RPCs are concurrent. Two handlers are invoked at the same time on two
separate threads. Each one should see a different active span and
context.
"""
interceptor = server_interceptor()
# Capture the currently active span in each thread
active_spans_in_handler = []
latch = get_latch(2)
def handler(request, context):
latch()
active_spans_in_handler.append(trace.get_current_span())
return b""
with futures.ThreadPoolExecutor(max_workers=2) as executor:
server = grpc.server(
executor,
options=(("grpc.so_reuseport", 0), ),
interceptors=[interceptor],
)
server.add_generic_rpc_handlers((UnaryUnaryRpcHandler(handler), ))
port = server.add_insecure_port("[::]:0")
channel = grpc.insecure_channel(f"localhost:{port:d}")
try:
server.start()
# Interleave calls so spans are active on each thread at the same
# time
with futures.ThreadPoolExecutor(max_workers=2) as tpe:
f1 = tpe.submit(
channel.unary_unary("TestServicer/handler"), b"")
f2 = tpe.submit(
channel.unary_unary("TestServicer/handler"), b"")
futures.wait((f1, f2))
finally:
server.stop(None)
self.assertEqual(len(active_spans_in_handler), 2)
# pylint:disable=unbalanced-tuple-unpacking
span1, span2 = active_spans_in_handler
# Spans should belong to separate traces
self.assertNotEqual(span1.context.span_id, span2.context.span_id)
self.assertNotEqual(span1.context.trace_id, span2.context.trace_id)
for span in (span1, span2):
# each should be a root span
self.assertIsNone(span2.parent)
# check attributes
self.assertSpanHasAttributes(
span,
{
SpanAttributes.NET_PEER_IP:
"[::1]",
SpanAttributes.NET_PEER_NAME:
"localhost",
SpanAttributes.RPC_METHOD:
"handler",
SpanAttributes.RPC_SERVICE:
"TestServicer",
SpanAttributes.RPC_SYSTEM:
"grpc",
SpanAttributes.RPC_GRPC_STATUS_CODE:
grpc.StatusCode.OK.value[0],
},
)
import sys
import re
import random
import json
from itertools import groupby
import tornado.ioloop
import tornado.web
from concurrent import futures
import path, tools
from configer import conf
MAX_WORKERS = 16
executor = futures.ThreadPoolExecutor(max_workers=MAX_WORKERS)
@conf.register_my_setup(level=2)
def set_up():
''' erase all nodes
this function maybe called for hot deployment
'''
Router.mapper = []
Router.mapper_sentry = {}
Router.last_sentry = {}
global logger
logger = tools.Log().getLog()
#automic scan dirs and initial all node
files_list = os.listdir(path._BIZ_PATH)
files_list = set(
def get_thread_pool(self):
"""Override this method to customize how the thread pool is created"""
return futures.ThreadPoolExecutor(max_workers=self.cfg.threads)
def generate_new_device_map(local_ip, ip_range=(6, 128)):
devices_map = {}
subnet_ip = local_ip.split(".")[0:3]
subnet_ip = ".".join(subnet_ip)
logging.info("Scanning attached devices")
scanned = ["%s.%i" % (subnet_ip, i) for i in range(*ip_range)]
urls = ["http://%s" % str(s) for s in scanned]
# We can use a with statement to ensure threads are cleaned up promptly
with futures.ThreadPoolExecutor(max_workers=128) as executor:
# Start the load operations and mark each future with its URL
fs = [executor.submit(scan_one_device, url) for url in urls]
for f in concurrent.futures.as_completed(fs):
try:
id, ip = f.result()
if id is None:
continue
devices_map[id] = {
"ip": ip,
"status": "Software broken",
"id": id
}
except Exception as e:
logging.error("Error whilst pinging url")
logging.error(traceback.format_exc(e))
if len(devices_map) < 1:
logging.warning("No device detected")
return devices_map
logging.info("Detected %i devices:\n%s" %
(len(devices_map), str(devices_map.keys())))
# We can use a with statement to ensure threads are cleaned up promptly
with futures.ThreadPoolExecutor(max_workers=128) as executor:
# Start the load operations and mark each future with its URL
fs = {}
for id in devices_map.keys():
fs[executor.submit(update_dev_map_wrapped, devices_map, id)] = id
for f in concurrent.futures.as_completed(fs):
id = fs[f]
try:
data = f.result()
devices_map[id].update(data)
except Exception as e:
logging.error("Could not get data from device %s :" % id)
logging.error(traceback.format_exc(e))
# Adds the active_branch to devices_,map
with futures.ThreadPoolExecutor(max_workers=128) as executor:
# Start the load operations and mark each future with its URL
fs = {}
for id in devices_map.keys():
fs[executor.submit(update_dev_map_wrapped,
devices_map,
id,
what='device/active_branch',
port='8888')] = id
for f in concurrent.futures.as_completed(fs):
id = fs[f]
try:
data = f.result()
devices_map[id].update(data)
except Exception as e:
logging.error("Could not get data from device %s :" % id)
logging.error(traceback.format_exc(e))
# Adds the check_update to devices_,map
with futures.ThreadPoolExecutor(max_workers=128) as executor:
# Start the load operations and mark each future with its URL
fs = {}
for id in devices_map.keys():
fs[executor.submit(update_dev_map_wrapped,
devices_map,
id,
what='device/check_update',
port='8888')] = id
for f in concurrent.futures.as_completed(fs):
id = fs[f]
try:
data = f.result()
devices_map[id].update(data)
except Exception as e:
logging.error("Could not get data from device %s :" % id)
logging.error(traceback.format_exc(e))
return devices_map
def init_rpc_server(self):
server = grpc.server(futures.ThreadPoolExecutor(max_workers=2))
add_NodeServicer_to_server(self, server)
server.add_insecure_port(f'localhost:{self.port}')
server.start()
server.wait_for_termination()
def start_yatai_service_grpc_server(db_url, repo_base_url, grpc_port, ui_port,
with_ui, s3_endpoint_url):
from bentoml.yatai.yatai_service_impl import YataiService
yatai_service = YataiService(
db_url=db_url,
repo_base_url=repo_base_url,
s3_endpoint_url=s3_endpoint_url,
)
server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
add_YataiServicer_to_server(yatai_service, server)
debug_mode = get_debug_mode()
if debug_mode:
try:
logger.debug('Enabling gRPC server reflection for debugging')
from grpc_reflection.v1alpha import reflection
from bentoml.yatai.proto import yatai_service_pb2
SERVICE_NAMES = (
yatai_service_pb2.DESCRIPTOR.services_by_name['Yatai'].
full_name,
reflection.SERVICE_NAME,
)
reflection.enable_server_reflection(SERVICE_NAMES, server)
except ImportError:
logger.debug(
'Failed enabling gRPC server reflection, missing required package: '
'"pip install grpcio-reflection"')
server.add_insecure_port(f'[::]:{grpc_port}')
server.start()
if with_ui:
web_ui_log_path = os.path.join(
config("logging").get("BASE_LOG_DIR"),
config('logging').get("yatai_web_server_log_filename"),
)
ensure_node_available_or_raise()
yatai_grpc_server_addess = f'localhost:{grpc_port}'
async_start_yatai_service_web_ui(yatai_grpc_server_addess, ui_port,
web_ui_log_path, debug_mode)
# We don't import _echo function from click_utils because of circular dep
click.echo(
f'* Starting BentoML YataiService gRPC Server\n'
f'* Debug mode: { "on" if debug_mode else "off"}\n'
f'* Web UI: {f"running on http://127.0.0.1:{ui_port}" if with_ui else "off"}\n'
f'* Running on 127.0.0.1:{grpc_port} (Press CTRL+C to quit)\n'
f'* Usage:\n'
f'* Set config: `bentoml config set yatai_service.url=127.0.0.1:{grpc_port}`\n'
f'* Set env var: `export BENTOML__YATAI_SERVICE__URL=127.0.0.1:{grpc_port}`\n'
f'* Help and instructions: '
f'https://docs.bentoml.org/en/latest/guides/yatai_service.html\n'
f'{f"* Web server log can be found here: {web_ui_log_path}" if with_ui else ""}'
)
try:
while True:
time.sleep(_ONE_DAY_IN_SECONDS)
except KeyboardInterrupt:
logger.info("Terminating YataiService gRPC server..")
server.stop(grace=None)
def composite(sources, bounds, shape, target_crs, band_count):
"""Composite data from sources into a single raster covering bounds, but in
the target CRS."""
# avoid circular dependencies
from . import _nodata, get_resolution_in_meters, get_source, read_window
# TODO this belongs in render
if bounds.crs == target_crs:
canvas_bounds = bounds
else:
canvas_bounds = Bounds(
warp.transform_bounds(bounds.crs, target_crs, *bounds.bounds),
target_crs)
resolution = get_resolution_in_meters(bounds, shape)
sources = recipes.preprocess(sources, resolution=resolution)
def _read_window(source):
with get_source(source.url) as src:
LOG.info("Compositing %s (%s) as band %s", source.url, source.name,
source.band)
# read a window from the source data
# TODO ask for a buffer here, get back an updated bounding box
# reflecting it
try:
window_data = read_window(src, canvas_bounds, shape,
source.recipes)
except Exception as e:
LOG.exception("Error reading %s: %s", source.url, e)
return
return source, PixelCollection(window_data.data,
window_data.bounds, source.band)
# iterate over available sources, sorted by decreasing "quality"
with futures.ThreadPoolExecutor(max_workers=multiprocessing.cpu_count() *
5) as executor:
ws = executor.map(_read_window, sources)
sources_used = []
ws = recipes.postprocess(ws)
canvas_data = np.ma.zeros((band_count, ) + shape,
dtype=np.float32,
fill_value=_nodata(np.float32))
canvas_data.mask = True
canvas = PixelCollection(canvas_data, canvas_bounds)
for source, window_data in filter(None, ws):
window_data = recipes.apply(source.recipes, window_data, source=source)
if window_data.data is None:
continue
# paste the resulting data onto a canvas
canvas = paste(
PixelCollection(
window_data.data.astype(np.float32),
window_data.bounds,
window_data.band,
),
canvas,
)
sources_used.append(source)
if not canvas.data.mask.any():
# stop if all pixels are valid
break
return map(lambda s: (s.name, s.url), sources_used), canvas
def _sync(self, depsgraph):
stage = self.cached_stage.create()
UsdGeom.SetStageMetersPerUnit(stage, 1)
UsdGeom.SetStageUpAxis(stage, UsdGeom.Tokens.z)
root_prim = stage.GetPseudoRoot()
objects_len = sum(1 for _ in object.ObjectData.depsgraph_objects(depsgraph, use_scene_cameras=False))
objects_stage = Usd.Stage.CreateNew(str(get_temp_file(".usda")))
object_root_prim = objects_stage.GetPseudoRoot()
for i, obj_data in enumerate(object.ObjectData.depsgraph_objects_obj(depsgraph, use_scene_cameras=False)):
if self.render_engine.test_break():
return
self.notify_status(0.0, f"Syncing object {i}/{objects_len}: {obj_data.object.name}")
object.sync(object_root_prim, obj_data)
for prim in objects_stage.GetPseudoRoot().GetAllChildren():
override_prim = stage.OverridePrim(root_prim.GetPath().AppendChild(prim.GetName()))
override_prim.GetReferences().AddReference(objects_stage.GetRootLayer().realPath, prim.GetPath())
instance_len = sum(1 for _ in object.ObjectData.depsgraph_objects_inst(depsgraph, use_scene_cameras=False))
chunks = math.ceil(instance_len / CHUNK_COUNT)
chunks_data = {}
for i in range(chunks):
chunk_stage = Usd.Stage.CreateNew(str(get_temp_file(".usda")))
chunk_prim = stage.OverridePrim(f'/chunk_{i}')
chunks_data[i] = {'stage': chunk_stage, 'prim': chunk_prim}
objects_processed = 0
threadLock = threading.Lock()
def sync_chunk(idx, stage, prim):
nonlocal objects_processed
xform = UsdGeom.Xform.Define(stage, stage.GetPseudoRoot().GetPath().AppendChild(f'chunk_{idx}'))
obj_prim = xform.GetPrim()
for i, obj_data in enumerate(object.ObjectData.depsgraph_objects_inst(depsgraph, use_scene_cameras=False)):
if i >= (idx) * CHUNK_COUNT and i < (idx + 1) * CHUNK_COUNT:
with threadLock:
objects_processed += 1
self.notify_status(0.0, f"Syncing instances: {objects_processed} / {instance_len}")
object.sync(obj_prim, obj_data, objects_stage)
stage.SetDefaultPrim(obj_prim)
with futures.ThreadPoolExecutor() as executor:
chunk_sync = []
for idx in chunks_data:
chunk_sync.append(executor.submit(sync_chunk, idx,
chunks_data[idx]['stage'],
chunks_data[idx]["prim"]))
for idx, future in enumerate(futures.wait(chunk_sync)):
if idx == 0:
for i in chunks_data:
chunk_prim = stage.GetPrimAtPath(chunks_data[i]["prim"].GetPath())
chunk_prim.GetReferences().AddReference(chunks_data[i]['stage'].GetRootLayer().realPath)
pass
if depsgraph.scene.world is not None:
world.sync(root_prim, depsgraph.scene.world)
object.sync(stage.GetPseudoRoot(), object.ObjectData.from_object(depsgraph.scene.camera),
scene=depsgraph.scene)
async def searcher_async(self):
with futures.ThreadPoolExecutor() as executor:
fut = executor.submit(self.searcher)
return await asyncio.wrap_future(fut)
def _handle_training(ctx: Context) -> None:
with futures.ThreadPoolExecutor() as pool:
trainings = [
i.result() for i in [
pool.submit(Training.from_training_scene, j)
for j in _iter_training_images()
]
]
races_with_score = sorted(
((i, i.score(ctx)) for i in race.find(ctx)),
key=lambda x: x[1],
reverse=True,
)
trainings_with_score = [(i, i.score(ctx)) for i in trainings]
trainings_with_score = sorted(trainings_with_score,
key=lambda x: x[1],
reverse=True)
expected_score = ctx.expected_score()
LOGGER.info("expected score:\t%2.2f", expected_score)
for r, s in races_with_score:
LOGGER.info("score:\trace:\t%2.2f:\t%s", s, r)
for t, s in trainings_with_score:
LOGGER.info("score:\ttraining:\t%2.2f:\t%s", s, t)
training, training_score = trainings_with_score[0]
if races_with_score:
r, s = races_with_score[0]
if (s > expected_score and
s > training_score) or (ctx.fan_count < ctx.target_fan_count
and r.estimate_order(ctx) <= 3):
# go to race
action.wait_tap_image(templates.RETURN_BUTTON)
action.wait_tap_image(templates.SINGLE_MODE_COMMAND_RACE)
tmpl, _ = action.wait_image(
templates.SINGLE_MODE_RACE_START_BUTTON,
templates.SINGLE_MODE_CONTINUOUS_RACE_TITLE,
)
if tmpl.name == templates.SINGLE_MODE_CONTINUOUS_RACE_TITLE:
if ctx.continuous_race_count() >= 3:
action.wait_tap_image(templates.GREEN_OK_BUTTON)
else:
# continuous race count incorrect, evaluate again:
ctx.race_turns.update(
range(ctx.turn_count() - 3, ctx.turn_count()))
action.wait_tap_image(templates.CANCEL_BUTTON)
action.wait_tap_image(
templates.SINGLE_MODE_COMMAND_TRAINING)
_handle_training(ctx)
return
_choose_race(ctx, r)
_handle_race(ctx, r)
return
if training_score < expected_score:
# not worth, go rest
action.tap_image(templates.RETURN_BUTTON)
action.wait_image(templates.SINGLE_MODE_COMMAND_TRAINING)
if action.tap_image(templates.SINGLE_MODE_COMMAND_HEALTH_CARE):
return
if ctx.mood < ctx.MOOD_VERY_GOOD:
tmpl, pos = action.wait_image(
templates.SINGLE_MODE_COMMAND_GO_OUT,
templates.SINGLE_MODE_COMMAND_SUMMER_REST,
)
action.tap(pos)
action.wait_image_disappear(tmpl)
else:
tmpl, pos = (action.wait_image(
templates.SINGLE_MODE_REST,
templates.SINGLE_MODE_COMMAND_SUMMER_REST,
) if ctx.vitality < 0.8 else action.wait_image(
templates.SINGLE_MODE_COMMAND_GO_OUT,
templates.SINGLE_MODE_COMMAND_SUMMER_REST,
))
action.tap(pos)
action.wait_image_disappear(tmpl)
time.sleep(0.5)
if action.count_image(templates.SINGLE_MODE_GO_OUT_MENU_TITLE):
options_with_score = sorted(
[(i, i.score(ctx))
for i in go_out.Option.from_menu(template.screenshot())],
key=lambda x: x[1],
)
for option, score in options_with_score:
LOGGER.info("go out option:\t%s:\tscore:%.2f", option, score)
action.tap(options_with_score[0][0].position)
return
x, y = training.confirm_position
if trainings[-1] != training:
action.tap((x, y))
time.sleep(0.1)
action.tap((x, y))
param = param_value[0]
value = param_value[1]
if value[-1] != '9':
newvalue = value[:-1] + str(int(value[-1]) + 1)
else:
newvalue = value[:-1] + '0'
newurl = url.replace(param + '=' + value, param + '=' + newvalue)
if "^" not in url:
# get request
rsp = requests.get(newurl)
else:
# post request
post_url = newurl.split("^")[0]
data = newurl.split("^")[1]
rsp = requests.post(post_url,
data=data.encode("utf-8"),
verify=False,
timeout=10)
if rsp.status_code == 200 and not rsp.history:
string_to_write = ("Congratulations! unauthorize vul may exist:" +
url + "the vul param is:" + param + "\n")
CLIOutput().good_print(string_to_write)
with open("%s/result.txt" % current_dir, "a+") as f:
f.write(string_to_write)
from concurrent import futures
with futures.ThreadPoolExecutor(max_workers=15) as executor:
executor.map(check, check_url_list)
def dataframe_to_arrays(df, schema, preserve_index, nthreads=1, columns=None):
if columns is None:
columns = df.columns
column_names = []
index_columns = []
index_column_names = []
type = None
if preserve_index:
n = len(getattr(df.index, 'levels', [df.index]))
index_columns.extend(df.index.get_level_values(i) for i in range(n))
columns_to_convert = []
convert_types = []
if not df.columns.is_unique:
raise ValueError('Duplicate column names found: {}'.format(
list(df.columns)))
for name in columns:
col = df[name]
name = _column_name_to_strings(name)
if schema is not None:
field = schema.field_by_name(name)
type = getattr(field, "type", None)
columns_to_convert.append(col)
convert_types.append(type)
column_names.append(name)
for i, column in enumerate(index_columns):
columns_to_convert.append(column)
convert_types.append(None)
name = _index_level_name(column, i, column_names)
index_column_names.append(name)
# NOTE(wesm): If nthreads=None, then we use a heuristic to decide whether
# using a thread pool is worth it. Currently the heuristic is whether the
# nrows > 100 * ncols.
if nthreads is None:
nrows, ncols = len(df), len(df.columns)
if nrows > ncols * 100:
nthreads = pa.cpu_count()
else:
nthreads = 1
def convert_column(col, ty):
return pa.array(col, from_pandas=True, type=ty)
if nthreads == 1:
arrays = [
convert_column(c, t)
for c, t in zip(columns_to_convert, convert_types)
]
else:
from concurrent import futures
with futures.ThreadPoolExecutor(nthreads) as executor:
arrays = list(
executor.map(convert_column, columns_to_convert,
convert_types))
types = [x.type for x in arrays]
metadata = construct_metadata(df, column_names, index_columns,
index_column_names, preserve_index, types)
names = column_names + index_column_names
return names, arrays, metadata
def serve():
server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
launchtor_pb2_grpc.add_TorRequesterServicer_to_server(TorRequester(), server)
server.add_insecure_port('[::]:'+tor_grpc_port)
server.start()
server.wait_for_termination()
def register_tests(self):
if self.tests:
return self.tests
if not self.rebuild():
raise RuntimeError("Could not rebuild GStreamer unit tests")
self.load_tests_info()
mesontests = self.get_meson_tests()
to_inspect = []
all_sublaunchers_tests = set()
for test in mesontests:
sublauncher_tests = self.setup_tests_from_sublauncher(test)
if sublauncher_tests:
all_sublaunchers_tests |= sublauncher_tests
continue
binary = test['cmd'][0]
test_info = self.check_binary_ts(binary)
if test_info is True:
continue
elif test_info is None:
test_info = self.check_binary_ts(binary)
if test_info is None:
raise RuntimeError("Test binary %s does not exist"
" even after a full rebuild" % binary)
with open(binary, 'rb') as f:
if b"gstcheck" not in f.read():
self.tests_info[binary] = [0, []]
continue
to_inspect.append(test)
if to_inspect:
executor = conc.ThreadPoolExecutor(
max_workers=self.options.num_jobs)
tmp = []
for test in to_inspect:
tmp.append(executor.submit(self._list_gst_check_tests, test))
for e in tmp:
e.result()
for test in mesontests:
name = self.get_test_name(test)
if name in all_sublaunchers_tests:
continue
gst_tests = self.tests_info[test['cmd'][0]][1]
if os.path.basename(test['cmd'][0]) in \
['gst-tester-1.0', 'gst-tester-1.0.exe']:
fpath = test['cmd'][1]
self.add_test(
GstValidateSimpleTest(fpath, name, self.options,
self.reporter))
continue
if not gst_tests:
child_env = self.get_child_env(name)
self.add_test(
GstCheckTest(name, self.options, self.reporter, test,
child_env))
else:
for ltest in gst_tests:
name = self.get_test_name(test) + '.' + ltest
child_env = self.get_child_env(name, ltest)
self.add_test(
GstCheckTest(name, self.options, self.reporter, test,
child_env))
self.save_tests_info()
self._registered = True
return self.tests
