def test_send_msg_slack(self):
self.args.slack_usr = '******'
ingest_job = IngestJob(self.args)
assert ingest_job.slack_obj is not None
msg = 'test_message_' + datetime.now().strftime("%Y-%m-%d %H:%M:%S")
ingest_job.send_msg(msg, send_slack=True)
log_fname = ingest_job.get_log_fname()
os.remove(log_fname)
def test_send_msg(self):
ingest_job = IngestJob(self.args)
msg = 'test_message_' + datetime.now().strftime("%Y-%m-%d %H:%M:%S")
ingest_job.send_msg(msg)
log_fname = ingest_job.get_log_fname()
with open(log_fname) as f:
log_data = f.readlines()
assert msg in log_data[-1]
os.remove(log_fname)
def per_channel_ingest(args, channel, threads=8):
args.channel = channel
ingest_job = IngestJob(args)
# extract img_size and datatype to check inputs (by actually reading the data)
# this can take a while, as we actually load in the first image slice,
# so we should store this first slice so we don't have to load it again when we later read the entire chunk in z
# we don't do this for render data source because we get the image size and attributes from the render metadata and the # of bits aren't in the metadata or render
if ingest_job.datasource != 'render':
im_width, im_height, im_datatype = ingest_job.get_img_info(
ingest_job.z_range[0])
# we do this before creating boss resources that could be inaccurate
try:
assert ingest_job.img_size[0] == im_width and ingest_job.img_size[
1] == im_height and ingest_job.datatype == im_datatype
except AssertionError:
ingest_job.send_msg('Mismatch between image file and input parameters. Determined image width: {}, height: {}, datatype: {}'.format(
im_width, im_height, im_datatype))
raise ValueError('Image attributes do not match arguments')
# create or get the boss resources for the data
get_only = not ingest_job.create_resources
boss_res_params = BossResParams(ingest_job)
boss_res_params.get_resources(get_only=get_only)
# we just create the resources, don't do anything else
if ingest_job.create_resources:
ingest_job.send_msg('{} Resources set up. Collection: {}, Experiment: {}, Channel: {}'.format(
get_formatted_datetime(), ingest_job.coll_name, ingest_job.exp_name, ingest_job.ch_name))
return 0
else:
ingest_job.send_msg('{} Starting ingest for Collection: {}, Experiment: {}, Channel: {}, Z: {z[0]},{z[1]}'.format(
get_formatted_datetime(), ingest_job.coll_name, ingest_job.exp_name, ingest_job.ch_name, z=ingest_job.z_range))
# we begin the ingest here:
stride_x = 1024
stride_y = 1024
stride_z = 16
x_buckets = get_supercube_lims(ingest_job.x_extent, stride_x)
y_buckets = get_supercube_lims(ingest_job.y_extent, stride_y)
z_buckets = get_supercube_lims(ingest_job.z_range, stride_z)
with ThreadPool(threads) as pool:
# load images files in stacks of 16 at a time into numpy array
for _, z_slices in z_buckets.items():
# read images into numpy array
im_array = ingest_job.read_img_stack(z_slices)
z_rng = [z_slices[0] - ingest_job.offsets[2],
z_slices[-1] + 1 - ingest_job.offsets[2]]
# slice into np array blocks
for _, y_slices in y_buckets.items():
y_rng = [y_slices[0], y_slices[-1] + 1]
ingest_block_partial = partial(
ingest_block, x_buckets=x_buckets, boss_res_params=boss_res_params, ingest_job=ingest_job,
y_rng=y_rng, z_rng=z_rng, im_array=im_array)
pool.map(ingest_block_partial, x_buckets.keys())
# checking data posted correctly for an entire z slice
assert_equal(boss_res_params, ingest_job, ingest_job.z_range)
ch_link = (
'https://ndwebtools.neurodata.io/channel_detail/{}/{}/{}/').format(ingest_job.coll_name, ingest_job.exp_name, ingest_job.ch_name)
ingest_job.send_msg('{} Finished z slices {} for Collection: {}, Experiment: {}, Channel: {}\nThere were {} read failures and {} POST failures.\nView properties of channel and start downsample job on ndwebtools: {}'.format(
get_formatted_datetime(),
ingest_job.z_range, ingest_job.coll_name, ingest_job.exp_name, ingest_job.ch_name,
ingest_job.num_READ_failures, ingest_job.num_POST_failures, ch_link), send_slack=True)
return 0
