def test_mapping_api(client: TestClient):
key = create_api_client('user1', 'sirius_cybernetics_gpp', INGEST_JOBS_API)
request = Mapping(name="foo", description="bar", resource_spec="blah")
response: CreateMappingResponse = client.post(url="/api/ingest/mappings",
data=request.json(),
headers={API_KEY_NAME: key})
assert response.status_code == 200
response = client.get(url="/api/ingest/mappings/" + "foo",
headers={API_KEY_NAME: key})
mapping = Mapping(
**response.json()) # first item because we have a version tag in there
assert mapping.name == "foo"
def test_auth_session(sample_file):
ingestor = MappedHD5Ingestor(Mapping(**mapping_dict), sample_file,
"test_root")
for name, doc in ingestor.generate_docstream():
if name == 'start':
assert doc['auth_session'] == []
continue
ingestor = MappedHD5Ingestor(Mapping(**mapping_dict),
sample_file,
"test_root",
auth_session=['bealine1', 'users1'])
for name, doc in ingestor.generate_docstream():
if name == 'start':
assert doc['auth_session'] == ['bealine1', 'users1']
continue
def test_timestamp_error(sample_file):
mapping = {
"name": "test name",
"description": "test descriptions",
"version": "42",
"resource_spec": "MultiKeySlice",
"md_mappings": [],
"stream_mappings": {
"do_not_cross": {
"time_stamp":
"/does/not/exist",
"mapping_fields": [{
"field": "/exchange/data",
"external": True
}, {
"field": "/process/acquisition/sample_position_x",
"description": "tile_xmovedist"
}]
}
}
}
ingestor = MappedHD5Ingestor(Mapping(**mapping), sample_file, "test_root")
list(ingestor.generate_docstream())
assert "Error fetching timestamp" in ingestor.issues[0]
def test_hdf5_mapped_ingestor(sample_file, tmp_path):
ingestor = MappedHD5Ingestor(Mapping(**mapping_dict),
sample_file,
"test_root",
thumbs_root=tmp_path)
run_cache = SingleRunCache()
descriptors = []
result_events = []
result_datums = []
start_found = False
stop_found = False
run_uid = ""
for name, doc in ingestor.generate_docstream():
run_cache.callback(name, doc)
if name == "start":
assert doc[
":measurement:sample:name"] == "my sample", "metadata in start doc"
assert doc["projections"][0][
'name'] == "foo_bar", "projection is in start doc"
start_found = True
run_uid = doc['uid']
continue
if name == "descriptor":
descriptors.append(doc)
continue
if name == "resource":
doc["spec"] == mapping_dict["resource_spec"]
continue
if name == "datum":
result_datums.append(doc)
continue
if name == "resource":
result_events.append(doc)
continue
if name == "event":
result_events.append(doc)
if name == "stop":
stop_found = True
assert doc["num_events"]["primary"] == num_frames_primary
# assert doc["num_events"]["darks"] == num_frames_darks
continue
assert start_found, "a start document was produced"
assert stop_found, "a stop document was produced"
assert len(descriptors) == 2, "return two descriptors"
assert descriptors[0]["name"] == "primary", "first descriptor is primary"
assert descriptors[1]["name"] == "darks", "second descriptor is darks"
assert len(
descriptors[0]["data_keys"].keys()) == 2, "primary has two data_keys"
assert len(result_datums) == num_frames_primary + num_frames_darks
assert len(result_events) == num_frames_primary + num_frames_darks
run = run_cache.retrieve()
stream = run["primary"].to_dask()
assert stream
dir = Path(tmp_path)
file = run_uid + ".png"
assert Path(dir / file).exists()
def test_ingest(sample_file, init_mongomock):
mapping = Mapping(**mapping_dict)
create_mapping("slartibartfast", mapping)
mapping = find_mapping("slartibartfast", "magrathia")
assert mapping.resource_spec == "MultiKeySlice", "test a field"
job = create_job("user1", sample_file.filename, "magrathia", ["deep_thought"])
start_uid = ingest("slartibartfast", job)
job = find_job(job.id)
assert job is not None
assert job.status == JobStatus.successful, f'injest completed {job.status_history}'
assert bluesky_context.db['run_start'].find_one({"uid": start_uid}) is not None, "job wrote start doc"
def gen_ev_docs(scm: ScicatTomo):
with open(
'/home/dylan/work/als-computing/splash-ingest/.scratch/832Mapping.json',
'r') as json_file:
data = json.load(json_file)
map = Mapping(**data)
with h5py.File('/home/dylan/data/beamlines/als832/20210204_172932_ddd.h5',
'r') as h5_file:
ingestor = MappedHD5Ingestor(
map, h5_file, 'root', '/home/dylan/data/beamlines/als832/thumbs')
for name, doc in ingestor.generate_docstream():
if 'start' in name:
doRaw(doc, scm)
if 'descriptor' in name:
pprint(doc)
def test_mapped_ingestor_bad_metadata_field(sample_file):
mapping_dict_bad_metadata_field = {
"name": "test name",
"description": "test descriptions",
"version": "42",
"resource_spec": "MultiKeySlice",
"md_mappings": [
{
"field": "raise_exception"
},
],
"stream_mappings": {}
}
ingestor = MappedHD5Ingestor(Mapping(**mapping_dict_bad_metadata_field),
sample_file, "test_root")
list(ingestor.generate_docstream())
assert "Error finding mapping" in ingestor.issues[0]
def test_mapped_ingestor_bad_stream_field(sample_file):
mapping_dict_bad_stream_field = {
"name": "test name",
"description": "test descriptions",
"version": "42",
"resource_spec": "MultiKeySlice",
"md_mappings": [{
"field": "/measurement/sample/name"
}],
"stream_mappings": {
"primary": {
"time_stamp": "/does/not/exist",
"mapping_fields": [{
"field": "raise_exception"
}]
},
}
}
ingestor = MappedHD5Ingestor(Mapping(**mapping_dict_bad_stream_field),
sample_file, "test_root")
list(ingestor.generate_docstream())
assert "Error finding mapping" in ingestor.issues[0]
def test_build_mapping():
mapping = Mapping(**mapping_dict)
assert mapping.name == 'test name'
assert mapping.stream_mappings['primary'].mapping_fields[
0].field == '/exchange/data'
def test_calc_num_events(sample_file):
stream_mapping = Mapping(
**mapping_dict).stream_mappings["primary"].mapping_fields
num_events = calc_num_events(stream_mapping, sample_file)
assert num_events == num_frames_primary, "primary stream has same number events as primary stream frames"
assert calc_num_events({}, sample_file) == 0, "no fields returns none"
def find_mapping(submitter, mapping_id: str) -> Mapping:
mapping_dict = service_context.ingest_mappings.find_one({"name": mapping_id})
if mapping_dict is None:
return None
revision = mapping_dict.pop('revision')
return Mapping(**mapping_dict)