def load_kwargs(hdf5, default_raw):
kwargs = [{
'filetype': 'HDF5',
'path': hdf5.filename
}, {
'filetype': 'RAW',
'path': default_raw._path,
'nav_shape': default_raw.shape.nav,
'sig_shape': default_raw.shape.sig,
'dtype': default_raw.dtype
}, {
'filetype': 'memory',
'data': np.ones((3, 4, 5, 6))
}]
testdata_path = get_testdata_path()
blo_path = os.path.join(testdata_path, 'default.blo')
if os.path.isfile(blo_path):
kwargs.append({
'filetype': 'BLO',
'path': blo_path,
})
dm_files = list(sorted(glob(os.path.join(testdata_path, 'dm', '*.dm4'))))
if dm_files:
kwargs.append({'filetype': 'dm', 'files': dm_files})
empad_path = os.path.join(testdata_path, 'EMPAD',
'acquisition_12_pretty.xml')
if os.path.isfile(empad_path):
kwargs.append({'filetype': 'EMPAD', 'path': empad_path})
frms6_path = os.path.join(testdata_path, 'frms6',
'C16_15_24_151203_019.hdr')
if os.path.isfile(frms6_path):
kwargs.append({'filetype': 'frms6', 'path': frms6_path})
k2is_path = os.path.join(testdata_path, 'Capture52', 'Capture52_.gtg')
if os.path.isfile(k2is_path):
kwargs.append({'filetype': 'k2is', 'path': k2is_path})
mib_path = os.path.join(testdata_path, 'default.mib')
if os.path.isfile(mib_path):
kwargs.append({
'filetype': 'mib',
'path': mib_path,
'nav_shape': (32, 32)
})
mrc_path = os.path.join(testdata_path, 'mrc', '20200821_92978_movie.mrc')
if os.path.isfile(mrc_path):
kwargs.append({'filetype': 'mrc', 'path': mrc_path})
seq_path = os.path.join(testdata_path, 'default.seq')
if os.path.isfile(seq_path):
kwargs.append({
'filetype': 'seq',
'path': seq_path,
'nav_shape': (8, 8)
})
ser_path = os.path.join(testdata_path, 'default.ser')
if os.path.isfile(ser_path):
kwargs.append({'filetype': 'ser', 'path': ser_path})
return kwargs
import numpy as np
import pytest
from libertem.io.dataset.mib import MIBDataSet
from libertem.job.masks import ApplyMasksJob
from libertem.job.raw import PickFrameJob
from libertem.udf.raw import PickUDF
from libertem.executor.inline import InlineJobExecutor
from libertem.analysis.raw import PickFrameAnalysis
from libertem.common import Slice, Shape
from libertem.io.dataset.base import TilingScheme
from utils import dataset_correction_verification, get_testdata_path
MIB_TESTDATA_PATH = os.path.join(get_testdata_path(), 'default.mib')
HAVE_MIB_TESTDATA = os.path.exists(MIB_TESTDATA_PATH)
pytestmark = pytest.mark.skipif(not HAVE_MIB_TESTDATA,
reason="need .mib testdata") # NOQA
@pytest.fixture
def default_mib(lt_ctx):
scan_size = (32, 32)
ds = MIBDataSet(path=MIB_TESTDATA_PATH, scan_size=scan_size)
ds = ds.initialize(lt_ctx.executor)
return ds
def test_detect(lt_ctx):
from libertem.common import Shape
from libertem.common.buffers import reshaped_view
from libertem.udf.sumsigudf import SumSigUDF
from libertem.udf.raw import PickUDF
from libertem.io.dataset.base import TilingScheme, BufferedBackend, MMapBackend, DirectBackend
from libertem.io.corrections import CorrectionSet
from utils import get_testdata_path, ValidationUDF
import defusedxml.ElementTree as ET
try:
import pims
except ModuleNotFoundError:
pims = None
SEQ_TESTDATA_PATH = os.path.join(get_testdata_path(), 'default.seq')
HAVE_SEQ_TESTDATA = os.path.exists(SEQ_TESTDATA_PATH)
needsdata = pytest.mark.skipif(not HAVE_SEQ_TESTDATA,
reason="need .seq testdata")
@pytest.fixture
def default_seq(lt_ctx):
nav_shape = (8, 8)
ds = lt_ctx.load(
"seq",
path=SEQ_TESTDATA_PATH,
nav_shape=nav_shape,
io_backend=MMapBackend(),
)
def test_datadir_inspection():
print("get_testdata_path():", get_testdata_path())
print(list(sorted(glob.glob(get_testdata_path() + '/*'))))
print(list(sorted(glob.glob(TMP_TESTDATA_PATH + '/*'))))
assert HAVE_TMP_TESTDATA
import os
import glob
import nbformat
import pytest
import numpy as np
from nbconvert.preprocessors import ExecutePreprocessor
from libertem.web.notebook_generator.notebook_generator import notebook_generator
from utils import get_testdata_path
from temp_utils import _get_hdf5_params, create_random_hdf5
pytestmark = [pytest.mark.functional]
TMP_TESTDATA_PATH = os.path.join(get_testdata_path(), 'temp_data')
HAVE_TMP_TESTDATA = os.path.exists(TMP_TESTDATA_PATH)
def test_ring_default(hdf5_ds_2, tmpdir_factory, lt_ctx, local_cluster_url):
datadir = tmpdir_factory.mktemp('template_tests')
conn = {'connection': {'type': 'tcp', 'address': local_cluster_url}}
path = hdf5_ds_2.path
dataset = _get_hdf5_params(path)
analysis = [{
"analysisType": "APPLY_RING_MASK",
"parameters": {
'shape': 'ring',
'cx': 8,
from libertem.io.dataset.blo import BloDataSet
from libertem.io.dataset.base import TilingScheme, BufferedBackend, MMapBackend
from libertem.common import Shape
from libertem.common.buffers import reshaped_view
from libertem.udf.sumsigudf import SumSigUDF
from libertem.udf.raw import PickUDF
from utils import dataset_correction_verification, get_testdata_path, ValidationUDF
try:
import hyperspy.api as hs
except ModuleNotFoundError:
hs = None
BLO_TESTDATA_PATH = os.path.join(get_testdata_path(), 'default.blo')
HAVE_BLO_TESTDATA = os.path.exists(BLO_TESTDATA_PATH)
pytestmark = pytest.mark.skipif(not HAVE_BLO_TESTDATA, reason="need .blo testdata") # NOQA
@pytest.fixture()
def default_blo(lt_ctx):
ds = lt_ctx.load(
"blo",
path=str(BLO_TESTDATA_PATH),
io_backend=MMapBackend(),
)
return ds
from libertem.io.dataset.k2is import K2ISDataSet
from libertem.executor.inline import InlineJobExecutor
from libertem.analysis.raw import PickFrameAnalysis
from libertem.common.buffers import BufferWrapper
from libertem.udf import UDF
from libertem.udf.raw import PickUDF
from libertem.udf.masks import ApplyMasksUDF
from libertem.io.dataset.base import TilingScheme, BufferedBackend, MMapBackend
from libertem.common import Shape
from libertem.common.buffers import reshaped_view
from libertem import masks
from utils import dataset_correction_verification, get_testdata_path, ValidationUDF
K2IS_TESTDATA_PATH = os.path.join(get_testdata_path(), 'Capture52', 'Capture52_.gtg')
K2IS_TESTDATA_RAW = os.path.join(
get_testdata_path(),
'Capture52',
'Capture52_.gtg_(34, 35, 1860, 2048)_uint16.raw'
)
HAVE_K2IS_TESTDATA = os.path.exists(K2IS_TESTDATA_PATH)
HAVE_K2IS_RAWDATA = os.path.exists(K2IS_TESTDATA_RAW)
pytestmark = pytest.mark.skipif(not HAVE_K2IS_TESTDATA, reason="need K2IS testdata") # NOQA
@pytest.fixture
def default_k2is(lt_ctx):
ds = lt_ctx.load(
"k2is",
from libertem.analysis.sum import SumAnalysis
from libertem.udf.sumsigudf import SumSigUDF
from libertem.io.dataset.base import TilingScheme, BufferedBackend, MMapBackend
from libertem.common import Shape
from libertem.common.buffers import reshaped_view
from libertem.udf.raw import PickUDF
from utils import (dataset_correction_verification, get_testdata_path,
ValidationUDF, FakeBackend)
try:
import stemtool
except ModuleNotFoundError:
stemtool = None
FRMS6_TESTDATA_PATH = os.path.join(get_testdata_path(), 'frms6',
'C16_15_24_151203_019.hdr')
HAVE_FRMS6_TESTDATA = os.path.exists(FRMS6_TESTDATA_PATH)
pytestmark = pytest.mark.skipif(not HAVE_FRMS6_TESTDATA,
reason="need frms6 testdata") # NOQA
@pytest.fixture
def default_frms6(lt_ctx):
ds = lt_ctx.load(
"frms6",
path=FRMS6_TESTDATA_PATH,
io_backend=MMapBackend(),
)
return ds
from libertem.udf import UDF
from libertem.udf.sum import SumUDF
from libertem.udf.sumsigudf import SumSigUDF
from libertem.contrib.daskadapter import make_dask_array
from libertem.io.dataset.base import TilingScheme
from libertem.common import Shape
from libertem_live.detectors.merlin import MerlinDataSource, MerlinControl
from libertem_live.detectors.merlin.sim import (CameraSim, ServerThreadMixin,
StopException, TriggerClient,
UndeadException)
from utils import get_testdata_path
MIB_TESTDATA_PATH = os.path.join(get_testdata_path(), 'default.mib')
HAVE_MIB_TESTDATA = os.path.exists(MIB_TESTDATA_PATH)
PTYCHO_TESTDATA_PATH = os.path.join(get_testdata_path(), '20200518 165148',
'default.hdr')
HAVE_PTYCHO_TESTDATA = os.path.exists(PTYCHO_TESTDATA_PATH)
pytestmark = [
pytest.mark.skipif(not HAVE_MIB_TESTDATA, reason="need .mib testdata"),
pytest.mark.data,
]
def run_camera_sim(*args, **kwargs):
server = CameraSim(*args,
host='127.0.0.1',
import os
import glob
import hashlib
import numpy as np
import pytest
from libertem.io.dataset.dm import DMDataSet
from libertem.udf.sum import SumUDF
from utils import dataset_correction_verification, get_testdata_path
DM_TESTDATA_PATH = os.path.join(get_testdata_path(), 'dm')
HAVE_DM_TESTDATA = os.path.exists(DM_TESTDATA_PATH)
pytestmark = pytest.mark.skipif(not HAVE_DM_TESTDATA,
reason="need .dm4 testdata") # NOQA
@pytest.fixture
def default_dm(lt_ctx):
files = list(sorted(glob.glob(os.path.join(DM_TESTDATA_PATH, '*.dm4'))))
ds = lt_ctx.load("dm", files=files)
return ds
@pytest.fixture
def dm_stack_of_3d(lt_ctx):
files = list(
sorted(glob.glob(os.path.join(DM_TESTDATA_PATH, '3D', '*.dm3'))))
ds = lt_ctx.load("dm", files=files)
import pytest
import numpy as np
from libertem.job.masks import ApplyMasksJob
from libertem.job.raw import PickFrameJob
from libertem.executor.inline import InlineJobExecutor
from libertem.analysis.raw import PickFrameAnalysis
from libertem.io.dataset.base import DataSetException, TilingScheme
from libertem.io.dataset.empad import EMPADDataSet
from libertem.common import Slice, Shape
from utils import _mk_random
from utils import dataset_correction_verification, get_testdata_path
EMPAD_TESTDATA_PATH = os.path.join(get_testdata_path(), 'EMPAD')
EMPAD_RAW = os.path.join(EMPAD_TESTDATA_PATH, 'scan_11_x4_y4.raw')
EMPAD_XML = os.path.join(EMPAD_TESTDATA_PATH, 'acquisition_12_pretty.xml')
EMPAD_XML_2 = os.path.join(EMPAD_TESTDATA_PATH, 'acquisition_12_pretty.xml')
HAVE_EMPAD_TESTDATA = os.path.exists(EMPAD_RAW) and os.path.exists(EMPAD_XML)
pytestmark = pytest.mark.skipif(not HAVE_EMPAD_TESTDATA,
reason="need EMPAD testdata") # NOQA
@pytest.fixture
def default_empad():
executor = InlineJobExecutor()
ds = EMPADDataSet(path=EMPAD_XML, )
ds = ds.initialize(executor)
yield ds
from libertem.io.dataset.k2is import K2ISDataSet
from libertem.executor.inline import InlineJobExecutor
from libertem.analysis.raw import PickFrameAnalysis
from libertem.common.buffers import BufferWrapper
from libertem.udf import UDF
from libertem.udf.raw import PickUDF
from libertem.udf.sumsigudf import SumSigUDF
from libertem.udf.masks import ApplyMasksUDF
from libertem.io.dataset.base import TilingScheme, BufferedBackend, MMapBackend
from libertem.common import Shape
from libertem.common.buffers import reshaped_view
from libertem import masks
from utils import dataset_correction_verification, get_testdata_path, ValidationUDF
K2IS_TESTDATA_DIR = os.path.join(get_testdata_path(), 'Capture52')
K2IS_TESTDATA_PATH = os.path.join(K2IS_TESTDATA_DIR, 'Capture52_.gtg')
K2IS_TESTDATA_RAW = os.path.join(
K2IS_TESTDATA_DIR, 'Capture52_.gtg_(34, 35, 1860, 2048)_uint16.raw')
HAVE_K2IS_TESTDATA = os.path.exists(K2IS_TESTDATA_PATH)
HAVE_K2IS_RAWDATA = os.path.exists(K2IS_TESTDATA_RAW)
pytestmark = pytest.mark.skipif(not HAVE_K2IS_TESTDATA,
reason="need K2IS testdata") # NOQA
@pytest.fixture
def default_k2is(lt_ctx):
ds = lt_ctx.load(
"k2is",
path=K2IS_TESTDATA_PATH,
import os
import hashlib
import numpy as np
import pytest
from libertem.io.dataset.mrc import MRCDataSet
from utils import dataset_correction_verification, get_testdata_path
MRC_TESTDATA_PATH = os.path.join(
get_testdata_path(),
'mrc',
'20200821_92978_movie.mrc',
)
HAVE_MRC_TESTDATA = os.path.exists(MRC_TESTDATA_PATH)
print(MRC_TESTDATA_PATH)
pytestmark = pytest.mark.skipif(not HAVE_MRC_TESTDATA,
reason="need .mrc testdata") # NOQA
@pytest.fixture
def default_mrc(lt_ctx):
ds = lt_ctx.load("mrc", path=MRC_TESTDATA_PATH)
return ds
def test_simple_open(default_mrc):
assert tuple(default_mrc.shape) == (4, 1024, 1024)
import numpy as np
import pytest
from PIL import Image
from libertem.io.dataset.tvips import TVIPSDataSet, get_filenames
from libertem.udf.raw import PickUDF
from libertem.common import Shape
from libertem.udf.sumsigudf import SumSigUDF
from libertem.io.dataset.base import (TilingScheme, BufferedBackend,
MMapBackend, DirectBackend)
from libertem.common.buffers import reshaped_view
from utils import dataset_correction_verification, get_testdata_path, ValidationUDF
TVIPS_TESTDATA_PATH = os.path.join(get_testdata_path(), 'TVIPS',
'rec_20200623_080237_000.tvips')
HAVE_TVIPS_TESTDATA = os.path.exists(TVIPS_TESTDATA_PATH)
TVIPS_REFERENCE_DATA_PATTERN = os.path.join(get_testdata_path(), 'TVIPS',
'rec_20200623_080237_*.tif')
needsdata = pytest.mark.skipif(not HAVE_TVIPS_TESTDATA,
reason="need .tvips testdata") # NOQA
def get_reference_files():
files = glob.glob(TVIPS_REFERENCE_DATA_PATTERN)
return list(sorted(files, key=lambda fn: int(fn[-6:-4])))
from libertem.udf.sumsigudf import SumSigUDF
from libertem.io.dataset.base import BufferedBackend, TilingScheme
from libertem.common import Shape
from libertem.common.buffers import reshaped_view
from libertem.udf.raw import PickUDF
from utils import dataset_correction_verification, get_testdata_path, ValidationUDF
try:
import mrcfile
except ModuleNotFoundError:
mrcfile = None
MRC_TESTDATA_PATH = os.path.join(
get_testdata_path(), 'mrc', '20200821_92978_movie.mrc',
)
HAVE_MRC_TESTDATA = os.path.exists(MRC_TESTDATA_PATH)
pytestmark = pytest.mark.skipif(not HAVE_MRC_TESTDATA, reason="need .mrc testdata") # NOQA
@pytest.fixture
def default_mrc(lt_ctx):
ds = lt_ctx.load("mrc", path=MRC_TESTDATA_PATH)
return ds
@pytest.fixture
def buffered_mrc(lt_ctx):
# FIXME: not yet supported
from libertem.udf.sumsigudf import SumSigUDF
from libertem.io.dataset.base import TilingScheme, BufferedBackend, MMapBackend, DirectBackend
from libertem.common import Shape
from libertem.common.buffers import reshaped_view
from libertem.udf.raw import PickUDF
from utils import (dataset_correction_verification, get_testdata_path, ValidationUDF,
FakeBackend)
try:
import stemtool
except ModuleNotFoundError:
stemtool = None
FRMS6_TESTDATA_PATH = os.path.join(get_testdata_path(), 'frms6', 'C16_15_24_151203_019.hdr')
HAVE_FRMS6_TESTDATA = os.path.exists(FRMS6_TESTDATA_PATH)
needsdata = pytest.mark.skipif(not HAVE_FRMS6_TESTDATA, reason="need frms6 testdata") # NOQA
@pytest.fixture
def default_frms6(lt_ctx):
ds = lt_ctx.load(
"frms6",
path=FRMS6_TESTDATA_PATH,
io_backend=MMapBackend(),
)
return ds
