def load_kwargs(hdf5, default_raw): kwargs = [{ 'filetype': 'HDF5', 'path': hdf5.filename }, { 'filetype': 'RAW', 'path': default_raw._path, 'nav_shape': default_raw.shape.nav, 'sig_shape': default_raw.shape.sig, 'dtype': default_raw.dtype }, { 'filetype': 'memory', 'data': np.ones((3, 4, 5, 6)) }] testdata_path = get_testdata_path() blo_path = os.path.join(testdata_path, 'default.blo') if os.path.isfile(blo_path): kwargs.append({ 'filetype': 'BLO', 'path': blo_path, }) dm_files = list(sorted(glob(os.path.join(testdata_path, 'dm', '*.dm4')))) if dm_files: kwargs.append({'filetype': 'dm', 'files': dm_files}) empad_path = os.path.join(testdata_path, 'EMPAD', 'acquisition_12_pretty.xml') if os.path.isfile(empad_path): kwargs.append({'filetype': 'EMPAD', 'path': empad_path}) frms6_path = os.path.join(testdata_path, 'frms6', 'C16_15_24_151203_019.hdr') if os.path.isfile(frms6_path): kwargs.append({'filetype': 'frms6', 'path': frms6_path}) k2is_path = os.path.join(testdata_path, 'Capture52', 'Capture52_.gtg') if os.path.isfile(k2is_path): kwargs.append({'filetype': 'k2is', 'path': k2is_path}) mib_path = os.path.join(testdata_path, 'default.mib') if os.path.isfile(mib_path): kwargs.append({ 'filetype': 'mib', 'path': mib_path, 'nav_shape': (32, 32) }) mrc_path = os.path.join(testdata_path, 'mrc', '20200821_92978_movie.mrc') if os.path.isfile(mrc_path): kwargs.append({'filetype': 'mrc', 'path': mrc_path}) seq_path = os.path.join(testdata_path, 'default.seq') if os.path.isfile(seq_path): kwargs.append({ 'filetype': 'seq', 'path': seq_path, 'nav_shape': (8, 8) }) ser_path = os.path.join(testdata_path, 'default.ser') if os.path.isfile(ser_path): kwargs.append({'filetype': 'ser', 'path': ser_path}) return kwargs
import numpy as np import pytest from libertem.io.dataset.mib import MIBDataSet from libertem.job.masks import ApplyMasksJob from libertem.job.raw import PickFrameJob from libertem.udf.raw import PickUDF from libertem.executor.inline import InlineJobExecutor from libertem.analysis.raw import PickFrameAnalysis from libertem.common import Slice, Shape from libertem.io.dataset.base import TilingScheme from utils import dataset_correction_verification, get_testdata_path MIB_TESTDATA_PATH = os.path.join(get_testdata_path(), 'default.mib') HAVE_MIB_TESTDATA = os.path.exists(MIB_TESTDATA_PATH) pytestmark = pytest.mark.skipif(not HAVE_MIB_TESTDATA, reason="need .mib testdata") # NOQA @pytest.fixture def default_mib(lt_ctx): scan_size = (32, 32) ds = MIBDataSet(path=MIB_TESTDATA_PATH, scan_size=scan_size) ds = ds.initialize(lt_ctx.executor) return ds def test_detect(lt_ctx):
from libertem.common import Shape from libertem.common.buffers import reshaped_view from libertem.udf.sumsigudf import SumSigUDF from libertem.udf.raw import PickUDF from libertem.io.dataset.base import TilingScheme, BufferedBackend, MMapBackend, DirectBackend from libertem.io.corrections import CorrectionSet from utils import get_testdata_path, ValidationUDF import defusedxml.ElementTree as ET try: import pims except ModuleNotFoundError: pims = None SEQ_TESTDATA_PATH = os.path.join(get_testdata_path(), 'default.seq') HAVE_SEQ_TESTDATA = os.path.exists(SEQ_TESTDATA_PATH) needsdata = pytest.mark.skipif(not HAVE_SEQ_TESTDATA, reason="need .seq testdata") @pytest.fixture def default_seq(lt_ctx): nav_shape = (8, 8) ds = lt_ctx.load( "seq", path=SEQ_TESTDATA_PATH, nav_shape=nav_shape, io_backend=MMapBackend(), )
def test_datadir_inspection(): print("get_testdata_path():", get_testdata_path()) print(list(sorted(glob.glob(get_testdata_path() + '/*')))) print(list(sorted(glob.glob(TMP_TESTDATA_PATH + '/*')))) assert HAVE_TMP_TESTDATA
import os import glob import nbformat import pytest import numpy as np from nbconvert.preprocessors import ExecutePreprocessor from libertem.web.notebook_generator.notebook_generator import notebook_generator from utils import get_testdata_path from temp_utils import _get_hdf5_params, create_random_hdf5 pytestmark = [pytest.mark.functional] TMP_TESTDATA_PATH = os.path.join(get_testdata_path(), 'temp_data') HAVE_TMP_TESTDATA = os.path.exists(TMP_TESTDATA_PATH) def test_ring_default(hdf5_ds_2, tmpdir_factory, lt_ctx, local_cluster_url): datadir = tmpdir_factory.mktemp('template_tests') conn = {'connection': {'type': 'tcp', 'address': local_cluster_url}} path = hdf5_ds_2.path dataset = _get_hdf5_params(path) analysis = [{ "analysisType": "APPLY_RING_MASK", "parameters": { 'shape': 'ring', 'cx': 8,
from libertem.io.dataset.blo import BloDataSet from libertem.io.dataset.base import TilingScheme, BufferedBackend, MMapBackend from libertem.common import Shape from libertem.common.buffers import reshaped_view from libertem.udf.sumsigudf import SumSigUDF from libertem.udf.raw import PickUDF from utils import dataset_correction_verification, get_testdata_path, ValidationUDF try: import hyperspy.api as hs except ModuleNotFoundError: hs = None BLO_TESTDATA_PATH = os.path.join(get_testdata_path(), 'default.blo') HAVE_BLO_TESTDATA = os.path.exists(BLO_TESTDATA_PATH) pytestmark = pytest.mark.skipif(not HAVE_BLO_TESTDATA, reason="need .blo testdata") # NOQA @pytest.fixture() def default_blo(lt_ctx): ds = lt_ctx.load( "blo", path=str(BLO_TESTDATA_PATH), io_backend=MMapBackend(), ) return ds
from libertem.io.dataset.k2is import K2ISDataSet from libertem.executor.inline import InlineJobExecutor from libertem.analysis.raw import PickFrameAnalysis from libertem.common.buffers import BufferWrapper from libertem.udf import UDF from libertem.udf.raw import PickUDF from libertem.udf.masks import ApplyMasksUDF from libertem.io.dataset.base import TilingScheme, BufferedBackend, MMapBackend from libertem.common import Shape from libertem.common.buffers import reshaped_view from libertem import masks from utils import dataset_correction_verification, get_testdata_path, ValidationUDF K2IS_TESTDATA_PATH = os.path.join(get_testdata_path(), 'Capture52', 'Capture52_.gtg') K2IS_TESTDATA_RAW = os.path.join( get_testdata_path(), 'Capture52', 'Capture52_.gtg_(34, 35, 1860, 2048)_uint16.raw' ) HAVE_K2IS_TESTDATA = os.path.exists(K2IS_TESTDATA_PATH) HAVE_K2IS_RAWDATA = os.path.exists(K2IS_TESTDATA_RAW) pytestmark = pytest.mark.skipif(not HAVE_K2IS_TESTDATA, reason="need K2IS testdata") # NOQA @pytest.fixture def default_k2is(lt_ctx): ds = lt_ctx.load( "k2is",
from libertem.analysis.sum import SumAnalysis from libertem.udf.sumsigudf import SumSigUDF from libertem.io.dataset.base import TilingScheme, BufferedBackend, MMapBackend from libertem.common import Shape from libertem.common.buffers import reshaped_view from libertem.udf.raw import PickUDF from utils import (dataset_correction_verification, get_testdata_path, ValidationUDF, FakeBackend) try: import stemtool except ModuleNotFoundError: stemtool = None FRMS6_TESTDATA_PATH = os.path.join(get_testdata_path(), 'frms6', 'C16_15_24_151203_019.hdr') HAVE_FRMS6_TESTDATA = os.path.exists(FRMS6_TESTDATA_PATH) pytestmark = pytest.mark.skipif(not HAVE_FRMS6_TESTDATA, reason="need frms6 testdata") # NOQA @pytest.fixture def default_frms6(lt_ctx): ds = lt_ctx.load( "frms6", path=FRMS6_TESTDATA_PATH, io_backend=MMapBackend(), ) return ds
from libertem.udf import UDF from libertem.udf.sum import SumUDF from libertem.udf.sumsigudf import SumSigUDF from libertem.contrib.daskadapter import make_dask_array from libertem.io.dataset.base import TilingScheme from libertem.common import Shape from libertem_live.detectors.merlin import MerlinDataSource, MerlinControl from libertem_live.detectors.merlin.sim import (CameraSim, ServerThreadMixin, StopException, TriggerClient, UndeadException) from utils import get_testdata_path MIB_TESTDATA_PATH = os.path.join(get_testdata_path(), 'default.mib') HAVE_MIB_TESTDATA = os.path.exists(MIB_TESTDATA_PATH) PTYCHO_TESTDATA_PATH = os.path.join(get_testdata_path(), '20200518 165148', 'default.hdr') HAVE_PTYCHO_TESTDATA = os.path.exists(PTYCHO_TESTDATA_PATH) pytestmark = [ pytest.mark.skipif(not HAVE_MIB_TESTDATA, reason="need .mib testdata"), pytest.mark.data, ] def run_camera_sim(*args, **kwargs): server = CameraSim(*args, host='127.0.0.1',
import os import glob import hashlib import numpy as np import pytest from libertem.io.dataset.dm import DMDataSet from libertem.udf.sum import SumUDF from utils import dataset_correction_verification, get_testdata_path DM_TESTDATA_PATH = os.path.join(get_testdata_path(), 'dm') HAVE_DM_TESTDATA = os.path.exists(DM_TESTDATA_PATH) pytestmark = pytest.mark.skipif(not HAVE_DM_TESTDATA, reason="need .dm4 testdata") # NOQA @pytest.fixture def default_dm(lt_ctx): files = list(sorted(glob.glob(os.path.join(DM_TESTDATA_PATH, '*.dm4')))) ds = lt_ctx.load("dm", files=files) return ds @pytest.fixture def dm_stack_of_3d(lt_ctx): files = list( sorted(glob.glob(os.path.join(DM_TESTDATA_PATH, '3D', '*.dm3')))) ds = lt_ctx.load("dm", files=files)
import pytest import numpy as np from libertem.job.masks import ApplyMasksJob from libertem.job.raw import PickFrameJob from libertem.executor.inline import InlineJobExecutor from libertem.analysis.raw import PickFrameAnalysis from libertem.io.dataset.base import DataSetException, TilingScheme from libertem.io.dataset.empad import EMPADDataSet from libertem.common import Slice, Shape from utils import _mk_random from utils import dataset_correction_verification, get_testdata_path EMPAD_TESTDATA_PATH = os.path.join(get_testdata_path(), 'EMPAD') EMPAD_RAW = os.path.join(EMPAD_TESTDATA_PATH, 'scan_11_x4_y4.raw') EMPAD_XML = os.path.join(EMPAD_TESTDATA_PATH, 'acquisition_12_pretty.xml') EMPAD_XML_2 = os.path.join(EMPAD_TESTDATA_PATH, 'acquisition_12_pretty.xml') HAVE_EMPAD_TESTDATA = os.path.exists(EMPAD_RAW) and os.path.exists(EMPAD_XML) pytestmark = pytest.mark.skipif(not HAVE_EMPAD_TESTDATA, reason="need EMPAD testdata") # NOQA @pytest.fixture def default_empad(): executor = InlineJobExecutor() ds = EMPADDataSet(path=EMPAD_XML, ) ds = ds.initialize(executor) yield ds
from libertem.io.dataset.k2is import K2ISDataSet from libertem.executor.inline import InlineJobExecutor from libertem.analysis.raw import PickFrameAnalysis from libertem.common.buffers import BufferWrapper from libertem.udf import UDF from libertem.udf.raw import PickUDF from libertem.udf.sumsigudf import SumSigUDF from libertem.udf.masks import ApplyMasksUDF from libertem.io.dataset.base import TilingScheme, BufferedBackend, MMapBackend from libertem.common import Shape from libertem.common.buffers import reshaped_view from libertem import masks from utils import dataset_correction_verification, get_testdata_path, ValidationUDF K2IS_TESTDATA_DIR = os.path.join(get_testdata_path(), 'Capture52') K2IS_TESTDATA_PATH = os.path.join(K2IS_TESTDATA_DIR, 'Capture52_.gtg') K2IS_TESTDATA_RAW = os.path.join( K2IS_TESTDATA_DIR, 'Capture52_.gtg_(34, 35, 1860, 2048)_uint16.raw') HAVE_K2IS_TESTDATA = os.path.exists(K2IS_TESTDATA_PATH) HAVE_K2IS_RAWDATA = os.path.exists(K2IS_TESTDATA_RAW) pytestmark = pytest.mark.skipif(not HAVE_K2IS_TESTDATA, reason="need K2IS testdata") # NOQA @pytest.fixture def default_k2is(lt_ctx): ds = lt_ctx.load( "k2is", path=K2IS_TESTDATA_PATH,
import os import hashlib import numpy as np import pytest from libertem.io.dataset.mrc import MRCDataSet from utils import dataset_correction_verification, get_testdata_path MRC_TESTDATA_PATH = os.path.join( get_testdata_path(), 'mrc', '20200821_92978_movie.mrc', ) HAVE_MRC_TESTDATA = os.path.exists(MRC_TESTDATA_PATH) print(MRC_TESTDATA_PATH) pytestmark = pytest.mark.skipif(not HAVE_MRC_TESTDATA, reason="need .mrc testdata") # NOQA @pytest.fixture def default_mrc(lt_ctx): ds = lt_ctx.load("mrc", path=MRC_TESTDATA_PATH) return ds def test_simple_open(default_mrc): assert tuple(default_mrc.shape) == (4, 1024, 1024)
import numpy as np import pytest from PIL import Image from libertem.io.dataset.tvips import TVIPSDataSet, get_filenames from libertem.udf.raw import PickUDF from libertem.common import Shape from libertem.udf.sumsigudf import SumSigUDF from libertem.io.dataset.base import (TilingScheme, BufferedBackend, MMapBackend, DirectBackend) from libertem.common.buffers import reshaped_view from utils import dataset_correction_verification, get_testdata_path, ValidationUDF TVIPS_TESTDATA_PATH = os.path.join(get_testdata_path(), 'TVIPS', 'rec_20200623_080237_000.tvips') HAVE_TVIPS_TESTDATA = os.path.exists(TVIPS_TESTDATA_PATH) TVIPS_REFERENCE_DATA_PATTERN = os.path.join(get_testdata_path(), 'TVIPS', 'rec_20200623_080237_*.tif') needsdata = pytest.mark.skipif(not HAVE_TVIPS_TESTDATA, reason="need .tvips testdata") # NOQA def get_reference_files(): files = glob.glob(TVIPS_REFERENCE_DATA_PATTERN) return list(sorted(files, key=lambda fn: int(fn[-6:-4])))
from libertem.udf.sumsigudf import SumSigUDF from libertem.io.dataset.base import BufferedBackend, TilingScheme from libertem.common import Shape from libertem.common.buffers import reshaped_view from libertem.udf.raw import PickUDF from utils import dataset_correction_verification, get_testdata_path, ValidationUDF try: import mrcfile except ModuleNotFoundError: mrcfile = None MRC_TESTDATA_PATH = os.path.join( get_testdata_path(), 'mrc', '20200821_92978_movie.mrc', ) HAVE_MRC_TESTDATA = os.path.exists(MRC_TESTDATA_PATH) pytestmark = pytest.mark.skipif(not HAVE_MRC_TESTDATA, reason="need .mrc testdata") # NOQA @pytest.fixture def default_mrc(lt_ctx): ds = lt_ctx.load("mrc", path=MRC_TESTDATA_PATH) return ds @pytest.fixture def buffered_mrc(lt_ctx): # FIXME: not yet supported
from libertem.udf.sumsigudf import SumSigUDF from libertem.io.dataset.base import TilingScheme, BufferedBackend, MMapBackend, DirectBackend from libertem.common import Shape from libertem.common.buffers import reshaped_view from libertem.udf.raw import PickUDF from utils import (dataset_correction_verification, get_testdata_path, ValidationUDF, FakeBackend) try: import stemtool except ModuleNotFoundError: stemtool = None FRMS6_TESTDATA_PATH = os.path.join(get_testdata_path(), 'frms6', 'C16_15_24_151203_019.hdr') HAVE_FRMS6_TESTDATA = os.path.exists(FRMS6_TESTDATA_PATH) needsdata = pytest.mark.skipif(not HAVE_FRMS6_TESTDATA, reason="need frms6 testdata") # NOQA @pytest.fixture def default_frms6(lt_ctx): ds = lt_ctx.load( "frms6", path=FRMS6_TESTDATA_PATH, io_backend=MMapBackend(), ) return ds