def configure_step(self):
"""Configure build
- build Libint wrapper
- generate Makefile
"""
known_types = ['popt', 'psmp']
if self.cfg['type'] not in known_types:
raise EasyBuildError("Unknown build type specified: '%s', known types are %s",
self.cfg['type'], known_types)
# correct start dir, if needed
# recent CP2K versions have a 'cp2k' dir in the unpacked 'cp2k' dir
cp2k_path = os.path.join(self.cfg['start_dir'], 'cp2k')
if os.path.exists(cp2k_path):
self.cfg['start_dir'] = cp2k_path
self.log.info("Corrected start_dir to %s" % self.cfg['start_dir'])
# set compilers options according to toolchain config
# full debug: -g -traceback -check all -fp-stack-check
# -g links to mpi debug libs
if self.toolchain.options['debug']:
self.debug = '-g'
self.log.info("Debug build")
if self.toolchain.options['pic']:
self.fpic = "-fPIC"
self.log.info("Using fPIC")
# report on extra flags being used
if self.cfg['extracflags']:
self.log.info("Using extra CFLAGS: %s" % self.cfg['extracflags'])
if self.cfg['extradflags']:
self.log.info("Using extra DFLAGS: %s" % self.cfg['extradflags'])
# lib(x)smm support
libsmm = get_software_root('libsmm')
libxsmm = get_software_root('libxsmm')
if libxsmm:
self.cfg.update('extradflags', '-D__LIBXSMM')
self.libsmm = '-lxsmm -lxsmmf'
self.log.debug('Using libxsmm %s' % libxsmm)
elif libsmm:
libsmms = glob.glob(os.path.join(libsmm, 'lib', 'libsmm_*nn.a'))
dfs = [os.path.basename(os.path.splitext(x)[0]).replace('lib', '-D__HAS_') for x in libsmms]
moredflags = ' ' + ' '.join(dfs)
self.cfg.update('extradflags', moredflags)
self.libsmm = ' '.join(libsmms)
self.log.debug('Using libsmm %s (extradflags %s)' % (self.libsmm, moredflags))
# obtain list of modinc's to use
if self.cfg["modinc"]:
self.modincpath = self.prepmodinc()
# set typearch
self.typearch = "Linux-x86-64-%s" % self.toolchain.name
# extra make instructions
self.make_instructions = '' # "graphcon.o: graphcon.F\n\t$(FC) -c $(FCFLAGS2) $<\n"
# compiler toolchain specific configuration
comp_fam = self.toolchain.comp_family()
if comp_fam == toolchain.INTELCOMP:
options = self.configure_intel_based()
elif comp_fam == toolchain.GCC:
options = self.configure_GCC_based()
else:
raise EasyBuildError("Don't know how to tweak configuration for compiler family %s" % comp_fam)
# BLAS/LAPACK/FFTW
if get_software_root('imkl'):
options = self.configure_MKL(options)
else:
# BLAS
if get_software_root('ACML'):
options = self.configure_ACML(options)
else:
options = self.configure_BLAS_lib(options)
# FFTW (no MKL involved)
if 'fftw3' in os.getenv('LIBFFT', ''):
options = self.configure_FFTW3(options)
# LAPACK
if os.getenv('LIBLAPACK_MT', None) is not None:
options = self.configure_LAPACK(options)
if os.getenv('LIBSCALAPACK', None) is not None:
options = self.configure_ScaLAPACK(options)
# PLUMED
plumed = get_software_root('PLUMED')
if self.cfg['plumed'] and not plumed:
raise EasyBuildError("The PLUMED module needs to be loaded to build CP2K with PLUMED support")
# enable PLUMED support if PLUMED is listed as a dependency
# and PLUMED support is either explicitly enabled (plumed = True) or unspecified ('plumed' not defined)
if plumed and (self.cfg['plumed'] or self.cfg['plumed'] is None):
options['LIBS'] += ' -lplumed'
options['DFLAGS'] += ' -D__PLUMED2'
# ELPA
elpa = get_software_root('ELPA')
if elpa:
options['LIBS'] += ' -lelpa'
elpa_inc_dir = os.path.join(elpa, 'include', 'elpa-%s' % get_software_version('ELPA'), 'modules')
options['FCFLAGSOPT'] += ' -I%s ' % elpa_inc_dir
if LooseVersion(self.version) >= LooseVersion('6.1'):
elpa_ver = ''.join(get_software_version('ELPA').split('.')[:2])
options['DFLAGS'] += ' -D__ELPA=%s' % elpa_ver
elpa_inc_dir = os.path.join(elpa, 'include', 'elpa-%s' % get_software_version('ELPA'), 'elpa')
options['FCFLAGSOPT'] += ' -I%s ' % elpa_inc_dir
else:
options['DFLAGS'] += ' -D__ELPA3'
# CUDA
cuda = get_software_root('CUDA')
if cuda:
options['DFLAGS'] += ' -D__ACC -D__DBCSR_ACC'
options['LIBS'] += ' -lcudart -lcublas -lcufft -lrt'
options['NVCC'] = ' nvcc'
# avoid group nesting
options['LIBS'] = options['LIBS'].replace('-Wl,--start-group', '').replace('-Wl,--end-group', '')
options['LIBS'] = "-Wl,--start-group %s -Wl,--end-group" % options['LIBS']
# specify correct location for 'data' directory in final installation
options['DATA_DIR'] = os.path.join(self.installdir, 'data')
# create arch file using options set
archfile = os.path.join(self.cfg['start_dir'], 'arch', '%s.%s' % (self.typearch, self.cfg['type']))
txt = self._generate_makefile(options)
write_file(archfile, txt)
self.log.info("Content of makefile (%s):\n%s" % (archfile, txt))
def get_restore_response(domain,
couch_user,
app_id=None,
since=None,
version='1.0',
state=None,
items=False,
force_cache=False,
cache_timeout=None,
overwrite_cache=False,
as_user=None,
device_id=None,
user_id=None,
openrosa_version=None,
skip_fixtures=False,
auth_type=None):
"""
:param domain: Domain being restored from
:param couch_user: User performing restore
:param app_id: App ID of the app making the request
:param since: ID of current sync log used to generate incremental sync
:param version: Version of the sync response required
:param state: Hash value of the current database of cases on the device for consistency checking
:param items: Include item count if True
:param force_cache: Force response to be cached
:param cache_timeout: Override the default cache timeout of 1 hour.
:param overwrite_cache: Ignore cached response if True
:param as_user: Username of user to generate restore for (if different from current user)
:param device_id: ID of device performing restore
:param user_id: ID of user performing restore (used in case of deleted user with same username)
:param openrosa_version:
:param skip_fixtures: Do not include fixtures in sync payload
:param auth_type: The type of auth that was used to authenticate the request.
Used to determine if the request is coming from an actual user or as part of some automation.
:return: Tuple of (http response, timing context or None)
"""
if user_id and user_id != couch_user.user_id:
# sync with a user that has been deleted but a new
# user was created with the same username and password
from couchforms.openrosa_response import get_simple_response_xml
from couchforms.openrosa_response import ResponseNature
response = get_simple_response_xml(
'Attempt to sync with invalid user.',
ResponseNature.OTA_RESTORE_ERROR)
return HttpResponse(response,
content_type="text/xml; charset=utf-8",
status=412), None
is_demo_restore = couch_user.is_commcare_user() and couch_user.is_demo_user
if is_demo_restore:
# if user is in demo-mode, return demo restore
return demo_user_restore_response(couch_user), None
uses_login_as = bool(as_user)
as_user_obj = CouchUser.get_by_username(as_user) if uses_login_as else None
if uses_login_as and not as_user_obj:
msg = _('Invalid restore as user {}').format(as_user)
return HttpResponse(msg, status=401), None
is_permitted, message = is_permitted_to_restore(
domain,
couch_user,
as_user_obj,
)
if not is_permitted:
return HttpResponse(message, status=401), None
restore_user = get_restore_user(domain, couch_user, as_user_obj)
if not restore_user:
return HttpResponse('Could not find user', status=404), None
project = Domain.get_by_name(domain)
async_restore_enabled = (toggles.ASYNC_RESTORE.enabled(domain)
and openrosa_version and
LooseVersion(openrosa_version) >= LooseVersion(
OPENROSA_VERSION_MAP['ASYNC_RESTORE']))
# Ensure fixtures are included if sync is full rather than incremental
if not since:
skip_fixtures = False
app = get_app_cached(domain, app_id) if app_id else None
restore_config = RestoreConfig(
project=project,
restore_user=restore_user,
params=RestoreParams(
sync_log_id=since,
version=version,
state_hash=state,
include_item_count=items,
app=app,
device_id=device_id,
openrosa_version=openrosa_version,
),
cache_settings=RestoreCacheSettings(force_cache=force_cache
or async_restore_enabled,
cache_timeout=cache_timeout,
overwrite_cache=overwrite_cache),
is_async=async_restore_enabled,
skip_fixtures=skip_fixtures,
auth_type=auth_type)
return restore_config.get_response(), restore_config.timing_context
import traceback
from distutils.version import LooseVersion
try:
# Imported as a dependency for dopy
import six
HAS_SIX = True
except ImportError:
HAS_SIX = False
HAS_DOPY = False
try:
import dopy
from dopy.manager import DoError, DoManager
if LooseVersion(dopy.__version__) >= LooseVersion('0.3.2'):
HAS_DOPY = True
except ImportError:
pass
from ansible.module_utils.basic import AnsibleModule
class TimeoutError(Exception):
def __init__(self, msg, id_):
super(TimeoutError, self).__init__(msg)
self.id = id_
class JsonfyMixIn(object):
from functools import partial
from astropy import units as u
import auromat.fits
from auromat.solving.masking import maskStarfield
from auromat.util.image import readFocalLength35mm, loadImage, saveImage
from auromat.util.os import makedirs
import sys
from distutils.version import LooseVersion
from threading import Thread
try:
import pyfits
'OPTIONAL'
PYFITS_TOO_OLD = LooseVersion(pyfits.__version__) < '3.1.0'
except:
# if unavailable
PYFITS_TOO_OLD = True
"""
A general purpose astrometry solving module. It has functionality which
allows to solve images that not only contain stars but also other elements.
As it was developed for solving images taken from the ISS towards the earth
(meaning that images contain parts of the earth and may contain spacecraft structures)
it may be biased towards it, yet it very likely is suitable for other kinds
of images as well.
"""
def solveImages(imagePaths,
channel=None,
def _apply_mask_and_get_affinity(seeds, niimg, radius, allow_overlap,
mask_img=None):
seeds = list(seeds)
affine = get_affine(niimg)
# Compute world coordinates of all in-mask voxels.
if mask_img is not None:
mask_img = check_niimg_3d(mask_img)
mask_img = image.resample_img(mask_img, target_affine=affine,
target_shape=niimg.shape[:3],
interpolation='nearest')
mask, _ = masking._load_mask_img(mask_img)
mask_coords = list(zip(*np.where(mask != 0)))
X = masking._apply_mask_fmri(niimg, mask_img)
else:
mask_coords = list(np.ndindex(niimg.shape[:3]))
X = niimg.get_data().reshape([-1, niimg.shape[3]]).T
# For each seed, get coordinates of nearest voxel
nearests = []
for sx, sy, sz in seeds:
nearest = np.round(coord_transform(sx, sy, sz, np.linalg.inv(affine)))
nearest = nearest.astype(int)
nearest = (nearest[0], nearest[1], nearest[2])
try:
nearests.append(mask_coords.index(nearest))
except ValueError:
nearests.append(None)
mask_coords = np.asarray(list(zip(*mask_coords)))
mask_coords = coord_transform(mask_coords[0], mask_coords[1],
mask_coords[2], affine)
mask_coords = np.asarray(mask_coords).T
if (radius is not None and
LooseVersion(sklearn.__version__) < LooseVersion('0.16')):
# Fix for scikit learn versions below 0.16. See
# https://github.com/scikit-learn/scikit-learn/issues/4072
radius += 1e-6
clf = neighbors.NearestNeighbors(radius=radius)
A = clf.fit(mask_coords).radius_neighbors_graph(seeds)
A = A.tolil()
for i, nearest in enumerate(nearests):
if nearest is None:
continue
A[i, nearest] = True
# Include the voxel containing the seed itself if not masked
mask_coords = mask_coords.astype(int).tolist()
for i, seed in enumerate(seeds):
try:
A[i, mask_coords.index(seed)] = True
except ValueError:
# seed is not in the mask
pass
if not allow_overlap:
if np.any(A.sum(axis=0) >= 2):
raise ValueError('Overlap detected between spheres')
return X, A
from typing import Tuple
import ci_sdr
import torch
from torch_complex.tensor import ComplexTensor
from typeguard import check_argument_types
from espnet2.enh.decoder.abs_decoder import AbsDecoder
from espnet2.enh.encoder.abs_encoder import AbsEncoder
from espnet2.enh.encoder.conv_encoder import ConvEncoder
from espnet2.enh.separator.abs_separator import AbsSeparator
from espnet2.torch_utils.device_funcs import force_gatherable
from espnet2.train.abs_espnet_model import AbsESPnetModel
is_torch_1_3_plus = LooseVersion(torch.__version__) >= LooseVersion("1.3.0")
ALL_LOSS_TYPES = (
# mse_loss(predicted_mask, target_label)
"mask_mse",
# mse_loss(enhanced_magnitude_spectrum, target_magnitude_spectrum)
"magnitude",
# mse_loss(enhanced_complex_spectrum, target_complex_spectrum)
"spectrum",
# log_mse_loss(enhanced_complex_spectrum, target_complex_spectrum)
"spectrum_log",
# si_snr(enhanced_waveform, target_waveform)
"si_snr",
# ci_sdr(enhanced_waveform, target_waveform)
"ci_sdr",
)
EPS = torch.finfo(torch.get_default_dtype()).eps
self._fixed_image, self._moving_image)
return super(FLIRTRPT, self)._post_run_hook(runtime)
class ApplyXFMInputSpecRPT(nrc.SVGReportCapableInputSpec,
fsl.preprocess.ApplyXFMInputSpec):
pass
class ApplyXFMRPT(FLIRTRPT, fsl.ApplyXFM):
input_spec = ApplyXFMInputSpecRPT
output_spec = FLIRTOutputSpecRPT
if LooseVersion("0.0.0") < fs.Info.looseversion() < LooseVersion("6.0.0"):
_BBRegisterInputSpec = fs.preprocess.BBRegisterInputSpec
else:
_BBRegisterInputSpec = fs.preprocess.BBRegisterInputSpec6
class BBRegisterInputSpecRPT(nrc.SVGReportCapableInputSpec,
_BBRegisterInputSpec):
# Adds default=True, usedefault=True
out_lta_file = traits.Either(traits.Bool, File, default=True, usedefault=True,
argstr="--lta %s", min_ver='5.2.0',
desc="write the transformation matrix in LTA format")
class BBRegisterOutputSpecRPT(reporting.ReportCapableOutputSpec,
fs.preprocess.BBRegisterOutputSpec):
def skip_if_np_lt(ver_str: str, *args, reason: Optional[str] = None):
if reason is None:
reason = f"NumPy {ver_str} or greater required"
return pytest.mark.skipif(_np_version < LooseVersion(ver_str),
*args,
reason=reason)
if i % 100 == 99: # print every 100 mini-batches
print('[%5d]' % (i + 1))
if used_gpu_count:
print("Total GPU usage (MB): ",
calculate_gpu_usage(gpus), " / ",
used_gpu_count * total_mem)
print('All data was processed.')
time_elapsed = time.time() - start_time
print('Completed in {:.0f}h {:.0f}m {:.0f}s'.format(
time_elapsed // 3600, (time_elapsed % 3600) // 60, time_elapsed % 60))
if __name__ == "__main__":
assert LooseVersion(torch.__version__) >= LooseVersion('0.4.0'), \
'PyTorch>=0.4.0 is required'
parser = argparse.ArgumentParser(
description='Testing Modular '
'Segmentation Network Architectures for IVSLAM.')
parser.add_argument(
"--cfg",
default="config/ade20k-resnet50dilated-ppm_deepsup.yaml",
metavar="FILE",
help="path to config file",
type=str)
parser.add_argument("--gpus",
default="0",
help="gpus to use, e.g. 0-3 or 0,1,2,3")
parser.add_argument("opts",
Mask R-CNN
The main Mask R-CNN model implementation.
Copyright (c) 2017 Matterport, Inc.
Licensed under the MIT License (see LICENSE for details)
Written by Waleed Abdulla
"""
import tensorflow as tf
import keras
import keras.layers as KL
import keras.models as KM
# Requires TensorFlow 1.3+ and Keras 2.0.8+.
from distutils.version import LooseVersion
assert LooseVersion(tf.__version__) >= LooseVersion("1.3")
assert LooseVersion(keras.__version__) >= LooseVersion('2.0.8')
############################################################
# Utility Functions
############################################################
def log(text, array=None):
"""Prints a text message. And, optionally, if a Numpy array is provided it
prints it's shape, min, and max values.
"""
if array is not None:
text = text.ljust(25)
text += ("shape: {:20} ".format(str(array.shape)))
if array.size:
version = getattr(sys.modules[mod_name], "__VERSION__")
if version:
from distutils.version import LooseVersion
if LooseVersion(version) >= LooseVersion(min_version):
return mod
return False
# TODO:
# remove when gh-24839 is fixed.
# this affects numpy 1.16 and pytables 3.4.4
tables = safe_import("tables")
xfail_non_writeable = pytest.mark.xfail(
tables and LooseVersion(np.__version__) >= LooseVersion("1.16")
and LooseVersion(tables.__version__) < LooseVersion("3.5.1"),
reason=("gh-25511, gh-24839. pytables needs a "
"release beyond 3.4.4 to support numpy 1.16.x"),
)
def _skip_if_no_mpl():
mod = safe_import("matplotlib")
if mod:
mod.use("Agg")
else:
return True
def _skip_if_has_locale():
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
from distutils.version import LooseVersion
import tensorflow
# Eager Mode has been introduced in TF 1.7.0
if LooseVersion(tensorflow.__version__) >= LooseVersion('1.7.0'):
from tensorflow.python.eager import context
_has_eager = True
else:
_has_eager = False
def _executing_eagerly():
"""Returns true if eager execution is supported and enabled."""
return _has_eager and context.executing_eagerly()
def _cache(f):
cache = dict()
def wrapper(*args):
def configure_intel_based(self):
"""Configure for Intel based toolchains"""
# based on guidelines available at
# http://software.intel.com/en-us/articles/build-cp2k-using-intel-fortran-compiler-professional-edition/
intelurl = ''.join(["http://software.intel.com/en-us/articles/",
"build-cp2k-using-intel-fortran-compiler-professional-edition/"])
options = self.configure_common()
extrainc = ''
if self.modincpath:
extrainc = '-I%s' % self.modincpath
options.update({
# -Vaxlib : older options
'FREE': '-fpp -free',
# SAFE = -assume protect_parens -fp-model precise -ftz # causes problems, so don't use this
'SAFE': '-assume protect_parens -no-unroll-aggressive',
'INCFLAGS': '$(DFLAGS) -I$(INTEL_INC) -I$(INTEL_INCF) %s' % extrainc,
'LDFLAGS': '$(INCFLAGS) ',
'OBJECTS_ARCHITECTURE': 'machine_intel.o',
})
options['DFLAGS'] += ' -D__INTEL'
options['FCFLAGSOPT'] += ' $(INCFLAGS) -heap-arrays 64'
options['FCFLAGSOPT2'] += ' $(INCFLAGS) -heap-arrays 64'
ifortver = LooseVersion(get_software_version('ifort'))
# -i-static has been deprecated prior to 2013, but was still usable. From 2015 it is not.
if ifortver < LooseVersion("2013"):
options['LDFLAGS'] += ' -i-static '
else:
options['LDFLAGS'] += ' -static-intel '
# Otherwise it fails on linking, since there are 2 definitions of main
if LooseVersion(self.version) >= LooseVersion('4.1'):
options['LDFLAGS'] += ' -nofor-main '
failmsg = "CP2K won't build correctly with the Intel %%s compilers prior to %%s, see %s" % intelurl
if ifortver >= LooseVersion("2011") and ifortver < LooseVersion("2012"):
# don't allow using Intel compiler 2011 prior to release 8, because of known issue (see Intel URL)
if ifortver >= LooseVersion("2011.8"):
# add additional make instructions to Makefile
self.make_instructions += "et_coupling.o: et_coupling.F\n\t$(FC) -c $(FCFLAGS2) $<\n"
self.make_instructions += "qs_vxc_atom.o: qs_vxc_atom.F\n\t$(FC) -c $(FCFLAGS2) $<\n"
else:
raise EasyBuildError(failmsg, "v12", "v2011.8")
elif ifortver >= LooseVersion("11"):
if LooseVersion(get_software_version('ifort')) >= LooseVersion("11.1.072"):
self.make_instructions += "qs_vxc_atom.o: qs_vxc_atom.F\n\t$(FC) -c $(FCFLAGS2) $<\n"
else:
raise EasyBuildError(failmsg, "v11", "v11.1.072")
else:
raise EasyBuildError("Intel compilers version %s not supported yet.", ifortver)
return options
def configure_common(self):
"""Common configuration for all toolchains"""
# openmp introduces 2 major differences
# -automatic is default: -noautomatic -auto-scalar
# some mem-bandwidth optimisation
if self.cfg['type'] == 'psmp':
self.openmp = self.toolchain.get_flag('openmp')
# determine which opt flags to use
if self.cfg['typeopt']:
optflags = 'OPT'
regflags = 'OPT2'
else:
optflags = 'NOOPT'
regflags = 'NOOPT'
# make sure a MPI-2 able MPI lib is used
mpi2 = False
if hasattr(self.toolchain, 'MPI_FAMILY') and self.toolchain.MPI_FAMILY is not None:
known_mpi2_fams = [toolchain.MPICH, toolchain.MPICH2, toolchain.MVAPICH2, toolchain.OPENMPI,
toolchain.INTELMPI]
mpi_fam = self.toolchain.mpi_family()
if mpi_fam in known_mpi2_fams:
mpi2 = True
self.log.debug("Determined MPI2 compatibility based on MPI toolchain component: %s" % mpi_fam)
else:
self.log.debug("Cannot determine MPI2 compatibility based on MPI toolchain component: %s" % mpi_fam)
else:
# can't use toolchain.mpi_family, because of system toolchain
mpi2libs = ['impi', 'MVAPICH2', 'OpenMPI', 'MPICH2', 'MPICH']
for mpi2lib in mpi2libs:
if get_software_root(mpi2lib):
mpi2 = True
self.log.debug("Determined MPI2 compatibility based on loaded MPI module: %s")
else:
self.log.debug("MPI-2 supporting MPI library %s not loaded.")
if not mpi2:
raise EasyBuildError("CP2K needs MPI-2, no known MPI-2 supporting library loaded?")
cppflags = os.getenv('CPPFLAGS')
ldflags = os.getenv('LDFLAGS')
cflags = os.getenv('CFLAGS')
fflags = os.getenv('FFLAGS')
fflags_lowopt = re.sub('-O[0-9]', '-O1', fflags)
options = {
'CC': os.getenv('MPICC'),
'CPP': '',
'FC': '%s %s' % (os.getenv('MPIF90'), self.openmp),
'LD': '%s %s' % (os.getenv('MPIF90'), self.openmp),
'AR': 'ar -r',
'CPPFLAGS': '',
'FPIC': self.fpic,
'DEBUG': self.debug,
'FCFLAGS': '$(FCFLAGS%s)' % optflags,
'FCFLAGS2': '$(FCFLAGS%s)' % regflags,
'CFLAGS': ' %s %s %s $(FPIC) $(DEBUG) %s ' % (cflags, cppflags, ldflags, self.cfg['extracflags']),
'DFLAGS': ' -D__parallel -D__BLACS -D__SCALAPACK -D__FFTSG %s' % self.cfg['extradflags'],
'LIBS': os.getenv('LIBS', ''),
'FCFLAGSNOOPT': '$(DFLAGS) $(CFLAGS) -O0 $(FREE) $(FPIC) $(DEBUG)',
'FCFLAGSOPT': '%s $(FREE) $(SAFE) $(FPIC) $(DEBUG)' % fflags,
'FCFLAGSOPT2': '%s $(FREE) $(SAFE) $(FPIC) $(DEBUG)' % fflags_lowopt,
}
libint = get_software_root('LibInt')
if libint:
options['DFLAGS'] += ' -D__LIBINT'
libintcompiler = "%s %s" % (os.getenv('CC'), os.getenv('CFLAGS'))
# Build libint-wrapper, if required
libint_wrapper = ''
# required for old versions of GCC
if not self.compilerISO_C_BINDING:
options['DFLAGS'] += ' -D__HAS_NO_ISO_C_BINDING'
# determine path for libint_tools dir
libinttools_paths = ['libint_tools', 'tools/hfx_tools/libint_tools']
libinttools_path = None
for path in libinttools_paths:
path = os.path.join(self.cfg['start_dir'], path)
if os.path.isdir(path):
libinttools_path = path
change_dir(libinttools_path)
if not libinttools_path:
raise EasyBuildError("No libinttools dir found")
# build libint wrapper
cmd = "%s -c libint_cpp_wrapper.cpp -I%s/include" % (libintcompiler, libint)
if not run_cmd(cmd, log_all=True, simple=True):
raise EasyBuildError("Building the libint wrapper failed")
libint_wrapper = '%s/libint_cpp_wrapper.o' % libinttools_path
# determine Libint libraries based on major version number
libint_maj_ver = get_software_version('Libint').split('.')[0]
if libint_maj_ver == '1':
libint_libs = "$(LIBINTLIB)/libderiv.a $(LIBINTLIB)/libint.a $(LIBINTLIB)/libr12.a"
elif libint_maj_ver == '2':
libint_libs = "$(LIBINTLIB)/libint2.a"
else:
raise EasyBuildError("Don't know how to handle libint version %s", libint_maj_ver)
self.log.info("Using Libint version %s" % (libint_maj_ver))
options['LIBINTLIB'] = '%s/lib' % libint
options['LIBS'] += ' %s -lstdc++ %s' % (libint_libs, libint_wrapper)
else:
# throw a warning, since CP2K without Libint doesn't make much sense
self.log.warning("Libint module not loaded, so building without Libint support")
libxc = get_software_root('libxc')
if libxc:
cur_libxc_version = get_software_version('libxc')
if LooseVersion(self.version) >= LooseVersion('6.1'):
libxc_min_version = '4.0.3'
options['DFLAGS'] += ' -D__LIBXC'
else:
libxc_min_version = '2.0.1'
options['DFLAGS'] += ' -D__LIBXC2'
if LooseVersion(cur_libxc_version) < LooseVersion(libxc_min_version):
raise EasyBuildError("This version of CP2K is not compatible with libxc < %s" % libxc_min_version)
if LooseVersion(cur_libxc_version) >= LooseVersion('4.0.3'):
# cfr. https://www.cp2k.org/howto:compile#k_libxc_optional_wider_choice_of_xc_functionals
options['LIBS'] += ' -L%s/lib -lxcf03 -lxc' % libxc
elif LooseVersion(cur_libxc_version) >= LooseVersion('2.2'):
options['LIBS'] += ' -L%s/lib -lxcf90 -lxc' % libxc
else:
options['LIBS'] += ' -L%s/lib -lxc' % libxc
self.log.info("Using Libxc-%s" % cur_libxc_version)
else:
self.log.info("libxc module not loaded, so building without libxc support")
return options
use_only_mean_in_flow=True,
stochastic_duration_predictor_kernel_size=3,
stochastic_duration_predictor_dropout_rate=0.5,
stochastic_duration_predictor_flows=2,
stochastic_duration_predictor_dds_conv_layers=3,
)
defaults.update(kwargs)
return defaults
# NOTE(kan-bayashi): first forward requires jit compile
# so a little bit more time is needed to run. Therefore,
# here we extend execution timeout from 2 sec to 5 sec.
@pytest.mark.execution_timeout(5)
@pytest.mark.skipif(
LooseVersion(torch.__version__) < LooseVersion("1.4"),
reason="Pytorch >= 1.4 is required.",
)
@pytest.mark.skipif(
"1.6" in torch.__version__,
reason="group conv in pytorch 1.6 has an issue. "
"See https://github.com/pytorch/pytorch/issues/42446.",
)
@torch.no_grad()
@pytest.mark.parametrize(
"model_dict",
[
({}),
({"text_encoder_positionwise_layer_type": "linear"}),
({"text_encoder_positionwise_layer_type": "conv1d-linear"}),
({"text_encoder_normalize_before": False}),
"doc": [
"Sphinx==2.1.2",
"sphinx-rtd-theme>=0.2.4",
"sphinx-argparse>=0.2.5",
"commonmark==0.8.1",
"recommonmark>=0.4.0",
"travis-sphinx>=2.0.1",
"nbsphinx>=0.4.2",
"sphinx-markdown-tables>=0.0.12",
],
}
try:
# NOTE(kamo): These packages are not listed if installing from the PyPI server
import torch
if LooseVersion(torch.__version__) >= LooseVersion("1.1.0"):
requirements["install"].append("torch_optimizer")
if LooseVersion(torch.__version__) >= LooseVersion("1.7.1"):
requirements["install"].append("torchaudio==0.7.2")
elif LooseVersion(torch.__version__) >= LooseVersion("1.7.0"):
requirements["install"].append("torchaudio==0.7.0")
elif LooseVersion(torch.__version__) >= LooseVersion("1.6.0"):
# Due to https://github.com/pytorch/pytorch/issues/42213,
# use torchaudio.functional.istft instead of torch.functional.istft
requirements["install"].append("torchaudio==0.6.0")
elif LooseVersion(torch.__version__) >= LooseVersion("1.5.1"):
requirements["install"].append("torchaudio==0.5.1")
elif LooseVersion(torch.__version__) >= LooseVersion("1.5.0"):
requirements["install"].append("torchaudio==0.5.0")
elif LooseVersion(torch.__version__) >= LooseVersion("1.4.0"):
# pip<1.4 doesn't have the --pre flag, and will thus attempt to install alpha
# and beta versions of Buildbot. Prevent that from happening.
VERSION_MSG = """
This is a pre-release version of Buildbot, which can only be installed with
pip-1.4 or later Try installing the latest stable version of Buildbot instead:
pip install buildbot==0.8.12
See https://pypi.python.org/pypi/buildbot to verify the current stable version.
"""
if 'a' in version or 'b' in version:
try:
pip_dist = pkg_resources.get_distribution('pip')
except pkg_resources.DistributionNotFound:
pip_dist = None
if pip_dist:
if LooseVersion(pip_dist.version) < LooseVersion('1.4'):
raise RuntimeError(VERSION_MSG)
if sys.version_info[:2] == (2, 6):
# Twisted-15.4.0, txaio >=2.3.0 and autobahn >=0.13.0 don't support Python
# 2.6 anymore
twisted_ver = ">= 14.0.1, < 15.4.0"
autobahn_ver = ">= 0.10.2, < 0.13.0"
txaio_ver = "== 2.2.2"
else:
twisted_ver = ">= 14.0.1"
autobahn_ver = ">= 0.16.0"
txaio_ver = ">= 2.2.2"
bundle_version = version.split("-")[0]
def terraform_version_supported(min_version, max_version):
# The Terraform version is of the format "Terraform v0.0.0"
version = check_output(['terraform', 'version'
]).split()[1].decode().replace('v', '', 1)
return LooseVersion(min_version) <= LooseVersion(version) < LooseVersion(
max_version)
import argparse
import os
import subprocess
import sys
from distutils.version import LooseVersion
import numpy as np
import pyspark
import pyspark.sql.types as T
from pyspark import SparkConf
from pyspark.ml.evaluation import MulticlassClassificationEvaluator
if LooseVersion(pyspark.__version__) < LooseVersion('3.0.0'):
from pyspark.ml.feature import OneHotEncoderEstimator as OneHotEncoder
else:
from pyspark.ml.feature import OneHotEncoder
from pyspark.sql import SparkSession
from pyspark.sql.functions import udf
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
import horovod.spark.torch as hvd
from horovod.spark.common.backend import SparkBackend
from horovod.spark.common.store import Store
parser = argparse.ArgumentParser(
description='PyTorch Spark MNIST Example',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('--master', help='spark master to connect to')
assert not cluster.workers
cluster.scale(2)
await cluster
assert len(cluster.workers) == 2
assert len(cluster.scheduler.nthreads) == 2
cluster.scale(1)
await cluster
assert len(cluster.workers) == 1
@pytest.mark.xfail(
sys.version_info >= (3, 8) and LooseVersion(tornado.version) < "6.0.3",
reason="Known issue with Python 3.8 and Tornado < 6.0.3. "
"See https://github.com/tornadoweb/tornado/pull/2683.",
strict=True,
)
def test_silent_startup():
code = """if 1:
from time import sleep
from distributed import LocalCluster
if __name__ == "__main__":
with LocalCluster(1, dashboard_address=None, scheduler_port=0):
sleep(.1)
"""
out = subprocess.check_output([sys.executable, "-Wi", "-c", code],
target = 'android'
for known in PLATFORMS:
if target.startswith(known):
target = known
if target not in PLATFORMS:
target = 'linux'
# For mac, ensure extensions are built for macos 10.9 when compiling on a
# 10.9 system or above, overriding distuitls behaviour which is to target
# the version that python was built for. This may be overridden by setting
# MACOSX_DEPLOYMENT_TARGET before calling setup.py
if target == 'darwin':
if 'MACOSX_DEPLOYMENT_TARGET' not in os.environ:
current_system = LooseVersion(platform.mac_ver()[0])
python_target = LooseVersion(
get_config_var('MACOSX_DEPLOYMENT_TARGET'))
if python_target < '10.9' and current_system >= '10.9':
os.environ['MACOSX_DEPLOYMENT_TARGET'] = '10.9'
if target in ['linux', 'cygwin']:
from distutils import sysconfig
cvars = sysconfig.get_config_vars()
if hasattr(sysconfig,
'_config_vars') and sysconfig._config_vars is not None:
if 'OPT' in cvars:
sysconfig._config_vars['OPT'] = cvars['OPT'].replace(
'-Wstrict-prototypes', '')
sysconfig._config_vars['OPT'] = cvars['OPT'].replace(
:author: Pieter De Baets (Ghent University)
:author: Jens Timmerman (Ghent University)
"""
import os
from distutils.version import LooseVersion
from socket import gethostname
# note: release candidates should be versioned as a pre-release, e.g. "1.1rc1"
# 1.1-rc1 would indicate a post-release, i.e., and update of 1.1, so beware!
#
# important note: dev versions should follow the 'X.Y.Z.dev0' format
# see https://www.python.org/dev/peps/pep-0440/#developmental-releases
# recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like
# UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0'
# This causes problems further up the dependency chain...
VERSION = LooseVersion('3.7.1')
UNKNOWN = 'UNKNOWN'
def get_git_revision():
"""
Returns the git revision (e.g. aab4afc016b742c6d4b157427e192942d0e131fe),
or UNKNOWN is getting the git revision fails
relies on GitPython (see http://gitorious.org/git-python)
"""
try:
import git
except ImportError:
return UNKNOWN
try:
#!/usr/bin/env python3
import os.path
import tensorflow as tf
import helper
import warnings
from distutils.version import LooseVersion
import project_tests as tests
# Check TensorFlow Version
assert LooseVersion(tf.__version__) >= LooseVersion(
'1.0'
), 'Please use TensorFlow version 1.0 or newer. You are using {}'.format(
tf.__version__)
print('TensorFlow Version: {}'.format(tf.__version__))
# Check for a GPU
if not tf.test.gpu_device_name():
warnings.warn(
'No GPU found. Please use a GPU to train your neural network.')
else:
print('Default GPU Device: {}'.format(tf.test.gpu_device_name()))
def load_vgg(sess, vgg_path):
"""
Load Pretrained the Fully Convolutional VGG Model into TensorFlow.
:param sess: TensorFlow Session
:param vgg_path: Path to vgg folder, containing "variables/" and "saved_model.pb"
:return: Tuple of Tensors from VGG model (image_input, keep_prob, layer3_out, layer4_out, layer7_out)
"""
vgg_tag = 'vgg16'
class TestPyXDSMViewer(unittest.TestCase):
def test_pyxdsm_output_sides(self):
"""Makes XDSM for the Sellar problem"""
p = om.Problem()
p.model = model = SellarNoDerivatives()
model.add_design_var('z', lower=np.array([-10.0, 0.0]), upper=np.array([10.0, 10.0]),
indices=np.arange(2, dtype=int))
model.add_design_var('x', lower=0.0, upper=10.0)
model.add_objective('obj')
model.add_constraint('con1', equals=np.zeros(1))
model.add_constraint('con2', upper=0.0)
p.setup()
p.final_setup()
# Write output (outputs on the left)
filename = 'xdsm_outputs_on_the_left'
write_xdsm(p, filename=filename, out_format=PYXDSM_OUT, show_browser=SHOW, quiet=QUIET, output_side='left')
# Check if file was created
self.assertTrue(os.path.isfile(filename + '.' + PYXDSM_OUT))
filename = 'xdsm_outputs_on_the_right'
# Write output (all outputs on the right)
write_xdsm(p, filename=filename, out_format=PYXDSM_OUT, show_browser=SHOW, quiet=QUIET, output_side='right')
# Check if file was created
self.assertTrue(os.path.isfile(filename + '.' + PYXDSM_OUT))
filename = 'xdsm_outputs_side_mixed'
# Write output (outputs mixed)
write_xdsm(p, filename=filename, out_format=PYXDSM_OUT, show_browser=SHOW, quiet=QUIET,
output_side={'optimization': 'left', 'default': 'right'})
# Check if file was created
self.assertTrue(os.path.isfile(filename + '.' + PYXDSM_OUT))
def test_pyxdsm_case_reading(self):
"""
Writes a recorder file, and the XDSM writer makes the diagram based on the SQL file
and not the Problem instance.
"""
import openmdao.api as om
filename = 'xdsm_from_sql'
case_recording_filename = filename + '.sql'
p = om.Problem()
p.model = model = SellarNoDerivatives()
model.add_design_var('z', lower=np.array([-10.0, 0.0]),
upper=np.array([10.0, 10.0]), indices=np.arange(2, dtype=int))
model.add_design_var('x', lower=0.0, upper=10.0)
model.add_objective('obj')
model.add_constraint('con1', equals=np.zeros(1))
model.add_constraint('con2', upper=0.0)
recorder = om.SqliteRecorder(case_recording_filename)
p.driver.add_recorder(recorder)
p.setup()
p.final_setup()
# Write output
write_xdsm(case_recording_filename, filename=filename, out_format='tex', show_browser=False, quiet=QUIET)
# Check if file was created
self.assertTrue(os.path.isfile(case_recording_filename))
self.assertTrue(os.path.isfile(filename + '.tex'))
# Check that there are no errors when running from the command line with a recording.
check_call('openmdao xdsm --no_browser {}'.format(case_recording_filename))
def test_pyxdsm_sellar_no_recurse(self):
"""Makes XDSM for the Sellar problem, with no recursion."""
filename = 'xdsm1'
p = om.Problem()
p.model = model = SellarNoDerivatives()
model.add_design_var('z', lower=np.array([-10.0, 0.0]), upper=np.array([10.0, 10.0]),
indices=np.arange(2, dtype=int))
model.add_design_var('x', lower=0.0, upper=10.0)
model.add_objective('obj')
model.add_constraint('con1', equals=np.zeros(1))
model.add_constraint('con2', upper=0.0)
p.setup()
p.final_setup()
# Write output
write_xdsm(p, filename=filename, out_format=PYXDSM_OUT, show_browser=SHOW, recurse=False, quiet=QUIET)
# Check if file was created
self.assertTrue(os.path.isfile(filename + '.' + PYXDSM_OUT))
def test_pyxdsm_pdf(self):
"""
Makes an XDSM of the Sphere test case. It also adds a design variable, constraint and
objective.
"""
class Square(om.ExplicitComponent):
def setup(self):
self.add_input('x', np.array([1.5, 1.5]))
self.add_output('f', 0.0)
self.declare_partials('f', 'x', method='fd', form='central', step=1e-4)
def compute(self, inputs, outputs, discrete_inputs=None, discrete_outputs=None):
x = inputs['x']
outputs['f'] = sum(x ** 2)
x0 = np.array([1.2, 1.5])
filename = 'xdsm2'
p = om.Problem()
indeps = p.model.add_subsystem('indeps', om.IndepVarComp(), promotes=['*'])
indeps.add_output('x', list(x0))
p.model.add_subsystem('sphere', Square(), promotes=['*'])
p.model.add_subsystem('con', om.ExecComp('c=sum(x)', x=np.ones(2)), promotes=['*'])
p.driver = om.ScipyOptimizeDriver()
p.model.add_design_var('x')
p.model.add_objective('f')
p.model.add_constraint('c', lower=1.0)
p.setup()
p.final_setup()
# requesting 'pdf', but if 'pdflatex' is not found we will only get 'tex'
pdflatex = find_executable('pdflatex')
# Write output
write_xdsm(p, filename=filename, out_format='pdf', show_browser=SHOW, quiet=QUIET)
# Check if file was created
self.assertTrue(os.path.isfile(filename + '.tex'))
# Check if PDF was created (only if pdflatex is installed)
self.assertTrue(not pdflatex or os.path.isfile(filename + '.pdf'))
@unittest.SkipTest
def test_pyxdsm_tikz_content(self):
# Check if TiKZ file was created.
# Compare the content of the sample below and the newly created TiKZ file.
# The texts are cleaned up before comparison from whitespaces, line ends and comments
# Despite this different versions of pyxdsm *might* produce different outputs. If the test fails for you,
# it might be that the program still produces the correct output.
# The ordering of some lines might be different at different runs, but the test should work regardless
# of these changes.
# The sample text should be replaced if changes in the formatting or in the test problem are made.
sample_tikz_txt = r"""
%%% Preamble Requirements %%%
% \usepackage{geometry}
% \usepackage{amsfonts}
% \usepackage{amsmath}
% \usepackage{amssymb}
% \usepackage{tikz}
% Optional packages such as sfmath set through python interface
% \usepackage{sfmath}
% \usetikzlibrary{arrows,chains,positioning,scopes,shapes.geometric,shapes.misc,shadows}
%%% End Preamble Requirements %%%
\input{"path/to/diagram_styles"}
\begin{tikzpicture}
\matrix[MatrixSetup]{
%Row 0
\node [Optimization] (Driver) {$\text{\text{0, 10 $ \rightarrow $ 1: Driver}}$};&
&
&
&
&
&
&
\\
%Row 1
&
\node [Function] (indeps) {$\begin{array}{c}\text{1: indeps} \\ \text{\textit{IndepVarComp}}\end{array}$};&
&
\node [DataInter] (indeps-cycle@d1) {$\begin{array}{c}x, z\end{array}$};&
\node [DataInter] (indeps-cycle@d2) {$\begin{array}{c}z\end{array}$};&
\node [DataInter] (indeps-obj@cmp) {$\begin{array}{c}x, z\end{array}$};&
&
\\
%Row 2
&
&
\node [MDA] (cycle@solver) {$\text{\text{2, 5 $ \rightarrow $ 3: NL: NLBGS}}$};&
\node [DataInter] (cycle@solver-cycle@d1) {$\begin{array}{c}y2^{t}\end{array}$};&
\node [DataInter] (cycle@solver-cycle@d2) {$\begin{array}{c}y1^{t}\end{array}$};&
&
&
\\
%Row 3
&
&
\node [DataInter] (cycle@d1-cycle@solver) {$\begin{array}{c}y1\end{array}$};&
\node [Function] (cycle@d1) {$\begin{array}{c}\text{3: d1} \\ \text{\textit{SellarDis1}}\end{array}$};&
\node [DataInter] (cycle@d1-cycle@d2) {$\begin{array}{c}y1\end{array}$};&
\node [DataInter] (cycle@d1-obj@cmp) {$\begin{array}{c}y1\end{array}$};&
\node [DataInter] (cycle@d1-con@cmp1) {$\begin{array}{c}y1\end{array}$};&
\\
%Row 4
&
&
\node [DataInter] (cycle@d2-cycle@solver) {$\begin{array}{c}y2\end{array}$};&
\node [DataInter] (cycle@d2-cycle@d1) {$\begin{array}{c}y2\end{array}$};&
\node [Function] (cycle@d2) {$\begin{array}{c}\text{4: d2} \\ \text{\textit{SellarDis2}}\end{array}$};&
\node [DataInter] (cycle@d2-obj@cmp) {$\begin{array}{c}y2\end{array}$};&
&
\node [DataInter] (cycle@d2-con@cmp2) {$\begin{array}{c}y2\end{array}$};\\
%Row 5
&
&
&
&
&
\node [Function] (obj@cmp) {$\begin{array}{c}\text{6: obj\_cmp} \\ \text{\textit{ExecComp}}\end{array}$};&
&
\\
%Row 6
&
&
&
&
&
&
\node [Function] (con@cmp1) {$\begin{array}{c}\text{7: con\_cmp1} \\ \text{\textit{ExecComp}}\end{array}$};&
\\
%Row 7
&
&
&
&
&
&
&
\node [Function] (con@cmp2) {$\begin{array}{c}\text{8: con\_cmp2} \\ \text{\textit{ExecComp}}\end{array}$};\\
};
% XDSM process chains
{ [start chain=process]
\begin{pgfonlayer}{process}
\chainin (Driver);
\chainin (indeps) [join=by ProcessHV];
\chainin (cycle@solver) [join=by ProcessHV];
\chainin (obj@cmp) [join=by ProcessHV];
\chainin (con@cmp1) [join=by ProcessHV];
\chainin (con@cmp2) [join=by ProcessHV];
\chainin (Driver) [join=by ProcessHV];
\end{pgfonlayer}
}{ [start chain=process]
\begin{pgfonlayer}{process}
\chainin (cycle@solver);
\chainin (cycle@d1) [join=by ProcessHV];
\chainin (cycle@d2) [join=by ProcessHV];
\chainin (cycle@solver) [join=by ProcessHV];
\end{pgfonlayer}
}
\begin{pgfonlayer}{data}
\path
% Horizontal edges
(cycle@solver) edge [DataLine] (cycle@solver-cycle@d2)
(cycle@d1) edge [DataLine] (cycle@d1-cycle@solver)
(cycle@solver) edge [DataLine] (cycle@solver-cycle@d1)
(cycle@d2) edge [DataLine] (cycle@d2-cycle@solver)
(cycle@d1) edge [DataLine] (cycle@d1-obj@cmp)
(cycle@d1) edge [DataLine] (cycle@d1-con@cmp1)
(cycle@d1) edge [DataLine] (cycle@d1-cycle@d2)
(cycle@d2) edge [DataLine] (cycle@d2-obj@cmp)
(cycle@d2) edge [DataLine] (cycle@d2-con@cmp2)
(cycle@d2) edge [DataLine] (cycle@d2-cycle@d1)
(indeps) edge [DataLine] (indeps-cycle@d1)
(indeps) edge [DataLine] (indeps-obj@cmp)
(indeps) edge [DataLine] (indeps-cycle@d2)
% Vertical edges
(cycle@solver-cycle@d2) edge [DataLine] (cycle@d2)
(cycle@d1-cycle@solver) edge [DataLine] (cycle@solver)
(cycle@solver-cycle@d1) edge [DataLine] (cycle@d1)
(cycle@d2-cycle@solver) edge [DataLine] (cycle@solver)
(cycle@d1-obj@cmp) edge [DataLine] (obj@cmp)
(cycle@d1-con@cmp1) edge [DataLine] (con@cmp1)
(cycle@d1-cycle@d2) edge [DataLine] (cycle@d2)
(cycle@d2-obj@cmp) edge [DataLine] (obj@cmp)
(cycle@d2-con@cmp2) edge [DataLine] (con@cmp2)
(cycle@d2-cycle@d1) edge [DataLine] (cycle@d1)
(indeps-cycle@d1) edge [DataLine] (cycle@d1)
(indeps-obj@cmp) edge [DataLine] (obj@cmp)
(indeps-cycle@d2) edge [DataLine] (cycle@d2);
\end{pgfonlayer}
\end{tikzpicture}"""
def filter_lines(lns):
# Empty lines are excluded.
# Leading and trailing whitespaces are removed
# Comments are removed.
return [ln.strip() for ln in lns if ln.strip() and not ln.strip().startswith('%')]
filename = 'pyxdsm_mda_tikz'
out_format = PYXDSM_OUT
p = om.Problem(model=SellarMDA())
p.setup()
p.final_setup()
# Write output
write_xdsm(p, filename=filename, out_format=out_format, quiet=QUIET, show_browser=SHOW, include_solver=True)
# Check if file was created
tikz_file = filename + '.tikz'
self.assertTrue(os.path.isfile(tikz_file))
sample_lines = sample_tikz_txt.split('\n')
sample_lines = filter_lines(sample_lines)
with open(tikz_file, "r") as f:
new_lines = filter_lines(f.readlines())
no_match_sample = [] # Sample text
no_match_new = [] # New text
for new_line, sample_line in zip(new_lines, sample_lines):
if new_line.startswith(r"\input{"):
continue
if new_line != sample_line: # else everything is okay
# This can be because of the different ordering of lines or because of an error.
no_match_sample.append(new_line)
no_match_new.append(sample_line)
# Sort both sets of suspicious lines
no_match_sample.sort()
no_match_new.sort()
for sample_line, new_line in zip(no_match_sample, no_match_new):
# Now the lines should match, if only the ordering was different
self.assertEqual(sample_line, new_line)
# To be sure, check the length, otherwise a missing last line could get unnoticed because of using zip
self.assertEqual(len(new_lines), len(sample_lines))
def test_pyxdsm_identical_relative_names(self):
class TimeComp(om.ExplicitComponent):
def setup(self):
self.add_input('t_initial', val=0.)
self.add_input('t_duration', val=1.)
self.add_output('time', shape=(2,))
def compute(self, inputs, outputs):
t_initial = inputs['t_initial']
t_duration = inputs['t_duration']
outputs['time'][0] = t_initial
outputs['time'][1] = t_initial + t_duration
class Phase(om.Group):
def setup(self):
super(Phase, self).setup()
indep = om.IndepVarComp()
for var in ['t_initial', 't_duration']:
indep.add_output(var, val=1.0)
self.add_subsystem('time_extents', indep, promotes_outputs=['*'])
time_comp = TimeComp()
self.add_subsystem('time', time_comp)
self.connect('t_initial', 'time.t_initial')
self.connect('t_duration', 'time.t_duration')
self.set_order(['time_extents', 'time'])
p = om.Problem()
p.driver = om.ScipyOptimizeDriver()
orbit_phase = Phase()
p.model.add_subsystem('orbit_phase', orbit_phase)
systems_phase = Phase()
p.model.add_subsystem('systems_phase', systems_phase)
systems_phase = Phase()
p.model.add_subsystem('extra_phase', systems_phase)
p.model.add_design_var('orbit_phase.t_initial')
p.model.add_design_var('orbit_phase.t_duration')
p.model.add_objective('systems_phase.time.time')
p.setup()
p.run_model()
# Test non unique local names
filename = 'pyxdsm_identical_rel_names'
write_xdsm(p, filename, out_format=PYXDSM_OUT, quiet=QUIET, show_browser=SHOW)
self.assertTrue(os.path.isfile(filename + '.' + PYXDSM_OUT))
# Check formatting
# Max character box formatting
filename = 'pyxdsm_cut_char'
write_xdsm(p, filename, out_format=PYXDSM_OUT, quiet=QUIET, show_browser=SHOW, box_stacking='cut_chars',
box_width=15)
self.assertTrue(os.path.isfile(filename + '.' + PYXDSM_OUT))
# Cut characters box formatting
filename = 'pyxdsm_max_chars'
write_xdsm(p, filename, out_format=PYXDSM_OUT, quiet=True, show_browser=SHOW, box_stacking='max_chars',
box_width=15)
self.assertTrue(os.path.isfile(filename + '.' + PYXDSM_OUT))
def test_no_indepvarcomps(self):
p = om.Problem()
p.model = model = SellarNoDerivatives()
model.add_design_var('z', lower=np.array([-10.0, 0.0]), upper=np.array([10.0, 10.0]),
indices=np.arange(2, dtype=int))
# model.add_design_var('x', lower=0.0, upper=10.0)
model.add_objective('obj')
model.add_constraint('con1', equals=np.zeros(1))
model.add_constraint('con2', upper=0.0)
p.setup()
p.final_setup()
filename = 'pyxdsm_no_indepvarcomps'
write_xdsm(p, filename=filename, out_format='pdf', show_browser=SHOW, quiet=True, include_indepvarcomps=False)
self.assertTrue(os.path.isfile(filename + '.' + PYXDSM_OUT))
def test_model_path_and_recursion(self):
import openmdao.api as om
p = om.Problem()
model = p.model
group = model.add_subsystem('G1', om.Group(), promotes=['*'])
group2 = model.add_subsystem('G2', om.Group())
group.add_subsystem('ground', om.IndepVarComp('V', 0., units='V'))
group.add_subsystem('source', om.IndepVarComp('I', 0.1, units='A'))
group2.add_subsystem('source2', om.IndepVarComp('I', 0.1, units='A'))
group.add_subsystem('circuit', Circuit())
group.connect('source.I', 'circuit.I_in')
group.connect('ground.V', 'circuit.Vg')
model.add_design_var('ground.V')
model.add_design_var('source.I')
model.add_objective('circuit.D1.I')
p.setup()
# set some initial guesses
p['circuit.n1.V'] = 10.
p['circuit.n2.V'] = 1.
p.run_model()
# No model path, no recursion
write_xdsm(p, 'xdsm_circuit', out_format=PYXDSM_OUT, quiet=QUIET, show_browser=SHOW, recurse=False)
self.assertTrue(os.path.isfile('xdsm_circuit.' + PYXDSM_OUT))
# Model path given + recursion
write_xdsm(p, 'xdsm_circuit2', out_format=PYXDSM_OUT, quiet=QUIET, show_browser=SHOW, recurse=True,
model_path='G2', include_external_outputs=False)
self.assertTrue(os.path.isfile('xdsm_circuit2.' + PYXDSM_OUT))
# Model path given + no recursion
write_xdsm(p, 'xdsm_circuit3', out_format=PYXDSM_OUT, quiet=QUIET, show_browser=SHOW, recurse=False,
model_path='G1')
self.assertTrue(os.path.isfile('xdsm_circuit3.' + PYXDSM_OUT))
# Invalid model path, should raise error
with self.assertRaises(ValueError):
write_xdsm(p, 'xdsm_circuit4', out_format='tex', quiet=QUIET, show_browser=SHOW, recurse=False,
model_path='G3')
def test_pyxdsm_solver(self):
import openmdao.api as om
out_format = PYXDSM_OUT
p = om.Problem()
p.model = model = SellarNoDerivatives()
model.nonlinear_solver = om.NonlinearBlockGS()
p.driver = om.ScipyOptimizeDriver()
p.model.add_objective('obj')
p.setup()
p.run_model()
filename = 'pyxdsm_solver'
# Write output
write_xdsm(p, filename=filename, out_format=out_format, quiet=QUIET, show_browser=SHOW, include_solver=True)
# Check if file was created
self.assertTrue(os.path.isfile(filename + '.' + out_format))
filename = 'pyxdsm_solver2'
# Write output
write_xdsm(p, filename=filename, out_format=out_format, quiet=QUIET, show_browser=SHOW, include_solver=True,
recurse=False)
# Check if file was created
self.assertTrue(os.path.isfile(filename + '.' + out_format))
def test_pyxdsm_mda(self):
filename = 'pyxdsm_mda'
out_format = PYXDSM_OUT
p = om.Problem(model=SellarMDA())
p.setup()
p.final_setup()
# Write output
write_xdsm(p, filename=filename, out_format=out_format, quiet=QUIET, show_browser=SHOW, include_solver=True)
# Check if file was created
self.assertTrue(os.path.isfile(filename + '.' + out_format))
def test_pyxdsm_mdf(self):
filename = 'pyxdsm_mdf'
out_format = PYXDSM_OUT
p = om.Problem(model=SellarMDA())
model = p.model
p.driver = om.ScipyOptimizeDriver()
model.add_design_var('z', lower=np.array([-10.0, 0.0]),
upper=np.array([10.0, 10.0]), indices=np.arange(2, dtype=int))
model.add_design_var('x', lower=0.0, upper=10.0)
model.add_objective('obj')
model.add_constraint('con1', equals=np.zeros(1))
model.add_constraint('con2', upper=0.0)
p.setup()
p.final_setup()
# Write output
write_xdsm(p, filename=filename, out_format=out_format, quiet=QUIET, show_browser=SHOW, include_solver=True)
# Check if file was created
self.assertTrue(os.path.isfile(filename + '.' + out_format))
def test_parallel(self):
import openmdao.api as om
class SellarMDA(om.Group):
"""
Group containing the Sellar MDA.
"""
def setup(self):
indeps = self.add_subsystem('indeps', om.IndepVarComp(), promotes=['*'])
indeps.add_output('x', 1.0)
indeps.add_output('z', np.array([5.0, 2.0]))
cycle = self.add_subsystem('cycle', om.ParallelGroup(), promotes=['*'])
cycle.add_subsystem('d1', SellarDis1(), promotes_inputs=['x', 'z', 'y2'], promotes_outputs=['y1'])
cycle.add_subsystem('d2', SellarDis2(), promotes_inputs=['z', 'y1'], promotes_outputs=['y2'])
# Nonlinear Block Gauss Seidel is a gradient free solver
cycle.nonlinear_solver = om.NonlinearBlockGS()
self.add_subsystem('obj_cmp', om.ExecComp('obj = x**2 + z[1] + y1 + exp(-y2)',
z=np.array([0.0, 0.0]), x=0.0),
promotes=['x', 'z', 'y1', 'y2', 'obj'])
self.add_subsystem('con_cmp1', om.ExecComp('con1 = 3.16 - y1'), promotes=['con1', 'y1'])
self.add_subsystem('con_cmp2', om.ExecComp('con2 = y2 - 24.0'), promotes=['con2', 'y2'])
filename = 'pyxdsm_parallel'
out_format = PYXDSM_OUT
p = om.Problem(model=SellarMDA())
model = p.model
p.driver = om.ScipyOptimizeDriver()
model.add_design_var('z', lower=np.array([-10.0, 0.0]), upper=np.array([10.0, 10.0]),
indices=np.arange(2, dtype=int))
model.add_design_var('x', lower=0.0, upper=10.0)
model.add_objective('obj')
model.add_constraint('con1', equals=np.zeros(1))
model.add_constraint('con2', upper=0.0)
p.setup()
p.final_setup()
# Write output
write_xdsm(p, filename=filename, out_format=out_format, quiet=QUIET, show_browser=SHOW, show_parallel=True)
# Check if file was created
self.assertTrue(os.path.isfile(filename + '.' + out_format))
def test_execcomp(self):
filename = 'pyxdsm_execcomp'
out_format = PYXDSM_OUT
p = om.Problem()
indeps = p.model.add_subsystem('indeps', om.IndepVarComp(), promotes=['*'])
indeps.add_output('x')
p.model.add_subsystem('C1', om.ExecComp(['y=2.0*x+1.'], x=2.0), promotes=['*'])
p.driver = om.ScipyOptimizeDriver()
p.model.add_design_var('x', lower=0.0, upper=10.0)
p.model.add_objective('y')
p.setup()
# Conclude setup but don't run model.
p.final_setup()
write_xdsm(p, filename=filename, out_format=out_format, quiet=QUIET, show_browser=SHOW, show_parallel=True)
# Check if file was created
self.assertTrue(os.path.isfile(filename + '.' + out_format))
# Including the expression from the ExecComp formatted in LaTeX
write_xdsm(p, filename=filename + "2", out_format=out_format, quiet=QUIET, show_browser=SHOW,
show_parallel=True, equations=True)
# Check if file was created
self.assertTrue(os.path.isfile(filename + '.' + out_format))
def test_doe(self):
filename = 'pyxdsm_doe'
out_format = PYXDSM_OUT
p = om.Problem()
indeps = p.model.add_subsystem('indeps', om.IndepVarComp(), promotes=['*'])
indeps.add_output('x')
p.model.add_subsystem('C1', om.ExecComp(['y=2.0*x+1.'], x=2.0), promotes=['*'])
p.driver = om.DOEDriver()
p.model.add_design_var('x', lower=0.0, upper=10.0)
p.model.add_objective('y')
p.setup()
# Conclude setup but don't run model.
p.final_setup()
write_xdsm(p, filename=filename, out_format=out_format, quiet=QUIET, show_browser=SHOW, show_parallel=True)
# Check if file was created
self.assertTrue(os.path.isfile(filename + '.' + out_format))
def test_meta_model(self):
import openmdao.api as om
from openmdao.components.tests.test_meta_model_structured_comp import SampleMap
filename = 'pyxdsm_meta_model'
out_format = PYXDSM_OUT
model = om.Group()
ivc = om.IndepVarComp()
mapdata = SampleMap()
params = mapdata.param_data
x, y, z = params
outs = mapdata.output_data
z = outs[0]
ivc.add_output('x', x['default'], units=x['units'])
ivc.add_output('y', y['default'], units=y['units'])
ivc.add_output('z', z['default'], units=z['units'])
model.add_subsystem('des_vars', ivc, promotes=["*"])
comp = om.MetaModelStructuredComp(method='slinear', extrapolate=True)
for param in params:
comp.add_input(param['name'], param['default'], param['values'])
for out in outs:
comp.add_output(out['name'], out['default'], out['values'])
model.add_subsystem('comp', comp, promotes=["*"])
p = om.Problem(model)
p.setup()
p.final_setup()
write_xdsm(p, filename=filename, out_format=out_format, quiet=QUIET, show_browser=SHOW, show_parallel=True)
# Check if file was created
self.assertTrue(os.path.isfile(filename + '.' + out_format))
@unittest.skipUnless(LooseVersion(om_version) >= LooseVersion('3.2'), 'Auto-IVC introduced in OpenMDAO 3.2')
def test_auto_ivc(self):
"""
Tests a model with automatically added IndepVarComp.
"""
filename = 'xdsm_auto_ivc'
p = om.Problem()
p.model.add_subsystem('paraboloid',
om.ExecComp('f = (x-3)**2 + x*y + (y+4)**2 - 3'),
promotes_inputs=['x', 'y'])
# setup the optimization
p.driver = om.ScipyOptimizeDriver()
p.driver.options['optimizer'] = 'SLSQP'
p.model.add_design_var('x', lower=-50, upper=50)
p.model.add_design_var('y', lower=-50, upper=50)
p.model.add_objective('paraboloid.f')
p.setup()
p.final_setup()
# Write output
write_xdsm(p, filename=filename, out_format=PYXDSM_OUT, show_browser=SHOW, quiet=QUIET,
include_indepvarcomps=False) # Not showing the Auto IVC
# Check if file was created
self.assertTrue(os.path.isfile(filename + '.tex'))
write_xdsm(p, filename=filename + '2', out_format=PYXDSM_OUT, show_browser=SHOW, quiet=QUIET,
include_indepvarcomps=True) # Showing the Auto IVC
# Check if file was created
self.assertTrue(os.path.isfile(filename + '2.tex'))
print(
'Epoch {:>3}/{} Batch {:>4}/{} - Training Loss: {:>6.3f} - Validation loss: {:>6.3f}'
.format(epoch_i, epochs, batch_i,
len(train_source) // batch_size, loss,
validation_loss[0]))
# 保存模型
saver = tf.train.Saver()
saver.save(sess, checkpoint)
print('Model Trained and Saved') # 构造graph
if __name__ == '__main__':
# Check TensorFlow Version
assert LooseVersion(tf.__version__) >= LooseVersion(
'1.1'), 'Please use TensorFlow version 1.1 or newer'
print('TensorFlow Version: {}'.format(tf.__version__))
with open('data/letters_source.txt', 'r', encoding='utf-8') as fr:
source_data = fr.read()
with open('data/letters_target.txt', 'r', encoding='utf-8') as fr:
target_data = fr.read()
# 构造映射表
source_int_to_letter, source_letter_to_int = extract_character_vocab(
source_data)
target_int_to_letter, target_letter_to_int = extract_character_vocab(
target_data)
# 对字母进行转换
source_int = [[
source_letter_to_int.get(letter, source_letter_to_int['<UNK>'])
for letter in line
class TestXDSMjsViewer(unittest.TestCase):
def test_xdsmjs(self):
"""
Makes XDSMjs input file for the Sellar problem.
Data is in a separate JSON file.
"""
filename = 'xdsmjs' # this name is needed for XDSMjs
p = om.Problem()
p.model = model = SellarNoDerivatives()
model.add_design_var('z', lower=np.array([-10.0, 0.0]), upper=np.array([10.0, 10.0]),
indices=np.arange(2, dtype=int))
model.add_design_var('x', lower=0.0, upper=10.0)
model.add_objective('obj')
model.add_constraint('con1', equals=np.zeros(1))
model.add_constraint('con2', upper=0.0)
p.setup()
p.final_setup()
# Write output
write_xdsm(p, filename=filename, out_format='html', subs=(), show_browser=SHOW, embed_data=False)
# Check if file was created
self.assertTrue(os.path.isfile(filename + '.json'))
self.assertTrue(os.path.isfile(filename + '.html'))
def test_xdsmjs_embed_data(self):
"""
Makes XDSMjs HTML file for the Sellar problem.
Data is embedded into the HTML file.
"""
filename = 'xdsmjs_embedded' # this name is needed for XDSMjs
p = om.Problem()
p.model = model = SellarNoDerivatives()
model.add_design_var('z', lower=np.array([-10.0, 0.0]), upper=np.array([10.0, 10.0]),
indices=np.arange(2, dtype=int))
model.add_design_var('x', lower=0.0, upper=10.0)
model.add_objective('obj')
model.add_constraint('con1', equals=np.zeros(1))
model.add_constraint('con2', upper=0.0)
p.setup()
p.final_setup()
# Write output
write_xdsm(p, filename=filename, out_format='html', subs=(), show_browser=SHOW, embed_data=True)
# Check if file was created
self.assertTrue(os.path.isfile(filename + '.html'))
def test_xdsmjs_embeddable(self):
"""
Makes XDSMjs HTML file for the Sellar problem.
The HTML file is embeddable (no head and body tags).
"""
filename = 'xdsmjs_embeddable' # this name is needed for XDSMjs
p = om.Problem()
p.model = model = SellarNoDerivatives()
model.add_design_var('z', lower=np.array([-10.0, 0.0]), upper=np.array([10.0, 10.0]),
indices=np.arange(2, dtype=int))
model.add_design_var('x', lower=0.0, upper=10.0)
model.add_objective('obj')
model.add_constraint('con1', equals=np.zeros(1))
model.add_constraint('con2', upper=0.0)
p.setup()
p.final_setup()
# Write output
write_xdsm(p, filename=filename, out_format='html', subs=(), show_browser=SHOW, embed_data=True,
embeddable=True)
# Check if file was created
self.assertTrue(os.path.isfile(filename + '.html'))
def test_html_writer_dct(self):
"""
Makes XDSMjs input file.
Data is in a dictionary
"""
filename = 'xdsmjs2' # this name is needed for XDSMjs
data = {
"nodes": [{"id": "Opt", "name": "Optimization", "type": "optimization"},
{"id": "MDA", "name": "MDA", "type": "mda"},
{"id": "DA1", "name": "Analysis 1"},
{"id": "DA2", "name": "Analysis 2"},
{"id": "DA3", "name": "Analysis 3"},
{"id": "Func", "name": "Functions"}
],
"edges": [{"from": "Opt", "to": "DA1", "name": "x_0,x_1"},
{"from": "DA1", "to": "DA3", "name": "x_share"},
{"from": "DA3", "to": "DA1", "name": "y_1^2"},
{"from": "MDA", "to": "DA1", "name": "x_2"},
{"from": "Func", "to": "Opt", "name": "f,c"},
{"from": "_U_", "to": "DA1", "name": "x_0"},
{"from": "DA3", "to": "_U_", "name": "y_0"}
],
"workflow": ["Opt", ["MDA", "DA1", "DA2", "DA3"], "Func"]
}
outfile = filename + '.html'
write_html(outfile=outfile, source_data=data)
self.assertTrue(os.path.isfile(outfile))
def test_html_writer_str(self):
"""
Makes XDSMjs input file.
Data is a string.
"""
filename = 'xdsmjs4' # this name is needed for XDSMjs
data = ("{'nodes': [{'type': 'optimization', 'id': 'Opt', 'name': 'Optimization'}, "
"{'type': 'mda', 'id': 'MDA', 'name': 'MDA'}, {'id': 'DA1', 'name': 'Analysis 1'}, "
"{'id': 'DA2', 'name': 'Analysis 2'}, {'id': 'DA3', 'name': 'Analysis 3'}, "
"{'id': 'Func', 'name': 'Functions'}], "
"'edges': [{'to': 'DA1', 'from': 'Opt', 'name': 'x_0,x_1'}, "
"{'to': 'DA3', 'from': 'DA1', 'name': 'x_share'}, "
"{'to': 'DA1', 'from': 'DA3', 'name': 'y_1^2'}, "
"{'to': 'DA1', 'from': 'MDA', 'name': 'x_2'}, "
"{'to': 'Opt', 'from': 'Func', 'name': 'f,c'}, "
"{'to': 'DA1', 'from': '_U_', 'name': 'x_0'}, "
"{'to': '_U_', 'from': 'DA3', 'name': 'y_0'}], "
"'workflow': ['Opt', ['MDA', 'DA1', 'DA2', 'DA3'], 'Func']}")
outfile = filename + '.html'
write_html(outfile=outfile, source_data=data)
self.assertTrue(os.path.isfile(outfile))
def test_pyxdsm_identical_relative_names(self):
class TimeComp(om.ExplicitComponent):
def setup(self):
self.add_input('t_initial', val=0.)
self.add_input('t_duration', val=1.)
self.add_output('time', shape=(2,))
def compute(self, inputs, outputs):
t_initial = inputs['t_initial']
t_duration = inputs['t_duration']
outputs['time'][0] = t_initial
outputs['time'][1] = t_initial + t_duration
class Phase(om.Group):
def setup(self):
super(Phase, self).setup()
indep = om.IndepVarComp()
for var in ['t_initial', 't_duration']:
indep.add_output(var, val=1.0)
self.add_subsystem('time_extents', indep, promotes_outputs=['*'])
time_comp = TimeComp()
self.add_subsystem('time', time_comp)
self.connect('t_initial', 'time.t_initial')
self.connect('t_duration', 'time.t_duration')
self.set_order(['time_extents', 'time'])
p = om.Problem()
p.driver = om.ScipyOptimizeDriver()
orbit_phase = Phase()
p.model.add_subsystem('orbit_phase', orbit_phase)
systems_phase = Phase()
p.model.add_subsystem('systems_phase', systems_phase)
systems_phase = Phase()
p.model.add_subsystem('extra_phase', systems_phase)
p.model.add_design_var('orbit_phase.t_initial')
p.model.add_design_var('orbit_phase.t_duration')
p.model.add_objective('systems_phase.time.time')
p.setup()
p.run_model()
write_xdsm(p, 'xdsmjs_orbit', out_format='html', show_browser=SHOW)
self.assertTrue(os.path.isfile('xdsmjs_orbit.html'))
def test_xdsmjs_mda(self):
filename = 'xdsmjs_mda'
out_format = 'html'
prob = om.Problem(model=SellarMDA())
prob.setup()
prob.final_setup()
# Write output
write_xdsm(prob, filename=filename, out_format=out_format, quiet=QUIET, show_browser=SHOW, embed_data=True,
embeddable=True, include_solver=True)
# Check if file was created
self.assertTrue(os.path.isfile(filename + '.' + out_format))
def test_xdsmjs_mdf(self):
filename = 'xdsmjs_mdf'
out_format = 'html'
p = om.Problem(model=SellarMDA())
model = p.model
p.driver = om.ScipyOptimizeDriver()
model.add_design_var('z', lower=np.array([-10.0, 0.0]), upper=np.array([10.0, 10.0]),
indices=np.arange(2, dtype=int))
model.add_design_var('x', lower=0.0, upper=10.0)
model.add_objective('obj')
model.add_constraint('con1', equals=np.zeros(1))
model.add_constraint('con2', upper=0.0)
p.setup()
p.final_setup()
# Write output
write_xdsm(p, filename=filename, out_format=out_format, show_browser=SHOW, include_solver=True)
# Check if file was created
self.assertTrue(os.path.isfile(filename + '.' + out_format))
def test_xdsm_solver(self):
import openmdao.api as om
filename = 'xdsmjs_solver'
out_format = 'html'
p = om.Problem(model=SellarNoDerivatives())
p.model.nonlinear_solver = om.NonlinearBlockGS()
p.driver = om.ScipyOptimizeDriver()
p.model.add_objective('obj')
p.setup()
p.run_model()
# Write output
write_xdsm(p, filename=filename, out_format=out_format, show_browser=SHOW, include_solver=True)
# Check if file was created
self.assertTrue(os.path.isfile(filename + '.' + out_format))
def test_parallel(self):
import openmdao.api as om
class SellarMDA(om.Group):
"""
Group containing the Sellar MDA.
"""
def setup(self):
indeps = self.add_subsystem('indeps', om.IndepVarComp(), promotes=['*'])
indeps.add_output('x', 1.0)
indeps.add_output('z', np.array([5.0, 2.0]))
cycle = self.add_subsystem('cycle', om.ParallelGroup(), promotes=['*'])
cycle.add_subsystem('d1', SellarDis1(), promotes_inputs=['x', 'z', 'y2'], promotes_outputs=['y1'])
cycle.add_subsystem('d2', SellarDis2(), promotes_inputs=['z', 'y1'], promotes_outputs=['y2'])
# Nonlinear Block Gauss Seidel is a gradient free solver
cycle.nonlinear_solver = om.NonlinearBlockGS()
self.add_subsystem('obj_cmp', om.ExecComp('obj = x**2 + z[1] + y1 + exp(-y2)',
z=np.array([0.0, 0.0]), x=0.0),
promotes=['x', 'z', 'y1', 'y2', 'obj'])
self.add_subsystem('con_cmp1', om.ExecComp('con1 = 3.16 - y1'), promotes=['con1', 'y1'])
self.add_subsystem('con_cmp2', om.ExecComp('con2 = y2 - 24.0'), promotes=['con2', 'y2'])
filename = 'xdsmjs_parallel'
out_format = 'html'
p = om.Problem(model=SellarMDA())
model = p.model
p.driver = om.ScipyOptimizeDriver()
model.add_design_var('z', lower=np.array([-10.0, 0.0]), upper=np.array([10.0, 10.0]),
indices=np.arange(2, dtype=int))
model.add_design_var('x', lower=0.0, upper=10.0)
model.add_objective('obj')
model.add_constraint('con1', equals=np.zeros(1))
model.add_constraint('con2', upper=0.0)
p.setup()
p.final_setup()
# Write output
write_xdsm(p, filename=filename, out_format=out_format, quiet=QUIET, show_browser=SHOW, show_parallel=True)
# Check if file was created
self.assertTrue(os.path.isfile(filename + '.' + out_format))
def test_execcomp(self):
filename = 'xdsmjs_execcomp'
out_format = 'html'
p = om.Problem()
indeps = p.model.add_subsystem('indeps', om.IndepVarComp(), promotes=['*'])
indeps.add_output('x')
p.model.add_subsystem('C1', om.ExecComp(['y=2.0*x+1.'], x=2.0), promotes=['*'])
p.driver = om.ScipyOptimizeDriver()
p.model.add_design_var('x', lower=0.0, upper=10.0)
p.model.add_objective('y')
p.setup()
# Conclude setup but don't run model.
p.final_setup()
write_xdsm(p, filename=filename, out_format=out_format, quiet=QUIET, show_browser=SHOW, show_parallel=True)
# Check if file was created
self.assertTrue(os.path.isfile(filename + '.' + out_format))
def test_doe(self):
filename = 'xdsmjs_doe'
out_format = 'html'
p = om.Problem()
indeps = p.model.add_subsystem('indeps', om.IndepVarComp(), promotes=['*'])
indeps.add_output('x')
p.model.add_subsystem('C1', om.ExecComp(['y=2.0*x+1.'], x=2.0), promotes=['*'])
p.driver = om.DOEDriver()
p.model.add_design_var('x', lower=0.0, upper=10.0)
p.model.add_objective('y')
p.setup()
# Conclude setup but don't run model.
p.final_setup()
write_xdsm(p, filename=filename, out_format=out_format, quiet=QUIET, show_browser=SHOW, show_parallel=True)
# Check if file was created
self.assertTrue(os.path.isfile(filename + '.' + out_format))
def test_meta_model(self):
import openmdao.api as om
from openmdao.components.tests.test_meta_model_structured_comp import SampleMap
filename = 'xdsmjs_meta_model'
out_format = 'html'
model = om.Group()
ivc = om.IndepVarComp()
mapdata = SampleMap()
params = mapdata.param_data
x, y, z = params
outs = mapdata.output_data
z = outs[0]
ivc.add_output('x', x['default'], units=x['units'])
ivc.add_output('y', y['default'], units=y['units'])
ivc.add_output('z', z['default'], units=z['units'])
model.add_subsystem('des_vars', ivc, promotes=["*"])
comp = om.MetaModelStructuredComp(method='slinear', extrapolate=True)
for param in params:
comp.add_input(param['name'], param['default'], param['values'])
for out in outs:
comp.add_output(out['name'], out['default'], out['values'])
model.add_subsystem('comp', comp, promotes=["*"])
p = om.Problem(model)
p.setup()
p.final_setup()
write_xdsm(p, filename=filename, out_format=out_format, quiet=QUIET, show_browser=SHOW, show_parallel=True)
# Check if file was created
self.assertTrue(os.path.isfile(filename + '.' + out_format))
def test_circuit_recurse(self):
# Implicit component is also tested here
import openmdao.api as om
p = om.Problem()
model = p.model
model.add_subsystem('ground', om.IndepVarComp('V', 0., units='V'))
model.add_subsystem('source', om.IndepVarComp('I', 0.1, units='A'))
model.add_subsystem('circuit', Circuit())
model.connect('source.I', 'circuit.I_in')
model.connect('ground.V', 'circuit.Vg')
model.add_design_var('ground.V')
model.add_design_var('source.I')
model.add_objective('circuit.D1.I')
p.setup()
# set some initial guesses
p['circuit.n1.V'] = 10.
p['circuit.n2.V'] = 1.
p.run_model()
write_xdsm(p, 'xdsmjs_circuit', out_format='html', quiet=QUIET, show_browser=SHOW, recurse=True)
self.assertTrue(os.path.isfile('xdsmjs_circuit' + '.html'))
def test_legend_and_class_names(self):
import openmdao.api as om
p = om.Problem()
model = p.model
model.add_subsystem('ground', om.IndepVarComp('V', 0., units='V'))
model.add_subsystem('source', om.IndepVarComp('I', 0.1, units='A'))
model.add_subsystem('circuit', Circuit())
model.connect('source.I', 'circuit.I_in')
model.connect('ground.V', 'circuit.Vg')
model.add_design_var('ground.V')
model.add_design_var('source.I')
model.add_objective('circuit.D1.I')
p.setup()
# set some initial guesses
p['circuit.n1.V'] = 10.
p['circuit.n2.V'] = 1.
p.run_model()
write_xdsm(p, 'xdsmjs_circuit_legend', out_format='html', quiet=QUIET, show_browser=SHOW, recurse=True,
legend=True)
self.assertTrue(os.path.isfile('xdsmjs_circuit_legend' + '.html'))
write_xdsm(p, 'xdsmjs_circuit_class_names', out_format='html', quiet=QUIET, show_browser=SHOW, recurse=True,
class_names=True)
self.assertTrue(os.path.isfile('xdsmjs_circuit_class_names' + '.html'))
def test_xdsmjs_right_outputs(self):
"""Makes XDSM for the Sellar problem"""
filename = 'xdsmjs_outputs_on_the_right'
p = om.Problem()
p.model = model = SellarNoDerivatives()
model.add_design_var('z', lower=np.array([-10.0, 0.0]), upper=np.array([10.0, 10.0]),
indices=np.arange(2, dtype=int))
model.add_design_var('x', lower=0.0, upper=10.0)
model.add_objective('obj')
model.add_constraint('con1', equals=np.zeros(1))
model.add_constraint('con2', upper=0.0)
p.setup()
p.final_setup()
msg = 'Right side outputs not implemented for XDSMjs.'
# Write output
with assert_warning(Warning, msg):
write_xdsm(p, filename=filename, out_format='html', show_browser=SHOW, quiet=QUIET, output_side='right')
# Check if file was created
self.assertTrue(os.path.isfile(filename + '.html'))
def test_wrong_out_format(self):
"""Incorrect output format error."""
filename = 'xdsm_wrong_format' # this name is needed for XDSMjs
prob = om.Problem()
prob.model = SellarNoDerivatives()
prob.setup()
prob.final_setup()
# no output checking, just make sure no exceptions raised
with self.assertRaises(ValueError):
write_xdsm(prob, filename=filename, out_format='jpg', subs=(), show_browser=SHOW)
def test_command(self):
"""
Check that there are no errors when running from the command line with a script.
"""
from openmdao.test_suite.scripts import sellar
filename = os.path.abspath(sellar.__file__)
check_call('openmdao xdsm --no_browser {}'.format(filename))
@unittest.skipUnless(LooseVersion(om_version) >= LooseVersion('3.2'), 'Auto-IVC introduced in OpenMDAO 3.2')
def test_auto_ivc(self):
"""
Tests a model with automatically added IndepVarComp.
"""
filename = 'xdsm_auto_ivc'
p = om.Problem()
p.model.add_subsystem('paraboloid',
om.ExecComp('f = (x-3)**2 + x*y + (y+4)**2 - 3'),
promotes_inputs=['x', 'y'])
# setup the optimization
p.driver = om.ScipyOptimizeDriver()
p.driver.options['optimizer'] = 'SLSQP'
p.model.add_design_var('x', lower=-50, upper=50)
p.model.add_design_var('y', lower=-50, upper=50)
p.model.add_objective('paraboloid.f')
p.setup()
p.final_setup()
# Write output
write_xdsm(p, filename=filename, out_format='html', show_browser=SHOW, quiet=QUIET,
include_indepvarcomps=False) # Not showing the Auto IVC
# Check if file was created
self.assertTrue(os.path.isfile(filename + '.html'))
write_xdsm(p, filename=filename + '2', out_format='html', show_browser=SHOW, quiet=QUIET,
include_indepvarcomps=True) # Showing the Auto IVC
# Check if file was created
self.assertTrue(os.path.isfile(filename + '2.html'))
from distutils.version import LooseVersion
from typing import Sequence
from typing import Union
import torch
from espnet2.asr.specaug.abs_specaug import AbsSpecAug
from espnet2.layers.mask_along_axis import MaskAlongAxis
from espnet2.layers.time_warp import TimeWarp
if LooseVersion(torch.__version__) >= LooseVersion("1.1"):
DEFAULT_TIME_WARP_MODE = "bicubic"
else:
# pytorch1.0 doesn't implement bicubic
DEFAULT_TIME_WARP_MODE = "bilinear"
class SpecAug(AbsSpecAug):
"""Implementation of SpecAug.
Reference:
Daniel S. Park et al.
"SpecAugment: A Simple Data
Augmentation Method for Automatic Speech Recognition"
.. warning::
When using cuda mode, time_warp doesn't have reproducibility
due to `torch.nn.functional.interpolate`.
"""
# Add any Sphinx extension module names here, as strings. They can be extensions
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest',
'sphinx.ext.intersphinx', 'sphinx.ext.todo',
'sphinx.ext.pngmath', 'sphinx.ext.viewcode', 'sphinx.ext.autosummary',
'sphinx.ext.inheritance_diagram',
'matplotlib.sphinxext.plot_directive',
'matplotlib.sphinxext.only_directives',
'ipython_console_highlighting',
'ipython_directive',
'numpy_ext.numpydoc']
# plot_directive is broken on old matplotlib
from matplotlib import __version__ as mpl_version
from distutils.version import LooseVersion
if LooseVersion(mpl_version) < LooseVersion('1.0.1'):
extensions.remove('matplotlib.sphinxext.plot_directive')
extensions.append('numpy_ext.plot_directive')
# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']
# The suffix of source filenames.
source_suffix = '.rst'
# The encoding of source files.
#source_encoding = 'utf-8-sig'
# The master toctree document.
master_doc = 'index'
import re
import warnings
from distutils.version import LooseVersion
import numpy as np
import matplotlib
from matplotlib import ticker
from matplotlib.transforms import Bbox, TransformedBbox, Affine2D
mpl_version = LooseVersion(matplotlib.__version__) # noqa
from ...core.util import basestring, _getargspec
from ...element import Raster, RGB
def wrap_formatter(formatter):
"""
Wraps formatting function or string in
appropriate matplotlib formatter type.
"""
if isinstance(formatter, ticker.Formatter):
return formatter
elif callable(formatter):
args = [arg for arg in _getargspec(formatter).args if arg != 'self']
wrapped = formatter
if len(args) == 1:
def wrapped(val, pos=None):
return formatter(val)
return ticker.FuncFormatter(wrapped)
def tf_concat(axis, values, **kwargs):
if TENSORFLOW_VERSION >= LooseVersion('1.0'):
return tf.concat(values, axis, **kwargs)
else:
return tf.concat(axis, values, **kwargs)
