def filter_product(self, data, sap_code, base_version, version='latest'):
"""Find product information from a feed dump given a single sap_code, base version and optional version."""
product = {'version': '0.0.1'}
channels = self.env.get('channels').split(',')
#12 inputs to ccpinfo dict testing w BridgeCC
for channel in data['channel']:
if channel['name'] not in channels:
continue
for prod in channel['products']['product']:
if prod['id'] != sap_code:
continue
if base_version and prod['platforms']['platform'][0]['languageSet'][0].get('baseVersion') != base_version:
continue
if 'version' not in prod:
self.output('product has no version: {}'.format(prod['displayName']))
continue
if version == "latest":
if LV(prod['version']) > LV(product['version']):
product = prod
else:
if prod['version'] == version:
product = prod
if 'platforms' not in product:
return None
return product
def compile(cls):
cdict = super().compile()
if (cdict.get("provider_type", "")) == "":
cdict.pop("provider_type", "")
if (cdict.get("value_type", "")) == "":
cdict.pop("value_type", "")
CALM_VERSION = Version.get_version("Calm")
if LV(CALM_VERSION) < LV("3.2.0"):
value_type = cdict.pop("value_type")
cdict["attrs"]["value_type"] = value_type
else:
value_type = cdict.get("value_type", "IP")
if value_type == "VM":
account = cdict["attrs"]["account_reference"]
account_name = account["name"]
account_data = Cache.get_entity_data(
entity_type=CACHE.ENTITY.ACCOUNT, name=account_name)
if not account_data:
LOG.error("Account {} not found".format(account_name))
sys.exit(-1)
provider_type = account_data["provider_type"]
if provider_type not in ["nutanix_pc", "vmware"]:
LOG.error("Provider {} not supported for endpoints".format(
provider_type))
sys.exit(-1)
cdict["provider_type"] = provider_type.upper()
return cdict
def get_cache_tables(cls, sync_version=False):
"""returns tables used for cache purpose"""
db = get_db_handle()
db_tables = db.registered_tables
# Get calm version from api only if necessary
calm_version = CALM_VERSION
if sync_version or (not calm_version):
context = get_context()
server_config = context.get_server_config()
client = get_client_handle_obj(
server_config["pc_ip"],
server_config["pc_port"],
auth=(server_config["pc_username"], server_config["pc_password"]),
)
res, err = client.version.get_calm_version()
if err:
LOG.error("Failed to get version")
sys.exit(err["error"])
calm_version = res.content.decode("utf-8")
cache_tables = {}
for table in db_tables:
if hasattr(table, "__cache_type__") and (
LV(calm_version) >= LV(table.feature_min_version)
):
cache_tables[table.__cache_type__] = table
return cache_tables
def invoke(self, ctx):
if not ctx.protected_args:
return super(FeatureFlagMixin, self).invoke(ctx)
cmd_name = ctx.protected_args[0]
feature_min_version = self.feature_version_map.get(cmd_name, "")
if feature_min_version:
calm_version = Version.get_version("Calm")
if not calm_version:
LOG.error("Calm version not found. Please update cache")
sys.exit(-1)
if LV(calm_version) >= LV(feature_min_version):
return super().invoke(ctx)
else:
LOG.warning(
"Please update Calm (v{} -> >=v{}) to use this command.".
format(calm_version, feature_min_version))
return None
else:
return super().invoke(ctx)
def get_validators_with_defaults(schema_props):
validators = {}
defaults = {}
display_map = bidict()
for name, props in schema_props.items():
calm_version = Version.get_version("Calm")
# dev machines do not follow standard version protocols. Avoid matching there
attribute_min_version = str(props.get("x-calm-dsl-min-version", ""))
if not calm_version:
# Raise warning and set default to 2.9.0
calm_version = "2.9.0"
# If attribute version is less than calm version, ignore it
if attribute_min_version and LV(attribute_min_version) > LV(calm_version):
continue
ValidatorType, is_array, default = get_validator_details(schema_props, name)
attr_name = props.get("x-calm-dsl-display-name", name)
validators[attr_name] = (ValidatorType, is_array)
defaults[attr_name] = default
display_map[attr_name] = name
return validators, defaults, display_map
def create_environment_payload(UserEnvironment, metadata=dict()):
"""
Creates environment payload
Args:
UserEnvironment(object): Environment object
metadata (dict) : Metadata for environment
Returns:
response(tuple): tuple consisting of environment payload object and error
"""
err = {"error": "", "code": -1}
if UserEnvironment is None:
err["error"] = "Given environment is empty."
return None, err
if not isinstance(UserEnvironment, EnvironmentType):
err["error"] = "Given environment is not of type Environment"
return None, err
spec = {
"name": UserEnvironment.__name__,
"description": UserEnvironment.__doc__ or "",
"resources": UserEnvironment,
}
env_project = metadata.get("project_reference", {}).get("name", "")
if not env_project:
ContextObj = get_context()
project_config = ContextObj.get_project_config()
env_project = project_config["name"]
project_cache_data = Cache.get_entity_data(
entity_type=CACHE.ENTITY.PROJECT, name=env_project)
if not project_cache_data:
LOG.error("Project {} not found.".format(env_project))
sys.exit("Project {} not found.".format(env_project))
metadata_payload = {
"spec_version": 1,
"kind": "environment",
"name": UserEnvironment.__name__,
"uuid": str(uuid.uuid4()),
}
calm_version = Version.get_version("Calm")
if LV(calm_version) >= LV("3.2.0"):
metadata_payload["project_reference"] = {
"kind": "project",
"name": project_cache_data["name"],
"uuid": project_cache_data["uuid"],
}
UserEnvironmentPayload = _environment_payload()
UserEnvironmentPayload.metadata = metadata_payload
UserEnvironmentPayload.spec = spec
return UserEnvironmentPayload, None
def compile(cls):
cdict = super().compile()
cdict["account_reference_list"] = []
cdict["subnet_reference_list"] = []
cdict["external_network_list"] = []
cdict["default_subnet_reference"] = {}
CALM_VERSION = Version.get_version("Calm")
default_subnet_reference = None
# Populate accounts
provider_list = cdict.pop("provider_list", [])
for provider_obj in provider_list:
provider_data = provider_obj.get_dict()
if provider_obj.type == "nutanix_pc":
if "subnet_reference_list" in provider_data:
cdict["subnet_reference_list"].extend(
provider_data["subnet_reference_list"])
if "external_network_list" in provider_data:
for _network in provider_data["external_network_list"]:
_network.pop("kind", None)
cdict["external_network_list"].append(_network)
if "default_subnet_reference" in provider_data:
# From 3.2, only subnets from local account can be marked as default
if provider_data.get("subnet_reference_list"
) or LV(CALM_VERSION) < LV("3.2.0"):
cdict["default_subnet_reference"] = provider_data[
"default_subnet_reference"]
if "account_reference" in provider_data:
cdict["account_reference_list"].append(
provider_data["account_reference"])
quotas = cdict.pop("quotas", None)
if quotas:
project_resources = []
for qk, qv in quotas.items():
if qk != "VCPUS":
qv *= 1073741824
project_resources.append({"limit": qv, "resource_type": qk})
cdict["resource_domain"] = {"resources": project_resources}
# pop out unnecessary attibutes
cdict.pop("environment_definition_list", None)
# empty dict is not accepted for default_environment_reference
default_env = cdict.get("default_environment_reference")
if not default_env:
cdict.pop("default_environment_reference", None)
if not cdict.get("default_subnet_reference"):
cdict.pop("default_subnet_reference", None)
return cdict
def validate_version():
# At initializing dsl, version might not found in cache
calm_version = Version.get_version("Calm")
if calm_version:
if LV(calm_version) < LV(LATEST_VERIFIED_VERSION):
LOG.warning(
"Calm server version ({}) is less than verified version. ({})."
.format(calm_version, LATEST_VERIFIED_VERSION))
def test_endpoint_validation_and_type_update2(self, EndpointPayload):
"""
test_endpoint_name_validations
"""
client = get_api_client()
endpoint = copy.deepcopy(change_uuids(EndpointPayload, {}))
# set values and credentials to empty
CALM_VERSION = Version.get_version("Calm")
if LV(CALM_VERSION) < LV("3.3.0"):
message = (
"Name can contain only alphanumeric, underscores, hyphens and spaces"
)
else:
message = "Name can contain only unicode characters, underscores, hyphens and spaces"
endpoint["spec"]["name"] = "ep-$.-name1" + str(uuid.uuid4())[-10:]
# Endpoint Create
res, err = client.endpoint.create(endpoint)
if not err:
pytest.fail(
"Endpoint created successfully with unsupported name formats")
assert err.get("code", 0) == 422
assert message in res.text
endpoint["spec"]["name"] = "endpoint_" + str(uuid.uuid4())[-10:]
res, err = client.endpoint.create(endpoint)
if err:
pytest.fail("[{}] - {}".format(err["code"], err["error"]))
ep = res.json()
ep_uuid = ep["metadata"]["uuid"]
ep_name = ep["spec"]["name"]
print(">> Endpoint created: {}".format(ep_name))
del ep["status"]
ep["spec"]["name"] = "-test_ep_name_" + str(uuid.uuid4())[-10:]
if LV(CALM_VERSION) < LV("3.3.0"):
message = (
"Names can only start with alphanumeric characters or underscore (_)"
)
else:
message = "Names can only start with unicode characters or underscore (_)"
res, err = client.endpoint.update(ep_uuid, ep)
if not err:
pytest.fail(
"Endpoint updated successfully with unsupported name formats")
assert err.get("code", 0) == 422
assert message in res.text
# delete the endpoint
_, err = client.endpoint.delete(ep_uuid)
if err:
pytest.fail("[{}] - {}".format(err["code"], err["error"]))
else:
print("endpoint {} deleted".format(ep_name))
def new_version(self):
"""
compare current version with version in db
"""
self.remote_version = DB.get_config('version_remote')
self.message = DB.get_config('version_msg')
if not self.remote_version:
return False
elif LV(self.local_version) < LV(self.remote_version):
return True
else:
return False
def run(self, uuid, payload):
from calm.dsl.store.version import Version
calm_version = Version.get_version("Calm")
runbook_run_api = self.RUN
if LV(calm_version) >= LV("3.3.2"):
runbook_run_api = self.EXECUTE
return self.connection._call(
runbook_run_api.format(uuid),
verify=False,
request_json=payload,
method=REQUEST.METHOD.POST,
)
def get_api_obj(cls):
"""returns object to call ahv provider specific apis"""
client = get_api_client()
calm_version = Version.get_version("Calm")
api_handlers = AWSBase.api_handlers
latest_version = "0"
for version in api_handlers.keys():
if LV(version) <= LV(calm_version) and LV(latest_version) < LV(
version):
latest_version = version
api_handler = api_handlers[latest_version]
return api_handler(client.connection)
def translateAndCapitalizeNamespaces(self, text):
"""
Makes sure that localized namespace names are used.
"""
# arz uses english stylish codes
if self.site.sitename() == 'wikipedia:arz':
return text
family = self.site.family
# wiki links aren't parsed here.
exceptions = ['nowiki', 'comment', 'math', 'pre']
for nsNumber in self.site.namespaces():
if nsNumber in (0, 2, 3):
# skip main (article) namespace
# skip user namespace, maybe gender is used
continue
# a clone is needed. Won't change the namespace dict
namespaces = list(self.site.namespace(nsNumber, all=True))
thisNs = namespaces.pop(0)
if nsNumber == 6 and family.name == 'wikipedia':
if self.site.lang in ('en', 'fr') and \
LV(self.site.version()) >= LV('1.14'):
# do not change "Image" on en-wiki and fr-wiki
assert u'Image' in namespaces
namespaces.remove(u'Image')
if self.site.lang == 'hu':
# do not change "Kép" on hu-wiki
assert u'Kép' in namespaces
namespaces.remove(u'Kép')
elif self.site.lang == 'pt':
# bug #3346901 should be implemented
continue
# lowerspaced and underscored namespaces
for i in range(len(namespaces)):
item = namespaces[i].replace(' ', '[ _]')
item = u'[%s%s]' % (item[0], item[0].lower()) + item[1:]
namespaces[i] = item
namespaces.append(thisNs[0].lower() + thisNs[1:])
if thisNs and namespaces:
text = pywikibot.replaceExcept(
text,
r'\[\[\s*(%s) *:(?P<nameAndLabel>.*?)\]\]'
% '|'.join(namespaces),
r'[[%s:\g<nameAndLabel>]]' % thisNs,
exceptions)
return text
def create_environment_payload(UserEnvironment):
err = {"error": "", "code": -1}
if UserEnvironment is None:
err["error"] = "Given environment is empty."
return None, err
if not isinstance(UserEnvironment, EnvironmentType):
err["error"] = "Given environment is not of type Environment"
return None, err
spec = {
"name": UserEnvironment.__name__,
"description": UserEnvironment.__doc__ or "",
"resources": UserEnvironment,
}
ContextObj = get_context()
project_config = ContextObj.get_project_config()
project_cache_data = Cache.get_entity_data(
entity_type=CACHE.ENTITY.PROJECT, name=project_config["name"])
if not project_cache_data:
LOG.error("Project {} not found.".format(project_config["name"]))
sys.exit(-1)
metadata = {
"spec_version": 1,
"kind": "environment",
"name": UserEnvironment.__name__,
}
calm_version = Version.get_version("Calm")
if LV(calm_version) >= LV("3.2.0"):
metadata["project_reference"] = {
"kind": "project",
"name": project_cache_data["name"],
"uuid": project_cache_data["uuid"],
}
UserEnvironmentPayload = _environment_payload()
UserEnvironmentPayload.metadata = metadata
UserEnvironmentPayload.spec = spec
return UserEnvironmentPayload, None
def convert(self, value, param, ctx):
if self.feature_min_version:
calm_version = Version.get_version("Calm")
if not calm_version:
LOG.error("Calm version not found. Please update cache")
sys.exit(-1)
# TODO add the pc version to warning also
if LV(calm_version) < LV(self.feature_min_version):
LOG.error(
"Calm {} does not support '{}' option. Please upgrade server to Calm {}"
.format(calm_version, param.name,
self.feature_min_version))
sys.exit(-1)
# Add validation for file types etc.
return value
def get_package_version():
try:
proc = subprocess.run(['radicale', '--version'],
stdout=subprocess.PIPE, check=True)
output = proc.stdout.decode('utf-8')
except subprocess.CalledProcessError:
return None
package_version = str(output.strip())
return LV(package_version)
def get_vmware_vm_data_with_version_filtering(vm_data):
"""returns instance_data_according_to_version_filter"""
CALM_VERSION = Version.get_version("Calm")
instance_id = vm_data["instance_id"]
instance_name = vm_data["instance_name"]
if LV(CALM_VERSION) >= LV("3.3.0"):
hostname = vm_data["guest_hostname"]
address = ",".join(vm_data["guest_ipaddress"])
vcpus = vm_data["cpu"]
sockets = vm_data["num_vcpus_per_socket"]
memory = int(vm_data["memory"]) // 1024
guest_family = vm_data.get("guest_family", "")
template = vm_data.get("is_template", False)
else:
hostname = vm_data["guest.hostName"]
address = ",".join(vm_data["guest.ipAddress"])
vcpus = vm_data["config.hardware.numCPU"]
sockets = vm_data["config.hardware.numCoresPerSocket"]
memory = int(vm_data["config.hardware.memoryMB"]) // 1024
guest_family = vm_data.get("guest.guestFamily", "")
template = vm_data.get("config.template", False)
return (
instance_id,
instance_name,
hostname,
address,
vcpus,
sockets,
memory,
guest_family,
template,
)
def test_versions(self):
import tensorflow as tf
if mod_version:
if not nvidia:
self.assertEqual(LV(tf.__version__), LV(mod_version))
else:
print('NOTE: running NVIDIA container, skipping some tests...')
if is_tf2:
import tensorflow.keras as keras
else:
import keras
self.assertGreaterEqual(LV(keras.__version__), LV("2.0"))
if is_tf2:
self.assertGreaterEqual(LV(keras.__version__), LV("2.2.4"))
if expect_horovod:
import horovod
import horovod.tensorflow as hvd
self.assertGreaterEqual(LV(horovod.__version__), LV("0.18.2"))
def __eq__(self, other):
if not isinstance(other, InstalledRpm):
return False
if self.name != other.name:
raise ValueError(
'Cannot compare packages with differing names {0} != {1}'.
format(self.name, other.name))
if (not self._distribution) != (not other._distribution):
raise ValueError(
'Cannot compare packages that one has distribution while the other does not {0} != {1}'
.format(self.package, other.package))
eq_ret = (type(self) == type(other)
and LV(self.epoch) == LV(other.epoch)
and LV(self.version) == LV(other.version)
and LV(self.release) == LV(other.release))
if self._distribution:
return eq_ret and LV(self._distribution) == LV(other._distribution)
else:
return eq_ret
def __lt__(self, other):
if not isinstance(other, InstalledRpm):
return False
if self == other:
return False
self_ep, other_ep = LV(self.epoch), LV(other.epoch)
if self_ep != other_ep:
return self_ep < other_ep
self_v, other_v = LV(self.version), LV(other.version)
if self_v != other_v:
return self_v < other_v
self_rl, other_rl = LV(self._release_sep), LV(other._release_sep)
if self_rl != other_rl:
return self_rl < other_rl
# If we reach this point, the self == other test has determined that
# we have a _distribution, so we rely on that.
return LV(self._distribution) < LV(other._distribution)
from keras.layers import LSTM, CuDNNLSTM
from keras.utils import to_categorical
from distutils.version import LooseVersion as LV
from keras import __version__
from keras import backend as K
from sklearn.model_selection import train_test_split
import os
import sys
import numpy as np
print('Using Keras version:', __version__, 'backend:', K.backend())
assert (LV(__version__) >= LV("2.0.0"))
# If we are using TensorFlow as the backend, we can use TensorBoard to
# visualize our progress during training.
if K.backend() == "tensorflow":
import tensorflow as tf
from keras.callbacks import TensorBoard
import os, datetime
logdir = os.path.join(
os.getcwd(), "logs",
"20ng-cnn-" + datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S'))
print('TensorBoard log directory:', logdir)
try:
os.makedirs(logdir)
callbacks = [TensorBoard(log_dir=logdir)]
from distutils.version import LooseVersion as LV
import os
import horovod.torch as hvd
torch.manual_seed(42)
if torch.cuda.is_available():
device = torch.device('cuda')
else:
device = torch.device('cpu')
hvd.init()
torch.cuda.set_device(hvd.local_rank())
if hvd.rank() == 0:
print('Using PyTorch version:', torch.__version__, ' Device:', device)
assert (LV(torch.__version__) >= LV("1.0.0"))
subpath = 'dogs-vs-cats/train-2000'
if 'DATADIR' in os.environ:
DATADIR = os.environ['DATADIR']
else:
DATADIR = "/scratch/project_2003747/data/"
datapath = os.path.join(DATADIR, subpath)
if hvd.rank() == 0:
print('Reading data from path:', datapath)
(nimages_train, nimages_validation, nimages_test) = (2000, 1000, 22000)
def _data_path(path=None,
force_update=False,
update_path=True,
download=True,
name=None,
check_version=False,
return_version=False,
archive_name=None):
"""Aux function."""
key = {
'fake': 'MNE_DATASETS_FAKE_PATH',
'misc': 'MNE_DATASETS_MISC_PATH',
'sample': 'MNE_DATASETS_SAMPLE_PATH',
'spm': 'MNE_DATASETS_SPM_FACE_PATH',
'somato': 'MNE_DATASETS_SOMATO_PATH',
'brainstorm': 'MNE_DATASETS_BRAINSTORM_PATH',
'testing': 'MNE_DATASETS_TESTING_PATH',
'multimodal': 'MNE_DATASETS_MULTIMODAL_PATH',
}[name]
path = _get_path(path, key, name)
# To update the testing or misc dataset, push commits, then make a new
# release on GitHub. Then update the "releases" variable:
releases = dict(testing='0.25', misc='0.1')
# And also update the "hashes['testing']" variable below.
# To update any other dataset, update the data archive itself (upload
# an updated version) and update the hash.
archive_names = dict(
misc='mne-misc-data-%s.tar.gz' % releases['misc'],
sample='MNE-sample-data-processed.tar.gz',
somato='MNE-somato-data.tar.gz',
spm='MNE-spm-face.tar.gz',
testing='mne-testing-data-%s.tar.gz' % releases['testing'],
multimodal='MNE-multimodal-data.tar.gz',
fake='foo.tgz',
)
if archive_name is not None:
archive_names.update(archive_name)
folder_names = dict(
brainstorm='MNE-brainstorm-data',
fake='foo',
misc='MNE-misc-data',
sample='MNE-sample-data',
somato='MNE-somato-data',
multimodal='MNE-multimodal-data',
spm='MNE-spm-face',
testing='MNE-testing-data',
)
urls = dict(
brainstorm='https://mne-tools.s3.amazonaws.com/datasets/'
'MNE-brainstorm-data/%s',
fake='https://github.com/mne-tools/mne-testing-data/raw/master/'
'datasets/%s',
misc='https://codeload.github.com/mne-tools/mne-misc-data/'
'tar.gz/%s' % releases['misc'],
sample="https://mne-tools.s3.amazonaws.com/datasets/%s",
somato='https://mne-tools.s3.amazonaws.com/datasets/%s',
spm='https://mne-tools.s3.amazonaws.com/datasets/%s',
testing='https://codeload.github.com/mne-tools/mne-testing-data/'
'tar.gz/%s' % releases['testing'],
multimodal='https://ndownloader.figshare.com/files/5999598',
)
hashes = dict(
brainstorm=None,
fake='3194e9f7b46039bb050a74f3e1ae9908',
misc='f0708d8914cf2692fee7b6c9f105e71c',
sample='1d5da3a809fded1ef5734444ab5bf857',
somato='f3e3a8441477bb5bacae1d0c6e0964fb',
spm='f61041e3f3f2ba0def8a2ca71592cc41',
testing='217aed43e361c86b622dc0363ae3cef4',
multimodal='26ec847ae9ab80f58f204d09e2c08367',
)
folder_origs = dict( # not listed means None
misc='mne-misc-data-%s' % releases['misc'],
testing='mne-testing-data-%s' % releases['testing'],
)
folder_name = folder_names[name]
archive_name = archive_names[name]
hash_ = hashes[name]
url = urls[name]
folder_orig = folder_origs.get(name, None)
if '%s' in url:
url = url % archive_name
folder_path = op.join(path, folder_name)
if name == 'brainstorm':
extract_path = folder_path
folder_path = op.join(folder_path, archive_names[name].split('.')[0])
rm_archive = False
martinos_path = '/cluster/fusion/sample_data/' + archive_name
neurospin_path = '/neurospin/tmp/gramfort/' + archive_name
if not op.exists(folder_path) and not download:
return ''
if not op.exists(folder_path) or force_update:
if name == 'brainstorm':
if '--accept-brainstorm-license' in sys.argv:
answer = 'y'
else:
answer = input('%sAgree (y/[n])? ' % _bst_license_text)
if answer.lower() != 'y':
raise RuntimeError('You must agree to the license to use this '
'dataset')
logger.info('Downloading or reinstalling '
'data archive %s at location %s' % (archive_name, path))
if op.exists(martinos_path):
archive_name = martinos_path
elif op.exists(neurospin_path):
archive_name = neurospin_path
else:
archive_name = op.join(path, archive_name)
rm_archive = True
fetch_archive = True
if op.exists(archive_name):
msg = ('Archive already exists. Overwrite it (y/[n])? ')
answer = input(msg)
if answer.lower() == 'y':
os.remove(archive_name)
else:
fetch_archive = False
if fetch_archive:
_fetch_file(url,
archive_name,
print_destination=False,
hash_=hash_)
if op.exists(folder_path):
def onerror(func, path, exc_info):
"""Deal with access errors (e.g. testing dataset read-only)."""
# Is the error an access error ?
do = False
if not os.access(path, os.W_OK):
perm = os.stat(path).st_mode | stat.S_IWUSR
os.chmod(path, perm)
do = True
if not os.access(op.dirname(path), os.W_OK):
dir_perm = (os.stat(op.dirname(path)).st_mode
| stat.S_IWUSR)
os.chmod(op.dirname(path), dir_perm)
do = True
if do:
func(path)
else:
raise
shutil.rmtree(folder_path, onerror=onerror)
logger.info('Decompressing the archive: %s' % archive_name)
logger.info('(please be patient, this can take some time)')
for ext in ['gz', 'bz2']: # informed guess (and the only 2 options).
try:
if name != 'brainstorm':
extract_path = path
tf = tarfile.open(archive_name, 'r:%s' % ext)
tf.extractall(path=extract_path)
tf.close()
break
except tarfile.ReadError as err:
logger.info('%s is %s trying "bz2"' % (archive_name, err))
if folder_orig is not None:
shutil.move(op.join(path, folder_orig), folder_path)
if rm_archive:
os.remove(archive_name)
path = _do_path_update(path, update_path, key, name)
path = op.join(path, folder_name)
# compare the version of the dataset and mne
data_version = _dataset_version(path, name)
try:
from distutils.version import LooseVersion as LV
except:
warn('Could not determine %s dataset version; dataset could '
'be out of date. Please install the "distutils" package.' % name)
else: # 0.7 < 0.7.git shoud be False, therefore strip
if check_version and LV(data_version) < LV(mne_version.strip('.git')):
warn('The {name} dataset (version {current}) is older than '
'mne-python (version {newest}). If the examples fail, '
'you may need to update the {name} dataset by using '
'mne.datasets.{name}.data_path(force_update=True)'.format(
name=name, current=data_version, newest=mne_version))
return (path, data_version) if return_version else path
def _data_path(path=None,
force_update=False,
update_path=True,
download=True,
name=None,
check_version=True,
verbose=None):
"""Aux function
"""
key = {
'sample': 'MNE_DATASETS_SAMPLE_PATH',
'spm': 'MNE_DATASETS_SPM_FACE_PATH'
}[name]
if path is None:
# use an intelligent guess if it's not defined
def_path = op.realpath(
op.join(op.dirname(__file__), '..', '..', 'examples'))
path = get_config(key, def_path)
# use the same for all datasets
if not os.path.exists(path):
path = def_path
if not isinstance(path, string_types):
raise ValueError('path must be a string or None')
if name == 'sample':
archive_name = "MNE-sample-data-processed.tar.gz"
url = "ftp://surfer.nmr.mgh.harvard.edu/pub/data/MNE/" + archive_name
folder_name = "MNE-sample-data"
folder_path = op.join(path, folder_name)
rm_archive = False
elif name == 'spm':
archive_name = 'MNE-spm-face.tar.bz2'
url = 'ftp://surfer.nmr.mgh.harvard.edu/pub/data/MNE/' + archive_name
folder_name = "MNE-spm-face"
folder_path = op.join(path, folder_name)
rm_archive = False
else:
raise ValueError('Sorry, the dataset "%s" is not known.' % name)
martinos_path = '/cluster/fusion/sample_data/' + archive_name
neurospin_path = '/neurospin/tmp/gramfort/' + archive_name
if not op.exists(folder_path) and not download:
return ''
if not op.exists(folder_path) or force_update:
logger.info(
'Downloading or reinstalling data archive %s at location %s' %
(archive_name, path))
if op.exists(martinos_path):
archive_name = martinos_path
elif op.exists(neurospin_path):
archive_name = neurospin_path
else:
archive_name = op.join(path, archive_name)
rm_archive = True
fetch_archive = True
if op.exists(archive_name):
msg = ('Archive already exists. Overwrite it (y/[n])? ')
answer = raw_input(msg)
if answer.lower() == 'y':
os.remove(archive_name)
else:
fetch_archive = False
if fetch_archive:
_fetch_file(url, archive_name, print_destination=False)
if op.exists(folder_path):
shutil.rmtree(folder_path)
logger.info('Decompressing the archive: ' + archive_name)
logger.info('... please be patient, this can take some time')
for ext in ['gz', 'bz2']: # informed guess (and the only 2 options).
try:
tarfile.open(archive_name, 'r:%s' % ext).extractall(path=path)
break
except tarfile.ReadError as err:
logger.info('%s is %s trying "bz2"' % (archive_name, err))
if rm_archive:
os.remove(archive_name)
path = op.abspath(path)
if update_path is None:
if get_config(key, '') != path:
update_path = True
msg = ('Do you want to set the path:\n %s\nas the default '
'sample dataset path in the mne-python config [y]/n? ' %
path)
answer = raw_input(msg)
if answer.lower() == 'n':
update_path = False
else:
update_path = False
if update_path is True:
set_config(key, path)
path = op.join(path, folder_name)
# compare the version of the Sample dataset and mne
data_version = _dataset_version(path, name)
try:
from distutils.version import LooseVersion as LV
except:
warn('Could not determine sample dataset version; dataset could\n'
'be out of date. Please install the "distutils" package.')
else: # 0.7 < 0.7.git shoud be False, therefore strip
if check_version and LV(data_version) < LV(mne_version.strip('.git')):
warn('The {name} dataset (version {current}) is older than '
'mne-python (version {newest}). If the examples fail, '
'you may need to update the {name} dataset by using '
'mne.datasets.{name}.data_path(force_update=True)'.format(
name=name, current=data_version, newest=mne_version))
return path
def get_accounts(name, filter_by, limit, offset, quiet, all_items,
account_type):
"""Get the accounts, optionally filtered by a string"""
client = get_api_client()
calm_version = Version.get_version("Calm")
params = {"length": limit, "offset": offset}
filter_query = ""
if name:
filter_query = get_name_query([name])
if filter_by:
filter_query = filter_query + ";(" + filter_by + ")"
if account_type:
filter_query += ";(type=={})".format(",type==".join(account_type))
if all_items:
filter_query += get_states_filter(ACCOUNT.STATES)
# Remove PE accounts for versions >= 2.9.0 (TODO move to constants)
if LV(calm_version) >= LV("2.9.0"):
filter_query += ";type!=nutanix"
if filter_query.startswith(";"):
filter_query = filter_query[1:]
if filter_query:
params["filter"] = filter_query
res, err = client.account.list(params)
if err:
ContextObj = get_context()
server_config = ContextObj.get_server_config()
pc_ip = server_config["pc_ip"]
LOG.warning("Cannot fetch accounts from {}".format(pc_ip))
return
res = res.json()
total_matches = res["metadata"]["total_matches"]
if total_matches > limit:
LOG.warning(
"Displaying {} out of {} entities. Please use --limit and --offset option for more results."
.format(limit, total_matches))
json_rows = res["entities"]
if not json_rows:
click.echo(highlight_text("No account found !!!\n"))
return
if quiet:
for _row in json_rows:
row = _row["status"]
click.echo(highlight_text(row["name"]))
return
table = PrettyTable()
table.field_names = [
"NAME",
"ACCOUNT TYPE",
"STATE",
"OWNER",
"CREATED ON",
"LAST UPDATED",
"UUID",
]
for _row in json_rows:
row = _row["status"]
metadata = _row["metadata"]
creation_time = int(metadata["creation_time"]) // 1000000
last_update_time = int(metadata["last_update_time"]) // 1000000
if "owner_reference" in metadata:
owner_reference_name = metadata["owner_reference"]["name"]
else:
owner_reference_name = "-"
table.add_row([
highlight_text(row["name"]),
highlight_text(row["resources"]["type"]),
highlight_text(row["resources"]["state"]),
highlight_text(owner_reference_name),
highlight_text(time.ctime(creation_time)),
"{}".format(arrow.get(last_update_time).humanize()),
highlight_text(metadata["uuid"]),
])
click.echo(table)
class TestProjectCommands:
def setup_method(self):
""""Reset the context changes"""
ContextObj = get_context()
ContextObj.reset_configuration()
def teardown_method(self):
""""Reset the context changes"""
ContextObj = get_context()
ContextObj.reset_configuration()
def test_projects_list(self):
runner = CliRunner()
result = runner.invoke(cli, ["get", "projects"])
if result.exit_code:
cli_res_dict = {
"Output": result.output,
"Exception": str(result.exception)
}
LOG.debug("Cli Response: {}".format(
json.dumps(cli_res_dict, indent=4, separators=(",", ": "))))
LOG.debug("Traceback: \n{}".format("".join(
traceback.format_tb(result.exc_info[2]))))
pytest.fail("Project Get failed")
LOG.info("Success")
def test_compile_project(self):
runner = CliRunner()
LOG.info("Compiling Project file at {}".format(DSL_PROJECT_PATH))
result = runner.invoke(
cli, ["compile", "project", "--file={}".format(DSL_PROJECT_PATH)])
if result.exit_code:
cli_res_dict = {
"Output": result.output,
"Exception": str(result.exception)
}
LOG.debug("Cli Response: {}".format(
json.dumps(cli_res_dict, indent=4, separators=(",", ": "))))
LOG.debug("Traceback: \n{}".format("".join(
traceback.format_tb(result.exc_info[2]))))
pytest.fail("Project compile command failed")
LOG.info("Success")
def test_project_crud(self):
"""
It will cover create/describe/update/delete/get commands on project
This test assumes users/groups mentioned in project file are already created
"""
# Create operation
self._test_project_create_using_dsl()
# Read operations
click.echo("")
self._test_project_describe_out_json()
click.echo("")
self._test_project_describe_out_text()
click.echo("")
self._test_project_list_name_filter()
# Update operations
click.echo("")
self._test_update_project_using_cli_switches()
click.echo("")
self._test_update_project_using_dsl_file()
# Delete operations
click.echo("")
self._test_project_delete()
def _test_project_create_using_dsl(self):
runner = CliRunner()
self.dsl_project_name = "Test_DSL_Project_{}".format(str(uuid.uuid4()))
LOG.info("Testing 'calm create project' command")
result = runner.invoke(
cli,
[
"create",
"project",
"--file={}".format(DSL_PROJECT_PATH),
"--name={}".format(self.dsl_project_name),
"--description='Test DSL Project to delete'",
],
)
if result.exit_code:
cli_res_dict = {
"Output": result.output,
"Exception": str(result.exception)
}
LOG.debug("Cli Response: {}".format(
json.dumps(cli_res_dict, indent=4, separators=(",", ": "))))
LOG.debug("Traceback: \n{}".format("".join(
traceback.format_tb(result.exc_info[2]))))
pytest.fail("Project creation from python file failed")
LOG.info("Success")
def _test_project_describe_out_text(self):
runner = CliRunner()
LOG.info("Testing 'calm describe project --out text' command")
result = runner.invoke(cli,
["describe", "project", self.dsl_project_name])
if result.exit_code:
cli_res_dict = {
"Output": result.output,
"Exception": str(result.exception)
}
LOG.debug("Cli Response: {}".format(
json.dumps(cli_res_dict, indent=4, separators=(",", ": "))))
LOG.debug("Traceback: \n{}".format("".join(
traceback.format_tb(result.exc_info[2]))))
pytest.fail("Project Get call failed")
project_name_str = "Name: {}".format(self.dsl_project_name)
assert project_name_str in result.output
LOG.info("Success")
def _test_project_describe_out_json(self):
runner = CliRunner()
LOG.info("Testing 'calm describe project --out json' command")
result = runner.invoke(
cli,
["describe", "project", self.dsl_project_name, "--out", "json"])
if result.exit_code:
cli_res_dict = {
"Output": result.output,
"Exception": str(result.exception)
}
LOG.debug("Cli Response: {}".format(
json.dumps(cli_res_dict, indent=4, separators=(",", ": "))))
LOG.debug("Traceback: \n{}".format("".join(
traceback.format_tb(result.exc_info[2]))))
pytest.fail("Project Get call failed")
project_name_str = '"name": "{}"'.format(self.dsl_project_name)
assert project_name_str in result.output
LOG.info("Success")
def _test_project_list_name_filter(self):
runner = CliRunner()
LOG.info("Testing 'calm get projects --name <project_name>' command")
result = runner.invoke(
cli, ["get", "projects", "--name", self.dsl_project_name])
if result.exit_code:
cli_res_dict = {
"Output": result.output,
"Exception": str(result.exception)
}
LOG.debug("Cli Response: {}".format(
json.dumps(cli_res_dict, indent=4, separators=(",", ": "))))
LOG.debug("Traceback: \n{}".format("".join(
traceback.format_tb(result.exc_info[2]))))
pytest.fail("Project list call failed")
assert self.dsl_project_name in result.output
LOG.info("Success")
def _test_update_project_using_cli_switches(self):
"""
Adds user to given project.
(User must be prsent in db)
"""
runner = CliRunner()
LOG.info("Testing 'calm update project' command using cli switches")
result = runner.invoke(
cli,
[
"update",
"project",
self.dsl_project_name,
"--add_user",
USER_NAME,
],
)
if result.exit_code:
cli_res_dict = {
"Output": result.output,
"Exception": str(result.exception)
}
LOG.debug("Cli Response: {}".format(
json.dumps(cli_res_dict, indent=4, separators=(",", ": "))))
LOG.debug("Traceback: \n{}".format("".join(
traceback.format_tb(result.exc_info[2]))))
pytest.fail("Project update call failed")
LOG.info("Success")
def _test_update_project_using_dsl_file(self):
"""
Removes user from given project.
(User must be prsent in db)
"""
runner = CliRunner()
LOG.info("Testing 'calm update project' command using dsl file")
result = runner.invoke(
cli,
[
"update",
"project",
self.dsl_project_name,
"--file={}".format(DSL_PROJECT_PATH),
],
)
if result.exit_code:
cli_res_dict = {
"Output": result.output,
"Exception": str(result.exception)
}
LOG.debug("Cli Response: {}".format(
json.dumps(cli_res_dict, indent=4, separators=(",", ": "))))
LOG.debug("Traceback: \n{}".format("".join(
traceback.format_tb(result.exc_info[2]))))
pytest.fail("Project update call failed")
LOG.info("Success")
def _test_project_delete(self):
runner = CliRunner()
LOG.info("Testing 'calm delete project' command")
result = runner.invoke(cli,
["delete", "project", self.dsl_project_name])
if result.exit_code:
cli_res_dict = {
"Output": result.output,
"Exception": str(result.exception)
}
LOG.debug("Cli Response: {}".format(
json.dumps(cli_res_dict, indent=4, separators=(",", ": "))))
LOG.debug("Traceback: \n{}".format("".join(
traceback.format_tb(result.exc_info[2]))))
pytest.fail("Project delete call failed")
LOG.info("Success")
@pytest.mark.skipif(LV(CALM_VERSION) >= LV("3.2.0"),
reason="Env creation changed in 3.2.0")
def test_project_with_env_create_and_delete(self):
"""
Describe and update flow are already checked in `test_project_crud`
It will test only create and delete flow on projects with environment
"""
runner = CliRunner()
self.dsl_project_name = "Test_DSL_Project_Env{}".format(
str(uuid.uuid4()))
LOG.info("Testing 'calm create project' command")
result = runner.invoke(
cli,
[
"create",
"project",
"--file={}".format(DSL_PROJECT_WITH_ENV_PATH),
"--name={}".format(self.dsl_project_name),
"--description='Test DSL Project with Env to delete'",
],
)
if result.exit_code:
cli_res_dict = {
"Output": result.output,
"Exception": str(result.exception)
}
LOG.debug("Cli Response: {}".format(
json.dumps(cli_res_dict, indent=4, separators=(",", ": "))))
LOG.debug("Traceback: \n{}".format("".join(
traceback.format_tb(result.exc_info[2]))))
pytest.fail("Project creation from python file failed")
LOG.info("Success")
self._test_project_delete()
# Restoring the metadata context
get_metadata_payload(__file__)
APPLICATION.STATES.ERROR,
]
# project constants
DSL_CONFIG = json.loads(read_local_file(".tests/config.json"))
PROJECT = DSL_CONFIG["PROJECTS"]["PROJECT1"]
PROJECT_NAME = PROJECT["NAME"]
ENV_NAME = PROJECT["ENVIRONMENTS"][0]["NAME"]
BP_LAUNCH_PROFILE_NAME = "AhvVmProfile"
# calm_version
CALM_VERSION = Version.get_version("Calm")
@pytest.mark.skipif(
LV(CALM_VERSION) < LV("3.2.0"),
reason="Tests are for env changes introduced in 3.2.0",
)
class TestBpCommands:
def setup_method(self):
"""Method to instantiate to created_bp_list and reset context"""
# Resetting context
ContextObj = get_context()
ContextObj.reset_configuration()
self.created_bp_list = []
self.created_app_list = []
def _wait_for_non_busy_state(self, app_name):
def _data_path(path=None, force_update=False, update_path=True, download=True,
name=None, check_version=True):
"""Aux function
"""
key = {'sample': 'MNE_DATASETS_SAMPLE_PATH',
'spm': 'MNE_DATASETS_SPM_FACE_PATH',
'somato': 'MNE_DATASETS_SOMATO_PATH',
'testing': 'MNE_DATASETS_TESTING_PATH',
}[name]
if path is None:
# use an intelligent guess if it's not defined
def_path = op.realpath(op.join(op.dirname(__file__),
'..', '..', 'examples'))
# backward compatibility
if get_config(key) is None:
key = 'MNE_DATA'
path = get_config(key, def_path)
# use the same for all datasets
if not op.exists(path) or not os.access(path, os.W_OK):
try:
os.mkdir(path)
except OSError:
try:
logger.info("Checking for dataset in '~/mne_data'...")
path = op.join(op.expanduser("~"), "mne_data")
if not op.exists(path):
logger.info("Trying to create "
"'~/mne_data' in home directory")
os.mkdir(path)
except OSError:
raise OSError("User does not have write permissions "
"at '%s', try giving the path as an argument"
" to data_path() where user has write "
"permissions, for ex:data_path"
"('/home/xyz/me2/')" % (path))
if not isinstance(path, string_types):
raise ValueError('path must be a string or None')
if name == 'sample':
archive_name = "MNE-sample-data-processed.tar.gz"
folder_name = "MNE-sample-data"
url = "ftp://surfer.nmr.mgh.harvard.edu/pub/data/MNE/" + archive_name
hash_ = '1bb9f993bfba2057e0039c306a717109'
elif name == 'spm':
archive_name = 'MNE-spm-face.tar.bz2'
folder_name = "MNE-spm-face"
url = 'ftp://surfer.nmr.mgh.harvard.edu/pub/data/MNE/' + archive_name
hash_ = '3e9e83c642136e5b720e2ecc5dcc3244'
elif name == 'somato':
archive_name = 'MNE-somato-data.tar.gz'
folder_name = "MNE-somato-data"
url = 'ftp://surfer.nmr.mgh.harvard.edu/pub/data/MNE/' + archive_name
hash_ = 'f3e3a8441477bb5bacae1d0c6e0964fb'
elif name == 'testing':
archive_name = 'MNE-testing-data.tar.gz'
folder_name = 'MNE-testing-data'
url = 'http://lester.ilabs.uw.edu/files/' + archive_name
hash_ = 'f66d60852e5f42a4940fb22bc1e92dc2'
else:
raise ValueError('Sorry, the dataset "%s" is not known.' % name)
folder_path = op.join(path, folder_name)
rm_archive = False
martinos_path = '/cluster/fusion/sample_data/' + archive_name
neurospin_path = '/neurospin/tmp/gramfort/' + archive_name
if not op.exists(folder_path) and not download:
return ''
if not op.exists(folder_path) or force_update:
logger.info('Downloading or reinstalling '
'data archive %s at location %s' % (archive_name, path))
if op.exists(martinos_path):
archive_name = martinos_path
elif op.exists(neurospin_path):
archive_name = neurospin_path
else:
archive_name = op.join(path, archive_name)
rm_archive = True
fetch_archive = True
if op.exists(archive_name):
msg = ('Archive already exists. Overwrite it (y/[n])? ')
answer = input(msg)
if answer.lower() == 'y':
os.remove(archive_name)
else:
fetch_archive = False
if fetch_archive:
_fetch_file(url, archive_name, print_destination=False,
hash_=hash_)
if op.exists(folder_path):
shutil.rmtree(folder_path)
logger.info('Decompressing the archive: %s' % archive_name)
logger.info('(please be patient, this can take some time)')
for ext in ['gz', 'bz2']: # informed guess (and the only 2 options).
try:
tarfile.open(archive_name, 'r:%s' % ext).extractall(path=path)
break
except tarfile.ReadError as err:
logger.info('%s is %s trying "bz2"' % (archive_name, err))
if rm_archive:
os.remove(archive_name)
path = op.abspath(path)
if update_path is None:
if get_config(key, '') != path:
update_path = True
msg = ('Do you want to set the path:\n %s\nas the default '
'sample dataset path in the mne-python config [y]/n? '
% path)
answer = input(msg)
if answer.lower() == 'n':
update_path = False
else:
update_path = False
if update_path is True:
set_config(key, path)
path = op.join(path, folder_name)
# compare the version of the Sample dataset and mne
data_version = _dataset_version(path, name)
try:
from distutils.version import LooseVersion as LV
except:
warn('Could not determine sample dataset version; dataset could\n'
'be out of date. Please install the "distutils" package.')
else: # 0.7 < 0.7.git shoud be False, therefore strip
if check_version and LV(data_version) < LV(mne_version.strip('.git')):
warn('The {name} dataset (version {current}) is older than '
'mne-python (version {newest}). If the examples fail, '
'you may need to update the {name} dataset by using '
'mne.datasets.{name}.data_path(force_update=True)'.format(
name=name, current=data_version, newest=mne_version))
return path
def _prepare(self):
""" Prepare for running the main loop.
Here we do some initialization like obtaining the startup info,
creating the GUI application wrapper, etc.
"""
# Reset debug status
self.debugger.writestatus()
# Get startup info (get a copy, or setting the new version wont trigger!)
while self.context._stat_startup.recv() is None:
time.sleep(0.02)
self.startup_info = startup_info = self.context._stat_startup.recv().copy()
# Set startup info (with additional info)
if sys.platform.startswith('java'):
import __builtin__ as builtins # Jython
else:
builtins = __builtins__
if not isinstance(builtins, dict):
builtins = builtins.__dict__
startup_info['builtins'] = [builtin for builtin in builtins.keys()]
startup_info['version'] = tuple(sys.version_info)
startup_info['keywords'] = keyword.kwlist
self.context._stat_startup.send(startup_info)
# Prepare the Python environment
self._prepare_environment(startup_info)
# Run startup code (before loading GUI toolkit or IPython
self._run_startup_code(startup_info)
# Write Python banner (to stdout)
thename = 'Python'
if sys.version_info[0] == 2:
thename = 'Legacy Python'
if '__pypy__' in sys.builtin_module_names:
thename = 'Pypy'
if sys.platform.startswith('java'):
thename = 'Jython'
# Jython cannot do struct.calcsize("P")
import java.lang
real_plat = java.lang.System.getProperty("os.name").lower()
plat = '%s/%s' % (sys.platform, real_plat)
elif sys.platform.startswith('win'):
NBITS = 8 * struct.calcsize("P")
plat = 'Windows (%i bits)' % NBITS
else:
NBITS = 8 * struct.calcsize("P")
plat = '%s (%i bits)' % (sys.platform, NBITS)
printDirect("%s %s on %s.\n" %
(thename, sys.version.split('[')[0].rstrip(), plat))
# Integrate GUI
guiName, guiError = self._integrate_gui(startup_info)
# Write pyzo part of banner (including what GUI loop is integrated)
if True:
pyzoBanner = 'This is the Pyzo interpreter'
if guiError:
pyzoBanner += '. ' + guiError + '\n'
elif guiName:
pyzoBanner += ' with integrated event loop for '
pyzoBanner += guiName + '.\n'
else:
pyzoBanner += '.\n'
printDirect(pyzoBanner)
# Try loading IPython
if startup_info.get('ipython', '').lower() in ('', 'no', 'false'):
self._ipython = None
else:
try:
self._load_ipyhon()
except Exception:
type, value, tb = sys.exc_info();
del tb
printDirect('IPython could not be loaded: %s\n' % str(value))
self._ipython = None
# Set prompts
sys.ps1 = PS1(self)
sys.ps2 = PS2(self)
# Notify about project path
projectPath = startup_info['projectPath']
if projectPath:
printDirect('Prepending the project path %r to sys.path\n' %
projectPath)
# Write tips message.
if self._ipython:
import IPython
printDirect("\nUsing IPython %s -- An enhanced Interactive Python.\n"
% IPython.__version__)
printDirect(
"? -> Introduction and overview of IPython's features.\n"
"%quickref -> Quick reference.\n"
"help -> Python's own help system.\n"
"object? -> Details about 'object', "
"use 'object??' for extra details.\n")
else:
printDirect("Type 'help' for help, " +
"type '?' for a list of *magic* commands.\n")
# Notify the running of the script
if self._scriptToRunOnStartup:
printDirect('\x1b[0;33mRunning script: "'+self._scriptToRunOnStartup+'"\x1b[0m\n')
# Prevent app nap on OSX 9.2 and up
# The _nope module is taken from MINRK's appnope package
if sys.platform == "darwin" and LV(platform.mac_ver()[0]) >= LV("10.9"):
from pyzokernel import _nope
_nope.nope()
# Setup post-mortem debugging via appropriately logged exceptions
class PMHandler(logging.Handler):
def emit(self, record):
if record.exc_info:
sys.last_type, sys.last_value, sys.last_traceback = record.exc_info
return record
#
root_logger = logging.getLogger()
if not root_logger.handlers:
root_logger.addHandler(logging.StreamHandler())
root_logger.addHandler(PMHandler())
def _data_path(path=None, force_update=False, update_path=True, download=True,
name=None, check_version=False, return_version=False,
archive_name=None):
"""Aux function."""
key = {
'fake': 'MNE_DATASETS_FAKE_PATH',
'misc': 'MNE_DATASETS_MISC_PATH',
'sample': 'MNE_DATASETS_SAMPLE_PATH',
'spm': 'MNE_DATASETS_SPM_FACE_PATH',
'somato': 'MNE_DATASETS_SOMATO_PATH',
'brainstorm': 'MNE_DATASETS_BRAINSTORM_PATH',
'testing': 'MNE_DATASETS_TESTING_PATH',
'multimodal': 'MNE_DATASETS_MULTIMODAL_PATH',
'visual_92_categories': 'MNE_DATASETS_VISUAL_92_CATEGORIES_PATH',
'mtrf': 'MNE_DATASETS_MTRF_PATH',
'fieldtrip_cmc': 'MNE_DATASETS_FIELDTRIP_CMC_PATH'
}[name]
path = _get_path(path, key, name)
# To update the testing or misc dataset, push commits, then make a new
# release on GitHub. Then update the "releases" variable:
releases = dict(testing='0.40', misc='0.3')
# And also update the "hashes['testing']" variable below.
# To update any other dataset, update the data archive itself (upload
# an updated version) and update the hash.
archive_names = dict(
misc='mne-misc-data-%s.tar.gz' % releases['misc'],
sample='MNE-sample-data-processed.tar.gz',
somato='MNE-somato-data.tar.gz',
spm='MNE-spm-face.tar.gz',
testing='mne-testing-data-%s.tar.gz' % releases['testing'],
multimodal='MNE-multimodal-data.tar.gz',
fake='foo.tgz',
visual_92_categories='MNE-visual_92_categories.tar.gz',
mtrf='mTRF_1.5.zip',
fieldtrip_cmc='SubjectCMC.zip'
)
if archive_name is not None:
archive_names.update(archive_name)
folder_names = dict(
brainstorm='MNE-brainstorm-data',
fake='foo',
misc='MNE-misc-data',
mtrf='mTRF_1.5',
sample='MNE-sample-data',
somato='MNE-somato-data',
multimodal='MNE-multimodal-data',
spm='MNE-spm-face',
testing='MNE-testing-data',
visual_92_categories='MNE-visual_92_categories-data',
fieldtrip_cmc='MNE-fieldtrip_cmc-data'
)
urls = dict(
brainstorm='https://mne-tools.s3.amazonaws.com/datasets/'
'MNE-brainstorm-data/%s',
fake='https://github.com/mne-tools/mne-testing-data/raw/master/'
'datasets/%s',
misc='https://codeload.github.com/mne-tools/mne-misc-data/'
'tar.gz/%s' % releases['misc'],
sample="https://mne-tools.s3.amazonaws.com/datasets/%s",
somato='https://mne-tools.s3.amazonaws.com/datasets/%s',
spm='https://mne-tools.s3.amazonaws.com/datasets/%s',
testing='https://codeload.github.com/mne-tools/mne-testing-data/'
'tar.gz/%s' % releases['testing'],
multimodal='https://ndownloader.figshare.com/files/5999598',
visual_92_categories='https://mne-tools.s3.amazonaws.com/datasets/%s',
mtrf="https://superb-dca2.dl.sourceforge.net/project/aespa/%s",
fieldtrip_cmc='ftp://ftp.fieldtriptoolbox.org/pub/fieldtrip/'
'tutorial/%s'
)
hashes = dict(
brainstorm=None,
fake='3194e9f7b46039bb050a74f3e1ae9908',
misc='d822a720ef94302467cb6ad1d320b669',
sample='1d5da3a809fded1ef5734444ab5bf857',
somato='f3e3a8441477bb5bacae1d0c6e0964fb',
spm='ecce87351d88def59d3d4cdc561e2a60',
testing='02796b3ab145ee9cad680a545563beb5',
multimodal='26ec847ae9ab80f58f204d09e2c08367',
visual_92_categories='46c7e590f4a48596441ce001595d5e58',
mtrf='273a390ebbc48da2c3184b01a82e4636',
fieldtrip_cmc='6f9fd6520f9a66e20994423808d2528c'
)
folder_origs = dict( # not listed means None
misc='mne-misc-data-%s' % releases['misc'],
testing='mne-testing-data-%s' % releases['testing'],
)
folder_name = folder_names[name]
archive_name = archive_names[name]
hash_ = hashes[name]
url = urls[name]
folder_orig = folder_origs.get(name, None)
if '%s' in url:
url = url % archive_name
folder_path = op.join(path, folder_name)
if name == 'brainstorm':
extract_path = folder_path
folder_path = op.join(folder_path, archive_names[name].split('.')[0])
rm_archive = False
martinos_path = '/cluster/fusion/sample_data/' + archive_name
neurospin_path = '/neurospin/tmp/gramfort/' + archive_name
if not op.exists(folder_path) and not download:
return ''
if not op.exists(folder_path) or force_update:
if name == 'brainstorm':
if '--accept-brainstorm-license' in sys.argv:
answer = 'y'
else:
answer = input('%sAgree (y/[n])? ' % _bst_license_text)
if answer.lower() != 'y':
raise RuntimeError('You must agree to the license to use this '
'dataset')
logger.info('Downloading or reinstalling '
'data archive %s at location %s' % (archive_name, path))
if op.exists(martinos_path):
archive_name = martinos_path
elif op.exists(neurospin_path):
archive_name = neurospin_path
else:
archive_name = op.join(path, archive_name)
rm_archive = True
fetch_archive = True
if op.exists(archive_name):
msg = ('Archive already exists. Overwrite it (y/[n])? ')
answer = input(msg)
if answer.lower() == 'y':
os.remove(archive_name)
else:
fetch_archive = False
if fetch_archive:
_fetch_file(url, archive_name, print_destination=False,
hash_=hash_)
if op.exists(folder_path):
def onerror(func, path, exc_info):
"""Deal with access errors (e.g. testing dataset read-only)."""
# Is the error an access error ?
do = False
if not os.access(path, os.W_OK):
perm = os.stat(path).st_mode | stat.S_IWUSR
os.chmod(path, perm)
do = True
if not os.access(op.dirname(path), os.W_OK):
dir_perm = (os.stat(op.dirname(path)).st_mode |
stat.S_IWUSR)
os.chmod(op.dirname(path), dir_perm)
do = True
if do:
func(path)
else:
raise
shutil.rmtree(folder_path, onerror=onerror)
logger.info('Decompressing the archive: %s' % archive_name)
logger.info('(please be patient, this can take some time)')
if name != 'brainstorm':
extract_path = path
if name == 'fieldtrip_cmc':
extract_path = folder_path
if archive_name.endswith('.zip'):
with zipfile.ZipFile(archive_name, 'r') as ff:
ff.extractall(extract_path)
else:
for ext in ['gz', 'bz2']: # informed guess
try:
tf = tarfile.open(archive_name, 'r:%s' % ext)
tf.extractall(path=extract_path)
tf.close()
break
except tarfile.ReadError as err:
logger.info('%s is %s trying "bz2"' % (archive_name, err))
if folder_orig is not None:
shutil.move(op.join(path, folder_orig), folder_path)
if rm_archive:
os.remove(archive_name)
path = _do_path_update(path, update_path, key, name)
path = op.join(path, folder_name)
# compare the version of the dataset and mne
data_version = _dataset_version(path, name)
try:
from distutils.version import LooseVersion as LV
except:
warn('Could not determine %s dataset version; dataset could '
'be out of date. Please install the "distutils" package.'
% name)
else: # 0.7 < 0.7.git shoud be False, therefore strip
if check_version and LV(data_version) < LV(mne_version.strip('.git')):
warn('The {name} dataset (version {current}) is older than '
'mne-python (version {newest}). If the examples fail, '
'you may need to update the {name} dataset by using '
'mne.datasets.{name}.data_path(force_update=True)'.format(
name=name, current=data_version, newest=mne_version))
return (path, data_version) if return_version else path
