from shutil import copyfile, rmtree
import re
from collections import namedtuple
import requests
Target = namedtuple('Target', 'path, filename')
customfiles = {
'agents/plugins/robotmk':
Target('agents/custom/robotmk-external/lib/bin', 'robotmk.py'),
'agents/plugins/robotmk-runner':
Target('agents/custom/robotmk-external/lib/bin', 'robotmk-runner.py'),
'agents/plugins/robotmk-runner':
Target('agents/custom/robotmk-external/lib/bin', 'robotmk-runner.pyw'),
}
rootpath = Path(os.path.dirname(os.path.realpath(__file__)))
def get_tag():
if '--tag' in sys.argv:
tag = sys.argv[sys.argv.index('--tag') + 1]
else:
ostream = os.popen('git describe --tags')
tag = ostream.read().strip()
# match semantic version
if not re.match(
'^v(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$',
tag):
print "ERROR: Last git tag does not match the expected version format! Exiting."
sys.exit(1)
return tag
def _write_voc_xml(self, i, outdir):
img_name = osp.splitext(self.image_index[i])[0]
f_xml = osp.join(outdir, img_name +
'.xml') #osp.splitext(self.image_index[i])[0]
boxes = self.gt_roidb[i]['boxes'].copy()
if len(boxes) == 0:
return
if self.cfg.data_format == 'LTWH':
boxes[:, 2:] += boxes[:, :2] - 1
cls = self.gt_roidb[i]['cls']
diff = self.gt_roidb[i]['diff']
Path(f_xml).parent.mkdir(parents=True, exist_ok=True)
doc = Document()
anno = doc.createElement('annotation')
doc.appendChild(anno)
fname = doc.createElement('filename')
fname_text = doc.createTextNode(self.image_index[i] +
self.cfg.image_ext)
fname.appendChild(fname_text)
anno.appendChild(fname)
size = doc.createElement('size')
anno.appendChild(size)
##需要修改的就是这部分，宽高
width = doc.createElement('width')
width.appendChild(doc.createTextNode(str(self.widths[i])))
height = doc.createElement('height')
height.appendChild(doc.createTextNode(str(self.heights[i])))
depth = doc.createElement('depth')
depth.appendChild(doc.createTextNode('3'))
size.appendChild(width)
size.appendChild(height)
size.appendChild(depth)
segmented = doc.createElement('segmented')
segmented.appendChild(doc.createTextNode('0'))
anno.appendChild(segmented)
##需要添加目标
for idx in range(boxes.shape[0]):
objects = doc.createElement('object')
anno.appendChild(objects)
object_name = doc.createElement('name')
object_name.appendChild(doc.createTextNode(cls[idx]))
objects.appendChild(object_name)
pose = doc.createElement('pose')
pose.appendChild(doc.createTextNode('Unspecified'))
objects.appendChild(pose)
truncated = doc.createElement('truncated')
truncated.appendChild(doc.createTextNode('0'))
objects.appendChild(truncated)
difficult = doc.createElement('difficult')
difficult.appendChild(doc.createTextNode(str(int(diff[idx]))))
objects.appendChild(difficult)
bndbox = doc.createElement('bndbox')
objects.appendChild(bndbox)
xmin = doc.createElement('xmin')
xmin.appendChild(
doc.createTextNode((str(int(np.round(boxes[idx][0]))))))
bndbox.appendChild(xmin)
ymin = doc.createElement('ymin')
ymin.appendChild(
doc.createTextNode((str(int(np.round(boxes[idx][1]))))))
bndbox.appendChild(ymin)
xmax = doc.createElement('xmax')
xmax.appendChild(
doc.createTextNode((str(int(np.round(boxes[idx][2]))))))
bndbox.appendChild(xmax)
ymax = doc.createElement('ymax')
ymax.appendChild(
doc.createTextNode((str(int(np.round(boxes[idx][3]))))))
bndbox.appendChild(ymax)
f = open(f_xml, 'w')
f.write(doc.toprettyxml(indent='\t'))
f.close()
def main():
print('TRAINS SDK setup process')
conf_file = Path(LOCAL_CONFIG_FILES[0]).absolute()
if conf_file.exists() and conf_file.is_file(
) and conf_file.stat().st_size > 0:
print('Configuration file already exists: {}'.format(str(conf_file)))
print('Leaving setup, feel free to edit the configuration file.')
return
print(description, end='')
parse_input = get_user_input()
credentials = None
api_host = None
web_server = None
# noinspection PyBroadException
try:
parsed = ConfigFactory.parse_string(parse_input)
if parsed:
# Take the credentials in raw form or from api section
credentials = get_parsed_field(parsed, ["credentials"])
api_host = get_parsed_field(parsed, ["api_server", "host"])
web_server = get_parsed_field(parsed, ["web_server"])
except Exception:
credentials = credentials or None
api_host = api_host or None
web_server = web_server or None
while not credentials or set(credentials) != {"access_key", "secret_key"}:
print(
'Could not parse credentials, please try entering them manually.')
credentials = read_manual_credentials()
print('Detected credentials key=\"{}\" secret=\"{}\"'.format(
credentials['access_key'], credentials['secret_key'][0:4] + "***"))
if api_host:
api_host = input_url('API Host', api_host)
else:
print(host_description)
api_host = input_url('API Host', '')
parsed_host = verify_url(api_host)
if parsed_host.netloc.startswith('demoapp.'):
# this is our demo server
api_host = parsed_host.scheme + "://" + parsed_host.netloc.replace(
'demoapp.', 'demoapi.', 1) + parsed_host.path
web_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path
files_host = parsed_host.scheme + "://" + parsed_host.netloc.replace(
'demoapp.', 'demofiles.', 1) + parsed_host.path
elif parsed_host.netloc.startswith('app.'):
# this is our application server
api_host = parsed_host.scheme + "://" + parsed_host.netloc.replace(
'app.', 'api.', 1) + parsed_host.path
web_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path
files_host = parsed_host.scheme + "://" + parsed_host.netloc.replace(
'app.', 'files.', 1) + parsed_host.path
elif parsed_host.netloc.startswith('demoapi.'):
print(
'{} is the api server, we need the web server. Replacing \'demoapi.\' with \'demoapp.\''
.format(parsed_host.netloc))
api_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path
web_host = parsed_host.scheme + "://" + parsed_host.netloc.replace(
'demoapi.', 'demoapp.', 1) + parsed_host.path
files_host = parsed_host.scheme + "://" + parsed_host.netloc.replace(
'demoapi.', 'demofiles.', 1) + parsed_host.path
elif parsed_host.netloc.startswith('api.'):
print(
'{} is the api server, we need the web server. Replacing \'api.\' with \'app.\''
.format(parsed_host.netloc))
api_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path
web_host = parsed_host.scheme + "://" + parsed_host.netloc.replace(
'api.', 'app.', 1) + parsed_host.path
files_host = parsed_host.scheme + "://" + parsed_host.netloc.replace(
'api.', 'files.', 1) + parsed_host.path
elif parsed_host.port == 8008:
api_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path
web_host = parsed_host.scheme + "://" + parsed_host.netloc.replace(
':8008', ':8080', 1) + parsed_host.path
files_host = parsed_host.scheme + "://" + parsed_host.netloc.replace(
':8008', ':8081', 1) + parsed_host.path
elif parsed_host.port == 8080:
print(
'Port 8080 is the web port. Using port 8008 for API Host and 8080 for Web Application Host'
)
api_host = parsed_host.scheme + "://" + parsed_host.netloc.replace(
':8080', ':8008', 1) + parsed_host.path
web_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path
files_host = parsed_host.scheme + "://" + parsed_host.netloc.replace(
':8080', ':8081', 1) + parsed_host.path
else:
api_host = ''
web_host = ''
files_host = ''
if not parsed_host.port:
print(
'Host port not detected, do you wish to use the default 8080 port n/[y]? ',
end='')
replace_port = input().lower()
if not replace_port or replace_port == 'y' or replace_port == 'yes':
api_host = parsed_host.scheme + "://" + parsed_host.netloc + ':8008' + parsed_host.path
web_host = parsed_host.scheme + "://" + parsed_host.netloc + ':8080' + parsed_host.path
files_host = parsed_host.scheme + "://" + parsed_host.netloc + ':8081' + parsed_host.path
elif not replace_port or replace_port.lower(
) == 'n' or replace_port.lower() == 'no':
web_host = input_host_port("Web", parsed_host)
api_host = input_host_port("API", parsed_host)
files_host = input_host_port("Files", parsed_host)
if not api_host:
api_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path
web_host = input_url('Web Application Host',
web_server if web_server else web_host)
files_host = input_url('File Store Host', files_host)
print(
'\nTRAINS Hosts configuration:\nWeb App: {}\nAPI: {}\nFile Store: {}\n'
.format(web_host, api_host, files_host))
retry = 1
max_retries = 2
while retry <= max_retries: # Up to 2 tries by the user
if verify_credentials(api_host, credentials):
break
retry += 1
if retry < max_retries + 1:
credentials = read_manual_credentials()
else:
print('Exiting setup without creating configuration file')
return
# noinspection PyBroadException
try:
default_sdk_conf = Path(__file__).parent.absolute() / 'sdk.conf'
with open(str(default_sdk_conf), 'rt') as f:
default_sdk = f.read()
except Exception:
print('Error! Could not read default configuration file')
return
# noinspection PyBroadException
try:
with open(str(conf_file), 'wt') as f:
header = '# TRAINS SDK configuration file\n' \
'api {\n' \
' # Notice: \'host\' is the api server (default port 8008), not the web server.\n' \
' api_server: %s\n' \
' web_server: %s\n' \
' files_server: %s\n' \
' # Credentials are generated using the webapp, %s/profile\n' \
' credentials {"access_key": "%s", "secret_key": "%s"}\n' \
'}\n' \
'sdk ' % (api_host, web_host, files_host,
web_host, credentials['access_key'], credentials['secret_key'])
f.write(header)
f.write(default_sdk)
except Exception:
print('Error! Could not write configuration file at: {}'.format(
str(conf_file)))
return
print('\nNew configuration stored in {}'.format(str(conf_file)))
print('TRAINS setup completed successfully.')
@njit(nogil=True)
def get_neighbors(rowptr, col, a):
# O(1)
return col[rowptr[a]:rowptr[a + 1]]
@njit(nogil=True, parallel=True)
def random_walk(rowptr, col, p, q, num_nodes, walks_per_node, walk_length):
walks = np.zeros((num_nodes * walks_per_node, walk_length), dtype=np.int64)
for i in prange(num_nodes):
for j in prange(walks_per_node):
walks[i * walks_per_node + j] = walk_from_start(
rowptr, col, i, walk_length, p, q)
return walks
if __name__ == '__main__':
data = Dataset(root=Path('../../dataset'), name='ogb')
tik = time.time()
random_walk(rowptr=data.adj_t.storage.rowptr().numpy(),
col=data.adj_t.storage.col().numpy(),
p=4,
q=0.5,
num_nodes=data.num_nodes,
walks_per_node=10,
walk_length=80)
tok = time.time()
print('{seconds:.2f} used'.format(seconds=tok - tik))
def _extract_to_cache(cls, cached_file, name):
"""
Extract cached file zip file to cache folder
:param str cached_file: local copy of archive file
:param str name: cache context
:return: cached folder containing the extracted archive content
"""
# only zip files
if not cached_file or not str(cached_file).lower().endswith('.zip'):
return cached_file
cached_folder = Path(cached_file).parent
archive_suffix = cached_file.rpartition(".")[0]
name = encode_string_to_filename(name)
target_folder = Path("{0}_artifacts_archive_{1}".format(
archive_suffix, name))
if target_folder.exists():
# noinspection PyBroadException
try:
target_folder.touch(exist_ok=True)
return target_folder
except Exception:
pass
base_logger = LoggerRoot.get_base_logger()
try:
temp_target_folder = cached_folder / "{0}_{1}_{2}".format(
target_folder.name,
time() * 1000,
str(random()).replace('.', ''))
temp_target_folder.mkdir(parents=True, exist_ok=True)
ZipFile(cached_file).extractall(path=temp_target_folder.as_posix())
# we assume we will have such folder if we already extract the zip file
# noinspection PyBroadException
try:
# if rename fails, it means that someone else already manged to extract the zip, delete the current
# folder and return the already existing cached zip folder
shutil.move(temp_target_folder.as_posix(),
target_folder.as_posix())
except Exception:
if target_folder.exists():
target_folder.touch(exist_ok=True)
else:
base_logger.warning("Failed renaming {0} to {1}".format(
temp_target_folder, target_folder))
try:
shutil.rmtree(temp_target_folder)
except Exception as ex:
base_logger.warning(
"Exception {}\nFailed deleting folder {}".format(
ex, temp_target_folder))
except Exception as ex:
# failed extracting zip file:
base_logger.warning(
"Exception {}\nFailed extracting zip file {}".format(
ex, cached_file))
# noinspection PyBroadException
try:
target_folder.rmdir()
except Exception:
pass
return cached_file
return target_folder
def _upload_data_audit_artifacts(self, name):
# type: (str) -> ()
logger = self._task.get_logger()
pd_artifact = self._artifacts_container.get(name)
pd_metadata = self._artifacts_container.get_metadata(name)
# remove from artifacts watch list
if name in self._unregister_request:
try:
self._unregister_request.remove(name)
except KeyError:
pass
self._artifacts_container.unregister_artifact(name)
if pd_artifact is None:
return
override_filename_ext_in_uri = self._save_format
override_filename_in_uri = name
fd, local_csv = mkstemp(prefix=quote(name, safe="") + '.',
suffix=override_filename_ext_in_uri)
os.close(fd)
local_csv = Path(local_csv)
pd_artifact.to_csv(local_csv.as_posix(),
index=False,
compression=self._compression)
current_sha2, file_sha2 = self.sha256sum(local_csv.as_posix(),
skip_header=32)
if name in self._last_artifacts_upload:
previous_sha2 = self._last_artifacts_upload[name]
if previous_sha2 == current_sha2:
# nothing to do, we can skip the upload
# noinspection PyBroadException
try:
local_csv.unlink()
except Exception:
pass
return
self._last_artifacts_upload[name] = current_sha2
# If old trains-server, upload as debug image
if not Session.check_min_api_version('2.3'):
logger.report_image(title='artifacts',
series=name,
local_path=local_csv.as_posix(),
delete_after_upload=True,
iteration=self._task.get_last_iteration(),
max_image_history=2)
return
# Find our artifact
artifact = None
for an_artifact in self._task_artifact_list:
if an_artifact.key == name:
artifact = an_artifact
break
file_size = local_csv.stat().st_size
# upload file
uri = self._upload_local_file(
local_csv,
name,
delete_after_upload=True,
override_filename=override_filename_in_uri,
override_filename_ext=override_filename_ext_in_uri)
# update task artifacts
with self._task_edit_lock:
if not artifact:
artifact = tasks.Artifact(key=name,
type=self._pd_artifact_type)
self._task_artifact_list.append(artifact)
artifact_type_data = tasks.ArtifactTypeData()
artifact_type_data.data_hash = current_sha2
artifact_type_data.content_type = "text/csv"
artifact_type_data.preview = str(
pd_artifact.__repr__()) + '\n\n' + self._get_statistics(
{name: pd_artifact})
artifact.type_data = artifact_type_data
artifact.uri = uri
artifact.content_size = file_size
artifact.hash = file_sha2
artifact.timestamp = int(time())
artifact.display_data = [
(str(k), str(v)) for k, v in pd_metadata.items()
] if pd_metadata else None
self._task.set_artifacts(self._task_artifact_list)
def update_weights_package(self,
weights_filenames=None,
weights_path=None,
upload_uri=None,
target_filename=None,
auto_delete_file=True,
iteration=None):
"""
Update the model weights from a locally stored model files (or directory containing multiple files).
Uploading the model is a background process, the call returns immediately.
:param weights_filenames: list of locally stored filenames (list of strings)
:type weights_filenames: list
:param weights_path: directory path to package (all the files in the directory will be uploaded)
:type weights_path: str
:param upload_uri: destination uri for model weights upload (default: previously used uri)
:param target_filename: the newly created filename in the destination uri location (default: weights_filename)
:param auto_delete_file: delete temporary file after uploading
:return: uploaded uri for the weights package
"""
# create list of files
if (not weights_filenames and not weights_path) or (weights_filenames
and weights_path):
raise ValueError(
'Model update weights package should get either directory path to pack or a list of files'
)
if not weights_filenames:
weights_filenames = list(
map(six.text_type,
Path(weights_path).glob('*')))
# create packed model from all the files
fd, zip_file = mkstemp(prefix='model_package.', suffix='.zip')
try:
with zipfile.ZipFile(zip_file,
'w',
allowZip64=True,
compression=zipfile.ZIP_STORED) as zf:
for filename in weights_filenames:
zf.write(filename, arcname=Path(filename).name)
finally:
os.close(fd)
# now we can delete the files (or path if provided)
if auto_delete_file:
def safe_remove(path, is_dir=False):
try:
(os.rmdir if is_dir else os.remove)(path)
except OSError:
self._log.info('Failed removing temporary {}'.format(path))
for filename in weights_filenames:
safe_remove(filename)
if weights_path:
safe_remove(weights_path, is_dir=True)
if target_filename and not target_filename.lower().endswith('.zip'):
target_filename += '.zip'
# and now we should upload the file, always delete the temporary zip file
comment = self.comment or ''
iteration_msg = 'snapshot {} stored'.format(str(weights_filenames))
if not comment.startswith('\n'):
comment = '\n' + comment
comment = iteration_msg + comment
self.comment = comment
uploaded_uri = self.update_weights(weights_filename=zip_file,
auto_delete_file=True,
upload_uri=upload_uri,
target_filename=target_filename
or 'model_package.zip',
iteration=iteration,
update_comment=False)
# set the model tag (by now we should have a model object) so we know we have packaged file
self._set_package_tag()
return uploaded_uri
FREQUENCY_858 = 858
LEVEL_OFFSET_858 = find_level_offset_by_frequency("DVBT_T2_FREQUENCY_LEVEL_OFFSET", 858.0)
LEVEL_50_858 = str("%.2f" % (-50 - LEVEL_OFFSET_858))
PARAMETER_LIST = [
[FFT_SIZE_32KE, MODULATION_256QAM, 'PP2', CODE_RATE_3_4, GUARD_G1_8, 3.0, None],
[FFT_SIZE_32KE, MODULATION_256QAM, 'PP4', CODE_RATE_2_3, GUARD_G1_16, 5.0, None],
[FFT_SIZE_32KE, MODULATION_256QAM, 'PP4', CODE_RATE_3_5, GUARD_G19_256, 7.0, None]
]
FREQUENCY_LIST = [
[FREQUENCY_474, LEVEL_OFFSET_474, LEVEL_50_474],
[FREQUENCY_666, LEVEL_OFFSET_666, LEVEL_50_666],
[FREQUENCY_858, LEVEL_OFFSET_858, LEVEL_50_858],
]
my_file = Path("../../ekt_json/dvbt2_65_co_channel_interference.json")
if my_file.exists():
pass
else:
dict_test_parame_result = {}
list_test_parame_result = []
for FREQUENCY in FREQUENCY_LIST:
list_test_result = []
for PARAMETER in PARAMETER_LIST:
list_test_result.append(
[PARAMETER[0], PARAMETER[1], PARAMETER[2], PARAMETER[3], PARAMETER[4], PARAMETER[5], PARAMETER[6]])
list_test_parame_result.append([FREQUENCY[0], FREQUENCY[1], FREQUENCY[2], list_test_result])
dict_test_parame_result["test_parame_result"] = list_test_parame_result
write_json_file("../../ekt_json/dvbt2_65_co_channel_interference.json",
dict_test_parame_result)
from __future__ import print_function
import os, sys, shutil, subprocess
import getpass
import datetime
import logging
import functools, contextlib
import email.parser
try:
from pathlib2 import Path
except ImportError:
from pathlib import Path
from clckwrkbdgr import xdg
import clckwrkbdgr.utils, clckwrkbdgr.fs
MAILBOX = Path(
os.environ.get('MAILPATH',
Path('/var/mail') / getpass.getuser()))
MAILBOX_BAK = xdg.save_cache_path() / 'mbox.{0}'.format(os.getpid())
def mail_command(commands):
MAIL_COMMAND = ['mail', '-N']
p = subprocess.Popen(MAIL_COMMAND,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE)
_, errors = p.communicate('\n'.join(commands).encode('utf-8', 'replace'))
if errors:
logging.error(errors)
return 0 == p.wait()
pass
if __name__ == "__main__":
if not len(argv) == 3:
raise IOError('Usage: ' + argv[0] +
' <save_location> <test.json file>')
jsonFile = argv[2]
savePath = argv[1]
else:
if not len(argv) == 2:
raise IOError
jsonFile = argv[1]
savePath = "./"
checkIfExists = Path(jsonFile)
if checkIfExists.is_file:
testFile = checkIfExists.open()
else:
raise IOError('Usage: ' + jsonFile + ' must be a .json file.')
data = json.load(testFile)
topics = data['topics']
process = ['rosbag', 'record', '-o']
process.append(savePath)
process = process + topics
output = subprocess.Popen(process)
storePid = open('bag.pid', 'w')
storePid.write(get_pid('record'))
storePid.close()
from pathlib2 import Path
dac_dir = Path(__file__).parent
trajector_dir = dac_dir / 'trajs'
project_dir = dac_dir.parent.parent
results_dir = project_dir / 'results'
if not trajector_dir.is_dir():
trajector_dir.mkdir()
if not results_dir.is_dir():
results_dir.mkdir()
"""
Created on Mon Apr 23 14:25:39 2018
@author: pipolose
"""
# from scipy.stats import chisquare
from scipy.stats import chi2_contingency
from scipy.stats import kruskal
import numpy as np
import pandas as pd
from polyML import polyssifier_3 as ps
from pathlib2 import Path
from collections import OrderedDict
do_save = False
out_dir = Path('/data2/polo/figures')
csv_dir = Path('/data2/polo/half_baked_data')
dem_csv = csv_dir / 'TON_baseline_demo.csv'
site_csv = csv_dir / 'TON_siteid.csv'
in_mat = Path().cwd().parent / 'VBM_controls' /\
'TON_log_deg_maps_local_gm_corrected.mat'
source = 'python'
subject_list = ps.load_subject_list(in_mat.as_posix(), source=source)
sites_df = pd.read_csv(site_csv, index_col='subjid')
dem_df = pd.read_csv(dem_csv, index_col='subjid')
dem_df['site'] = sites_df.loc[dem_df.index]
dem_df['count'] = 1
used_dem = dem_df.loc[subject_list]
def loopFuzzingAndReduction(options, buildInfo, collector, i): # pylint: disable=invalid-name,missing-docstring
tempDir = Path(tempfile.mkdtemp("loop" + str(i))) # pylint: disable=invalid-name
loop.many_timed_runs(options.targetTime, tempDir, buildInfo.mtrArgs,
collector, False)
#Author: Daniel M. Pelt #Contact: [email protected] #Website: http://dmpelt.github.io/foam_ct_phantom/ #License: MIT # #This file is part of foam_ct_phantom, a Python package for generating #foam-like phantoms for CT. #----------------------------------------------------------------------- import ctypes from pathlib2 import Path import sys, os import numpy as np if sys.platform == 'darwin': libpath = Path(__file__).parent / 'libccode_c.dylib' lib = ctypes.CDLL(str(libpath)) elif os.name == 'nt': libpath = Path( __file__).parent.parent.parent.parent / 'bin' / 'ccode_c.dll' if not libpath.exists(): libpath = Path(__file__).parent.parent / 'bin' / 'ccode_c.dll' lib = ctypes.WinDLL(str(libpath)) else: libpath = Path(__file__).parent / 'libccode_c.so' lib = ctypes.CDLL(str(libpath)) asuint = ctypes.c_uint32 asfloat = ctypes.c_float cfloatp = ctypes.POINTER(ctypes.c_float)
def get_files(path):
all_objects = Path(path).glob('**/*')
files = [str(p) for p in all_objects if p.is_file()]
return files
def run(output_directory='tmp/', if_file_exists='skip'):
"""Preprocess pipeline: filtering, standarization and whitening filter
This step (optionally) performs filtering on the data, standarizes it
and computes a whitening filter. Filtering and standarized data are
processed in chunks and written to disk.
Parameters
----------
output_directory: str, optional
Location to store results, relative to CONFIG.data.root_folder,
defaults to tmp/. See list of files in Notes section below.
if_file_exists: str, optional
One of 'overwrite', 'abort', 'skip'. Control de behavior for every
generated file. If 'overwrite' it replaces the files if any exist,
if 'abort' it raises a ValueError exception if any file exists,
if 'skip' it skips the operation (and loads the files) if any of them
exist
Returns
-------
standarized_path: str
Path to standarized data binary file
standarized_params: str
Path to standarized data parameters
channel_index: numpy.ndarray
Channel indexes
whiten_filter: numpy.ndarray
Whiten matrix
Notes
-----
Running the preprocessor will generate the followiing files in
CONFIG.data.root_folder/output_directory/:
* ``preprocess/filtered.bin`` - Filtered recordings
* ``preprocess/filtered.yaml`` - Filtered recordings metadata
* ``preprocess/standarized.bin`` - Standarized recordings
* ``preprocess/standarized.yaml`` - Standarized recordings metadata
* ``preprocess/whitening.npy`` - Whitening filter
Everything is run on CPU.
Examples
--------
.. literalinclude:: ../../examples/pipeline/preprocess.py
"""
logger = logging.getLogger(__name__)
CONFIG = read_config()
OUTPUT_DTYPE = CONFIG.preprocess.dtype
PROCESSES = CONFIG.resources.processes
logger.info(
'Output dtype for transformed data will be {}'.format(OUTPUT_DTYPE))
TMP = Path(CONFIG.data.root_folder, output_directory, 'preprocess/')
TMP.mkdir(parents=True, exist_ok=True)
TMP = str(TMP)
path = os.path.join(CONFIG.data.root_folder, CONFIG.data.recordings)
params = dict(dtype=CONFIG.recordings.dtype,
n_channels=CONFIG.recordings.n_channels,
data_order=CONFIG.recordings.order)
# filter and standarize
if CONFIG.preprocess.apply_filter:
filter_params = CONFIG.preprocess.filter
(standarized_path,
standarized_params) = butterworth(path,
params['dtype'],
params['n_channels'],
params['data_order'],
filter_params.low_pass_freq,
filter_params.high_factor,
filter_params.order,
CONFIG.recordings.sampling_rate,
CONFIG.resources.max_memory,
TMP,
OUTPUT_DTYPE,
standarize=True,
output_filename='standarized.bin',
if_file_exists=if_file_exists,
processes=PROCESSES)
# just standarize
else:
(standarized_path,
standarized_params) = standarize(path,
params['dtype'],
params['n_channels'],
params['data_order'],
CONFIG.recordings.sampling_rate,
CONFIG.resources.max_memory,
TMP,
OUTPUT_DTYPE,
output_filename='standarized.bin',
if_file_exists=if_file_exists,
processes=PROCESSES)
# TODO: remove whiten_filter out of output argument
whiten_filter = whiten.matrix(standarized_path,
standarized_params['dtype'],
standarized_params['n_channels'],
standarized_params['data_order'],
CONFIG.channel_index,
CONFIG.spike_size,
CONFIG.resources.max_memory,
TMP,
output_filename='whitening.npy',
if_file_exists=if_file_exists)
return str(standarized_path), standarized_params, whiten_filter
def upload_artifact(self,
name,
artifact_object=None,
metadata=None,
delete_after_upload=False,
auto_pickle=True):
# type: (str, Optional[object], Optional[dict], bool, bool) -> bool
if not Session.check_min_api_version('2.3'):
LoggerRoot.get_base_logger().warning(
'Artifacts not supported by your TRAINS-server version, '
'please upgrade to the latest server version')
return False
if name in self._artifacts_container:
raise ValueError(
"Artifact by the name of {} is already registered, use register_artifact"
.format(name))
# convert string to object if try is a file/folder (dont try to serialize long texts
if isinstance(artifact_object,
six.string_types) and len(artifact_object) < 2048:
# noinspection PyBroadException
try:
artifact_path = Path(artifact_object)
if artifact_path.exists():
artifact_object = artifact_path
elif '*' in artifact_object or '?' in artifact_object:
# hackish, detect wildcard in tr files
folder = Path('').joinpath(*artifact_path.parts[:-1])
if folder.is_dir() and folder.parts:
wildcard = artifact_path.parts[-1]
if list(Path(folder).rglob(wildcard)):
artifact_object = artifact_path
except Exception:
pass
artifact_type_data = tasks.ArtifactTypeData()
artifact_type_data.preview = ''
override_filename_in_uri = None
override_filename_ext_in_uri = None
uri = None
if np and isinstance(artifact_object, np.ndarray):
artifact_type = 'numpy'
artifact_type_data.content_type = 'application/numpy'
artifact_type_data.preview = str(artifact_object.__repr__())
override_filename_ext_in_uri = '.npz'
override_filename_in_uri = name + override_filename_ext_in_uri
fd, local_filename = mkstemp(prefix=quote(name, safe="") + '.',
suffix=override_filename_ext_in_uri)
os.close(fd)
np.savez_compressed(local_filename, **{name: artifact_object})
delete_after_upload = True
elif pd and isinstance(artifact_object, pd.DataFrame):
artifact_type = 'pandas'
artifact_type_data.content_type = 'text/csv'
artifact_type_data.preview = str(artifact_object.__repr__())
override_filename_ext_in_uri = self._save_format
override_filename_in_uri = name
fd, local_filename = mkstemp(prefix=quote(name, safe="") + '.',
suffix=override_filename_ext_in_uri)
os.close(fd)
artifact_object.to_csv(local_filename,
compression=self._compression)
delete_after_upload = True
elif isinstance(artifact_object, Image.Image):
artifact_type = 'image'
artifact_type_data.content_type = 'image/png'
desc = str(artifact_object.__repr__())
artifact_type_data.preview = desc[1:desc.find(' at ')]
override_filename_ext_in_uri = '.png'
override_filename_in_uri = name + override_filename_ext_in_uri
fd, local_filename = mkstemp(prefix=quote(name, safe="") + '.',
suffix=override_filename_ext_in_uri)
os.close(fd)
artifact_object.save(local_filename)
delete_after_upload = True
elif isinstance(artifact_object, dict):
artifact_type = 'JSON'
artifact_type_data.content_type = 'application/json'
preview = json.dumps(artifact_object, sort_keys=True, indent=4)
override_filename_ext_in_uri = '.json'
override_filename_in_uri = name + override_filename_ext_in_uri
fd, local_filename = mkstemp(prefix=quote(name, safe="") + '.',
suffix=override_filename_ext_in_uri)
os.write(fd, bytes(preview.encode()))
os.close(fd)
if len(preview) < self.max_preview_size_bytes:
artifact_type_data.preview = preview
else:
artifact_type_data.preview = '# full json too large to store, storing first {}kb\n{}'.format(
self.max_preview_size_bytes // 1024,
preview[:self.max_preview_size_bytes])
delete_after_upload = True
elif isinstance(artifact_object, (
Path,
pathlib_Path,
) if pathlib_Path is not None else (Path, )):
# check if single file
artifact_object = Path(artifact_object)
artifact_object.expanduser().absolute()
# noinspection PyBroadException
try:
create_zip_file = not artifact_object.is_file()
except Exception: # Hack for windows pathlib2 bug, is_file isn't valid.
create_zip_file = True
else: # We assume that this is not Windows os
if artifact_object.is_dir():
# change to wildcard
artifact_object /= '*'
if create_zip_file:
folder = Path('').joinpath(*artifact_object.parts[:-1])
if not folder.is_dir() or not folder.parts:
raise ValueError(
"Artifact file/folder '{}' could not be found".format(
artifact_object.as_posix()))
wildcard = artifact_object.parts[-1]
files = list(Path(folder).rglob(wildcard))
override_filename_ext_in_uri = '.zip'
override_filename_in_uri = folder.parts[
-1] + override_filename_ext_in_uri
fd, zip_file = mkstemp(
prefix=quote(folder.parts[-1], safe="") + '.',
suffix=override_filename_ext_in_uri)
try:
artifact_type_data.content_type = 'application/zip'
artifact_type_data.preview = 'Archive content {}:\n'.format(
artifact_object.as_posix())
with ZipFile(zip_file,
'w',
allowZip64=True,
compression=ZIP_DEFLATED) as zf:
for filename in sorted(files):
if filename.is_file():
relative_file_name = filename.relative_to(
folder).as_posix()
artifact_type_data.preview += '{} - {}\n'.format(
relative_file_name,
humanfriendly.format_size(
filename.stat().st_size))
zf.write(filename.as_posix(),
arcname=relative_file_name)
except Exception as e:
# failed uploading folder:
LoggerRoot.get_base_logger().warning(
'Exception {}\nFailed zipping artifact folder {}'.
format(folder, e))
return False
finally:
os.close(fd)
artifact_object = zip_file
artifact_type = 'archive'
artifact_type_data.content_type = mimetypes.guess_type(
artifact_object)[0]
local_filename = artifact_object
delete_after_upload = True
else:
if not artifact_object.is_file():
raise ValueError(
"Artifact file '{}' could not be found".format(
artifact_object.as_posix()))
override_filename_in_uri = artifact_object.parts[-1]
artifact_type_data.preview = '{} - {}\n'.format(
artifact_object,
humanfriendly.format_size(artifact_object.stat().st_size))
artifact_object = artifact_object.as_posix()
artifact_type = 'custom'
artifact_type_data.content_type = mimetypes.guess_type(
artifact_object)[0]
local_filename = artifact_object
elif (isinstance(artifact_object, six.string_types)
and urlparse(artifact_object).scheme in remote_driver_schemes):
# we should not upload this, just register
local_filename = None
uri = artifact_object
artifact_type = 'custom'
artifact_type_data.content_type = mimetypes.guess_type(
artifact_object)[0]
elif isinstance(artifact_object, six.string_types):
# if we got here, we should store it as text file.
artifact_type = 'string'
artifact_type_data.content_type = 'text/plain'
if len(artifact_object) < self.max_preview_size_bytes:
artifact_type_data.preview = artifact_object
else:
artifact_type_data.preview = '# full text too large to store, storing first {}kb\n{}'.format(
self.max_preview_size_bytes // 1024,
artifact_object[:self.max_preview_size_bytes])
delete_after_upload = True
override_filename_ext_in_uri = '.txt'
override_filename_in_uri = name + override_filename_ext_in_uri
fd, local_filename = mkstemp(prefix=quote(name, safe="") + '.',
suffix=override_filename_ext_in_uri)
os.close(fd)
# noinspection PyBroadException
try:
with open(local_filename, 'wt') as f:
f.write(artifact_object)
except Exception as ex:
# cleanup and raise exception
os.unlink(local_filename)
raise
elif auto_pickle:
# if we are here it means we do not know what to do with the object, so we serialize it with pickle.
artifact_type = 'pickle'
artifact_type_data.content_type = 'application/pickle'
# noinspection PyBroadException
try:
artifact_type_data.preview = str(
artifact_object.__repr__())[:self.max_preview_size_bytes]
except Exception:
artifact_type_data.preview = ''
delete_after_upload = True
override_filename_ext_in_uri = '.pkl'
override_filename_in_uri = name + override_filename_ext_in_uri
fd, local_filename = mkstemp(prefix=quote(name, safe="") + '.',
suffix=override_filename_ext_in_uri)
os.close(fd)
# noinspection PyBroadException
try:
with open(local_filename, 'wb') as f:
pickle.dump(artifact_object, f)
except Exception as ex:
# cleanup and raise exception
os.unlink(local_filename)
raise
else:
raise ValueError("Artifact type {} not supported".format(
type(artifact_object)))
# remove from existing list, if exists
for artifact in self._task_artifact_list:
if artifact.key == name:
if artifact.type == self._pd_artifact_type:
raise ValueError(
"Artifact of name {} already registered, "
"use register_artifact instead".format(name))
self._task_artifact_list.remove(artifact)
break
if not local_filename:
file_size = None
file_hash = None
else:
# check that the file to upload exists
local_filename = Path(local_filename).absolute()
if not local_filename.exists() or not local_filename.is_file():
LoggerRoot.get_base_logger().warning(
'Artifact upload failed, cannot find file {}'.format(
local_filename.as_posix()))
return False
file_hash, _ = self.sha256sum(local_filename.as_posix())
file_size = local_filename.stat().st_size
uri = self._upload_local_file(
local_filename,
name,
delete_after_upload=delete_after_upload,
override_filename=override_filename_in_uri,
override_filename_ext=override_filename_ext_in_uri)
timestamp = int(time())
artifact = tasks.Artifact(
key=name,
type=artifact_type,
uri=uri,
content_size=file_size,
hash=file_hash,
timestamp=timestamp,
type_data=artifact_type_data,
display_data=[(str(k), str(v))
for k, v in metadata.items()] if metadata else None)
# update task artifacts
with self._task_edit_lock:
self._task_artifact_list.append(artifact)
self._task.set_artifacts(self._task_artifact_list)
return True
def _default_settings():
# type: () -> cmk.ec.settings.Settings
"""Returns default EC settings. This function should vanish in the long run!"""
return cmk.ec.settings.settings('', Path(cmk.utils.paths.omd_root),
Path(cmk.utils.paths.default_config_dir),
[''])
def update_weights(self,
weights_filename=None,
upload_uri=None,
target_filename=None,
auto_delete_file=True,
register_uri=None,
iteration=None,
update_comment=True):
"""
Update the model weights from a locally stored model filename.
Uploading the model is a background process, the call returns immediately.
:param weights_filename: locally stored filename to be uploaded as is
:param upload_uri: destination uri for model weights upload (default: previously used uri)
:param target_filename: the newly created filename in the destination uri location (default: weights_filename)
:param auto_delete_file: delete temporary file after uploading
:param register_uri: register an already uploaded weights file (uri must be valid)
:param update_comment: if True, model comment will be updated with local weights file location (provenance)
:return: uploaded uri
"""
def delete_previous_weights_file(filename=weights_filename):
try:
if filename:
os.remove(filename)
except OSError:
self._log.debug('Failed removing temporary file %s' % filename)
# test if we can update the model
if self.id and self.published:
raise ValueError('Model is published and cannot be changed')
if (not weights_filename and not register_uri) or (weights_filename
and register_uri):
raise ValueError(
'Model update must have either local weights file to upload, '
'or pre-uploaded register_uri, never both')
# only upload if we are connected to a task
if not self._task:
raise Exception('Missing a task for this model')
if weights_filename is not None:
# make sure we delete the previous file, if it exists
if self._model_local_filename != weights_filename:
delete_previous_weights_file(self._model_local_filename)
# store temp filename for deletion next time, if needed
if auto_delete_file:
self._model_local_filename = weights_filename
# make sure the created model is updated:
model = self._get_force_base_model()
if not model:
raise ValueError('Failed creating internal output model')
# select the correct file extension based on the framework, or update the framework based on the file extension
framework, file_ext = Framework._get_file_ext(
framework=self._get_model_data().framework,
filename=weights_filename or register_uri)
if weights_filename:
target_filename = target_filename or Path(weights_filename).name
if not target_filename.lower().endswith(file_ext):
target_filename += file_ext
# set target uri for upload (if specified)
if upload_uri:
self.set_upload_destination(upload_uri)
# let us know the iteration number, we put it in the comment section for now.
if update_comment:
comment = self.comment or ''
iteration_msg = 'snapshot {} stored'.format(weights_filename
or register_uri)
if not comment.startswith('\n'):
comment = '\n' + comment
comment = iteration_msg + comment
else:
comment = None
# if we have no output destination, just register the local model file
if weights_filename and not self.upload_storage_uri and not self._task.storage_uri:
register_uri = weights_filename
weights_filename = None
auto_delete_file = False
self._log.info(
'No output storage destination defined, registering local model %s'
% register_uri)
# start the upload
if weights_filename:
if not model.upload_storage_uri:
self.set_upload_destination(self.upload_storage_uri
or self._task.storage_uri)
output_uri = model.update_and_upload(
model_file=weights_filename,
task_id=self._task.id,
async_enable=True,
target_filename=target_filename,
framework=self.framework or framework,
comment=comment,
cb=delete_previous_weights_file if auto_delete_file else None,
iteration=iteration or self._task.get_last_iteration(),
)
elif register_uri:
register_uri = StorageHelper.conform_url(register_uri)
output_uri = model.update(uri=register_uri,
task_id=self._task.id,
framework=framework,
comment=comment)
else:
output_uri = None
# make sure that if we are in dev move we report that we are training (not debugging)
self._task._output_model_updated()
return output_uri
def main(sample, dry_run, limit, no_limit, resume,
template_path, database_path, config_path,
output_format):
"""
Mailmerge is a simple, command line mail merge tool.
For examples and formatting features, see:
https://github.com/awdeorio/mailmerge
"""
# We need an argument for each command line option. That also means a lot
# of local variables.
# pylint: disable=too-many-arguments, too-many-locals
# Convert paths from string to Path objects
# https://github.com/pallets/click/issues/405
template_path = Path(template_path)
database_path = Path(database_path)
config_path = Path(config_path)
# Make sure input files exist and provide helpful prompts
check_input_files(template_path, database_path, config_path, sample)
# Calculate start and stop indexes. Start and stop are zero-based. The
# input --resume is one-based.
start = resume - 1
stop = None if no_limit else resume - 1 + limit
# Run
message_num = 1 + start
try:
template_message = TemplateMessage(template_path)
csv_database = read_csv_database(database_path)
sendmail_client = SendmailClient(config_path, dry_run)
for _, row in enumerate_range(csv_database, start, stop):
sender, recipients, message = template_message.render(row)
sendmail_client.sendmail(sender, recipients, message)
print_bright_white_on_cyan(
">>> message {message_num}"
.format(message_num=message_num),
output_format,
)
print_message(message, output_format)
print_bright_white_on_cyan(
">>> message {message_num} sent"
.format(message_num=message_num),
output_format,
)
message_num += 1
except MailmergeError as error:
hint_text = '\nHint: "--resume {}"'.format(message_num)
sys.exit(
"Error on message {message_num}\n"
"{error}"
"{hint}"
.format(
message_num=message_num,
error=error,
hint=(hint_text if message_num > 1 else ""),
)
)
# Hints for user
if not no_limit:
print(
">>> Limit was {limit} message{pluralizer}. "
"To remove the limit, use the --no-limit option."
.format(limit=limit, pluralizer=("" if limit == 1 else "s"))
)
if dry_run:
print(
">>> This was a dry run. "
"To send messages, use the --no-dry-run option."
)
from django.test.client import RequestFactory
from django.test.utils import override_settings
from bedrock.base.urlresolvers import reverse
from mock import patch, Mock
from nose.tools import eq_, ok_
from pathlib2 import Path
from pyquery import PyQuery as pq
from rna.models import Release, Note
from bedrock.firefox.firefox_details import FirefoxDesktop
from bedrock.mozorg.tests import TestCase
from bedrock.releasenotes import views
from bedrock.thunderbird.details import ThunderbirdDesktop
DATA_PATH = str(Path(__file__).parent / 'data')
firefox_desktop = FirefoxDesktop(json_dir=DATA_PATH)
thunderbird_desktop = ThunderbirdDesktop(json_dir=DATA_PATH)
class TestRNAViews(TestCase):
def setUp(self):
self.factory = RequestFactory()
self.request = self.factory.get('/')
self.render_patch = patch(
'bedrock.releasenotes.views.l10n_utils.render')
self.mock_render = self.render_patch.start()
self.mock_render.return_value.has_header.return_value = False
def tearDown(self):
def post(self, input_format, output_format):
"""Convert from one RST format to another.
Usage example:
curl -XPOST "http://localhost:5000/convert/rs3/dis" -F [email protected]
"""
input_file = get_input_file(request)
if input_file is None:
res = jsonify(
error=("Please upload a file using the key "
"'input' or the form field 'input'. "
"Used file keys: {}. Used form fields: {}"
).format(request.files.keys(), request.form.keys()))
return cors_response(res, 500)
input_basename = Path(input_file.filename).stem
with tempfile.NamedTemporaryFile() as temp_inputfile:
input_file.save(temp_inputfile.name)
if input_format not in READ_FUNCTIONS:
res = jsonify(
error="Unknown input format: {}".format(input_format))
return cors_response(res, 400)
read_function = READ_FUNCTIONS[input_format]
try:
tree = read_function(temp_inputfile.name)
except Exception as err:
error_msg = u"{0} can't handle input file '{1}'. Got: {2}".format(
read_function, input_file.filename, err)
res = jsonify(error=error_msg,
traceback=traceback.format_exc())
return cors_response(res, 500)
with tempfile.NamedTemporaryFile() as temp_outputfile:
if output_format not in WRITE_FUNCTIONS:
res = jsonify(
error="Unknown output format: {}".format(output_format))
return cors_response(res, 400)
write_function = WRITE_FUNCTIONS[output_format]
try:
write_function(tree, output_file=temp_outputfile.name)
except Exception as err:
error_msg = (
u"{writer} can't convert ParentedTree to {output_format}. "
"Input file '{input_file}'. Got: {error}").format(
writer=write_function,
output_format=output_format,
input_file=input_file.filename,
error=err)
res = jsonify(error=error_msg,
traceback=traceback.format_exc())
return cors_response(res, 500)
output_filename = "{0}.{1}".format(input_basename, output_format)
res = send_file(temp_outputfile.name,
as_attachment=True,
attachment_filename=output_filename)
return cors_response(res)
def model_function(paras, run_id=None):
# input:
# paras ... list of model parameters scaled to their range;
# values for all N model parameters have to be provided
# example:
# [ x1, x2, x3, x4, .... ]
# run_id ... optional name of this run (to, e.g., print or store in a file)
# example:
# run_aset_001
# output:
# model output in dictionary
# example:
# model['out'] = 7.4
if not (run_id is None):
print("Run ID: ", run_id)
# ---------------
# derive some parameters
# ---------------
dict_dparas = {}
dict_dparas['sum_x05_x06'] = paras[4] + paras[
5] # MAX_MELT_FACTOR > MIN_MELT_FACTOR
dict_dparas['sum_x09_x10'] = paras[8] + paras[
9] # SNOW_SWI_MAX > SNOW_SWI_MIN
dict_dparas['half_x20'] = paras[
19] * 0.5 * 1000 # half the value but in [mm] not [m]
dict_dparas['half_x21'] = paras[
20] * 0.5 * 1000 # half the value but in [mm] not [m]
# ---------------
# paste all paras into template files
# ---------------
# ex.: string = "parameter x01 = {par[x01]} and another parameter x02 = {par[x02]}"
# keys = ['x01','x02']
# vals = [1.0,3.0]
# string.format(par=dict(zip(keys,vals)))
#
# --> 'parameter x01 = 1.0 and another parameter x02 = 3.0'
#
# to replace patterns: {par[x01]} by parameter value paras[0]
# {par[x02]} by parameter value paras[1]
# ...
if len(paras) > 9 and len(paras) < 100:
keys_paras = ["x{:02d}".format(ii) for ii in range(1, len(paras) + 1)]
elif len(paras) > 99 and len(paras) < 1000:
keys_paras = ["x{:03d}" for ii in range(1, len(paras) + 1)]
elif len(paras) <= 9:
keys_paras = ["x" + str(ii) for ii in range(1, len(paras) + 1)]
else:
raise ValueError("More than 999 parameters are not implemented yet!")
vals_paras = paras
dict_paras = dict(zip(keys_paras, vals_paras))
# fill in to templates
# templates need to have patterns:
# {par[x01]}, {par[x02]}, ... for parameters
# {dpar[something]}, {dpar[somethingelse]}, ... for derived parameters
# ---------------
# create a run folder
# ---------------
tmp_folder = "/tmp/eee-analysis/" + str(
run_id) # "/tmp/juletest" # TODO a generic folder name in /tmp
raven_exe_name = os.path.abspath(dir_path + "/../" +
"examples/raven-hmets/model/Raven.exe")
raven_obs_folder = os.path.abspath(dir_path + "/../" +
"examples/raven-hmets/model/data_obs")
if os.path.exists(tmp_folder):
shutil.rmtree(tmp_folder)
# all RAVEN setup files
writeString(Path(tmp_folder, "raven_hmets.rvi"),
RVI.format(par=dict_paras, dpar=dict_dparas))
writeString(Path(tmp_folder, "raven_hmets.rvp"),
RVP.format(par=dict_paras, dpar=dict_dparas))
writeString(Path(tmp_folder, "raven_hmets.rvh"),
RVH.format(par=dict_paras, dpar=dict_dparas))
writeString(Path(tmp_folder, "raven_hmets.rvt"),
RVT.format(par=dict_paras, dpar=dict_dparas))
writeString(Path(tmp_folder, "raven_hmets.rvc"),
RVC.format(par=dict_paras, dpar=dict_dparas))
# link executable
if not (os.path.exists(
str(Path(tmp_folder, os.path.basename(raven_exe_name))))):
print("from: ", os.path.realpath(raven_exe_name))
print("to: ", str(Path(tmp_folder,
os.path.basename(raven_exe_name))))
os.symlink(os.path.realpath(raven_exe_name),
str(Path(tmp_folder, os.path.basename(raven_exe_name))))
# link observations folder
if not (os.path.exists(
str(Path(tmp_folder, os.path.basename(raven_obs_folder))))):
os.symlink(os.path.realpath(raven_obs_folder),
str(Path(tmp_folder, os.path.basename(raven_obs_folder))))
# create ouput folder
out_folder = str(Path(tmp_folder, "output"))
os.makedirs(out_folder)
# ---------------
# run the model with these input rv* files
# ---------------
cmd = [
str(Path(tmp_folder, os.path.basename(raven_exe_name))),
str(Path(tmp_folder, "raven_hmets")), "-o",
str(Path(tmp_folder, "output")) + '/'
]
print("run cmd: ", ' '.join(cmd))
process = subprocess.Popen(cmd, stdout=subprocess.PIPE)
print("")
print("Raven standard output:")
for line in process.stdout:
print(">>> ", line.rstrip()) # rstrip removes trailing \n
if not (os.path.exists(str(Path(tmp_folder, "output",
"Diagnostics.csv")))):
print("")
print("ERROR: No Diagnostics.csv produced")
print("")
print("Raven error file content:")
ff = open(str(Path(tmp_folder, "output", "Raven_errors.txt")), "r")
lines = ff.readlines()
ff.close()
for line in lines:
print(">>> ", line.rstrip()) # rstrip removes trailing \n
raise ValueError(
"ERROR: No Diagnostics.csv produced (scroll up to see content of error file)"
)
model = {}
# ---------------
# extract model output: Diagnostics: NSE
# ---------------
model['nse'] = 0.0
ff = open(str(Path(tmp_folder, "output", "Diagnostics.csv")), "r")
lines = ff.readlines()
ff.close()
nse = np.float(lines[-1].strip().split(',')[2])
print("NSE: ", nse)
model['nse'] = nse
print("")
# ---------------
# extract model output: Hydrographs: simulated Q
# ---------------
model['Q'] = 0.0
warmup = 2 * 365 # 1 # model timestep 1 day and want to skip 2 years # first day 1991-01-01 00:00:00.00 (checked)
model['Q'] = np.transpose(
fread(str(Path(tmp_folder, "output", "Hydrographs.csv")),
skip=warmup + 1,
cskip=4,
nc=1))[0]
print("Q: ", model['Q'][0:4], "...", model['Q'][-4:])
print("Q_range: [", np.min(model['Q']), ",", np.max(model['Q']),
"]")
print("shape Q: ", np.shape(model['Q']))
print("")
# ---------------
# extract model output: BETWEEN_PONDED_WATER_AND_SOIL[0]_Daily_Average_BySubbasin.csv: accumulated infiltration volume
# ---------------
model['infiltration'] = 0.0
warmup = 2 * 365 # 1 # model timestep 1 day and want to skip 2 years # first day 1990-12-31 00:00:00.00 (checked) But all timesteps are shifted by 1 day...
#
# de-accumulated infiltration volume
model['infiltration'] = np.transpose(
fread(str(
Path(
tmp_folder, "output",
"BETWEEN_PONDED_WATER_AND_SOIL[0]_Daily_Average_BySubbasin.csv"
)),
skip=warmup,
cskip=2,
nc=1))[0]
model['infiltration'] = np.diff(model['infiltration'])
print("Infiltration I: ", model['infiltration'][0:4], "...",
model['infiltration'][-4:])
print("I_range: [", np.min(model['infiltration']), ",",
np.max(model['infiltration']), "]")
print("shape I: ", np.shape(model['infiltration']))
print("")
# ---------------
# cleanup
# ---------------
if os.path.exists(tmp_folder):
shutil.rmtree(tmp_folder)
return model
def matrix(path_to_data,
dtype,
n_channels,
data_order,
channel_index,
spike_size,
max_memory,
output_path,
output_filename='whitening.npy',
if_file_exists='skip'):
"""Compute whitening filter using the first batch of the data
Parameters
----------
path_to_data: str
Path to recordings in binary format
dtype: str
Recordings dtype
n_channels: int
Number of channels in the recordings
data_order: str
Recordings order, one of ('channels', 'samples'). In a dataset with k
observations per channel and j channels: 'channels' means first k
contiguous observations come from channel 0, then channel 1, and so
on. 'sample' means first j contiguous data are the first observations
from all channels, then the second observations from all channels and
so on
channel_index: np.array
A matrix of size [n_channels, n_nieghbors], showing neighboring channel
information
spike_size: int
Spike size
max_memory: str
Max memory to use in each batch (e.g. 100MB, 1GB)
output_path: str
Where to store the whitenint gilter
output_filename: str, optional
Filename for the output data, defaults to whitening.npy
if_file_exists: str, optional
One of 'overwrite', 'abort', 'skip'. If 'overwrite' it replaces the
whitening filter if it exists, if 'abort' if raise a ValueError
exception if the file exists, if 'skip' if skips the operation if the
file exists
Returns
-------
standarized_path: str
Path to standarized recordings
standarized_params: dict
A dictionary with the parameters for the standarized recordings
(dtype, n_channels, data_order)
"""
logger = logging.getLogger(__name__)
# compute Q (using the first batchfor whitening
logger.info('Computing whitening matrix...')
bp = BatchProcessor(path_to_data, dtype, n_channels, data_order,
max_memory)
batches = bp.multi_channel()
first_batch = next(batches)
whiten_filter = _matrix(first_batch, channel_index, spike_size)
path_to_whitening_matrix = Path(output_path, output_filename)
save_numpy_object(whiten_filter,
path_to_whitening_matrix,
if_file_exists='overwrite',
name='whitening filter')
return whiten_filter
universal_newlines=True,
stderr=subprocess.STDOUT,
env=env,
)
except subprocess.CalledProcessError as exception:
assert not exception.returncode, exception.output
wheels = list(dest_path.glob("*.whl"))
assert len(wheels) == 1
wheel = wheels[0]
return wheel
return _wheel
THIS_PROJECT_ROOT = Path(__file__).resolve().parents[3]
@pytest.fixture(scope="session")
def tox_wheel(wheel):
return wheel(THIS_PROJECT_ROOT)
@pytest.fixture(scope="session")
def magic_non_canonical_wheel(wheel, tmp_path_factory):
magic_proj = tmp_path_factory.mktemp("magic")
(magic_proj / "setup.py").write_text(
"from setuptools import setup\nsetup(name='com.magic.this-is-fun')")
return wheel(magic_proj)
def locale_paths(tmp_path, monkeypatch):
monkeypatch.setattr(cmk.utils.paths, "locale_dir",
Path("%s/locale" % cmk_path()))
monkeypatch.setattr(cmk.utils.paths, "local_locale_dir",
tmp_path / "locale")
def get_stderr_path_with_postfix(self, postfix):
return Path("{}_{}".format(
self.__syslog_ng_paths["file_paths"]["stderr"], postfix))
def _get_reader(reader_report, cls):
path = reader_report.simulation.config["output"]["output_dir"]
ext = FORMAT_TO_EXT[reader_report.config.get("format", "HDF5")]
file_name = reader_report.config.get("file_name", reader_report.name) + ext
path = str(Path(path, file_name))
return cls(path)
def main(args):
sys.path.append(str(Path(__file__).parent))
checkpoint_path = Path(args.checkpoint_dir)
checkpoint_path.mkdir(exist_ok=True)
logger = utils.setup_logger(__name__,
os.path.join(args.checkpoint_dir, 'train.log'))
utils.read_config_file(args.config)
utils.config.update(args.__dict__)
logger.debug('Running with config %s', utils.config)
configure(os.path.join('runs', args.expname))
# Let's use Amazon S3
s3 = boto3.resource(
's3') #s3 = boto3.client('s3', profile_name='signal-rnd')
mybucket = s3.Bucket('data.data-science.signal')
myfolder = 'summaries-segmentation'
start = timer()
print 'Loading word vectors....'
if not args.test:
#key = myfolder + utils.config['word2vecfile']
#word2vec = gensim.models.KeyedVectors.load_word2vec_format(mybucket.Object(key).get()['Body'].read(), binary=True)
#word2vec = gensim.models.KeyedVectors.load_word2vec_format(io.BytesIO(mybucket.Object(key).get()['Body'].read()), binary=True)
word2vec = gensim.models.KeyedVectors.load_word2vec_format(
utils.config['word2vecfile'], binary=True)
#response = urllib2.urlopen('https://drive.google.com/file/d/0B7XkCwpI5KDYNlNUTTlSS21pQmM/edit?usp=sharing')
#word2vec = gensim.models.KeyedVectors.load_word2vec_format(response.read(), binary=True)
#mybucket.Object(key).download_file('GoogleNews_vectors')
#word2vec = gensim.models.KeyedVectors.load_word2vec_format('GoogleNews_vectors', binary=True)
else:
word2vec = None
word2vec_done = timer()
print 'Loading word2vec ellapsed: ' + str(word2vec_done -
start) + ' seconds'
print 'Loading samples....'
if not args.infer:
if args.wiki:
if (args.wiki_folder):
signal_training = True
dataset_path = args.wiki_folder #Path(args.wiki_folder)
os.makedirs('.' +
args.wiki_folder) #to keep the container tidy
else:
signal_training = False
dataset_path = Path(utils.config['wikidataset'])
#dataset_path = Path(utils.config['wikidataset'])
train_dataset = WikipediaDataSet(
str(dataset_path) + '/train',
word2vec=word2vec,
folder=signal_training,
high_granularity=args.high_granularity)
dev_dataset = WikipediaDataSet(
str(dataset_path) + '/dev',
word2vec=word2vec,
folder=signal_training,
high_granularity=args.high_granularity)
test_dataset = WikipediaDataSet(
str(dataset_path) + '/test',
word2vec=word2vec,
folder=signal_training,
high_granularity=args.high_granularity)
else:
dataset_path = utils.config['choidataset']
train_dataset = ChoiDataset(dataset_path, word2vec)
dev_dataset = ChoiDataset(dataset_path, word2vec)
test_dataset = ChoiDataset(dataset_path, word2vec)
train_dl = DataLoader(train_dataset,
batch_size=args.bs,
collate_fn=collate_fn,
shuffle=True,
num_workers=args.num_workers)
dev_dl = DataLoader(dev_dataset,
batch_size=args.test_bs,
collate_fn=collate_fn,
shuffle=False,
num_workers=args.num_workers)
test_dl = DataLoader(test_dataset,
batch_size=args.test_bs,
collate_fn=collate_fn,
shuffle=False,
num_workers=args.num_workers)
samples_done = timer()
print 'Samples pulled successfully into container in: ' + str(
samples_done - word2vec_done) + ' seconds'
assert bool(args.model) ^ bool(
args.load_from) # exactly one of them must be set
if args.model:
model = import_model(args.model)
elif args.load_from:
key = myfolder + args.load_from
#model = torch.load(mybucket.Object(key).get()['Body'].read())
#fileobj = io.BytesIO()
#mybucket.Object(key).download_fileobj(fileobj)
mybucket.Object(key).download_file('trained_model')
#with open(args.load_from, 'rb') as f:
with open('trained_model', 'rb') as f:
model = torch.load(f)
model.train()
model = maybe_cuda(model)
optimizer = torch.optim.Adam(model.parameters(), lr=1e-3)
if not args.infer:
best_val_pk = 1.0
for j in range(args.epochs):
train(model, args, j, train_dl, logger, optimizer)
model_name = 'model{:03d}.t7'.format(j)
with (checkpoint_path /
'model{:03d}.t7'.format(j)).open('wb') as f:
torch.save(model, f)
val_pk, threshold = validate(model, args, j, dev_dl, logger)
if val_pk < best_val_pk:
test_pk = test(model, args, j, test_dl, logger, threshold)
logger.debug(
colored(
'Current best model from epoch {} with p_k {} and threshold {}'
.format(j, test_pk, threshold), 'green'))
best_val_pk = val_pk
with (checkpoint_path /
'best_model.t7'.format(j)).open('wb') as f:
torch.save(model, f)
key = myfolder + '/results/trained_model.t7'
mybucket.Object(key).upload_file(
str(checkpoint_path) + '/' + model_name)
key = myfolder + '/results/best_model.t7'
mybucket.Object(key).upload_file(
str(checkpoint_path) + '/best_model.t7')
key = myfolder + '/results/train.log'
mybucket.Object(key).upload_file(str(checkpoint_path) + '/train.log')
else:
test_dataset = WikipediaDataSet(args.infer,
word2vec=word2vec,
high_granularity=args.high_granularity)
test_dl = DataLoader(test_dataset,
batch_size=args.test_bs,
collate_fn=collate_fn,
shuffle=False,
num_workers=args.num_workers)
print test(model, args, 0, test_dl, logger, 0.4)
def main():
parser = ArgumentParser()
parser.add_argument(
"--run",
help="Run the autoscaler after wizard finished",
action="store_true",
default=False,
)
parser.add_argument(
"--remote",
help="Run the autoscaler as a service, launch on the `services` queue",
action="store_true",
default=False,
)
args = parser.parse_args()
if running_remotely():
hyper_params = AwsAutoScaler.Settings().as_dict()
configurations = AwsAutoScaler.Configuration().as_dict()
else:
print(
"AWS Autoscaler setup wizard\n"
"---------------------------\n"
"Follow the wizard to configure your AWS auto-scaler service.\n"
"Once completed, you will be able to view and change the configuration in the trains-server web UI.\n"
"It means there is no need to worry about typos or mistakes :)\n")
config_file = Path(CONF_FILE).absolute()
if config_file.exists() and input_bool(
"Load configurations from config file '{}' [Y/n]? ".format(
str(CONF_FILE)),
default=True,
):
with config_file.open("r") as f:
conf = yaml.load(f, Loader=yaml.SafeLoader)
hyper_params = conf["hyper_params"]
configurations = conf["configurations"]
else:
configurations, hyper_params = run_wizard()
# noinspection PyBroadException
try:
with config_file.open("w+") as f:
conf = {
"hyper_params": hyper_params,
"configurations": configurations,
}
yaml.safe_dump(conf, f)
except Exception:
print(
"Error! Could not write configuration file at: {}".format(
str(CONF_FILE)))
return
task = Task.init(project_name="DevOps",
task_name="AWS Auto-Scaler",
task_type=Task.TaskTypes.service)
task.connect(hyper_params)
task.connect_configuration(configurations)
if args.remote or args.run:
print("Running AWS auto-scaler as a service\nExecution log {}".format(
task.get_output_log_web_page()))
if args.remote:
# if we are running remotely enqueue this run, and leave the process
# the trains-agent services will pick it up and execute it for us.
task.execute_remotely(queue_name='services')
autoscaler = AwsAutoScaler(hyper_params, configurations)
if running_remotely() or args.run:
autoscaler.start()
