def __init__(self,
path: pathlib2.Path,
depth: int = None,
root: pathlib2.Path = None,
children: Union[List, None] = None) -> None:
"""FileNode Initializer
:param path: file path
:type path: pathlib2.Path
:param depth: depth of file/directory relative to root, defaults to None
:type depth: int, optional
:param root: root path, can be None, defaults to None
:type root: pathlib2.Path, optional
:param children: List of Node when current node is a directory, None if current node is a file, defaults to None
:type children: Union[List, None], optional
"""
self._path = path
self._root = root
self._relative_path = self._path.relative_to(
self._root) if self._root else None
self._depth = depth
self._filename = path.name
self._stat = path.stat()
self._children = children if children else []
self._id = UniqueFileIdentifier(self._path)
def rm_old_local_cached_dirs(cache_dir):
"""Remove old local cached directories, which were created four weeks ago.
Args:
cache_dir (Path): Full path to the cache directory
"""
assert isinstance(cache_dir, Path)
cache_dir = cache_dir.expanduser()
# This is in autobisectjs because it has a lock so we do not race while removing directories
# Adapted from http://stackoverflow.com/a/11337407
SECONDS_IN_A_DAY = 24 * 60 * 60
s3CacheObj = s3cache.S3Cache(compile_shell.S3_SHELL_CACHE_DIRNAME)
if s3CacheObj.connect():
NUMBER_OF_DAYS = 1 # EC2 VMs generally have less disk space for local shell caches
else:
NUMBER_OF_DAYS = 28
names = [cache_dir / x for x in cache_dir.iterdir()]
for name in names:
if name.is_dir():
timediff = time.mktime(time.gmtime()) - Path.stat(name).st_atime
if timediff > SECONDS_IN_A_DAY * NUMBER_OF_DAYS:
shutil.rmtree(str(name))
def __init__(self, path: pathlib2.Path):
"""Initialize UniqueFile Identifier by setting st_dev and st_ino
:param path: a file path of type pathlib2.Path
:type path: pathlib2.Path
"""
self.st_dev = path.stat().st_dev
self.st_ino = path.stat().st_ino
def _make_file_info(target: pathlib.Path,
arcname: Optional[str] = None) -> Dict[str, Any]:
f = {} # type: Dict[str, Any]
f['origin'] = str(target)
if arcname is not None:
f['filename'] = arcname
else:
f['filename'] = str(target)
if os.name == 'nt':
fstat = os.stat(str(target), follow_symlinks=False)
if target.is_symlink():
f['emptystream'] = False
f['attributes'] = fstat.st_file_attributes & FILE_ATTRIBUTE_WINDOWS_MASK # type: ignore # noqa
elif target.is_dir():
f['emptystream'] = True
f['attributes'] = fstat.st_file_attributes & FILE_ATTRIBUTE_WINDOWS_MASK # type: ignore # noqa
elif target.is_file():
f['emptystream'] = False
f['attributes'] = stat.FILE_ATTRIBUTE_ARCHIVE # type: ignore # noqa
f['uncompressed'] = fstat.st_size
else:
fstat = target.stat()
if target.is_symlink():
f['emptystream'] = False
f['attributes'] = stat.FILE_ATTRIBUTE_ARCHIVE # type: ignore # noqa
f['attributes'] |= FILE_ATTRIBUTE_UNIX_EXTENSION | (
stat.S_IFLNK << 16)
f['attributes'] |= (stat.S_IMODE(fstat.st_mode) << 16)
elif target.is_dir():
f['emptystream'] = True
f['attributes'] = stat.FILE_ATTRIBUTE_DIRECTORY # type: ignore # noqa
f['attributes'] |= FILE_ATTRIBUTE_UNIX_EXTENSION | (
stat.S_IFDIR << 16)
f['attributes'] |= (stat.S_IMODE(fstat.st_mode) << 16)
elif target.is_file():
f['emptystream'] = False
f['uncompressed'] = fstat.st_size
f['attributes'] = stat.FILE_ATTRIBUTE_ARCHIVE # type: ignore # noqa
f['attributes'] |= FILE_ATTRIBUTE_UNIX_EXTENSION | (
stat.S_IMODE(fstat.st_mode) << 16)
f['creationtime'] = target.stat().st_ctime
f['lastwritetime'] = target.stat().st_mtime
f['lastaccesstime'] = target.stat().st_atime
return f
def load_GPR_data(filepath, samplePoint=512):
dPosXYZs = []
radarData = []
try:
file = Path(filepath)
fileByte = file.stat().st_size
print(fileByte)
traceByte = 90 + samplePoint * 2
traceNum = int((fileByte - FILE_INFO_BYTE_NUM) / traceByte)
if isinstance(traceNum, int):
with open(file, 'rb') as f:
f.seek(FILE_INFO_BYTE_NUM)
for ele in range(0, traceNum):
fGPSOffset = struct.unpack('f', f.read(4))[0]
chReserve = struct.unpack('2c', f.read(1 * 2))
ucYear = struct.unpack('B', f.read(1))[0]
ucMonth = struct.unpack('B', f.read(1))[0]
ucDay = struct.unpack('B', f.read(1))[0]
ucHour = struct.unpack('B', f.read(1))[0]
ucMin = struct.unpack('B', f.read(1))[0]
ucSec = struct.unpack('B', f.read(1))[0]
usMilSec = struct.unpack('H', f.read(2))[0]
dPosXYZ = struct.unpack('3d', f.read(8 * 3))
dPosXYZs.append(dPosXYZ)
ucTrcCount = struct.unpack('BB', f.read(1 * 2))
ucVoltage = struct.unpack('BB', f.read(1 * 2))
fWheelOffset = struct.unpack('f', f.read(4))[0]
chPhotoName1 = struct.unpack('8c', f.read(1 * 8))
chPhotoName2 = struct.unpack('8c', f.read(1 * 8))
usMetalDiameter = struct.unpack('H', f.read(2))[0]
usMetalDepth = struct.unpack('H', f.read(2))[0]
bMetalFlag = struct.unpack('?', f.read(1))[0]
chMarkName = struct.unpack('17c', f.read(1 * 17))
fMarkHeight = struct.unpack('f', f.read(4))[0]
usMarkFlag = struct.unpack('H', f.read(2))[0]
data = struct.unpack(
str(samplePoint) + 'h', f.read(samplePoint * 2))
radarData.append(data)
dPosXYZs = np.array(dPosXYZs).T
return dPosXYZs, radarData
else:
return errorhandle.LOAD_GPR_SIZE_ERROR
except Exception as e:
print(e)
return errorhandle.LOAD_GPR_FAILURE
def _set_file_property(self, outfilename: pathlib.Path,
properties: Dict[str, Any]) -> None:
# creation time
creationtime = ArchiveTimestamp(
properties['lastwritetime']).totimestamp()
if creationtime is not None:
os.utime(str(outfilename), times=(creationtime, creationtime))
if os.name == 'posix':
st_mode = properties['posix_mode']
if st_mode is not None:
outfilename.chmod(st_mode)
return
# fallback: only set readonly if specified
if properties['readonly'] and not properties['is_directory']:
ro_mask = 0o777 ^ (stat.S_IWRITE | stat.S_IWGRP | stat.S_IWOTH)
outfilename.chmod(outfilename.stat().st_mode & ro_mask)
def get_coverage_build(dirpath, args):
"""Gets a coverage build from a specified server.
Args:
dirpath (Path): Directory in which build is to be downloaded in.
args (class): Command line arguments.
Returns:
Path: Path to the js coverage build
"""
RUN_COV_LOG.info("Downloading coverage build zip file into %s from %s",
str(dirpath), args.url)
with requests.get(args.url, stream=True) as f:
build_request_data = io.BytesIO(f.content)
RUN_COV_LOG.info("Extracting coverage build zip file...")
build_zip = zipfile.ZipFile(build_request_data)
extract_folder = dirpath / "cov-build"
extract_folder.mkdir(parents=True,
exist_ok=True) # Ensure this dir has been created
# In 3.5 <= Python < 3.6, .extractall does not automatically create intermediate folders that do not exist
build_zip.extractall(str(extract_folder.resolve()))
RUN_COV_LOG.info("Coverage build zip file extracted to this folder: %s",
extract_folder.resolve())
js_cov_bin_name = "js" + (".exe" if platform.system() == "Windows" else "")
js_cov_bin = extract_folder / "dist" / "bin" / js_cov_bin_name
Path.chmod(js_cov_bin,
Path.stat(js_cov_bin).st_mode
| 0o111) # Ensure the js binary is executable
assert js_cov_bin.is_file()
# Check that the binary is non-debug.
assert not queryBuildConfiguration(js_cov_bin, "debug")
assert queryBuildConfiguration(js_cov_bin, "coverage")
js_cov_fmconf = extract_folder / "dist" / "bin" / (js_cov_bin_name +
".fuzzmanagerconf")
assert js_cov_fmconf.is_file()
# Check that a coverage build with *.gcno files are present
js_cov_unified_gcno = extract_folder / "js" / "src" / "Unified_cpp_js_src0.gcno"
assert js_cov_unified_gcno.is_file()
return js_cov_bin
def GPRGPSReader(GPR_file, sample_point=1024):
'''
读取二进制文件.GPR中的GPS信息
.GPR文件结构如下：
文件头 1065字节
每道道头 90字节
每道数据 sample_point*2字节
GPS信息位于第15字节
输入GPR文件路径的str 或者 Path对象
返回每一道GPS的X Y Z坐标
'''
import struct
import matplotlib.pyplot as plt
import numpy as np
from pathlib2 import Path
file = Path(GPR_file)
dPosXYZs = []
file_byte = file.stat().st_size
file_info_byte = 1065
trace_byte = 90 + sample_point * 2
trace_num = (file_byte - file_info_byte) / trace_byte
assert (file_byte - file_info_byte) % trace_byte == 0 #不为0说明sample_point不对
with open(file, 'rb') as f:
# 整个数据开头有1065字节数据头 + 每道道头90 + 每道数据1024*2
f.seek(file_info_byte) #跳过数据开头冗余的1065字节
for trace_ii in np.arange(trace_num):
try:
fGPSOffset = struct.unpack('f', f.read(4))[0]
chReserve = struct.unpack('2c', f.read(1 * 2))
except:
print(trace_ii)
break
else:
ucYear = struct.unpack('B', f.read(1))[0]
ucMonth = struct.unpack('B', f.read(1))[0]
ucDay = struct.unpack('B', f.read(1))[0]
ucHour = struct.unpack('B', f.read(1))[0]
ucMin = struct.unpack('B', f.read(1))[0]
ucSec = struct.unpack('B', f.read(1))[0]
usMilSec = struct.unpack('H', f.read(2))[0]
# dPosX,dPosY,dPosZ = struct.unpack('3d',f.read(8*3))
dPosXYZ = struct.unpack('3d', f.read(8 * 3))
dPosXYZs.append(dPosXYZ)
ucTrcCount = struct.unpack('BB', f.read(1 * 2))
ucVoltage = struct.unpack('BB', f.read(1 * 2))
fWheelOffset = struct.unpack('f', f.read(4))[0]
chPhotoName1 = struct.unpack('8c', f.read(1 * 8))
chPhotoName2 = struct.unpack('8c', f.read(1 * 8))
usMetalDiameter = struct.unpack('H', f.read(2))[0]
usMetalDepth = struct.unpack('H', f.read(2))[0]
bMetalFlag = struct.unpack('?', f.read(1))[0]
chMarkName = struct.unpack('17c', f.read(1 * 17))
fMarkHeight = struct.unpack('f', f.read(4))[0]
usMarkFlag = struct.unpack('H', f.read(2))[0]
data = struct.unpack(
str(sample_point) + 'H', f.read(sample_point * 2))
dPosXYZs = np.array(dPosXYZs).T
return dPosXYZs
def main():
print('TRAINS SDK setup process')
conf_file = Path(LOCAL_CONFIG_FILES[0]).absolute()
if conf_file.exists() and conf_file.is_file() and conf_file.stat().st_size > 0:
print('Configuration file already exists: {}'.format(str(conf_file)))
print('Leaving setup, feel free to edit the configuration file.')
return
print(host_description, end='')
parsed_host = None
while not parsed_host:
parse_input = input()
if not parse_input:
parse_input = def_host
# noinspection PyBroadException
try:
if not parse_input.startswith('http://') and not parse_input.startswith('https://'):
parse_input = 'http://'+parse_input
parsed_host = urlparse(parse_input)
if parsed_host.scheme not in ('http', 'https'):
parsed_host = None
except Exception:
parsed_host = None
print('Could not parse url {}\nEnter your trains-server host: '.format(parse_input), end='')
if parsed_host.port == 8080:
# this is a docker 8080 is the web address, we need the api address, it is 8008
print('Port 8080 is the web port, we need the api port. Replacing 8080 with 8008')
api_host = parsed_host.scheme + "://" + parsed_host.netloc.replace(':8080', ':8008') + parsed_host.path
web_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path
elif parsed_host.netloc.startswith('demoapp.'):
print('{} is the web server, we need the api server. Replacing \'demoapp.\' with \'demoapi.\''.format(
parsed_host.netloc))
# this is our demo server
api_host = parsed_host.scheme + "://" + parsed_host.netloc.replace('demoapp.', 'demoapi.') + parsed_host.path
web_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path
elif parsed_host.netloc.startswith('app.'):
print('{} is the web server, we need the api server. Replacing \'app.\' with \'api.\''.format(
parsed_host.netloc))
# this is our application server
api_host = parsed_host.scheme + "://" + parsed_host.netloc.replace('app.', 'api.') + parsed_host.path
web_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path
elif parsed_host.port == 8008:
api_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path
web_host = parsed_host.scheme + "://" + parsed_host.netloc.replace(':8008', ':8080') + parsed_host.path
elif parsed_host.netloc.startswith('demoapi.'):
api_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path
web_host = parsed_host.scheme + "://" + parsed_host.netloc.replace('demoapi.', 'demoapp.') + parsed_host.path
elif parsed_host.netloc.startswith('api.'):
api_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path
web_host = parsed_host.scheme + "://" + parsed_host.netloc.replace('api.', 'app.') + parsed_host.path
else:
api_host = None
web_host = None
if not parsed_host.port:
print('Host port not detected, do you wish to use the default 8008 port n/[y]? ', end='')
replace_port = input().lower()
if not replace_port or replace_port == 'y' or replace_port == 'yes':
api_host = parsed_host.scheme + "://" + parsed_host.netloc + ':8008' + parsed_host.path
web_host = parsed_host.scheme + "://" + parsed_host.netloc + ':8080' + parsed_host.path
if not api_host:
api_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path
if not web_host:
web_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path
print('Host configured to: {}'.format(api_host))
print(description.format(web_host), end='')
parse_input = input()
# check if these are valid credentials
credentials = None
# noinspection PyBroadException
try:
parsed = ConfigFactory.parse_string(parse_input)
if parsed:
credentials = parsed.get("credentials", None)
except Exception:
credentials = None
if not credentials or set(credentials) != {"access_key", "secret_key"}:
print('Could not parse user credentials, try again one after the other.')
credentials = {}
# parse individual
print('Enter user access key: ', end='')
credentials['access_key'] = input()
print('Enter user secret: ', end='')
credentials['secret_key'] = input()
print('Detected credentials key=\"{}\" secret=\"{}\"'.format(credentials['access_key'],
credentials['secret_key'], ))
# noinspection PyBroadException
try:
default_sdk_conf = Path(__file__).parent.absolute() / 'sdk.conf'
with open(str(default_sdk_conf), 'rt') as f:
default_sdk = f.read()
except Exception:
print('Error! Could not read default configuration file')
return
# noinspection PyBroadException
try:
with open(str(conf_file), 'wt') as f:
header = '# TRAINS SDK configuration file\n' \
'api {\n' \
' # Notice: \'host\' is the api server (default port 8008), not the web server.\n' \
' host: %s\n' \
' # Credentials are generated in the webapp, %s/admin\n' \
' credentials {"access_key": "%s", "secret_key": "%s"}\n' \
'}\n' \
'sdk ' % (api_host, web_host, credentials['access_key'], credentials['secret_key'])
f.write(header)
f.write(default_sdk)
except Exception:
print('Error! Could not write configuration file at: {}'.format(str(conf_file)))
return
print('\nNew configuration stored in {}'.format(str(conf_file)))
print('TRAINS setup completed successfully.')
def main():
default_config_file = os.getenv(
LOCAL_CONFIG_FILE_OVERRIDE_VAR) or LOCAL_CONFIG_FILES[0]
p = argparse.ArgumentParser(description=__doc__)
p.add_argument(
"--file",
"-F",
help="Target configuration file path (default is %(default)s)",
default=default_config_file,
type=validate_file)
args = p.parse_args()
print('TRAINS SDK setup process')
conf_file = Path(args.file).absolute()
if conf_file.exists() and conf_file.is_file(
) and conf_file.stat().st_size > 0:
print('Configuration file already exists: {}'.format(str(conf_file)))
print('Leaving setup, feel free to edit the configuration file.')
return
print(description, end='')
parse_input = get_user_input()
credentials = None
api_host = None
web_server = None
# noinspection PyBroadException
try:
parsed = ConfigFactory.parse_string(parse_input)
if parsed:
# Take the credentials in raw form or from api section
credentials = get_parsed_field(parsed, ["credentials"])
api_host = get_parsed_field(parsed, ["api_server", "host"])
web_server = get_parsed_field(parsed, ["web_server"])
except Exception:
credentials = credentials or None
api_host = api_host or None
web_server = web_server or None
while not credentials or set(credentials) != {"access_key", "secret_key"}:
print(
'Could not parse credentials, please try entering them manually.')
credentials = read_manual_credentials()
print('Detected credentials key=\"{}\" secret=\"{}\"'.format(
credentials['access_key'], credentials['secret_key'][0:4] + "***"))
host_description = """
Editing configuration file: {CONFIG_FILE}
Enter the url of the trains-server's Web service, for example: {HOST}
""".format(
CONFIG_FILE=args.file,
HOST=def_host,
)
if api_host:
api_host = input_url('API Host', api_host)
else:
print(host_description)
api_host = input_url('API Host', '')
parsed_host = verify_url(api_host)
if parsed_host.netloc.startswith('demoapp.'):
# this is our demo server
api_host = parsed_host.scheme + "://" + parsed_host.netloc.replace(
'demoapp.', 'demoapi.', 1) + parsed_host.path
web_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path
files_host = parsed_host.scheme + "://" + parsed_host.netloc.replace(
'demoapp.', 'demofiles.', 1) + parsed_host.path
elif parsed_host.netloc.startswith('app.'):
# this is our application server
api_host = parsed_host.scheme + "://" + parsed_host.netloc.replace(
'app.', 'api.', 1) + parsed_host.path
web_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path
files_host = parsed_host.scheme + "://" + parsed_host.netloc.replace(
'app.', 'files.', 1) + parsed_host.path
elif parsed_host.netloc.startswith('demoapi.'):
print(
'{} is the api server, we need the web server. Replacing \'demoapi.\' with \'demoapp.\''
.format(parsed_host.netloc))
api_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path
web_host = parsed_host.scheme + "://" + parsed_host.netloc.replace(
'demoapi.', 'demoapp.', 1) + parsed_host.path
files_host = parsed_host.scheme + "://" + parsed_host.netloc.replace(
'demoapi.', 'demofiles.', 1) + parsed_host.path
elif parsed_host.netloc.startswith('api.'):
print(
'{} is the api server, we need the web server. Replacing \'api.\' with \'app.\''
.format(parsed_host.netloc))
api_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path
web_host = parsed_host.scheme + "://" + parsed_host.netloc.replace(
'api.', 'app.', 1) + parsed_host.path
files_host = parsed_host.scheme + "://" + parsed_host.netloc.replace(
'api.', 'files.', 1) + parsed_host.path
elif parsed_host.port == 8008:
api_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path
web_host = parsed_host.scheme + "://" + parsed_host.netloc.replace(
':8008', ':8080', 1) + parsed_host.path
files_host = parsed_host.scheme + "://" + parsed_host.netloc.replace(
':8008', ':8081', 1) + parsed_host.path
elif parsed_host.port == 8080:
print(
'Port 8080 is the web port. Using port 8008 for API Host and 8080 for Web Application Host'
)
api_host = parsed_host.scheme + "://" + parsed_host.netloc.replace(
':8080', ':8008', 1) + parsed_host.path
web_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path
files_host = parsed_host.scheme + "://" + parsed_host.netloc.replace(
':8080', ':8081', 1) + parsed_host.path
else:
api_host = ''
web_host = ''
files_host = ''
if not parsed_host.port:
print(
'Host port not detected, do you wish to use the default 8080 port n/[y]? ',
end='')
replace_port = input().lower()
if not replace_port or replace_port == 'y' or replace_port == 'yes':
api_host = parsed_host.scheme + "://" + parsed_host.netloc + ':8008' + parsed_host.path
web_host = parsed_host.scheme + "://" + parsed_host.netloc + ':8080' + parsed_host.path
files_host = parsed_host.scheme + "://" + parsed_host.netloc + ':8081' + parsed_host.path
elif not replace_port or replace_port.lower(
) == 'n' or replace_port.lower() == 'no':
web_host = input_host_port("Web", parsed_host)
api_host = input_host_port("API", parsed_host)
files_host = input_host_port("Files", parsed_host)
if not api_host:
api_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path
web_host = input_url('Web Application Host',
web_server if web_server else web_host)
files_host = input_url('File Store Host', files_host)
print(
'\nTRAINS Hosts configuration:\nWeb App: {}\nAPI: {}\nFile Store: {}\n'
.format(web_host, api_host, files_host))
retry = 1
max_retries = 2
while retry <= max_retries: # Up to 2 tries by the user
if verify_credentials(api_host, credentials):
break
retry += 1
if retry < max_retries + 1:
credentials = read_manual_credentials()
else:
print('Exiting setup without creating configuration file')
return
# noinspection PyBroadException
try:
default_sdk_conf = Path(__file__).parent.absolute() / 'sdk.conf'
with open(str(default_sdk_conf), 'rt') as f:
default_sdk = f.read()
except Exception:
print('Error! Could not read default configuration file')
return
# noinspection PyBroadException
try:
with open(str(conf_file), 'wt') as f:
header = '# TRAINS SDK configuration file\n' \
'api {\n' \
' # Notice: \'host\' is the api server (default port 8008), not the web server.\n' \
' api_server: %s\n' \
' web_server: %s\n' \
' files_server: %s\n' \
' # Credentials are generated using the webapp, %s/profile\n' \
' credentials {"access_key": "%s", "secret_key": "%s"}\n' \
'}\n' \
'sdk ' % (api_host, web_host, files_host,
web_host, credentials['access_key'], credentials['secret_key'])
f.write(header)
f.write(default_sdk)
except Exception:
print('Error! Could not write configuration file at: {}'.format(
str(conf_file)))
return
print('\nNew configuration stored in {}'.format(str(conf_file)))
print('TRAINS setup completed successfully.')
def main():
print('TRAINS-AGENT setup process')
conf_file = Path(LOCAL_CONFIG_FILES[0]).absolute()
if conf_file.exists() and conf_file.is_file(
) and conf_file.stat().st_size > 0:
print('Configuration file already exists: {}'.format(str(conf_file)))
print('Leaving setup, feel free to edit the configuration file.')
return
print(host_description)
web_host = input_url('Web Application Host', '')
parsed_host = verify_url(web_host)
if parsed_host.port == 8008:
print(
'Port 8008 is the api port. Replacing 8080 with 8008 for Web application'
)
api_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path
web_host = parsed_host.scheme + "://" + parsed_host.netloc.replace(
':8008', ':8080', 1) + parsed_host.path
files_host = parsed_host.scheme + "://" + parsed_host.netloc.replace(
':8008', ':8081', 1) + parsed_host.path
elif parsed_host.port == 8080:
api_host = parsed_host.scheme + "://" + parsed_host.netloc.replace(
':8080', ':8008', 1) + parsed_host.path
web_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path
files_host = parsed_host.scheme + "://" + parsed_host.netloc.replace(
':8080', ':8081', 1) + parsed_host.path
elif parsed_host.netloc.startswith('demoapp.'):
# this is our demo server
api_host = parsed_host.scheme + "://" + parsed_host.netloc.replace(
'demoapp.', 'demoapi.', 1) + parsed_host.path
web_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path
files_host = parsed_host.scheme + "://" + parsed_host.netloc.replace(
'demoapp.', 'demofiles.', 1) + parsed_host.path
elif parsed_host.netloc.startswith('app.'):
# this is our application server
api_host = parsed_host.scheme + "://" + parsed_host.netloc.replace(
'app.', 'api.', 1) + parsed_host.path
web_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path
files_host = parsed_host.scheme + "://" + parsed_host.netloc.replace(
'app.', 'files.', 1) + parsed_host.path
elif parsed_host.netloc.startswith('demoapi.'):
print(
'{} is the api server, we need the web server. Replacing \'demoapi.\' with \'demoapp.\''
.format(parsed_host.netloc))
api_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path
web_host = parsed_host.scheme + "://" + parsed_host.netloc.replace(
'demoapi.', 'demoapp.', 1) + parsed_host.path
files_host = parsed_host.scheme + "://" + parsed_host.netloc.replace(
'demoapi.', 'demofiles.', 1) + parsed_host.path
elif parsed_host.netloc.startswith('api.'):
print(
'{} is the api server, we need the web server. Replacing \'api.\' with \'app.\''
.format(parsed_host.netloc))
api_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path
web_host = parsed_host.scheme + "://" + parsed_host.netloc.replace(
'api.', 'app.', 1) + parsed_host.path
files_host = parsed_host.scheme + "://" + parsed_host.netloc.replace(
'api.', 'files.', 1) + parsed_host.path
else:
api_host = ''
web_host = ''
files_host = ''
if not parsed_host.port:
print(
'Host port not detected, do you wish to use the default 8008 port n/[y]? ',
end='')
replace_port = input().lower()
if not replace_port or replace_port == 'y' or replace_port == 'yes':
api_host = parsed_host.scheme + "://" + parsed_host.netloc + ':8008' + parsed_host.path
web_host = parsed_host.scheme + "://" + parsed_host.netloc + ':8080' + parsed_host.path
files_host = parsed_host.scheme + "://" + parsed_host.netloc + ':8081' + parsed_host.path
if not api_host:
api_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path
api_host = input_url('API Host', api_host)
files_host = input_url('File Store Host', files_host)
print(
'\nTRAINS Hosts configuration:\nAPI: {}\nWeb App: {}\nFile Store: {}\n'
.format(api_host, web_host, files_host))
while True:
print(description.format(web_host), end='')
parse_input = input()
# check if these are valid credentials
credentials = None
# noinspection PyBroadException
try:
parsed = ConfigFactory.parse_string(parse_input)
if parsed:
credentials = parsed.get("credentials", None)
except Exception:
credentials = None
if not credentials or set(credentials) != {"access_key", "secret_key"}:
print(
'Could not parse user credentials, try again one after the other.'
)
credentials = {}
# parse individual
print('Enter user access key: ', end='')
credentials['access_key'] = input()
print('Enter user secret: ', end='')
credentials['secret_key'] = input()
print('Detected credentials key=\"{}\" secret=\"{}\"'.format(
credentials['access_key'],
credentials['secret_key'],
))
from trains_agent.backend_api.session import Session
# noinspection PyBroadException
try:
print('Verifying credentials ...')
Session(api_key=credentials['access_key'],
secret_key=credentials['secret_key'],
host=api_host)
print('Credentials verified!')
break
except Exception:
print(
'Error: could not verify credentials: host={} access={} secret={}'
.format(api_host, credentials['access_key'],
credentials['secret_key']))
# get GIT User/Pass for cloning
print(
'Enter git username for repository cloning (leave blank for SSH key authentication): [] ',
end='')
git_user = input()
if git_user.strip():
print('Enter password for user \'{}\': '.format(git_user), end='')
git_pass = input()
print(
'Git repository cloning will be using user={} password={}'.format(
git_user, git_pass))
else:
git_user = None
git_pass = None
# noinspection PyBroadException
try:
conf_folder = Path(__file__).parent.absolute(
) / '..' / 'backend_api' / 'config' / 'default'
default_conf = ''
for conf in (
'agent.conf',
'sdk.conf',
):
conf_file_section = conf_folder / conf
with open(str(conf_file_section), 'rt') as f:
default_conf += conf.split('.')[0] + ' '
default_conf += f.read()
default_conf += '\n'
except Exception:
print('Error! Could not read default configuration file')
return
# noinspection PyBroadException
try:
with open(str(conf_file), 'wt') as f:
header = '# TRAINS-AGENT configuration file\n' \
'api {\n' \
' api_server: %s\n' \
' web_server: %s\n' \
' files_server: %s\n' \
' # Credentials are generated using the webapp, %s/profile\n' \
' # Override with os environment: TRAINS_API_ACCESS_KEY / TRAINS_API_SECRET_KEY\n' \
' credentials {"access_key": "%s", "secret_key": "%s"}\n' \
'}\n\n' % (api_host, web_host, files_host,
web_host, credentials['access_key'], credentials['secret_key'])
f.write(header)
git_credentials = '# Set GIT user/pass credentials\n' \
'# leave blank for GIT SSH credentials\n' \
'agent.git_user=\"{}\"\n' \
'agent.git_pass=\"{}\"\n' \
'\n'.format(git_user or '', git_pass or '')
f.write(git_credentials)
f.write(default_conf)
except Exception:
print('Error! Could not write configuration file at: {}'.format(
str(conf_file)))
return
print('\nNew configuration stored in {}'.format(str(conf_file)))
print('TRAINS-AGENT setup completed successfully.')
def upload_artifact(self,
name,
artifact_object=None,
metadata=None,
delete_after_upload=False):
if not Session.check_min_api_version('2.3'):
LoggerRoot.get_base_logger().warning(
'Artifacts not supported by your TRAINS-server version, '
'please upgrade to the latest server version')
return False
if name in self._artifacts_container:
raise ValueError(
"Artifact by the name of {} is already registered, use register_artifact"
.format(name))
artifact_type_data = tasks.ArtifactTypeData()
override_filename_in_uri = None
override_filename_ext_in_uri = None
uri = None
if np and isinstance(artifact_object, np.ndarray):
artifact_type = 'numpy'
artifact_type_data.content_type = 'application/numpy'
artifact_type_data.preview = str(artifact_object.__repr__())
override_filename_ext_in_uri = '.npz'
override_filename_in_uri = name + override_filename_ext_in_uri
fd, local_filename = mkstemp(prefix=quote(name, safe="") + '.',
suffix=override_filename_ext_in_uri)
os.close(fd)
np.savez_compressed(local_filename, **{name: artifact_object})
delete_after_upload = True
elif pd and isinstance(artifact_object, pd.DataFrame):
artifact_type = 'pandas'
artifact_type_data.content_type = 'text/csv'
artifact_type_data.preview = str(artifact_object.__repr__())
override_filename_ext_in_uri = self._save_format
override_filename_in_uri = name
fd, local_filename = mkstemp(prefix=quote(name, safe="") + '.',
suffix=override_filename_ext_in_uri)
os.close(fd)
artifact_object.to_csv(local_filename,
compression=self._compression)
delete_after_upload = True
elif isinstance(artifact_object, Image.Image):
artifact_type = 'image'
artifact_type_data.content_type = 'image/png'
desc = str(artifact_object.__repr__())
artifact_type_data.preview = desc[1:desc.find(' at ')]
override_filename_ext_in_uri = '.png'
override_filename_in_uri = name + override_filename_ext_in_uri
fd, local_filename = mkstemp(prefix=quote(name, safe="") + '.',
suffix=override_filename_ext_in_uri)
os.close(fd)
artifact_object.save(local_filename)
delete_after_upload = True
elif isinstance(artifact_object, dict):
artifact_type = 'JSON'
artifact_type_data.content_type = 'application/json'
preview = json.dumps(artifact_object, sort_keys=True, indent=4)
override_filename_ext_in_uri = '.json'
override_filename_in_uri = name + override_filename_ext_in_uri
fd, local_filename = mkstemp(prefix=quote(name, safe="") + '.',
suffix=override_filename_ext_in_uri)
os.write(fd, bytes(preview.encode()))
os.close(fd)
artifact_type_data.preview = preview
delete_after_upload = True
elif isinstance(artifact_object, six.string_types) and urlparse(
artifact_object).scheme in remote_driver_schemes:
# we should not upload this, just register
local_filename = None
uri = artifact_object
artifact_type = 'custom'
artifact_type_data.content_type = mimetypes.guess_type(
artifact_object)[0]
elif isinstance(artifact_object, six.string_types + (Path, )):
# check if single file
artifact_object = Path(artifact_object)
artifact_object.expanduser().absolute()
try:
create_zip_file = not artifact_object.is_file()
except Exception: # Hack for windows pathlib2 bug, is_file isn't valid.
create_zip_file = True
else: # We assume that this is not Windows os
if artifact_object.is_dir():
# change to wildcard
artifact_object /= '*'
if create_zip_file:
folder = Path('').joinpath(*artifact_object.parts[:-1])
if not folder.is_dir() or not folder.parts:
raise ValueError(
"Artifact file/folder '{}' could not be found".format(
artifact_object.as_posix()))
wildcard = artifact_object.parts[-1]
files = list(Path(folder).rglob(wildcard))
override_filename_ext_in_uri = '.zip'
override_filename_in_uri = folder.parts[
-1] + override_filename_ext_in_uri
fd, zip_file = mkstemp(
prefix=quote(folder.parts[-1], safe="") + '.',
suffix=override_filename_ext_in_uri)
try:
artifact_type_data.content_type = 'application/zip'
artifact_type_data.preview = 'Archive content {}:\n'.format(
artifact_object.as_posix())
with ZipFile(zip_file,
'w',
allowZip64=True,
compression=ZIP_DEFLATED) as zf:
for filename in sorted(files):
if filename.is_file():
relative_file_name = filename.relative_to(
folder).as_posix()
artifact_type_data.preview += '{} - {}\n'.format(
relative_file_name,
humanfriendly.format_size(
filename.stat().st_size))
zf.write(filename.as_posix(),
arcname=relative_file_name)
except Exception as e:
# failed uploading folder:
LoggerRoot.get_base_logger().warning(
'Exception {}\nFailed zipping artifact folder {}'.
format(folder, e))
return None
finally:
os.close(fd)
artifact_object = zip_file
artifact_type = 'archive'
artifact_type_data.content_type = mimetypes.guess_type(
artifact_object)[0]
local_filename = artifact_object
delete_after_upload = True
else:
if not artifact_object.is_file():
raise ValueError(
"Artifact file '{}' could not be found".format(
artifact_object.as_posix()))
override_filename_in_uri = artifact_object.parts[-1]
artifact_object = artifact_object.as_posix()
artifact_type = 'custom'
artifact_type_data.content_type = mimetypes.guess_type(
artifact_object)[0]
local_filename = artifact_object
else:
raise ValueError("Artifact type {} not supported".format(
type(artifact_object)))
# remove from existing list, if exists
for artifact in self._task_artifact_list:
if artifact.key == name:
if artifact.type == self._pd_artifact_type:
raise ValueError(
"Artifact of name {} already registered, "
"use register_artifact instead".format(name))
self._task_artifact_list.remove(artifact)
break
if not local_filename:
file_size = None
file_hash = None
else:
# check that the file to upload exists
local_filename = Path(local_filename).absolute()
if not local_filename.exists() or not local_filename.is_file():
LoggerRoot.get_base_logger().warning(
'Artifact upload failed, cannot find file {}'.format(
local_filename.as_posix()))
return False
file_hash, _ = self.sha256sum(local_filename.as_posix())
file_size = local_filename.stat().st_size
uri = self._upload_local_file(
local_filename,
name,
delete_after_upload=delete_after_upload,
override_filename=override_filename_in_uri,
override_filename_ext=override_filename_ext_in_uri)
timestamp = int(time())
artifact = tasks.Artifact(
key=name,
type=artifact_type,
uri=uri,
content_size=file_size,
hash=file_hash,
timestamp=timestamp,
type_data=artifact_type_data,
display_data=[(str(k), str(v))
for k, v in metadata.items()] if metadata else None)
# update task artifacts
with self._task_edit_lock:
self._task_artifact_list.append(artifact)
self._task.set_artifacts(self._task_artifact_list)
return True
def _daemon(cls, jupyter_notebook_filename):
from trains import Task
# load jupyter notebook package
# noinspection PyBroadException
try:
# noinspection PyPackageRequirements
from nbconvert.exporters.script import ScriptExporter
_script_exporter = ScriptExporter()
except Exception:
return
# load pigar
# noinspection PyBroadException
try:
from ....utilities.pigar.reqs import get_installed_pkgs_detail, file_import_modules
from ....utilities.pigar.modules import ReqsModules
from ....utilities.pigar.log import logger
logger.setLevel(logging.WARNING)
except Exception:
file_import_modules = None
# load IPython
# noinspection PyBroadException
try:
# noinspection PyPackageRequirements
from IPython import get_ipython
except Exception:
# should not happen
get_ipython = None
# setup local notebook files
if jupyter_notebook_filename:
notebook = Path(jupyter_notebook_filename)
local_jupyter_filename = jupyter_notebook_filename
else:
notebook = None
fd, local_jupyter_filename = mkstemp(suffix='.ipynb')
os.close(fd)
last_update_ts = None
counter = 0
prev_script_hash = None
# noinspection PyBroadException
try:
from ....version import __version__
our_module = cls.__module__.split('.')[0], __version__
except Exception:
our_module = None
# noinspection PyBroadException
try:
import re
replace_ipython_pattern = re.compile(r'\n([ \t]*)get_ipython\(\)')
except Exception:
replace_ipython_pattern = None
# main observer loop, check if we need to exit
while not cls._exit_event.wait(timeout=0.):
# wait for timeout or sync event
cls._sync_event.wait(cls._sample_frequency if counter else cls.
_first_sample_frequency)
cls._sync_event.clear()
counter += 1
# noinspection PyBroadException
try:
# if there is no task connected, do nothing
task = Task.current_task()
if not task:
continue
script_code = None
fmodules = None
current_cell = None
# if we have a local file:
if notebook:
if not notebook.exists():
continue
# check if notebook changed
if last_update_ts is not None and notebook.stat(
).st_mtime - last_update_ts <= 0:
continue
last_update_ts = notebook.stat().st_mtime
else:
# serialize notebook to a temp file
if cls._jupyter_history_logger:
script_code, current_cell = cls._jupyter_history_logger.history_to_str(
)
else:
# noinspection PyBroadException
try:
# noinspection PyBroadException
try:
os.unlink(local_jupyter_filename)
except Exception:
pass
get_ipython().run_line_magic(
'history',
'-t -f {}'.format(local_jupyter_filename))
with open(local_jupyter_filename, 'r') as f:
script_code = f.read()
# load the modules
from ....utilities.pigar.modules import ImportedModules
fmodules = ImportedModules()
for nm in set(
[str(m).split('.')[0] for m in sys.modules]):
fmodules.add(nm, 'notebook', 0)
except Exception:
continue
# get notebook python script
if script_code is None:
script_code, _ = _script_exporter.from_filename(
local_jupyter_filename)
current_script_hash = hash(script_code + (current_cell or ''))
if prev_script_hash and prev_script_hash == current_script_hash:
continue
# remove ipython direct access from the script code
# we will not be able to run them anyhow
if replace_ipython_pattern:
script_code = replace_ipython_pattern.sub(
r'\n# \g<1>get_ipython()', script_code)
requirements_txt = ''
conda_requirements = ''
# parse jupyter python script and prepare pip requirements (pigar)
# if backend supports requirements
if file_import_modules and Session.check_min_api_version(
'2.2'):
if fmodules is None:
fmodules, _ = file_import_modules(
notebook.parts[-1] if notebook else 'notebook',
script_code)
if current_cell:
cell_fmodules, _ = file_import_modules(
notebook.parts[-1] if notebook else 'notebook',
current_cell)
# noinspection PyBroadException
try:
fmodules |= cell_fmodules
except Exception:
pass
# add current cell to the script
if current_cell:
script_code += '\n' + current_cell
fmodules = ScriptRequirements.add_trains_used_packages(
fmodules)
# noinspection PyUnboundLocalVariable
installed_pkgs = get_installed_pkgs_detail()
# make sure we are in installed packages
if our_module and (our_module[0] not in installed_pkgs):
installed_pkgs[our_module[0]] = our_module
# noinspection PyUnboundLocalVariable
reqs = ReqsModules()
for name in fmodules:
if name in installed_pkgs:
pkg_name, version = installed_pkgs[name]
reqs.add(pkg_name, version, fmodules[name])
requirements_txt, conda_requirements = ScriptRequirements.create_requirements_txt(
reqs)
# update script
prev_script_hash = current_script_hash
data_script = task.data.script
data_script.diff = script_code
data_script.requirements = {
'pip': requirements_txt,
'conda': conda_requirements
}
# noinspection PyProtectedMember
task._update_script(script=data_script)
# update requirements
# noinspection PyProtectedMember
task._update_requirements(requirements=requirements_txt)
except Exception:
pass
def main():
print('TRAINS-AGENT setup process')
conf_file = Path(LOCAL_CONFIG_FILES[0]).absolute()
if conf_file.exists() and conf_file.is_file(
) and conf_file.stat().st_size > 0:
print('Configuration file already exists: {}'.format(str(conf_file)))
print('Leaving setup, feel free to edit the configuration file.')
return
print(description, end='')
sentinel = ''
parse_input = '\n'.join(iter(input, sentinel))
credentials = None
api_server = None
web_server = None
# noinspection PyBroadException
try:
parsed = ConfigFactory.parse_string(parse_input)
if parsed:
# Take the credentials in raw form or from api section
credentials = get_parsed_field(parsed, ["credentials"])
api_server = get_parsed_field(parsed, ["api_server", "host"])
web_server = get_parsed_field(parsed, ["web_server"])
except Exception:
credentials = credentials or None
api_server = api_server or None
web_server = web_server or None
while not credentials or set(credentials) != {"access_key", "secret_key"}:
print(
'Could not parse credentials, please try entering them manually.')
credentials = read_manual_credentials()
print('Detected credentials key=\"{}\" secret=\"{}\"'.format(
credentials['access_key'], credentials['secret_key'][0:4] + "***"))
web_input = True
if web_server:
host = input_url('WEB Host', web_server)
elif api_server:
web_input = False
host = input_url('API Host', api_server)
else:
print(host_description)
host = input_url('WEB Host', '')
parsed_host = verify_url(host)
api_host, files_host, web_host = parse_host(parsed_host, allow_input=True)
# on of these two we configured
if not web_input:
web_host = input_url('Web Application Host', web_host)
else:
api_host = input_url('API Host', api_host)
files_host = input_url('File Store Host', files_host)
print(
'\nTRAINS Hosts configuration:\nWeb App: {}\nAPI: {}\nFile Store: {}\n'
.format(web_host, api_host, files_host))
retry = 1
max_retries = 2
while retry <= max_retries: # Up to 2 tries by the user
if verify_credentials(api_host, credentials):
break
retry += 1
if retry < max_retries + 1:
credentials = read_manual_credentials()
else:
print('Exiting setup without creating configuration file')
return
# get GIT User/Pass for cloning
print(
'Enter git username for repository cloning (leave blank for SSH key authentication): [] ',
end='')
git_user = input()
if git_user.strip():
print('Enter password for user \'{}\': '.format(git_user), end='')
git_pass = input()
print(
'Git repository cloning will be using user={} password={}'.format(
git_user, git_pass))
else:
git_user = None
git_pass = None
# get extra-index-url for pip installations
extra_index_urls = []
print(
'\nEnter additional artifact repository (extra-index-url) to use when installing python packages '
'(leave blank if not required):',
end='')
index_url = input().strip()
while index_url:
extra_index_urls.append(index_url)
print(
'Another artifact repository? (enter another url or leave blank if done):',
end='')
index_url = input().strip()
if len(extra_index_urls):
print("The following artifact repositories will be added:\n\t- {}".
format("\n\t- ".join(extra_index_urls)))
# noinspection PyBroadException
try:
conf_folder = Path(__file__).parent.absolute(
) / '..' / 'backend_api' / 'config' / 'default'
default_conf = ''
for conf in (
'agent.conf',
'sdk.conf',
):
conf_file_section = conf_folder / conf
with open(str(conf_file_section), 'rt') as f:
default_conf += conf.split('.')[0] + ' '
default_conf += f.read()
default_conf += '\n'
except Exception:
print('Error! Could not read default configuration file')
return
# noinspection PyBroadException
try:
with open(str(conf_file), 'wt') as f:
header = '# TRAINS-AGENT configuration file\n' \
'api {\n' \
' api_server: %s\n' \
' web_server: %s\n' \
' files_server: %s\n' \
' # Credentials are generated using the webapp, %s/profile\n' \
' # Override with os environment: TRAINS_API_ACCESS_KEY / TRAINS_API_SECRET_KEY\n' \
' credentials {"access_key": "%s", "secret_key": "%s"}\n' \
'}\n\n' % (api_host, web_host, files_host,
web_host, credentials['access_key'], credentials['secret_key'])
f.write(header)
git_credentials = '# Set GIT user/pass credentials\n' \
'# leave blank for GIT SSH credentials\n' \
'agent.git_user=\"{}\"\n' \
'agent.git_pass=\"{}\"\n' \
'\n'.format(git_user or '', git_pass or '')
f.write(git_credentials)
extra_index_str = '# extra_index_url: ["https://allegroai.jfrog.io/trainsai/api/pypi/public/simple"]\n' \
'agent.package_manager.extra_index_url= ' \
'[\n{}\n]\n\n'.format("\n".join(map("\"{}\"".format, extra_index_urls)))
f.write(extra_index_str)
f.write(default_conf)
except Exception:
print('Error! Could not write configuration file at: {}'.format(
str(conf_file)))
return
print('\nNew configuration stored in {}'.format(str(conf_file)))
print('TRAINS-AGENT setup completed successfully.')
assert Path(
args.model, cfg.invel + "_" +
str(i)).stat().st_size == cfg.x[i] * cfg.y[i] * cfg.z[
i] * cfg.nvar * nbit_float, f"Layer-{i}: Mismatch of mesh size"
if i + 1 < cfg.g:
# Check consistency of adjcent meshes
check_mesh_cont(Path(args.model, cfg.invel + "_" + str(i)),
Path(args.model, cfg.invel + "_" + str(i + 1)),
cfg.x[i], cfg.y[i], cfg.z[i])
# Topography
if cfg.intopo:
file_topo = Path(args.model, cfg.intopo)
nx, ny, pad = np.fromfile(file_topo, dtype='int32', count=3)
assert nx == cfg.x[0] and ny == cfg.y[0], f"Mismatch topography domain size"
assert (nx + 2 * pad) * (ny + 2 * pad) * nbit_float == file_topo.stat(
).st_size, f"Topography size does not match parameters"
# Receivers
if cfg.recvfile:
assert Path(args.model, cfg.recvfile).parent.exists(
), f"Receiver output directory does not exist"
assert cfg.recv_steps % (cfg.recv_stride * cfg.recv_cpu_buffer_size \
* cfg.recv_gpu_buffer_size * cfg.recv_num_writes) == 0, "Check divisibility of receiver writing"
assert cfg.recv_length <= len(
cfg.recv_coords), f"More receivers required than given"
# Source files in Ossian's format
if cfg.sourcefile:
assert Path(args.model,
cfg.sourcefile).parent.exists(), f"Source file doesn't exist"
assert cfg.src_steps % (cfg.src_stride * cfg.src_cpu_buffer_size \
def upload_artifact(self, name, artifact_object=None, metadata=None, delete_after_upload=False):
if not Session.check_min_api_version('2.3'):
LoggerRoot.get_base_logger().warning('Artifacts not supported by your TRAINS-server version, '
'please upgrade to the latest server version')
return False
if name in self._artifacts_dict:
raise ValueError("Artifact by the name of {} is already registered, use register_artifact".format(name))
artifact_type_data = tasks.ArtifactTypeData()
use_filename_in_uri = True
if np and isinstance(artifact_object, np.ndarray):
artifact_type = 'numpy'
artifact_type_data.content_type = 'application/numpy'
artifact_type_data.preview = str(artifact_object.__repr__())
fd, local_filename = mkstemp(suffix='.npz')
os.close(fd)
np.savez_compressed(local_filename, **{name: artifact_object})
delete_after_upload = True
use_filename_in_uri = False
elif pd and isinstance(artifact_object, pd.DataFrame):
artifact_type = 'pandas'
artifact_type_data.content_type = 'text/csv'
artifact_type_data.preview = str(artifact_object.__repr__())
fd, local_filename = mkstemp(suffix=self._save_format)
os.close(fd)
artifact_object.to_csv(local_filename, compression=self._compression)
delete_after_upload = True
use_filename_in_uri = False
elif isinstance(artifact_object, Image.Image):
artifact_type = 'image'
artifact_type_data.content_type = 'image/png'
desc = str(artifact_object.__repr__())
artifact_type_data.preview = desc[1:desc.find(' at ')]
fd, local_filename = mkstemp(suffix='.png')
os.close(fd)
artifact_object.save(local_filename)
delete_after_upload = True
use_filename_in_uri = False
elif isinstance(artifact_object, dict):
artifact_type = 'JSON'
artifact_type_data.content_type = 'application/json'
preview = json.dumps(artifact_object, sort_keys=True, indent=4)
fd, local_filename = mkstemp(suffix='.json')
os.write(fd, bytes(preview.encode()))
os.close(fd)
artifact_type_data.preview = preview
delete_after_upload = True
use_filename_in_uri = False
elif isinstance(artifact_object, six.string_types) or isinstance(artifact_object, Path):
if isinstance(artifact_object, Path):
artifact_object = artifact_object.as_posix()
artifact_type = 'custom'
artifact_type_data.content_type = mimetypes.guess_type(artifact_object)[0]
local_filename = artifact_object
else:
raise ValueError("Artifact type {} not supported".format(type(artifact_object)))
# remove from existing list, if exists
for artifact in self._task_artifact_list:
if artifact.key == name:
if artifact.type == self._pd_artifact_type:
raise ValueError("Artifact of name {} already registered, "
"use register_artifact instead".format(name))
self._task_artifact_list.remove(artifact)
break
# check that the file to upload exists
local_filename = Path(local_filename).absolute()
if not local_filename.exists() or not local_filename.is_file():
LoggerRoot.get_base_logger().warning('Artifact upload failed, cannot find file {}'.format(
local_filename.as_posix()))
return False
file_hash, _ = self.sha256sum(local_filename.as_posix())
timestamp = int(time())
file_size = local_filename.stat().st_size
uri = self._upload_local_file(local_filename, name,
delete_after_upload=delete_after_upload, use_filename=use_filename_in_uri)
artifact = tasks.Artifact(key=name, type=artifact_type,
uri=uri,
content_size=file_size,
hash=file_hash,
timestamp=timestamp,
type_data=artifact_type_data,
display_data=[(str(k), str(v)) for k, v in metadata.items()] if metadata else None)
# update task artifacts
with self._task_edit_lock:
self._task_artifact_list.append(artifact)
self._task.set_artifacts(self._task_artifact_list)
return True
def _daemon(cls, jupyter_notebook_filename):
from trains import Task
# load jupyter notebook package
# noinspection PyBroadException
try:
from nbconvert.exporters.script import ScriptExporter
_script_exporter = ScriptExporter()
except Exception:
return
# load pigar
# noinspection PyBroadException
try:
from pigar.reqs import get_installed_pkgs_detail, file_import_modules
from pigar.modules import ReqsModules
from pigar.log import logger
logger.setLevel(logging.WARNING)
except Exception:
file_import_modules = None
# main observer loop
notebook = Path(jupyter_notebook_filename)
last_update_ts = None
counter = 0
prev_script_hash = None
while True:
if cls._exit_event.wait(cls._sample_frequency if counter else cls.
_first_sample_frequency):
return
counter += 1
# noinspection PyBroadException
try:
if not notebook.exists():
continue
# check if notebook changed
if last_update_ts is not None and notebook.stat(
).st_mtime - last_update_ts <= 0:
continue
last_update_ts = notebook.stat().st_mtime
task = Task.current_task()
if not task:
continue
# get notebook python script
script_code, resources = _script_exporter.from_filename(
jupyter_notebook_filename)
current_script_hash = hash(script_code)
if prev_script_hash and prev_script_hash == current_script_hash:
continue
requirements_txt = ''
# parse jupyter python script and prepare pip requirements (pigar)
# if backend supports requirements
if file_import_modules and Session.api_version > '2.1':
fmodules, _ = file_import_modules(notebook.parts[-1],
script_code)
installed_pkgs = get_installed_pkgs_detail()
reqs = ReqsModules()
for name in fmodules:
if name in installed_pkgs:
pkg_name, version = installed_pkgs[name]
reqs.add(pkg_name, version, fmodules[name])
requirements_txt = ScriptRequirements.create_requirements_txt(
reqs)
# update script
prev_script_hash = current_script_hash
data_script = task.data.script
data_script.diff = script_code
data_script.requirements = {'pip': requirements_txt}
task._update_script(script=data_script)
# update requirements
if requirements_txt:
task._update_requirements(requirements=requirements_txt)
except Exception:
pass
def _daemon(cls, jupyter_notebook_filename):
from trains import Task
# load jupyter notebook package
# noinspection PyBroadException
try:
from nbconvert.exporters.script import ScriptExporter
_script_exporter = ScriptExporter()
except Exception:
return
# load pigar
# noinspection PyBroadException
try:
from pigar.reqs import get_installed_pkgs_detail, file_import_modules
from pigar.modules import ReqsModules
from pigar.log import logger
logger.setLevel(logging.WARNING)
except Exception:
file_import_modules = None
# load IPython
# noinspection PyBroadException
try:
from IPython import get_ipython
except Exception:
# should not happen
get_ipython = None
# setup local notebook files
if jupyter_notebook_filename:
notebook = Path(jupyter_notebook_filename)
local_jupyter_filename = jupyter_notebook_filename
else:
notebook = None
fd, local_jupyter_filename = mkstemp(suffix='.ipynb')
os.close(fd)
last_update_ts = None
counter = 0
prev_script_hash = None
# main observer loop, check if we need to exit
while not cls._exit_event.wait(timeout=0.):
# wait for timeout or sync event
cls._sync_event.wait(cls._sample_frequency if counter else cls.
_first_sample_frequency)
cls._sync_event.clear()
counter += 1
# noinspection PyBroadException
try:
# if there is no task connected, do nothing
task = Task.current_task()
if not task:
continue
# if we have a local file:
if notebook:
if not notebook.exists():
continue
# check if notebook changed
if last_update_ts is not None and notebook.stat(
).st_mtime - last_update_ts <= 0:
continue
last_update_ts = notebook.stat().st_mtime
else:
# serialize notebook to a temp file
# noinspection PyBroadException
try:
get_ipython().run_line_magic('notebook',
local_jupyter_filename)
except Exception as ex:
continue
# get notebook python script
script_code, resources = _script_exporter.from_filename(
local_jupyter_filename)
current_script_hash = hash(script_code)
if prev_script_hash and prev_script_hash == current_script_hash:
continue
requirements_txt = ''
conda_requirements = ''
# parse jupyter python script and prepare pip requirements (pigar)
# if backend supports requirements
if file_import_modules and Session.check_min_api_version(
'2.2'):
fmodules, _ = file_import_modules(notebook.parts[-1],
script_code)
fmodules = ScriptRequirements.add_trains_used_packages(
fmodules)
installed_pkgs = get_installed_pkgs_detail()
reqs = ReqsModules()
for name in fmodules:
if name in installed_pkgs:
pkg_name, version = installed_pkgs[name]
reqs.add(pkg_name, version, fmodules[name])
requirements_txt, conda_requirements = ScriptRequirements.create_requirements_txt(
reqs)
# update script
prev_script_hash = current_script_hash
data_script = task.data.script
data_script.diff = script_code
data_script.requirements = {
'pip': requirements_txt,
'conda': conda_requirements
}
task._update_script(script=data_script)
# update requirements
task._update_requirements(requirements=requirements_txt)
except Exception:
pass
def main():
print('TRAINS SDK setup process')
conf_file = Path(LOCAL_CONFIG_FILES[0]).absolute()
if conf_file.exists() and conf_file.is_file(
) and conf_file.stat().st_size > 0:
print('Configuration file already exists: {}'.format(str(conf_file)))
print('Leaving setup, feel free to edit the configuration file.')
return
print(host_description)
web_host = input_url('Web Application Host', '')
parsed_host = verify_url(web_host)
if parsed_host.port == 8008:
print(
'Port 8008 is the api port. Replacing 8080 with 8008 for Web application'
)
api_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path
web_host = parsed_host.scheme + "://" + parsed_host.netloc.replace(
':8008', ':8080', 1) + parsed_host.path
files_host = parsed_host.scheme + "://" + parsed_host.netloc.replace(
':8008', ':8081', 1) + parsed_host.path
elif parsed_host.port == 8080:
api_host = parsed_host.scheme + "://" + parsed_host.netloc.replace(
':8080', ':8008', 1) + parsed_host.path
web_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path
files_host = parsed_host.scheme + "://" + parsed_host.netloc.replace(
':8080', ':8081', 1) + parsed_host.path
elif parsed_host.netloc.startswith('demoapp.'):
# this is our demo server
api_host = parsed_host.scheme + "://" + parsed_host.netloc.replace(
'demoapp.', 'demoapi.', 1) + parsed_host.path
web_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path
files_host = parsed_host.scheme + "://" + parsed_host.netloc.replace(
'demoapp.', 'demofiles.', 1) + parsed_host.path
elif parsed_host.netloc.startswith('app.'):
# this is our application server
api_host = parsed_host.scheme + "://" + parsed_host.netloc.replace(
'app.', 'api.', 1) + parsed_host.path
web_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path
files_host = parsed_host.scheme + "://" + parsed_host.netloc.replace(
'app.', 'files.', 1) + parsed_host.path
elif parsed_host.netloc.startswith('demoapi.'):
print(
'{} is the api server, we need the web server. Replacing \'demoapi.\' with \'demoapp.\''
.format(parsed_host.netloc))
api_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path
web_host = parsed_host.scheme + "://" + parsed_host.netloc.replace(
'demoapi.', 'demoapp.', 1) + parsed_host.path
files_host = parsed_host.scheme + "://" + parsed_host.netloc.replace(
'demoapi.', 'demofiles.', 1) + parsed_host.path
elif parsed_host.netloc.startswith('api.'):
print(
'{} is the api server, we need the web server. Replacing \'api.\' with \'app.\''
.format(parsed_host.netloc))
api_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path
web_host = parsed_host.scheme + "://" + parsed_host.netloc.replace(
'api.', 'app.', 1) + parsed_host.path
files_host = parsed_host.scheme + "://" + parsed_host.netloc.replace(
'api.', 'files.', 1) + parsed_host.path
else:
api_host = ''
web_host = ''
files_host = ''
if not parsed_host.port:
print(
'Host port not detected, do you wish to use the default 8080 port n/[y]? ',
end='')
replace_port = input().lower()
if not replace_port or replace_port == 'y' or replace_port == 'yes':
api_host = parsed_host.scheme + "://" + parsed_host.netloc + ':8008' + parsed_host.path
web_host = parsed_host.scheme + "://" + parsed_host.netloc + ':8080' + parsed_host.path
files_host = parsed_host.scheme + "://" + parsed_host.netloc + ':8081' + parsed_host.path
elif not replace_port or replace_port.lower(
) == 'n' or replace_port.lower() == 'no':
web_host = input_host_port("Web", parsed_host)
api_host = input_host_port("API", parsed_host)
files_host = input_host_port("Files", parsed_host)
if not api_host:
api_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path
api_host = input_url('API Host', api_host)
files_host = input_url('File Store Host', files_host)
print(
'\nTRAINS Hosts configuration:\nWeb App: {}\nAPI: {}\nFile Store: {}\n'
.format(web_host, api_host, files_host))
while True:
print(description.format(web_host), end='')
parse_input = input()
# check if these are valid credentials
credentials = None
# noinspection PyBroadException
try:
parsed = ConfigFactory.parse_string(parse_input)
if parsed:
credentials = parsed.get("credentials", None)
except Exception:
credentials = None
if not credentials or set(credentials) != {"access_key", "secret_key"}:
print(
'Could not parse user credentials, try again one after the other.'
)
credentials = {}
# parse individual
print('Enter user access key: ', end='')
credentials['access_key'] = input()
print('Enter user secret: ', end='')
credentials['secret_key'] = input()
print('Detected credentials key=\"{}\" secret=\"{}\"'.format(
credentials['access_key'],
credentials['secret_key'],
))
from trains.backend_api.session import Session
# noinspection PyBroadException
try:
print('Verifying credentials ...')
Session(api_key=credentials['access_key'],
secret_key=credentials['secret_key'],
host=api_host)
print('Credentials verified!')
break
except Exception:
print(
'Error: could not verify credentials: host={} access={} secret={}'
.format(api_host, credentials['access_key'],
credentials['secret_key']))
# noinspection PyBroadException
try:
default_sdk_conf = Path(__file__).parent.absolute() / 'sdk.conf'
with open(str(default_sdk_conf), 'rt') as f:
default_sdk = f.read()
except Exception:
print('Error! Could not read default configuration file')
return
# noinspection PyBroadException
try:
with open(str(conf_file), 'wt') as f:
header = '# TRAINS SDK configuration file\n' \
'api {\n' \
' # Notice: \'host\' is the api server (default port 8008), not the web server.\n' \
' api_server: %s\n' \
' web_server: %s\n' \
' files_server: %s\n' \
' # Credentials are generated using the webapp, %s/profile\n' \
' credentials {"access_key": "%s", "secret_key": "%s"}\n' \
'}\n' \
'sdk ' % (api_host, web_host, files_host,
web_host, credentials['access_key'], credentials['secret_key'])
f.write(header)
f.write(default_sdk)
except Exception:
print('Error! Could not write configuration file at: {}'.format(
str(conf_file)))
return
print('\nNew configuration stored in {}'.format(str(conf_file)))
print('TRAINS setup completed successfully.')
def main():
default_config_file = os.getenv(LOCAL_CONFIG_FILE_OVERRIDE_VAR) or LOCAL_CONFIG_FILES[0]
p = argparse.ArgumentParser(description=__doc__)
p.add_argument(
"--file", "-F", help="Target configuration file path (default is %(default)s)",
default=default_config_file,
type=validate_file
)
args = p.parse_args()
print('TRAINS SDK setup process')
conf_file = Path(args.file).absolute()
if conf_file.exists() and conf_file.is_file() and conf_file.stat().st_size > 0:
print('Configuration file already exists: {}'.format(str(conf_file)))
print('Leaving setup, feel free to edit the configuration file.')
return
print(description, end='')
sentinel = ''
parse_input = '\n'.join(iter(input, sentinel))
credentials = None
api_server = None
web_server = None
# noinspection PyBroadException
try:
parsed = ConfigFactory.parse_string(parse_input)
if parsed:
# Take the credentials in raw form or from api section
credentials = get_parsed_field(parsed, ["credentials"])
api_server = get_parsed_field(parsed, ["api_server", "host"])
web_server = get_parsed_field(parsed, ["web_server"])
except Exception:
credentials = credentials or None
api_server = api_server or None
web_server = web_server or None
while not credentials or set(credentials) != {"access_key", "secret_key"}:
print('Could not parse credentials, please try entering them manually.')
credentials = read_manual_credentials()
print('Detected credentials key=\"{}\" secret=\"{}\"'.format(credentials['access_key'],
credentials['secret_key'][0:4] + "***"))
web_input = True
if web_server:
host = input_url('WEB Host', web_server)
elif api_server:
web_input = False
host = input_url('API Host', api_server)
else:
print(host_description.format(CONFIG_FILE=args.file, HOST=def_host,))
host = input_url('WEB Host', '')
parsed_host = verify_url(host)
api_host, files_host, web_host = parse_host(parsed_host, allow_input=True)
# on of these two we configured
if not web_input:
web_host = input_url('Web Application Host', web_host)
else:
api_host = input_url('API Host', api_host)
files_host = input_url('File Store Host', files_host)
print('\nTRAINS Hosts configuration:\nWeb App: {}\nAPI: {}\nFile Store: {}\n'.format(
web_host, api_host, files_host))
retry = 1
max_retries = 2
while retry <= max_retries: # Up to 2 tries by the user
if verify_credentials(api_host, credentials):
break
retry += 1
if retry < max_retries + 1:
credentials = read_manual_credentials()
else:
print('Exiting setup without creating configuration file')
return
# noinspection PyBroadException
try:
default_sdk_conf = Path(__file__).parent.absolute() / 'sdk.conf'
with open(str(default_sdk_conf), 'rt') as f:
default_sdk = f.read()
except Exception:
print('Error! Could not read default configuration file')
return
# noinspection PyBroadException
try:
with open(str(conf_file), 'wt') as f:
header = '# TRAINS SDK configuration file\n' \
'api {\n' \
' # Notice: \'host\' is the api server (default port 8008), not the web server.\n' \
' api_server: %s\n' \
' web_server: %s\n' \
' files_server: %s\n' \
' # Credentials are generated using the webapp, %s/profile\n' \
' credentials {"access_key": "%s", "secret_key": "%s"}\n' \
'}\n' \
'sdk ' % (api_host, web_host, files_host,
web_host, credentials['access_key'], credentials['secret_key'])
f.write(header)
f.write(default_sdk)
except Exception:
print('Error! Could not write configuration file at: {}'.format(str(conf_file)))
return
print('\nNew configuration stored in {}'.format(str(conf_file)))
print('TRAINS setup completed successfully.')
def upload_artifact(self,
name,
artifact_object=None,
metadata=None,
preview=None,
delete_after_upload=False,
auto_pickle=True):
# type: (str, Optional[object], Optional[dict], Optional[str], bool, bool) -> bool
if not Session.check_min_api_version('2.3'):
LoggerRoot.get_base_logger().warning(
'Artifacts not supported by your TRAINS-server version, '
'please upgrade to the latest server version')
return False
if name in self._artifacts_container:
raise ValueError(
"Artifact by the name of {} is already registered, use register_artifact"
.format(name))
# cast preview to string
if preview:
preview = str(preview)
# convert string to object if try is a file/folder (dont try to serialize long texts
if isinstance(artifact_object,
six.string_types) and len(artifact_object) < 2048:
# noinspection PyBroadException
try:
artifact_path = Path(artifact_object)
if artifact_path.exists():
artifact_object = artifact_path
elif '*' in artifact_object or '?' in artifact_object:
# hackish, detect wildcard in tr files
folder = Path('').joinpath(*artifact_path.parts[:-1])
if folder.is_dir() and folder.parts:
wildcard = artifact_path.parts[-1]
if list(Path(folder).rglob(wildcard)):
artifact_object = artifact_path
except Exception:
pass
artifact_type_data = tasks.ArtifactTypeData()
artifact_type_data.preview = ''
override_filename_in_uri = None
override_filename_ext_in_uri = None
uri = None
if np and isinstance(artifact_object, np.ndarray):
artifact_type = 'numpy'
artifact_type_data.content_type = 'application/numpy'
artifact_type_data.preview = preview or str(
artifact_object.__repr__())
override_filename_ext_in_uri = '.npz'
override_filename_in_uri = name + override_filename_ext_in_uri
fd, local_filename = mkstemp(prefix=quote(name, safe="") + '.',
suffix=override_filename_ext_in_uri)
os.close(fd)
np.savez_compressed(local_filename, **{name: artifact_object})
delete_after_upload = True
elif pd and isinstance(artifact_object, pd.DataFrame):
artifact_type = 'pandas'
artifact_type_data.content_type = 'text/csv'
artifact_type_data.preview = preview or str(
artifact_object.__repr__())
override_filename_ext_in_uri = self._save_format
override_filename_in_uri = name
fd, local_filename = mkstemp(prefix=quote(name, safe="") + '.',
suffix=override_filename_ext_in_uri)
os.close(fd)
artifact_object.to_csv(local_filename,
compression=self._compression)
delete_after_upload = True
elif isinstance(artifact_object, Image.Image):
artifact_type = 'image'
artifact_type_data.content_type = 'image/png'
desc = str(artifact_object.__repr__())
artifact_type_data.preview = preview or desc[1:desc.find(' at ')]
override_filename_ext_in_uri = '.png'
override_filename_in_uri = name + override_filename_ext_in_uri
fd, local_filename = mkstemp(prefix=quote(name, safe="") + '.',
suffix=override_filename_ext_in_uri)
os.close(fd)
artifact_object.save(local_filename)
delete_after_upload = True
elif isinstance(artifact_object, dict):
artifact_type = 'JSON'
artifact_type_data.content_type = 'application/json'
preview = preview or json.dumps(
artifact_object, sort_keys=True, indent=4)
override_filename_ext_in_uri = '.json'
override_filename_in_uri = name + override_filename_ext_in_uri
fd, local_filename = mkstemp(prefix=quote(name, safe="") + '.',
suffix=override_filename_ext_in_uri)
os.write(fd, bytes(preview.encode()))
os.close(fd)
if len(preview) < self.max_preview_size_bytes:
artifact_type_data.preview = preview
else:
artifact_type_data.preview = '# full json too large to store, storing first {}kb\n{}'.format(
self.max_preview_size_bytes // 1024,
preview[:self.max_preview_size_bytes])
delete_after_upload = True
elif isinstance(artifact_object, (
Path,
pathlib_Path,
) if pathlib_Path is not None else (Path, )):
# check if single file
artifact_object = Path(artifact_object)
artifact_object.expanduser().absolute()
# noinspection PyBroadException
try:
create_zip_file = not artifact_object.is_file()
except Exception: # Hack for windows pathlib2 bug, is_file isn't valid.
create_zip_file = True
else: # We assume that this is not Windows os
if artifact_object.is_dir():
# change to wildcard
artifact_object /= '*'
if create_zip_file:
folder = Path('').joinpath(*artifact_object.parts[:-1])
if not folder.is_dir() or not folder.parts:
raise ValueError(
"Artifact file/folder '{}' could not be found".format(
artifact_object.as_posix()))
wildcard = artifact_object.parts[-1]
files = list(Path(folder).rglob(wildcard))
override_filename_ext_in_uri = '.zip'
override_filename_in_uri = folder.parts[
-1] + override_filename_ext_in_uri
fd, zip_file = mkstemp(
prefix=quote(folder.parts[-1], safe="") + '.',
suffix=override_filename_ext_in_uri)
try:
artifact_type_data.content_type = 'application/zip'
archive_preview = 'Archive content {}:\n'.format(
artifact_object.as_posix())
with ZipFile(zip_file,
'w',
allowZip64=True,
compression=ZIP_DEFLATED) as zf:
for filename in sorted(files):
if filename.is_file():
relative_file_name = filename.relative_to(
folder).as_posix()
archive_preview += '{} - {}\n'.format(
relative_file_name,
humanfriendly.format_size(
filename.stat().st_size))
zf.write(filename.as_posix(),
arcname=relative_file_name)
except Exception as e:
# failed uploading folder:
LoggerRoot.get_base_logger().warning(
'Exception {}\nFailed zipping artifact folder {}'.
format(folder, e))
return False
finally:
os.close(fd)
artifact_type_data.preview = preview or archive_preview
artifact_object = zip_file
artifact_type = 'archive'
artifact_type_data.content_type = mimetypes.guess_type(
artifact_object)[0]
local_filename = artifact_object
delete_after_upload = True
else:
if not artifact_object.is_file():
raise ValueError(
"Artifact file '{}' could not be found".format(
artifact_object.as_posix()))
override_filename_in_uri = artifact_object.parts[-1]
artifact_type_data.preview = preview or '{} - {}\n'.format(
artifact_object,
humanfriendly.format_size(artifact_object.stat().st_size))
artifact_object = artifact_object.as_posix()
artifact_type = 'custom'
artifact_type_data.content_type = mimetypes.guess_type(
artifact_object)[0]
local_filename = artifact_object
elif (isinstance(artifact_object, six.string_types)
and len(artifact_object) < 4096
and urlparse(artifact_object).scheme in remote_driver_schemes):
# we should not upload this, just register
local_filename = None
uri = artifact_object
artifact_type = 'custom'
artifact_type_data.content_type = mimetypes.guess_type(
artifact_object)[0]
elif isinstance(artifact_object, six.string_types):
# if we got here, we should store it as text file.
artifact_type = 'string'
artifact_type_data.content_type = 'text/plain'
if preview:
artifact_type_data.preview = preview
elif len(artifact_object) < self.max_preview_size_bytes:
artifact_type_data.preview = artifact_object
else:
artifact_type_data.preview = '# full text too large to store, storing first {}kb\n{}'.format(
self.max_preview_size_bytes // 1024,
artifact_object[:self.max_preview_size_bytes])
delete_after_upload = True
override_filename_ext_in_uri = '.txt'
override_filename_in_uri = name + override_filename_ext_in_uri
fd, local_filename = mkstemp(prefix=quote(name, safe="") + '.',
suffix=override_filename_ext_in_uri)
os.close(fd)
# noinspection PyBroadException
try:
with open(local_filename, 'wt') as f:
f.write(artifact_object)
except Exception:
# cleanup and raise exception
os.unlink(local_filename)
raise
elif auto_pickle:
# if we are here it means we do not know what to do with the object, so we serialize it with pickle.
artifact_type = 'pickle'
artifact_type_data.content_type = 'application/pickle'
# noinspection PyBroadException
try:
artifact_type_data.preview = preview or str(
artifact_object.__repr__())[:self.max_preview_size_bytes]
except Exception:
artifact_type_data.preview = preview or ''
delete_after_upload = True
override_filename_ext_in_uri = '.pkl'
override_filename_in_uri = name + override_filename_ext_in_uri
fd, local_filename = mkstemp(prefix=quote(name, safe="") + '.',
suffix=override_filename_ext_in_uri)
os.close(fd)
# noinspection PyBroadException
try:
with open(local_filename, 'wb') as f:
pickle.dump(artifact_object, f)
except Exception:
# cleanup and raise exception
os.unlink(local_filename)
raise
else:
raise ValueError("Artifact type {} not supported".format(
type(artifact_object)))
# remove from existing list, if exists
for artifact in self._task_artifact_list:
if artifact.key == name:
if artifact.type == self._pd_artifact_type:
raise ValueError(
"Artifact of name {} already registered, "
"use register_artifact instead".format(name))
self._task_artifact_list.remove(artifact)
break
if not local_filename:
file_size = None
file_hash = None
else:
# check that the file to upload exists
local_filename = Path(local_filename).absolute()
if not local_filename.exists() or not local_filename.is_file():
LoggerRoot.get_base_logger().warning(
'Artifact upload failed, cannot find file {}'.format(
local_filename.as_posix()))
return False
file_hash, _ = self.sha256sum(local_filename.as_posix())
file_size = local_filename.stat().st_size
uri = self._upload_local_file(
local_filename,
name,
delete_after_upload=delete_after_upload,
override_filename=override_filename_in_uri,
override_filename_ext=override_filename_ext_in_uri)
timestamp = int(time())
artifact = tasks.Artifact(
key=name,
type=artifact_type,
uri=uri,
content_size=file_size,
hash=file_hash,
timestamp=timestamp,
type_data=artifact_type_data,
display_data=[(str(k), str(v))
for k, v in metadata.items()] if metadata else None)
# update task artifacts
with self._task_edit_lock:
self._task_artifact_list.append(artifact)
self._task.set_artifacts(self._task_artifact_list)
return True
def _upload_data_audit_artifacts(self, name):
logger = self._task.get_logger()
pd_artifact = self._artifacts_container.get(name)
pd_metadata = self._artifacts_container.get_metadata(name)
# remove from artifacts watch list
if name in self._unregister_request:
try:
self._unregister_request.remove(name)
except KeyError:
pass
self._artifacts_container.unregister_artifact(name)
if pd_artifact is None:
return
override_filename_ext_in_uri = self._save_format
override_filename_in_uri = name
fd, local_csv = mkstemp(prefix=quote(name, safe="") + '.',
suffix=override_filename_ext_in_uri)
os.close(fd)
local_csv = Path(local_csv)
pd_artifact.to_csv(local_csv.as_posix(),
index=False,
compression=self._compression)
current_sha2, file_sha2 = self.sha256sum(local_csv.as_posix(),
skip_header=32)
if name in self._last_artifacts_upload:
previous_sha2 = self._last_artifacts_upload[name]
if previous_sha2 == current_sha2:
# nothing to do, we can skip the upload
try:
local_csv.unlink()
except Exception:
pass
return
self._last_artifacts_upload[name] = current_sha2
# If old trains-server, upload as debug image
if not Session.check_min_api_version('2.3'):
logger.report_image(title='artifacts',
series=name,
local_path=local_csv.as_posix(),
delete_after_upload=True,
iteration=self._task.get_last_iteration(),
max_image_history=2)
return
# Find our artifact
artifact = None
for an_artifact in self._task_artifact_list:
if an_artifact.key == name:
artifact = an_artifact
break
file_size = local_csv.stat().st_size
# upload file
uri = self._upload_local_file(
local_csv,
name,
delete_after_upload=True,
override_filename=override_filename_in_uri,
override_filename_ext=override_filename_ext_in_uri)
# update task artifacts
with self._task_edit_lock:
if not artifact:
artifact = tasks.Artifact(key=name,
type=self._pd_artifact_type)
self._task_artifact_list.append(artifact)
artifact_type_data = tasks.ArtifactTypeData()
artifact_type_data.data_hash = current_sha2
artifact_type_data.content_type = "text/csv"
artifact_type_data.preview = str(
pd_artifact.__repr__()) + '\n\n' + self._get_statistics(
{name: pd_artifact})
artifact.type_data = artifact_type_data
artifact.uri = uri
artifact.content_size = file_size
artifact.hash = file_sha2
artifact.timestamp = int(time())
artifact.display_data = [
(str(k), str(v)) for k, v in pd_metadata.items()
] if pd_metadata else None
self._task.set_artifacts(self._task_artifact_list)
def main():
default_config_file = LOCAL_CONFIG_FILE_OVERRIDE_VAR.get()
if not default_config_file:
for f in LOCAL_CONFIG_FILES:
default_config_file = f
if os.path.exists(os.path.expanduser(os.path.expandvars(f))):
break
p = argparse.ArgumentParser(description=__doc__)
p.add_argument(
"--file",
"-F",
help="Target configuration file path (default is %(default)s)",
default=default_config_file,
type=validate_file)
args = p.parse_args()
print('ClearML SDK setup process')
conf_file = Path(os.path.expanduser(args.file)).absolute()
if conf_file.exists() and conf_file.is_file(
) and conf_file.stat().st_size > 0:
print('Configuration file already exists: {}'.format(str(conf_file)))
print('Leaving setup, feel free to edit the configuration file.')
return
print(description, end='')
sentinel = ''
parse_input = ''
# COLAB_GPU will always be available, even when running on CPU
if os.environ.get('COLAB_GPU'):
# When running from a colab instance and calling clearml-init
# colab will squish the api credentials into a single line
# The regex splits this single line based on 2 spaces or more
import re
api_input = input()
parse_input = '\n'.join(re.split(r" {2,}", api_input))
else:
for line in iter(input, sentinel):
parse_input += line + '\n'
if line.rstrip() == '}':
break
credentials = None
api_server = None
web_server = None
# noinspection PyBroadException
try:
parsed = ConfigFactory.parse_string(parse_input)
if parsed:
# Take the credentials in raw form or from api section
credentials = get_parsed_field(parsed, ["credentials"])
api_server = get_parsed_field(parsed, ["api_server", "host"])
web_server = get_parsed_field(parsed, ["web_server"])
except Exception:
credentials = credentials or None
api_server = api_server or None
web_server = web_server or None
while not credentials or set(credentials) != {"access_key", "secret_key"}:
print(
'Could not parse credentials, please try entering them manually.')
credentials = read_manual_credentials()
print('Detected credentials key=\"{}\" secret=\"{}\"'.format(
credentials['access_key'], credentials['secret_key'][0:4] + "***"))
web_input = True
if web_server:
host = input_url('WEB Host', web_server)
elif api_server:
web_input = False
host = input_url('API Host', api_server)
else:
print(host_description.format(
CONFIG_FILE=args.file,
HOST=def_host,
))
host = input_url('WEB Host', '')
parsed_host = verify_url(host)
api_host, files_host, web_host = parse_host(parsed_host, allow_input=True)
# on of these two we configured
if not web_input:
web_host = input_url('Web Application Host', web_host)
else:
if web_input is True and not web_host:
web_host = host
api_host = input_url('API Host', api_host)
files_host = input_url('File Store Host', files_host)
print(
'\nClearML Hosts configuration:\nWeb App: {}\nAPI: {}\nFile Store: {}\n'
.format(web_host, api_host, files_host))
retry = 1
max_retries = 2
while retry <= max_retries: # Up to 2 tries by the user
if verify_credentials(api_host, credentials):
break
retry += 1
if retry < max_retries + 1:
credentials = read_manual_credentials()
else:
print('Exiting setup without creating configuration file')
return
# noinspection PyBroadException
try:
default_sdk_conf = Path(
__file__).absolute().parents[2] / 'config/default/sdk.conf'
with open(str(default_sdk_conf), 'rt') as f:
default_sdk = f.read()
except Exception:
print('Error! Could not read default configuration file')
return
# noinspection PyBroadException
try:
with open(str(conf_file), 'wt') as f:
header = '# ClearML SDK configuration file\n' \
'api {\n' \
' # Notice: \'host\' is the api server (default port 8008), not the web server.\n' \
' api_server: %s\n' \
' web_server: %s\n' \
' files_server: %s\n' \
' # Credentials are generated using the webapp, %s/settings\n' \
' # Override with os environment: CLEARML_API_ACCESS_KEY / CLEARML_API_SECRET_KEY\n' \
' credentials {"access_key": "%s", "secret_key": "%s"}\n' \
'}\n' \
'sdk ' % (api_host, web_host, files_host,
web_host, credentials['access_key'], credentials['secret_key'])
f.write(header)
f.write(default_sdk)
except Exception:
print('Error! Could not write configuration file at: {}'.format(
str(conf_file)))
return
print('\nNew configuration stored in {}'.format(str(conf_file)))
print('ClearML setup completed successfully.')
