def __init__(self, path: pathlib2.Path, depth: int = None, root: pathlib2.Path = None, children: Union[List, None] = None) -> None: """FileNode Initializer :param path: file path :type path: pathlib2.Path :param depth: depth of file/directory relative to root, defaults to None :type depth: int, optional :param root: root path, can be None, defaults to None :type root: pathlib2.Path, optional :param children: List of Node when current node is a directory, None if current node is a file, defaults to None :type children: Union[List, None], optional """ self._path = path self._root = root self._relative_path = self._path.relative_to( self._root) if self._root else None self._depth = depth self._filename = path.name self._stat = path.stat() self._children = children if children else [] self._id = UniqueFileIdentifier(self._path)
def rm_old_local_cached_dirs(cache_dir): """Remove old local cached directories, which were created four weeks ago. Args: cache_dir (Path): Full path to the cache directory """ assert isinstance(cache_dir, Path) cache_dir = cache_dir.expanduser() # This is in autobisectjs because it has a lock so we do not race while removing directories # Adapted from http://stackoverflow.com/a/11337407 SECONDS_IN_A_DAY = 24 * 60 * 60 s3CacheObj = s3cache.S3Cache(compile_shell.S3_SHELL_CACHE_DIRNAME) if s3CacheObj.connect(): NUMBER_OF_DAYS = 1 # EC2 VMs generally have less disk space for local shell caches else: NUMBER_OF_DAYS = 28 names = [cache_dir / x for x in cache_dir.iterdir()] for name in names: if name.is_dir(): timediff = time.mktime(time.gmtime()) - Path.stat(name).st_atime if timediff > SECONDS_IN_A_DAY * NUMBER_OF_DAYS: shutil.rmtree(str(name))
def __init__(self, path: pathlib2.Path): """Initialize UniqueFile Identifier by setting st_dev and st_ino :param path: a file path of type pathlib2.Path :type path: pathlib2.Path """ self.st_dev = path.stat().st_dev self.st_ino = path.stat().st_ino
def _make_file_info(target: pathlib.Path, arcname: Optional[str] = None) -> Dict[str, Any]: f = {} # type: Dict[str, Any] f['origin'] = str(target) if arcname is not None: f['filename'] = arcname else: f['filename'] = str(target) if os.name == 'nt': fstat = os.stat(str(target), follow_symlinks=False) if target.is_symlink(): f['emptystream'] = False f['attributes'] = fstat.st_file_attributes & FILE_ATTRIBUTE_WINDOWS_MASK # type: ignore # noqa elif target.is_dir(): f['emptystream'] = True f['attributes'] = fstat.st_file_attributes & FILE_ATTRIBUTE_WINDOWS_MASK # type: ignore # noqa elif target.is_file(): f['emptystream'] = False f['attributes'] = stat.FILE_ATTRIBUTE_ARCHIVE # type: ignore # noqa f['uncompressed'] = fstat.st_size else: fstat = target.stat() if target.is_symlink(): f['emptystream'] = False f['attributes'] = stat.FILE_ATTRIBUTE_ARCHIVE # type: ignore # noqa f['attributes'] |= FILE_ATTRIBUTE_UNIX_EXTENSION | ( stat.S_IFLNK << 16) f['attributes'] |= (stat.S_IMODE(fstat.st_mode) << 16) elif target.is_dir(): f['emptystream'] = True f['attributes'] = stat.FILE_ATTRIBUTE_DIRECTORY # type: ignore # noqa f['attributes'] |= FILE_ATTRIBUTE_UNIX_EXTENSION | ( stat.S_IFDIR << 16) f['attributes'] |= (stat.S_IMODE(fstat.st_mode) << 16) elif target.is_file(): f['emptystream'] = False f['uncompressed'] = fstat.st_size f['attributes'] = stat.FILE_ATTRIBUTE_ARCHIVE # type: ignore # noqa f['attributes'] |= FILE_ATTRIBUTE_UNIX_EXTENSION | ( stat.S_IMODE(fstat.st_mode) << 16) f['creationtime'] = target.stat().st_ctime f['lastwritetime'] = target.stat().st_mtime f['lastaccesstime'] = target.stat().st_atime return f
def load_GPR_data(filepath, samplePoint=512): dPosXYZs = [] radarData = [] try: file = Path(filepath) fileByte = file.stat().st_size print(fileByte) traceByte = 90 + samplePoint * 2 traceNum = int((fileByte - FILE_INFO_BYTE_NUM) / traceByte) if isinstance(traceNum, int): with open(file, 'rb') as f: f.seek(FILE_INFO_BYTE_NUM) for ele in range(0, traceNum): fGPSOffset = struct.unpack('f', f.read(4))[0] chReserve = struct.unpack('2c', f.read(1 * 2)) ucYear = struct.unpack('B', f.read(1))[0] ucMonth = struct.unpack('B', f.read(1))[0] ucDay = struct.unpack('B', f.read(1))[0] ucHour = struct.unpack('B', f.read(1))[0] ucMin = struct.unpack('B', f.read(1))[0] ucSec = struct.unpack('B', f.read(1))[0] usMilSec = struct.unpack('H', f.read(2))[0] dPosXYZ = struct.unpack('3d', f.read(8 * 3)) dPosXYZs.append(dPosXYZ) ucTrcCount = struct.unpack('BB', f.read(1 * 2)) ucVoltage = struct.unpack('BB', f.read(1 * 2)) fWheelOffset = struct.unpack('f', f.read(4))[0] chPhotoName1 = struct.unpack('8c', f.read(1 * 8)) chPhotoName2 = struct.unpack('8c', f.read(1 * 8)) usMetalDiameter = struct.unpack('H', f.read(2))[0] usMetalDepth = struct.unpack('H', f.read(2))[0] bMetalFlag = struct.unpack('?', f.read(1))[0] chMarkName = struct.unpack('17c', f.read(1 * 17)) fMarkHeight = struct.unpack('f', f.read(4))[0] usMarkFlag = struct.unpack('H', f.read(2))[0] data = struct.unpack( str(samplePoint) + 'h', f.read(samplePoint * 2)) radarData.append(data) dPosXYZs = np.array(dPosXYZs).T return dPosXYZs, radarData else: return errorhandle.LOAD_GPR_SIZE_ERROR except Exception as e: print(e) return errorhandle.LOAD_GPR_FAILURE
def _set_file_property(self, outfilename: pathlib.Path, properties: Dict[str, Any]) -> None: # creation time creationtime = ArchiveTimestamp( properties['lastwritetime']).totimestamp() if creationtime is not None: os.utime(str(outfilename), times=(creationtime, creationtime)) if os.name == 'posix': st_mode = properties['posix_mode'] if st_mode is not None: outfilename.chmod(st_mode) return # fallback: only set readonly if specified if properties['readonly'] and not properties['is_directory']: ro_mask = 0o777 ^ (stat.S_IWRITE | stat.S_IWGRP | stat.S_IWOTH) outfilename.chmod(outfilename.stat().st_mode & ro_mask)
def get_coverage_build(dirpath, args): """Gets a coverage build from a specified server. Args: dirpath (Path): Directory in which build is to be downloaded in. args (class): Command line arguments. Returns: Path: Path to the js coverage build """ RUN_COV_LOG.info("Downloading coverage build zip file into %s from %s", str(dirpath), args.url) with requests.get(args.url, stream=True) as f: build_request_data = io.BytesIO(f.content) RUN_COV_LOG.info("Extracting coverage build zip file...") build_zip = zipfile.ZipFile(build_request_data) extract_folder = dirpath / "cov-build" extract_folder.mkdir(parents=True, exist_ok=True) # Ensure this dir has been created # In 3.5 <= Python < 3.6, .extractall does not automatically create intermediate folders that do not exist build_zip.extractall(str(extract_folder.resolve())) RUN_COV_LOG.info("Coverage build zip file extracted to this folder: %s", extract_folder.resolve()) js_cov_bin_name = "js" + (".exe" if platform.system() == "Windows" else "") js_cov_bin = extract_folder / "dist" / "bin" / js_cov_bin_name Path.chmod(js_cov_bin, Path.stat(js_cov_bin).st_mode | 0o111) # Ensure the js binary is executable assert js_cov_bin.is_file() # Check that the binary is non-debug. assert not queryBuildConfiguration(js_cov_bin, "debug") assert queryBuildConfiguration(js_cov_bin, "coverage") js_cov_fmconf = extract_folder / "dist" / "bin" / (js_cov_bin_name + ".fuzzmanagerconf") assert js_cov_fmconf.is_file() # Check that a coverage build with *.gcno files are present js_cov_unified_gcno = extract_folder / "js" / "src" / "Unified_cpp_js_src0.gcno" assert js_cov_unified_gcno.is_file() return js_cov_bin
def GPRGPSReader(GPR_file, sample_point=1024): ''' 读取二进制文件.GPR中的GPS信息 .GPR文件结构如下: 文件头 1065字节 每道道头 90字节 每道数据 sample_point*2字节 GPS信息位于第15字节 输入GPR文件路径的str 或者 Path对象 返回每一道GPS的X Y Z坐标 ''' import struct import matplotlib.pyplot as plt import numpy as np from pathlib2 import Path file = Path(GPR_file) dPosXYZs = [] file_byte = file.stat().st_size file_info_byte = 1065 trace_byte = 90 + sample_point * 2 trace_num = (file_byte - file_info_byte) / trace_byte assert (file_byte - file_info_byte) % trace_byte == 0 #不为0说明sample_point不对 with open(file, 'rb') as f: # 整个数据开头有1065字节数据头 + 每道道头90 + 每道数据1024*2 f.seek(file_info_byte) #跳过数据开头冗余的1065字节 for trace_ii in np.arange(trace_num): try: fGPSOffset = struct.unpack('f', f.read(4))[0] chReserve = struct.unpack('2c', f.read(1 * 2)) except: print(trace_ii) break else: ucYear = struct.unpack('B', f.read(1))[0] ucMonth = struct.unpack('B', f.read(1))[0] ucDay = struct.unpack('B', f.read(1))[0] ucHour = struct.unpack('B', f.read(1))[0] ucMin = struct.unpack('B', f.read(1))[0] ucSec = struct.unpack('B', f.read(1))[0] usMilSec = struct.unpack('H', f.read(2))[0] # dPosX,dPosY,dPosZ = struct.unpack('3d',f.read(8*3)) dPosXYZ = struct.unpack('3d', f.read(8 * 3)) dPosXYZs.append(dPosXYZ) ucTrcCount = struct.unpack('BB', f.read(1 * 2)) ucVoltage = struct.unpack('BB', f.read(1 * 2)) fWheelOffset = struct.unpack('f', f.read(4))[0] chPhotoName1 = struct.unpack('8c', f.read(1 * 8)) chPhotoName2 = struct.unpack('8c', f.read(1 * 8)) usMetalDiameter = struct.unpack('H', f.read(2))[0] usMetalDepth = struct.unpack('H', f.read(2))[0] bMetalFlag = struct.unpack('?', f.read(1))[0] chMarkName = struct.unpack('17c', f.read(1 * 17)) fMarkHeight = struct.unpack('f', f.read(4))[0] usMarkFlag = struct.unpack('H', f.read(2))[0] data = struct.unpack( str(sample_point) + 'H', f.read(sample_point * 2)) dPosXYZs = np.array(dPosXYZs).T return dPosXYZs
def main(): print('TRAINS SDK setup process') conf_file = Path(LOCAL_CONFIG_FILES[0]).absolute() if conf_file.exists() and conf_file.is_file() and conf_file.stat().st_size > 0: print('Configuration file already exists: {}'.format(str(conf_file))) print('Leaving setup, feel free to edit the configuration file.') return print(host_description, end='') parsed_host = None while not parsed_host: parse_input = input() if not parse_input: parse_input = def_host # noinspection PyBroadException try: if not parse_input.startswith('http://') and not parse_input.startswith('https://'): parse_input = 'http://'+parse_input parsed_host = urlparse(parse_input) if parsed_host.scheme not in ('http', 'https'): parsed_host = None except Exception: parsed_host = None print('Could not parse url {}\nEnter your trains-server host: '.format(parse_input), end='') if parsed_host.port == 8080: # this is a docker 8080 is the web address, we need the api address, it is 8008 print('Port 8080 is the web port, we need the api port. Replacing 8080 with 8008') api_host = parsed_host.scheme + "://" + parsed_host.netloc.replace(':8080', ':8008') + parsed_host.path web_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path elif parsed_host.netloc.startswith('demoapp.'): print('{} is the web server, we need the api server. Replacing \'demoapp.\' with \'demoapi.\''.format( parsed_host.netloc)) # this is our demo server api_host = parsed_host.scheme + "://" + parsed_host.netloc.replace('demoapp.', 'demoapi.') + parsed_host.path web_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path elif parsed_host.netloc.startswith('app.'): print('{} is the web server, we need the api server. Replacing \'app.\' with \'api.\''.format( parsed_host.netloc)) # this is our application server api_host = parsed_host.scheme + "://" + parsed_host.netloc.replace('app.', 'api.') + parsed_host.path web_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path elif parsed_host.port == 8008: api_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path web_host = parsed_host.scheme + "://" + parsed_host.netloc.replace(':8008', ':8080') + parsed_host.path elif parsed_host.netloc.startswith('demoapi.'): api_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path web_host = parsed_host.scheme + "://" + parsed_host.netloc.replace('demoapi.', 'demoapp.') + parsed_host.path elif parsed_host.netloc.startswith('api.'): api_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path web_host = parsed_host.scheme + "://" + parsed_host.netloc.replace('api.', 'app.') + parsed_host.path else: api_host = None web_host = None if not parsed_host.port: print('Host port not detected, do you wish to use the default 8008 port n/[y]? ', end='') replace_port = input().lower() if not replace_port or replace_port == 'y' or replace_port == 'yes': api_host = parsed_host.scheme + "://" + parsed_host.netloc + ':8008' + parsed_host.path web_host = parsed_host.scheme + "://" + parsed_host.netloc + ':8080' + parsed_host.path if not api_host: api_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path if not web_host: web_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path print('Host configured to: {}'.format(api_host)) print(description.format(web_host), end='') parse_input = input() # check if these are valid credentials credentials = None # noinspection PyBroadException try: parsed = ConfigFactory.parse_string(parse_input) if parsed: credentials = parsed.get("credentials", None) except Exception: credentials = None if not credentials or set(credentials) != {"access_key", "secret_key"}: print('Could not parse user credentials, try again one after the other.') credentials = {} # parse individual print('Enter user access key: ', end='') credentials['access_key'] = input() print('Enter user secret: ', end='') credentials['secret_key'] = input() print('Detected credentials key=\"{}\" secret=\"{}\"'.format(credentials['access_key'], credentials['secret_key'], )) # noinspection PyBroadException try: default_sdk_conf = Path(__file__).parent.absolute() / 'sdk.conf' with open(str(default_sdk_conf), 'rt') as f: default_sdk = f.read() except Exception: print('Error! Could not read default configuration file') return # noinspection PyBroadException try: with open(str(conf_file), 'wt') as f: header = '# TRAINS SDK configuration file\n' \ 'api {\n' \ ' # Notice: \'host\' is the api server (default port 8008), not the web server.\n' \ ' host: %s\n' \ ' # Credentials are generated in the webapp, %s/admin\n' \ ' credentials {"access_key": "%s", "secret_key": "%s"}\n' \ '}\n' \ 'sdk ' % (api_host, web_host, credentials['access_key'], credentials['secret_key']) f.write(header) f.write(default_sdk) except Exception: print('Error! Could not write configuration file at: {}'.format(str(conf_file))) return print('\nNew configuration stored in {}'.format(str(conf_file))) print('TRAINS setup completed successfully.')
def main(): default_config_file = os.getenv( LOCAL_CONFIG_FILE_OVERRIDE_VAR) or LOCAL_CONFIG_FILES[0] p = argparse.ArgumentParser(description=__doc__) p.add_argument( "--file", "-F", help="Target configuration file path (default is %(default)s)", default=default_config_file, type=validate_file) args = p.parse_args() print('TRAINS SDK setup process') conf_file = Path(args.file).absolute() if conf_file.exists() and conf_file.is_file( ) and conf_file.stat().st_size > 0: print('Configuration file already exists: {}'.format(str(conf_file))) print('Leaving setup, feel free to edit the configuration file.') return print(description, end='') parse_input = get_user_input() credentials = None api_host = None web_server = None # noinspection PyBroadException try: parsed = ConfigFactory.parse_string(parse_input) if parsed: # Take the credentials in raw form or from api section credentials = get_parsed_field(parsed, ["credentials"]) api_host = get_parsed_field(parsed, ["api_server", "host"]) web_server = get_parsed_field(parsed, ["web_server"]) except Exception: credentials = credentials or None api_host = api_host or None web_server = web_server or None while not credentials or set(credentials) != {"access_key", "secret_key"}: print( 'Could not parse credentials, please try entering them manually.') credentials = read_manual_credentials() print('Detected credentials key=\"{}\" secret=\"{}\"'.format( credentials['access_key'], credentials['secret_key'][0:4] + "***")) host_description = """ Editing configuration file: {CONFIG_FILE} Enter the url of the trains-server's Web service, for example: {HOST} """.format( CONFIG_FILE=args.file, HOST=def_host, ) if api_host: api_host = input_url('API Host', api_host) else: print(host_description) api_host = input_url('API Host', '') parsed_host = verify_url(api_host) if parsed_host.netloc.startswith('demoapp.'): # this is our demo server api_host = parsed_host.scheme + "://" + parsed_host.netloc.replace( 'demoapp.', 'demoapi.', 1) + parsed_host.path web_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path files_host = parsed_host.scheme + "://" + parsed_host.netloc.replace( 'demoapp.', 'demofiles.', 1) + parsed_host.path elif parsed_host.netloc.startswith('app.'): # this is our application server api_host = parsed_host.scheme + "://" + parsed_host.netloc.replace( 'app.', 'api.', 1) + parsed_host.path web_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path files_host = parsed_host.scheme + "://" + parsed_host.netloc.replace( 'app.', 'files.', 1) + parsed_host.path elif parsed_host.netloc.startswith('demoapi.'): print( '{} is the api server, we need the web server. Replacing \'demoapi.\' with \'demoapp.\'' .format(parsed_host.netloc)) api_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path web_host = parsed_host.scheme + "://" + parsed_host.netloc.replace( 'demoapi.', 'demoapp.', 1) + parsed_host.path files_host = parsed_host.scheme + "://" + parsed_host.netloc.replace( 'demoapi.', 'demofiles.', 1) + parsed_host.path elif parsed_host.netloc.startswith('api.'): print( '{} is the api server, we need the web server. Replacing \'api.\' with \'app.\'' .format(parsed_host.netloc)) api_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path web_host = parsed_host.scheme + "://" + parsed_host.netloc.replace( 'api.', 'app.', 1) + parsed_host.path files_host = parsed_host.scheme + "://" + parsed_host.netloc.replace( 'api.', 'files.', 1) + parsed_host.path elif parsed_host.port == 8008: api_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path web_host = parsed_host.scheme + "://" + parsed_host.netloc.replace( ':8008', ':8080', 1) + parsed_host.path files_host = parsed_host.scheme + "://" + parsed_host.netloc.replace( ':8008', ':8081', 1) + parsed_host.path elif parsed_host.port == 8080: print( 'Port 8080 is the web port. Using port 8008 for API Host and 8080 for Web Application Host' ) api_host = parsed_host.scheme + "://" + parsed_host.netloc.replace( ':8080', ':8008', 1) + parsed_host.path web_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path files_host = parsed_host.scheme + "://" + parsed_host.netloc.replace( ':8080', ':8081', 1) + parsed_host.path else: api_host = '' web_host = '' files_host = '' if not parsed_host.port: print( 'Host port not detected, do you wish to use the default 8080 port n/[y]? ', end='') replace_port = input().lower() if not replace_port or replace_port == 'y' or replace_port == 'yes': api_host = parsed_host.scheme + "://" + parsed_host.netloc + ':8008' + parsed_host.path web_host = parsed_host.scheme + "://" + parsed_host.netloc + ':8080' + parsed_host.path files_host = parsed_host.scheme + "://" + parsed_host.netloc + ':8081' + parsed_host.path elif not replace_port or replace_port.lower( ) == 'n' or replace_port.lower() == 'no': web_host = input_host_port("Web", parsed_host) api_host = input_host_port("API", parsed_host) files_host = input_host_port("Files", parsed_host) if not api_host: api_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path web_host = input_url('Web Application Host', web_server if web_server else web_host) files_host = input_url('File Store Host', files_host) print( '\nTRAINS Hosts configuration:\nWeb App: {}\nAPI: {}\nFile Store: {}\n' .format(web_host, api_host, files_host)) retry = 1 max_retries = 2 while retry <= max_retries: # Up to 2 tries by the user if verify_credentials(api_host, credentials): break retry += 1 if retry < max_retries + 1: credentials = read_manual_credentials() else: print('Exiting setup without creating configuration file') return # noinspection PyBroadException try: default_sdk_conf = Path(__file__).parent.absolute() / 'sdk.conf' with open(str(default_sdk_conf), 'rt') as f: default_sdk = f.read() except Exception: print('Error! Could not read default configuration file') return # noinspection PyBroadException try: with open(str(conf_file), 'wt') as f: header = '# TRAINS SDK configuration file\n' \ 'api {\n' \ ' # Notice: \'host\' is the api server (default port 8008), not the web server.\n' \ ' api_server: %s\n' \ ' web_server: %s\n' \ ' files_server: %s\n' \ ' # Credentials are generated using the webapp, %s/profile\n' \ ' credentials {"access_key": "%s", "secret_key": "%s"}\n' \ '}\n' \ 'sdk ' % (api_host, web_host, files_host, web_host, credentials['access_key'], credentials['secret_key']) f.write(header) f.write(default_sdk) except Exception: print('Error! Could not write configuration file at: {}'.format( str(conf_file))) return print('\nNew configuration stored in {}'.format(str(conf_file))) print('TRAINS setup completed successfully.')
def main(): print('TRAINS-AGENT setup process') conf_file = Path(LOCAL_CONFIG_FILES[0]).absolute() if conf_file.exists() and conf_file.is_file( ) and conf_file.stat().st_size > 0: print('Configuration file already exists: {}'.format(str(conf_file))) print('Leaving setup, feel free to edit the configuration file.') return print(host_description) web_host = input_url('Web Application Host', '') parsed_host = verify_url(web_host) if parsed_host.port == 8008: print( 'Port 8008 is the api port. Replacing 8080 with 8008 for Web application' ) api_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path web_host = parsed_host.scheme + "://" + parsed_host.netloc.replace( ':8008', ':8080', 1) + parsed_host.path files_host = parsed_host.scheme + "://" + parsed_host.netloc.replace( ':8008', ':8081', 1) + parsed_host.path elif parsed_host.port == 8080: api_host = parsed_host.scheme + "://" + parsed_host.netloc.replace( ':8080', ':8008', 1) + parsed_host.path web_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path files_host = parsed_host.scheme + "://" + parsed_host.netloc.replace( ':8080', ':8081', 1) + parsed_host.path elif parsed_host.netloc.startswith('demoapp.'): # this is our demo server api_host = parsed_host.scheme + "://" + parsed_host.netloc.replace( 'demoapp.', 'demoapi.', 1) + parsed_host.path web_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path files_host = parsed_host.scheme + "://" + parsed_host.netloc.replace( 'demoapp.', 'demofiles.', 1) + parsed_host.path elif parsed_host.netloc.startswith('app.'): # this is our application server api_host = parsed_host.scheme + "://" + parsed_host.netloc.replace( 'app.', 'api.', 1) + parsed_host.path web_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path files_host = parsed_host.scheme + "://" + parsed_host.netloc.replace( 'app.', 'files.', 1) + parsed_host.path elif parsed_host.netloc.startswith('demoapi.'): print( '{} is the api server, we need the web server. Replacing \'demoapi.\' with \'demoapp.\'' .format(parsed_host.netloc)) api_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path web_host = parsed_host.scheme + "://" + parsed_host.netloc.replace( 'demoapi.', 'demoapp.', 1) + parsed_host.path files_host = parsed_host.scheme + "://" + parsed_host.netloc.replace( 'demoapi.', 'demofiles.', 1) + parsed_host.path elif parsed_host.netloc.startswith('api.'): print( '{} is the api server, we need the web server. Replacing \'api.\' with \'app.\'' .format(parsed_host.netloc)) api_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path web_host = parsed_host.scheme + "://" + parsed_host.netloc.replace( 'api.', 'app.', 1) + parsed_host.path files_host = parsed_host.scheme + "://" + parsed_host.netloc.replace( 'api.', 'files.', 1) + parsed_host.path else: api_host = '' web_host = '' files_host = '' if not parsed_host.port: print( 'Host port not detected, do you wish to use the default 8008 port n/[y]? ', end='') replace_port = input().lower() if not replace_port or replace_port == 'y' or replace_port == 'yes': api_host = parsed_host.scheme + "://" + parsed_host.netloc + ':8008' + parsed_host.path web_host = parsed_host.scheme + "://" + parsed_host.netloc + ':8080' + parsed_host.path files_host = parsed_host.scheme + "://" + parsed_host.netloc + ':8081' + parsed_host.path if not api_host: api_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path api_host = input_url('API Host', api_host) files_host = input_url('File Store Host', files_host) print( '\nTRAINS Hosts configuration:\nAPI: {}\nWeb App: {}\nFile Store: {}\n' .format(api_host, web_host, files_host)) while True: print(description.format(web_host), end='') parse_input = input() # check if these are valid credentials credentials = None # noinspection PyBroadException try: parsed = ConfigFactory.parse_string(parse_input) if parsed: credentials = parsed.get("credentials", None) except Exception: credentials = None if not credentials or set(credentials) != {"access_key", "secret_key"}: print( 'Could not parse user credentials, try again one after the other.' ) credentials = {} # parse individual print('Enter user access key: ', end='') credentials['access_key'] = input() print('Enter user secret: ', end='') credentials['secret_key'] = input() print('Detected credentials key=\"{}\" secret=\"{}\"'.format( credentials['access_key'], credentials['secret_key'], )) from trains_agent.backend_api.session import Session # noinspection PyBroadException try: print('Verifying credentials ...') Session(api_key=credentials['access_key'], secret_key=credentials['secret_key'], host=api_host) print('Credentials verified!') break except Exception: print( 'Error: could not verify credentials: host={} access={} secret={}' .format(api_host, credentials['access_key'], credentials['secret_key'])) # get GIT User/Pass for cloning print( 'Enter git username for repository cloning (leave blank for SSH key authentication): [] ', end='') git_user = input() if git_user.strip(): print('Enter password for user \'{}\': '.format(git_user), end='') git_pass = input() print( 'Git repository cloning will be using user={} password={}'.format( git_user, git_pass)) else: git_user = None git_pass = None # noinspection PyBroadException try: conf_folder = Path(__file__).parent.absolute( ) / '..' / 'backend_api' / 'config' / 'default' default_conf = '' for conf in ( 'agent.conf', 'sdk.conf', ): conf_file_section = conf_folder / conf with open(str(conf_file_section), 'rt') as f: default_conf += conf.split('.')[0] + ' ' default_conf += f.read() default_conf += '\n' except Exception: print('Error! Could not read default configuration file') return # noinspection PyBroadException try: with open(str(conf_file), 'wt') as f: header = '# TRAINS-AGENT configuration file\n' \ 'api {\n' \ ' api_server: %s\n' \ ' web_server: %s\n' \ ' files_server: %s\n' \ ' # Credentials are generated using the webapp, %s/profile\n' \ ' # Override with os environment: TRAINS_API_ACCESS_KEY / TRAINS_API_SECRET_KEY\n' \ ' credentials {"access_key": "%s", "secret_key": "%s"}\n' \ '}\n\n' % (api_host, web_host, files_host, web_host, credentials['access_key'], credentials['secret_key']) f.write(header) git_credentials = '# Set GIT user/pass credentials\n' \ '# leave blank for GIT SSH credentials\n' \ 'agent.git_user=\"{}\"\n' \ 'agent.git_pass=\"{}\"\n' \ '\n'.format(git_user or '', git_pass or '') f.write(git_credentials) f.write(default_conf) except Exception: print('Error! Could not write configuration file at: {}'.format( str(conf_file))) return print('\nNew configuration stored in {}'.format(str(conf_file))) print('TRAINS-AGENT setup completed successfully.')
def upload_artifact(self, name, artifact_object=None, metadata=None, delete_after_upload=False): if not Session.check_min_api_version('2.3'): LoggerRoot.get_base_logger().warning( 'Artifacts not supported by your TRAINS-server version, ' 'please upgrade to the latest server version') return False if name in self._artifacts_container: raise ValueError( "Artifact by the name of {} is already registered, use register_artifact" .format(name)) artifact_type_data = tasks.ArtifactTypeData() override_filename_in_uri = None override_filename_ext_in_uri = None uri = None if np and isinstance(artifact_object, np.ndarray): artifact_type = 'numpy' artifact_type_data.content_type = 'application/numpy' artifact_type_data.preview = str(artifact_object.__repr__()) override_filename_ext_in_uri = '.npz' override_filename_in_uri = name + override_filename_ext_in_uri fd, local_filename = mkstemp(prefix=quote(name, safe="") + '.', suffix=override_filename_ext_in_uri) os.close(fd) np.savez_compressed(local_filename, **{name: artifact_object}) delete_after_upload = True elif pd and isinstance(artifact_object, pd.DataFrame): artifact_type = 'pandas' artifact_type_data.content_type = 'text/csv' artifact_type_data.preview = str(artifact_object.__repr__()) override_filename_ext_in_uri = self._save_format override_filename_in_uri = name fd, local_filename = mkstemp(prefix=quote(name, safe="") + '.', suffix=override_filename_ext_in_uri) os.close(fd) artifact_object.to_csv(local_filename, compression=self._compression) delete_after_upload = True elif isinstance(artifact_object, Image.Image): artifact_type = 'image' artifact_type_data.content_type = 'image/png' desc = str(artifact_object.__repr__()) artifact_type_data.preview = desc[1:desc.find(' at ')] override_filename_ext_in_uri = '.png' override_filename_in_uri = name + override_filename_ext_in_uri fd, local_filename = mkstemp(prefix=quote(name, safe="") + '.', suffix=override_filename_ext_in_uri) os.close(fd) artifact_object.save(local_filename) delete_after_upload = True elif isinstance(artifact_object, dict): artifact_type = 'JSON' artifact_type_data.content_type = 'application/json' preview = json.dumps(artifact_object, sort_keys=True, indent=4) override_filename_ext_in_uri = '.json' override_filename_in_uri = name + override_filename_ext_in_uri fd, local_filename = mkstemp(prefix=quote(name, safe="") + '.', suffix=override_filename_ext_in_uri) os.write(fd, bytes(preview.encode())) os.close(fd) artifact_type_data.preview = preview delete_after_upload = True elif isinstance(artifact_object, six.string_types) and urlparse( artifact_object).scheme in remote_driver_schemes: # we should not upload this, just register local_filename = None uri = artifact_object artifact_type = 'custom' artifact_type_data.content_type = mimetypes.guess_type( artifact_object)[0] elif isinstance(artifact_object, six.string_types + (Path, )): # check if single file artifact_object = Path(artifact_object) artifact_object.expanduser().absolute() try: create_zip_file = not artifact_object.is_file() except Exception: # Hack for windows pathlib2 bug, is_file isn't valid. create_zip_file = True else: # We assume that this is not Windows os if artifact_object.is_dir(): # change to wildcard artifact_object /= '*' if create_zip_file: folder = Path('').joinpath(*artifact_object.parts[:-1]) if not folder.is_dir() or not folder.parts: raise ValueError( "Artifact file/folder '{}' could not be found".format( artifact_object.as_posix())) wildcard = artifact_object.parts[-1] files = list(Path(folder).rglob(wildcard)) override_filename_ext_in_uri = '.zip' override_filename_in_uri = folder.parts[ -1] + override_filename_ext_in_uri fd, zip_file = mkstemp( prefix=quote(folder.parts[-1], safe="") + '.', suffix=override_filename_ext_in_uri) try: artifact_type_data.content_type = 'application/zip' artifact_type_data.preview = 'Archive content {}:\n'.format( artifact_object.as_posix()) with ZipFile(zip_file, 'w', allowZip64=True, compression=ZIP_DEFLATED) as zf: for filename in sorted(files): if filename.is_file(): relative_file_name = filename.relative_to( folder).as_posix() artifact_type_data.preview += '{} - {}\n'.format( relative_file_name, humanfriendly.format_size( filename.stat().st_size)) zf.write(filename.as_posix(), arcname=relative_file_name) except Exception as e: # failed uploading folder: LoggerRoot.get_base_logger().warning( 'Exception {}\nFailed zipping artifact folder {}'. format(folder, e)) return None finally: os.close(fd) artifact_object = zip_file artifact_type = 'archive' artifact_type_data.content_type = mimetypes.guess_type( artifact_object)[0] local_filename = artifact_object delete_after_upload = True else: if not artifact_object.is_file(): raise ValueError( "Artifact file '{}' could not be found".format( artifact_object.as_posix())) override_filename_in_uri = artifact_object.parts[-1] artifact_object = artifact_object.as_posix() artifact_type = 'custom' artifact_type_data.content_type = mimetypes.guess_type( artifact_object)[0] local_filename = artifact_object else: raise ValueError("Artifact type {} not supported".format( type(artifact_object))) # remove from existing list, if exists for artifact in self._task_artifact_list: if artifact.key == name: if artifact.type == self._pd_artifact_type: raise ValueError( "Artifact of name {} already registered, " "use register_artifact instead".format(name)) self._task_artifact_list.remove(artifact) break if not local_filename: file_size = None file_hash = None else: # check that the file to upload exists local_filename = Path(local_filename).absolute() if not local_filename.exists() or not local_filename.is_file(): LoggerRoot.get_base_logger().warning( 'Artifact upload failed, cannot find file {}'.format( local_filename.as_posix())) return False file_hash, _ = self.sha256sum(local_filename.as_posix()) file_size = local_filename.stat().st_size uri = self._upload_local_file( local_filename, name, delete_after_upload=delete_after_upload, override_filename=override_filename_in_uri, override_filename_ext=override_filename_ext_in_uri) timestamp = int(time()) artifact = tasks.Artifact( key=name, type=artifact_type, uri=uri, content_size=file_size, hash=file_hash, timestamp=timestamp, type_data=artifact_type_data, display_data=[(str(k), str(v)) for k, v in metadata.items()] if metadata else None) # update task artifacts with self._task_edit_lock: self._task_artifact_list.append(artifact) self._task.set_artifacts(self._task_artifact_list) return True
def _daemon(cls, jupyter_notebook_filename): from trains import Task # load jupyter notebook package # noinspection PyBroadException try: # noinspection PyPackageRequirements from nbconvert.exporters.script import ScriptExporter _script_exporter = ScriptExporter() except Exception: return # load pigar # noinspection PyBroadException try: from ....utilities.pigar.reqs import get_installed_pkgs_detail, file_import_modules from ....utilities.pigar.modules import ReqsModules from ....utilities.pigar.log import logger logger.setLevel(logging.WARNING) except Exception: file_import_modules = None # load IPython # noinspection PyBroadException try: # noinspection PyPackageRequirements from IPython import get_ipython except Exception: # should not happen get_ipython = None # setup local notebook files if jupyter_notebook_filename: notebook = Path(jupyter_notebook_filename) local_jupyter_filename = jupyter_notebook_filename else: notebook = None fd, local_jupyter_filename = mkstemp(suffix='.ipynb') os.close(fd) last_update_ts = None counter = 0 prev_script_hash = None # noinspection PyBroadException try: from ....version import __version__ our_module = cls.__module__.split('.')[0], __version__ except Exception: our_module = None # noinspection PyBroadException try: import re replace_ipython_pattern = re.compile(r'\n([ \t]*)get_ipython\(\)') except Exception: replace_ipython_pattern = None # main observer loop, check if we need to exit while not cls._exit_event.wait(timeout=0.): # wait for timeout or sync event cls._sync_event.wait(cls._sample_frequency if counter else cls. _first_sample_frequency) cls._sync_event.clear() counter += 1 # noinspection PyBroadException try: # if there is no task connected, do nothing task = Task.current_task() if not task: continue script_code = None fmodules = None current_cell = None # if we have a local file: if notebook: if not notebook.exists(): continue # check if notebook changed if last_update_ts is not None and notebook.stat( ).st_mtime - last_update_ts <= 0: continue last_update_ts = notebook.stat().st_mtime else: # serialize notebook to a temp file if cls._jupyter_history_logger: script_code, current_cell = cls._jupyter_history_logger.history_to_str( ) else: # noinspection PyBroadException try: # noinspection PyBroadException try: os.unlink(local_jupyter_filename) except Exception: pass get_ipython().run_line_magic( 'history', '-t -f {}'.format(local_jupyter_filename)) with open(local_jupyter_filename, 'r') as f: script_code = f.read() # load the modules from ....utilities.pigar.modules import ImportedModules fmodules = ImportedModules() for nm in set( [str(m).split('.')[0] for m in sys.modules]): fmodules.add(nm, 'notebook', 0) except Exception: continue # get notebook python script if script_code is None: script_code, _ = _script_exporter.from_filename( local_jupyter_filename) current_script_hash = hash(script_code + (current_cell or '')) if prev_script_hash and prev_script_hash == current_script_hash: continue # remove ipython direct access from the script code # we will not be able to run them anyhow if replace_ipython_pattern: script_code = replace_ipython_pattern.sub( r'\n# \g<1>get_ipython()', script_code) requirements_txt = '' conda_requirements = '' # parse jupyter python script and prepare pip requirements (pigar) # if backend supports requirements if file_import_modules and Session.check_min_api_version( '2.2'): if fmodules is None: fmodules, _ = file_import_modules( notebook.parts[-1] if notebook else 'notebook', script_code) if current_cell: cell_fmodules, _ = file_import_modules( notebook.parts[-1] if notebook else 'notebook', current_cell) # noinspection PyBroadException try: fmodules |= cell_fmodules except Exception: pass # add current cell to the script if current_cell: script_code += '\n' + current_cell fmodules = ScriptRequirements.add_trains_used_packages( fmodules) # noinspection PyUnboundLocalVariable installed_pkgs = get_installed_pkgs_detail() # make sure we are in installed packages if our_module and (our_module[0] not in installed_pkgs): installed_pkgs[our_module[0]] = our_module # noinspection PyUnboundLocalVariable reqs = ReqsModules() for name in fmodules: if name in installed_pkgs: pkg_name, version = installed_pkgs[name] reqs.add(pkg_name, version, fmodules[name]) requirements_txt, conda_requirements = ScriptRequirements.create_requirements_txt( reqs) # update script prev_script_hash = current_script_hash data_script = task.data.script data_script.diff = script_code data_script.requirements = { 'pip': requirements_txt, 'conda': conda_requirements } # noinspection PyProtectedMember task._update_script(script=data_script) # update requirements # noinspection PyProtectedMember task._update_requirements(requirements=requirements_txt) except Exception: pass
def main(): print('TRAINS-AGENT setup process') conf_file = Path(LOCAL_CONFIG_FILES[0]).absolute() if conf_file.exists() and conf_file.is_file( ) and conf_file.stat().st_size > 0: print('Configuration file already exists: {}'.format(str(conf_file))) print('Leaving setup, feel free to edit the configuration file.') return print(description, end='') sentinel = '' parse_input = '\n'.join(iter(input, sentinel)) credentials = None api_server = None web_server = None # noinspection PyBroadException try: parsed = ConfigFactory.parse_string(parse_input) if parsed: # Take the credentials in raw form or from api section credentials = get_parsed_field(parsed, ["credentials"]) api_server = get_parsed_field(parsed, ["api_server", "host"]) web_server = get_parsed_field(parsed, ["web_server"]) except Exception: credentials = credentials or None api_server = api_server or None web_server = web_server or None while not credentials or set(credentials) != {"access_key", "secret_key"}: print( 'Could not parse credentials, please try entering them manually.') credentials = read_manual_credentials() print('Detected credentials key=\"{}\" secret=\"{}\"'.format( credentials['access_key'], credentials['secret_key'][0:4] + "***")) web_input = True if web_server: host = input_url('WEB Host', web_server) elif api_server: web_input = False host = input_url('API Host', api_server) else: print(host_description) host = input_url('WEB Host', '') parsed_host = verify_url(host) api_host, files_host, web_host = parse_host(parsed_host, allow_input=True) # on of these two we configured if not web_input: web_host = input_url('Web Application Host', web_host) else: api_host = input_url('API Host', api_host) files_host = input_url('File Store Host', files_host) print( '\nTRAINS Hosts configuration:\nWeb App: {}\nAPI: {}\nFile Store: {}\n' .format(web_host, api_host, files_host)) retry = 1 max_retries = 2 while retry <= max_retries: # Up to 2 tries by the user if verify_credentials(api_host, credentials): break retry += 1 if retry < max_retries + 1: credentials = read_manual_credentials() else: print('Exiting setup without creating configuration file') return # get GIT User/Pass for cloning print( 'Enter git username for repository cloning (leave blank for SSH key authentication): [] ', end='') git_user = input() if git_user.strip(): print('Enter password for user \'{}\': '.format(git_user), end='') git_pass = input() print( 'Git repository cloning will be using user={} password={}'.format( git_user, git_pass)) else: git_user = None git_pass = None # get extra-index-url for pip installations extra_index_urls = [] print( '\nEnter additional artifact repository (extra-index-url) to use when installing python packages ' '(leave blank if not required):', end='') index_url = input().strip() while index_url: extra_index_urls.append(index_url) print( 'Another artifact repository? (enter another url or leave blank if done):', end='') index_url = input().strip() if len(extra_index_urls): print("The following artifact repositories will be added:\n\t- {}". format("\n\t- ".join(extra_index_urls))) # noinspection PyBroadException try: conf_folder = Path(__file__).parent.absolute( ) / '..' / 'backend_api' / 'config' / 'default' default_conf = '' for conf in ( 'agent.conf', 'sdk.conf', ): conf_file_section = conf_folder / conf with open(str(conf_file_section), 'rt') as f: default_conf += conf.split('.')[0] + ' ' default_conf += f.read() default_conf += '\n' except Exception: print('Error! Could not read default configuration file') return # noinspection PyBroadException try: with open(str(conf_file), 'wt') as f: header = '# TRAINS-AGENT configuration file\n' \ 'api {\n' \ ' api_server: %s\n' \ ' web_server: %s\n' \ ' files_server: %s\n' \ ' # Credentials are generated using the webapp, %s/profile\n' \ ' # Override with os environment: TRAINS_API_ACCESS_KEY / TRAINS_API_SECRET_KEY\n' \ ' credentials {"access_key": "%s", "secret_key": "%s"}\n' \ '}\n\n' % (api_host, web_host, files_host, web_host, credentials['access_key'], credentials['secret_key']) f.write(header) git_credentials = '# Set GIT user/pass credentials\n' \ '# leave blank for GIT SSH credentials\n' \ 'agent.git_user=\"{}\"\n' \ 'agent.git_pass=\"{}\"\n' \ '\n'.format(git_user or '', git_pass or '') f.write(git_credentials) extra_index_str = '# extra_index_url: ["https://allegroai.jfrog.io/trainsai/api/pypi/public/simple"]\n' \ 'agent.package_manager.extra_index_url= ' \ '[\n{}\n]\n\n'.format("\n".join(map("\"{}\"".format, extra_index_urls))) f.write(extra_index_str) f.write(default_conf) except Exception: print('Error! Could not write configuration file at: {}'.format( str(conf_file))) return print('\nNew configuration stored in {}'.format(str(conf_file))) print('TRAINS-AGENT setup completed successfully.')
assert Path( args.model, cfg.invel + "_" + str(i)).stat().st_size == cfg.x[i] * cfg.y[i] * cfg.z[ i] * cfg.nvar * nbit_float, f"Layer-{i}: Mismatch of mesh size" if i + 1 < cfg.g: # Check consistency of adjcent meshes check_mesh_cont(Path(args.model, cfg.invel + "_" + str(i)), Path(args.model, cfg.invel + "_" + str(i + 1)), cfg.x[i], cfg.y[i], cfg.z[i]) # Topography if cfg.intopo: file_topo = Path(args.model, cfg.intopo) nx, ny, pad = np.fromfile(file_topo, dtype='int32', count=3) assert nx == cfg.x[0] and ny == cfg.y[0], f"Mismatch topography domain size" assert (nx + 2 * pad) * (ny + 2 * pad) * nbit_float == file_topo.stat( ).st_size, f"Topography size does not match parameters" # Receivers if cfg.recvfile: assert Path(args.model, cfg.recvfile).parent.exists( ), f"Receiver output directory does not exist" assert cfg.recv_steps % (cfg.recv_stride * cfg.recv_cpu_buffer_size \ * cfg.recv_gpu_buffer_size * cfg.recv_num_writes) == 0, "Check divisibility of receiver writing" assert cfg.recv_length <= len( cfg.recv_coords), f"More receivers required than given" # Source files in Ossian's format if cfg.sourcefile: assert Path(args.model, cfg.sourcefile).parent.exists(), f"Source file doesn't exist" assert cfg.src_steps % (cfg.src_stride * cfg.src_cpu_buffer_size \
def upload_artifact(self, name, artifact_object=None, metadata=None, delete_after_upload=False): if not Session.check_min_api_version('2.3'): LoggerRoot.get_base_logger().warning('Artifacts not supported by your TRAINS-server version, ' 'please upgrade to the latest server version') return False if name in self._artifacts_dict: raise ValueError("Artifact by the name of {} is already registered, use register_artifact".format(name)) artifact_type_data = tasks.ArtifactTypeData() use_filename_in_uri = True if np and isinstance(artifact_object, np.ndarray): artifact_type = 'numpy' artifact_type_data.content_type = 'application/numpy' artifact_type_data.preview = str(artifact_object.__repr__()) fd, local_filename = mkstemp(suffix='.npz') os.close(fd) np.savez_compressed(local_filename, **{name: artifact_object}) delete_after_upload = True use_filename_in_uri = False elif pd and isinstance(artifact_object, pd.DataFrame): artifact_type = 'pandas' artifact_type_data.content_type = 'text/csv' artifact_type_data.preview = str(artifact_object.__repr__()) fd, local_filename = mkstemp(suffix=self._save_format) os.close(fd) artifact_object.to_csv(local_filename, compression=self._compression) delete_after_upload = True use_filename_in_uri = False elif isinstance(artifact_object, Image.Image): artifact_type = 'image' artifact_type_data.content_type = 'image/png' desc = str(artifact_object.__repr__()) artifact_type_data.preview = desc[1:desc.find(' at ')] fd, local_filename = mkstemp(suffix='.png') os.close(fd) artifact_object.save(local_filename) delete_after_upload = True use_filename_in_uri = False elif isinstance(artifact_object, dict): artifact_type = 'JSON' artifact_type_data.content_type = 'application/json' preview = json.dumps(artifact_object, sort_keys=True, indent=4) fd, local_filename = mkstemp(suffix='.json') os.write(fd, bytes(preview.encode())) os.close(fd) artifact_type_data.preview = preview delete_after_upload = True use_filename_in_uri = False elif isinstance(artifact_object, six.string_types) or isinstance(artifact_object, Path): if isinstance(artifact_object, Path): artifact_object = artifact_object.as_posix() artifact_type = 'custom' artifact_type_data.content_type = mimetypes.guess_type(artifact_object)[0] local_filename = artifact_object else: raise ValueError("Artifact type {} not supported".format(type(artifact_object))) # remove from existing list, if exists for artifact in self._task_artifact_list: if artifact.key == name: if artifact.type == self._pd_artifact_type: raise ValueError("Artifact of name {} already registered, " "use register_artifact instead".format(name)) self._task_artifact_list.remove(artifact) break # check that the file to upload exists local_filename = Path(local_filename).absolute() if not local_filename.exists() or not local_filename.is_file(): LoggerRoot.get_base_logger().warning('Artifact upload failed, cannot find file {}'.format( local_filename.as_posix())) return False file_hash, _ = self.sha256sum(local_filename.as_posix()) timestamp = int(time()) file_size = local_filename.stat().st_size uri = self._upload_local_file(local_filename, name, delete_after_upload=delete_after_upload, use_filename=use_filename_in_uri) artifact = tasks.Artifact(key=name, type=artifact_type, uri=uri, content_size=file_size, hash=file_hash, timestamp=timestamp, type_data=artifact_type_data, display_data=[(str(k), str(v)) for k, v in metadata.items()] if metadata else None) # update task artifacts with self._task_edit_lock: self._task_artifact_list.append(artifact) self._task.set_artifacts(self._task_artifact_list) return True
def _daemon(cls, jupyter_notebook_filename): from trains import Task # load jupyter notebook package # noinspection PyBroadException try: from nbconvert.exporters.script import ScriptExporter _script_exporter = ScriptExporter() except Exception: return # load pigar # noinspection PyBroadException try: from pigar.reqs import get_installed_pkgs_detail, file_import_modules from pigar.modules import ReqsModules from pigar.log import logger logger.setLevel(logging.WARNING) except Exception: file_import_modules = None # main observer loop notebook = Path(jupyter_notebook_filename) last_update_ts = None counter = 0 prev_script_hash = None while True: if cls._exit_event.wait(cls._sample_frequency if counter else cls. _first_sample_frequency): return counter += 1 # noinspection PyBroadException try: if not notebook.exists(): continue # check if notebook changed if last_update_ts is not None and notebook.stat( ).st_mtime - last_update_ts <= 0: continue last_update_ts = notebook.stat().st_mtime task = Task.current_task() if not task: continue # get notebook python script script_code, resources = _script_exporter.from_filename( jupyter_notebook_filename) current_script_hash = hash(script_code) if prev_script_hash and prev_script_hash == current_script_hash: continue requirements_txt = '' # parse jupyter python script and prepare pip requirements (pigar) # if backend supports requirements if file_import_modules and Session.api_version > '2.1': fmodules, _ = file_import_modules(notebook.parts[-1], script_code) installed_pkgs = get_installed_pkgs_detail() reqs = ReqsModules() for name in fmodules: if name in installed_pkgs: pkg_name, version = installed_pkgs[name] reqs.add(pkg_name, version, fmodules[name]) requirements_txt = ScriptRequirements.create_requirements_txt( reqs) # update script prev_script_hash = current_script_hash data_script = task.data.script data_script.diff = script_code data_script.requirements = {'pip': requirements_txt} task._update_script(script=data_script) # update requirements if requirements_txt: task._update_requirements(requirements=requirements_txt) except Exception: pass
def _daemon(cls, jupyter_notebook_filename): from trains import Task # load jupyter notebook package # noinspection PyBroadException try: from nbconvert.exporters.script import ScriptExporter _script_exporter = ScriptExporter() except Exception: return # load pigar # noinspection PyBroadException try: from pigar.reqs import get_installed_pkgs_detail, file_import_modules from pigar.modules import ReqsModules from pigar.log import logger logger.setLevel(logging.WARNING) except Exception: file_import_modules = None # load IPython # noinspection PyBroadException try: from IPython import get_ipython except Exception: # should not happen get_ipython = None # setup local notebook files if jupyter_notebook_filename: notebook = Path(jupyter_notebook_filename) local_jupyter_filename = jupyter_notebook_filename else: notebook = None fd, local_jupyter_filename = mkstemp(suffix='.ipynb') os.close(fd) last_update_ts = None counter = 0 prev_script_hash = None # main observer loop, check if we need to exit while not cls._exit_event.wait(timeout=0.): # wait for timeout or sync event cls._sync_event.wait(cls._sample_frequency if counter else cls. _first_sample_frequency) cls._sync_event.clear() counter += 1 # noinspection PyBroadException try: # if there is no task connected, do nothing task = Task.current_task() if not task: continue # if we have a local file: if notebook: if not notebook.exists(): continue # check if notebook changed if last_update_ts is not None and notebook.stat( ).st_mtime - last_update_ts <= 0: continue last_update_ts = notebook.stat().st_mtime else: # serialize notebook to a temp file # noinspection PyBroadException try: get_ipython().run_line_magic('notebook', local_jupyter_filename) except Exception as ex: continue # get notebook python script script_code, resources = _script_exporter.from_filename( local_jupyter_filename) current_script_hash = hash(script_code) if prev_script_hash and prev_script_hash == current_script_hash: continue requirements_txt = '' conda_requirements = '' # parse jupyter python script and prepare pip requirements (pigar) # if backend supports requirements if file_import_modules and Session.check_min_api_version( '2.2'): fmodules, _ = file_import_modules(notebook.parts[-1], script_code) fmodules = ScriptRequirements.add_trains_used_packages( fmodules) installed_pkgs = get_installed_pkgs_detail() reqs = ReqsModules() for name in fmodules: if name in installed_pkgs: pkg_name, version = installed_pkgs[name] reqs.add(pkg_name, version, fmodules[name]) requirements_txt, conda_requirements = ScriptRequirements.create_requirements_txt( reqs) # update script prev_script_hash = current_script_hash data_script = task.data.script data_script.diff = script_code data_script.requirements = { 'pip': requirements_txt, 'conda': conda_requirements } task._update_script(script=data_script) # update requirements task._update_requirements(requirements=requirements_txt) except Exception: pass
def main(): print('TRAINS SDK setup process') conf_file = Path(LOCAL_CONFIG_FILES[0]).absolute() if conf_file.exists() and conf_file.is_file( ) and conf_file.stat().st_size > 0: print('Configuration file already exists: {}'.format(str(conf_file))) print('Leaving setup, feel free to edit the configuration file.') return print(host_description) web_host = input_url('Web Application Host', '') parsed_host = verify_url(web_host) if parsed_host.port == 8008: print( 'Port 8008 is the api port. Replacing 8080 with 8008 for Web application' ) api_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path web_host = parsed_host.scheme + "://" + parsed_host.netloc.replace( ':8008', ':8080', 1) + parsed_host.path files_host = parsed_host.scheme + "://" + parsed_host.netloc.replace( ':8008', ':8081', 1) + parsed_host.path elif parsed_host.port == 8080: api_host = parsed_host.scheme + "://" + parsed_host.netloc.replace( ':8080', ':8008', 1) + parsed_host.path web_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path files_host = parsed_host.scheme + "://" + parsed_host.netloc.replace( ':8080', ':8081', 1) + parsed_host.path elif parsed_host.netloc.startswith('demoapp.'): # this is our demo server api_host = parsed_host.scheme + "://" + parsed_host.netloc.replace( 'demoapp.', 'demoapi.', 1) + parsed_host.path web_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path files_host = parsed_host.scheme + "://" + parsed_host.netloc.replace( 'demoapp.', 'demofiles.', 1) + parsed_host.path elif parsed_host.netloc.startswith('app.'): # this is our application server api_host = parsed_host.scheme + "://" + parsed_host.netloc.replace( 'app.', 'api.', 1) + parsed_host.path web_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path files_host = parsed_host.scheme + "://" + parsed_host.netloc.replace( 'app.', 'files.', 1) + parsed_host.path elif parsed_host.netloc.startswith('demoapi.'): print( '{} is the api server, we need the web server. Replacing \'demoapi.\' with \'demoapp.\'' .format(parsed_host.netloc)) api_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path web_host = parsed_host.scheme + "://" + parsed_host.netloc.replace( 'demoapi.', 'demoapp.', 1) + parsed_host.path files_host = parsed_host.scheme + "://" + parsed_host.netloc.replace( 'demoapi.', 'demofiles.', 1) + parsed_host.path elif parsed_host.netloc.startswith('api.'): print( '{} is the api server, we need the web server. Replacing \'api.\' with \'app.\'' .format(parsed_host.netloc)) api_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path web_host = parsed_host.scheme + "://" + parsed_host.netloc.replace( 'api.', 'app.', 1) + parsed_host.path files_host = parsed_host.scheme + "://" + parsed_host.netloc.replace( 'api.', 'files.', 1) + parsed_host.path else: api_host = '' web_host = '' files_host = '' if not parsed_host.port: print( 'Host port not detected, do you wish to use the default 8080 port n/[y]? ', end='') replace_port = input().lower() if not replace_port or replace_port == 'y' or replace_port == 'yes': api_host = parsed_host.scheme + "://" + parsed_host.netloc + ':8008' + parsed_host.path web_host = parsed_host.scheme + "://" + parsed_host.netloc + ':8080' + parsed_host.path files_host = parsed_host.scheme + "://" + parsed_host.netloc + ':8081' + parsed_host.path elif not replace_port or replace_port.lower( ) == 'n' or replace_port.lower() == 'no': web_host = input_host_port("Web", parsed_host) api_host = input_host_port("API", parsed_host) files_host = input_host_port("Files", parsed_host) if not api_host: api_host = parsed_host.scheme + "://" + parsed_host.netloc + parsed_host.path api_host = input_url('API Host', api_host) files_host = input_url('File Store Host', files_host) print( '\nTRAINS Hosts configuration:\nWeb App: {}\nAPI: {}\nFile Store: {}\n' .format(web_host, api_host, files_host)) while True: print(description.format(web_host), end='') parse_input = input() # check if these are valid credentials credentials = None # noinspection PyBroadException try: parsed = ConfigFactory.parse_string(parse_input) if parsed: credentials = parsed.get("credentials", None) except Exception: credentials = None if not credentials or set(credentials) != {"access_key", "secret_key"}: print( 'Could not parse user credentials, try again one after the other.' ) credentials = {} # parse individual print('Enter user access key: ', end='') credentials['access_key'] = input() print('Enter user secret: ', end='') credentials['secret_key'] = input() print('Detected credentials key=\"{}\" secret=\"{}\"'.format( credentials['access_key'], credentials['secret_key'], )) from trains.backend_api.session import Session # noinspection PyBroadException try: print('Verifying credentials ...') Session(api_key=credentials['access_key'], secret_key=credentials['secret_key'], host=api_host) print('Credentials verified!') break except Exception: print( 'Error: could not verify credentials: host={} access={} secret={}' .format(api_host, credentials['access_key'], credentials['secret_key'])) # noinspection PyBroadException try: default_sdk_conf = Path(__file__).parent.absolute() / 'sdk.conf' with open(str(default_sdk_conf), 'rt') as f: default_sdk = f.read() except Exception: print('Error! Could not read default configuration file') return # noinspection PyBroadException try: with open(str(conf_file), 'wt') as f: header = '# TRAINS SDK configuration file\n' \ 'api {\n' \ ' # Notice: \'host\' is the api server (default port 8008), not the web server.\n' \ ' api_server: %s\n' \ ' web_server: %s\n' \ ' files_server: %s\n' \ ' # Credentials are generated using the webapp, %s/profile\n' \ ' credentials {"access_key": "%s", "secret_key": "%s"}\n' \ '}\n' \ 'sdk ' % (api_host, web_host, files_host, web_host, credentials['access_key'], credentials['secret_key']) f.write(header) f.write(default_sdk) except Exception: print('Error! Could not write configuration file at: {}'.format( str(conf_file))) return print('\nNew configuration stored in {}'.format(str(conf_file))) print('TRAINS setup completed successfully.')
def main(): default_config_file = os.getenv(LOCAL_CONFIG_FILE_OVERRIDE_VAR) or LOCAL_CONFIG_FILES[0] p = argparse.ArgumentParser(description=__doc__) p.add_argument( "--file", "-F", help="Target configuration file path (default is %(default)s)", default=default_config_file, type=validate_file ) args = p.parse_args() print('TRAINS SDK setup process') conf_file = Path(args.file).absolute() if conf_file.exists() and conf_file.is_file() and conf_file.stat().st_size > 0: print('Configuration file already exists: {}'.format(str(conf_file))) print('Leaving setup, feel free to edit the configuration file.') return print(description, end='') sentinel = '' parse_input = '\n'.join(iter(input, sentinel)) credentials = None api_server = None web_server = None # noinspection PyBroadException try: parsed = ConfigFactory.parse_string(parse_input) if parsed: # Take the credentials in raw form or from api section credentials = get_parsed_field(parsed, ["credentials"]) api_server = get_parsed_field(parsed, ["api_server", "host"]) web_server = get_parsed_field(parsed, ["web_server"]) except Exception: credentials = credentials or None api_server = api_server or None web_server = web_server or None while not credentials or set(credentials) != {"access_key", "secret_key"}: print('Could not parse credentials, please try entering them manually.') credentials = read_manual_credentials() print('Detected credentials key=\"{}\" secret=\"{}\"'.format(credentials['access_key'], credentials['secret_key'][0:4] + "***")) web_input = True if web_server: host = input_url('WEB Host', web_server) elif api_server: web_input = False host = input_url('API Host', api_server) else: print(host_description.format(CONFIG_FILE=args.file, HOST=def_host,)) host = input_url('WEB Host', '') parsed_host = verify_url(host) api_host, files_host, web_host = parse_host(parsed_host, allow_input=True) # on of these two we configured if not web_input: web_host = input_url('Web Application Host', web_host) else: api_host = input_url('API Host', api_host) files_host = input_url('File Store Host', files_host) print('\nTRAINS Hosts configuration:\nWeb App: {}\nAPI: {}\nFile Store: {}\n'.format( web_host, api_host, files_host)) retry = 1 max_retries = 2 while retry <= max_retries: # Up to 2 tries by the user if verify_credentials(api_host, credentials): break retry += 1 if retry < max_retries + 1: credentials = read_manual_credentials() else: print('Exiting setup without creating configuration file') return # noinspection PyBroadException try: default_sdk_conf = Path(__file__).parent.absolute() / 'sdk.conf' with open(str(default_sdk_conf), 'rt') as f: default_sdk = f.read() except Exception: print('Error! Could not read default configuration file') return # noinspection PyBroadException try: with open(str(conf_file), 'wt') as f: header = '# TRAINS SDK configuration file\n' \ 'api {\n' \ ' # Notice: \'host\' is the api server (default port 8008), not the web server.\n' \ ' api_server: %s\n' \ ' web_server: %s\n' \ ' files_server: %s\n' \ ' # Credentials are generated using the webapp, %s/profile\n' \ ' credentials {"access_key": "%s", "secret_key": "%s"}\n' \ '}\n' \ 'sdk ' % (api_host, web_host, files_host, web_host, credentials['access_key'], credentials['secret_key']) f.write(header) f.write(default_sdk) except Exception: print('Error! Could not write configuration file at: {}'.format(str(conf_file))) return print('\nNew configuration stored in {}'.format(str(conf_file))) print('TRAINS setup completed successfully.')
def upload_artifact(self, name, artifact_object=None, metadata=None, preview=None, delete_after_upload=False, auto_pickle=True): # type: (str, Optional[object], Optional[dict], Optional[str], bool, bool) -> bool if not Session.check_min_api_version('2.3'): LoggerRoot.get_base_logger().warning( 'Artifacts not supported by your TRAINS-server version, ' 'please upgrade to the latest server version') return False if name in self._artifacts_container: raise ValueError( "Artifact by the name of {} is already registered, use register_artifact" .format(name)) # cast preview to string if preview: preview = str(preview) # convert string to object if try is a file/folder (dont try to serialize long texts if isinstance(artifact_object, six.string_types) and len(artifact_object) < 2048: # noinspection PyBroadException try: artifact_path = Path(artifact_object) if artifact_path.exists(): artifact_object = artifact_path elif '*' in artifact_object or '?' in artifact_object: # hackish, detect wildcard in tr files folder = Path('').joinpath(*artifact_path.parts[:-1]) if folder.is_dir() and folder.parts: wildcard = artifact_path.parts[-1] if list(Path(folder).rglob(wildcard)): artifact_object = artifact_path except Exception: pass artifact_type_data = tasks.ArtifactTypeData() artifact_type_data.preview = '' override_filename_in_uri = None override_filename_ext_in_uri = None uri = None if np and isinstance(artifact_object, np.ndarray): artifact_type = 'numpy' artifact_type_data.content_type = 'application/numpy' artifact_type_data.preview = preview or str( artifact_object.__repr__()) override_filename_ext_in_uri = '.npz' override_filename_in_uri = name + override_filename_ext_in_uri fd, local_filename = mkstemp(prefix=quote(name, safe="") + '.', suffix=override_filename_ext_in_uri) os.close(fd) np.savez_compressed(local_filename, **{name: artifact_object}) delete_after_upload = True elif pd and isinstance(artifact_object, pd.DataFrame): artifact_type = 'pandas' artifact_type_data.content_type = 'text/csv' artifact_type_data.preview = preview or str( artifact_object.__repr__()) override_filename_ext_in_uri = self._save_format override_filename_in_uri = name fd, local_filename = mkstemp(prefix=quote(name, safe="") + '.', suffix=override_filename_ext_in_uri) os.close(fd) artifact_object.to_csv(local_filename, compression=self._compression) delete_after_upload = True elif isinstance(artifact_object, Image.Image): artifact_type = 'image' artifact_type_data.content_type = 'image/png' desc = str(artifact_object.__repr__()) artifact_type_data.preview = preview or desc[1:desc.find(' at ')] override_filename_ext_in_uri = '.png' override_filename_in_uri = name + override_filename_ext_in_uri fd, local_filename = mkstemp(prefix=quote(name, safe="") + '.', suffix=override_filename_ext_in_uri) os.close(fd) artifact_object.save(local_filename) delete_after_upload = True elif isinstance(artifact_object, dict): artifact_type = 'JSON' artifact_type_data.content_type = 'application/json' preview = preview or json.dumps( artifact_object, sort_keys=True, indent=4) override_filename_ext_in_uri = '.json' override_filename_in_uri = name + override_filename_ext_in_uri fd, local_filename = mkstemp(prefix=quote(name, safe="") + '.', suffix=override_filename_ext_in_uri) os.write(fd, bytes(preview.encode())) os.close(fd) if len(preview) < self.max_preview_size_bytes: artifact_type_data.preview = preview else: artifact_type_data.preview = '# full json too large to store, storing first {}kb\n{}'.format( self.max_preview_size_bytes // 1024, preview[:self.max_preview_size_bytes]) delete_after_upload = True elif isinstance(artifact_object, ( Path, pathlib_Path, ) if pathlib_Path is not None else (Path, )): # check if single file artifact_object = Path(artifact_object) artifact_object.expanduser().absolute() # noinspection PyBroadException try: create_zip_file = not artifact_object.is_file() except Exception: # Hack for windows pathlib2 bug, is_file isn't valid. create_zip_file = True else: # We assume that this is not Windows os if artifact_object.is_dir(): # change to wildcard artifact_object /= '*' if create_zip_file: folder = Path('').joinpath(*artifact_object.parts[:-1]) if not folder.is_dir() or not folder.parts: raise ValueError( "Artifact file/folder '{}' could not be found".format( artifact_object.as_posix())) wildcard = artifact_object.parts[-1] files = list(Path(folder).rglob(wildcard)) override_filename_ext_in_uri = '.zip' override_filename_in_uri = folder.parts[ -1] + override_filename_ext_in_uri fd, zip_file = mkstemp( prefix=quote(folder.parts[-1], safe="") + '.', suffix=override_filename_ext_in_uri) try: artifact_type_data.content_type = 'application/zip' archive_preview = 'Archive content {}:\n'.format( artifact_object.as_posix()) with ZipFile(zip_file, 'w', allowZip64=True, compression=ZIP_DEFLATED) as zf: for filename in sorted(files): if filename.is_file(): relative_file_name = filename.relative_to( folder).as_posix() archive_preview += '{} - {}\n'.format( relative_file_name, humanfriendly.format_size( filename.stat().st_size)) zf.write(filename.as_posix(), arcname=relative_file_name) except Exception as e: # failed uploading folder: LoggerRoot.get_base_logger().warning( 'Exception {}\nFailed zipping artifact folder {}'. format(folder, e)) return False finally: os.close(fd) artifact_type_data.preview = preview or archive_preview artifact_object = zip_file artifact_type = 'archive' artifact_type_data.content_type = mimetypes.guess_type( artifact_object)[0] local_filename = artifact_object delete_after_upload = True else: if not artifact_object.is_file(): raise ValueError( "Artifact file '{}' could not be found".format( artifact_object.as_posix())) override_filename_in_uri = artifact_object.parts[-1] artifact_type_data.preview = preview or '{} - {}\n'.format( artifact_object, humanfriendly.format_size(artifact_object.stat().st_size)) artifact_object = artifact_object.as_posix() artifact_type = 'custom' artifact_type_data.content_type = mimetypes.guess_type( artifact_object)[0] local_filename = artifact_object elif (isinstance(artifact_object, six.string_types) and len(artifact_object) < 4096 and urlparse(artifact_object).scheme in remote_driver_schemes): # we should not upload this, just register local_filename = None uri = artifact_object artifact_type = 'custom' artifact_type_data.content_type = mimetypes.guess_type( artifact_object)[0] elif isinstance(artifact_object, six.string_types): # if we got here, we should store it as text file. artifact_type = 'string' artifact_type_data.content_type = 'text/plain' if preview: artifact_type_data.preview = preview elif len(artifact_object) < self.max_preview_size_bytes: artifact_type_data.preview = artifact_object else: artifact_type_data.preview = '# full text too large to store, storing first {}kb\n{}'.format( self.max_preview_size_bytes // 1024, artifact_object[:self.max_preview_size_bytes]) delete_after_upload = True override_filename_ext_in_uri = '.txt' override_filename_in_uri = name + override_filename_ext_in_uri fd, local_filename = mkstemp(prefix=quote(name, safe="") + '.', suffix=override_filename_ext_in_uri) os.close(fd) # noinspection PyBroadException try: with open(local_filename, 'wt') as f: f.write(artifact_object) except Exception: # cleanup and raise exception os.unlink(local_filename) raise elif auto_pickle: # if we are here it means we do not know what to do with the object, so we serialize it with pickle. artifact_type = 'pickle' artifact_type_data.content_type = 'application/pickle' # noinspection PyBroadException try: artifact_type_data.preview = preview or str( artifact_object.__repr__())[:self.max_preview_size_bytes] except Exception: artifact_type_data.preview = preview or '' delete_after_upload = True override_filename_ext_in_uri = '.pkl' override_filename_in_uri = name + override_filename_ext_in_uri fd, local_filename = mkstemp(prefix=quote(name, safe="") + '.', suffix=override_filename_ext_in_uri) os.close(fd) # noinspection PyBroadException try: with open(local_filename, 'wb') as f: pickle.dump(artifact_object, f) except Exception: # cleanup and raise exception os.unlink(local_filename) raise else: raise ValueError("Artifact type {} not supported".format( type(artifact_object))) # remove from existing list, if exists for artifact in self._task_artifact_list: if artifact.key == name: if artifact.type == self._pd_artifact_type: raise ValueError( "Artifact of name {} already registered, " "use register_artifact instead".format(name)) self._task_artifact_list.remove(artifact) break if not local_filename: file_size = None file_hash = None else: # check that the file to upload exists local_filename = Path(local_filename).absolute() if not local_filename.exists() or not local_filename.is_file(): LoggerRoot.get_base_logger().warning( 'Artifact upload failed, cannot find file {}'.format( local_filename.as_posix())) return False file_hash, _ = self.sha256sum(local_filename.as_posix()) file_size = local_filename.stat().st_size uri = self._upload_local_file( local_filename, name, delete_after_upload=delete_after_upload, override_filename=override_filename_in_uri, override_filename_ext=override_filename_ext_in_uri) timestamp = int(time()) artifact = tasks.Artifact( key=name, type=artifact_type, uri=uri, content_size=file_size, hash=file_hash, timestamp=timestamp, type_data=artifact_type_data, display_data=[(str(k), str(v)) for k, v in metadata.items()] if metadata else None) # update task artifacts with self._task_edit_lock: self._task_artifact_list.append(artifact) self._task.set_artifacts(self._task_artifact_list) return True
def _upload_data_audit_artifacts(self, name): logger = self._task.get_logger() pd_artifact = self._artifacts_container.get(name) pd_metadata = self._artifacts_container.get_metadata(name) # remove from artifacts watch list if name in self._unregister_request: try: self._unregister_request.remove(name) except KeyError: pass self._artifacts_container.unregister_artifact(name) if pd_artifact is None: return override_filename_ext_in_uri = self._save_format override_filename_in_uri = name fd, local_csv = mkstemp(prefix=quote(name, safe="") + '.', suffix=override_filename_ext_in_uri) os.close(fd) local_csv = Path(local_csv) pd_artifact.to_csv(local_csv.as_posix(), index=False, compression=self._compression) current_sha2, file_sha2 = self.sha256sum(local_csv.as_posix(), skip_header=32) if name in self._last_artifacts_upload: previous_sha2 = self._last_artifacts_upload[name] if previous_sha2 == current_sha2: # nothing to do, we can skip the upload try: local_csv.unlink() except Exception: pass return self._last_artifacts_upload[name] = current_sha2 # If old trains-server, upload as debug image if not Session.check_min_api_version('2.3'): logger.report_image(title='artifacts', series=name, local_path=local_csv.as_posix(), delete_after_upload=True, iteration=self._task.get_last_iteration(), max_image_history=2) return # Find our artifact artifact = None for an_artifact in self._task_artifact_list: if an_artifact.key == name: artifact = an_artifact break file_size = local_csv.stat().st_size # upload file uri = self._upload_local_file( local_csv, name, delete_after_upload=True, override_filename=override_filename_in_uri, override_filename_ext=override_filename_ext_in_uri) # update task artifacts with self._task_edit_lock: if not artifact: artifact = tasks.Artifact(key=name, type=self._pd_artifact_type) self._task_artifact_list.append(artifact) artifact_type_data = tasks.ArtifactTypeData() artifact_type_data.data_hash = current_sha2 artifact_type_data.content_type = "text/csv" artifact_type_data.preview = str( pd_artifact.__repr__()) + '\n\n' + self._get_statistics( {name: pd_artifact}) artifact.type_data = artifact_type_data artifact.uri = uri artifact.content_size = file_size artifact.hash = file_sha2 artifact.timestamp = int(time()) artifact.display_data = [ (str(k), str(v)) for k, v in pd_metadata.items() ] if pd_metadata else None self._task.set_artifacts(self._task_artifact_list)
def main(): default_config_file = LOCAL_CONFIG_FILE_OVERRIDE_VAR.get() if not default_config_file: for f in LOCAL_CONFIG_FILES: default_config_file = f if os.path.exists(os.path.expanduser(os.path.expandvars(f))): break p = argparse.ArgumentParser(description=__doc__) p.add_argument( "--file", "-F", help="Target configuration file path (default is %(default)s)", default=default_config_file, type=validate_file) args = p.parse_args() print('ClearML SDK setup process') conf_file = Path(os.path.expanduser(args.file)).absolute() if conf_file.exists() and conf_file.is_file( ) and conf_file.stat().st_size > 0: print('Configuration file already exists: {}'.format(str(conf_file))) print('Leaving setup, feel free to edit the configuration file.') return print(description, end='') sentinel = '' parse_input = '' # COLAB_GPU will always be available, even when running on CPU if os.environ.get('COLAB_GPU'): # When running from a colab instance and calling clearml-init # colab will squish the api credentials into a single line # The regex splits this single line based on 2 spaces or more import re api_input = input() parse_input = '\n'.join(re.split(r" {2,}", api_input)) else: for line in iter(input, sentinel): parse_input += line + '\n' if line.rstrip() == '}': break credentials = None api_server = None web_server = None # noinspection PyBroadException try: parsed = ConfigFactory.parse_string(parse_input) if parsed: # Take the credentials in raw form or from api section credentials = get_parsed_field(parsed, ["credentials"]) api_server = get_parsed_field(parsed, ["api_server", "host"]) web_server = get_parsed_field(parsed, ["web_server"]) except Exception: credentials = credentials or None api_server = api_server or None web_server = web_server or None while not credentials or set(credentials) != {"access_key", "secret_key"}: print( 'Could not parse credentials, please try entering them manually.') credentials = read_manual_credentials() print('Detected credentials key=\"{}\" secret=\"{}\"'.format( credentials['access_key'], credentials['secret_key'][0:4] + "***")) web_input = True if web_server: host = input_url('WEB Host', web_server) elif api_server: web_input = False host = input_url('API Host', api_server) else: print(host_description.format( CONFIG_FILE=args.file, HOST=def_host, )) host = input_url('WEB Host', '') parsed_host = verify_url(host) api_host, files_host, web_host = parse_host(parsed_host, allow_input=True) # on of these two we configured if not web_input: web_host = input_url('Web Application Host', web_host) else: if web_input is True and not web_host: web_host = host api_host = input_url('API Host', api_host) files_host = input_url('File Store Host', files_host) print( '\nClearML Hosts configuration:\nWeb App: {}\nAPI: {}\nFile Store: {}\n' .format(web_host, api_host, files_host)) retry = 1 max_retries = 2 while retry <= max_retries: # Up to 2 tries by the user if verify_credentials(api_host, credentials): break retry += 1 if retry < max_retries + 1: credentials = read_manual_credentials() else: print('Exiting setup without creating configuration file') return # noinspection PyBroadException try: default_sdk_conf = Path( __file__).absolute().parents[2] / 'config/default/sdk.conf' with open(str(default_sdk_conf), 'rt') as f: default_sdk = f.read() except Exception: print('Error! Could not read default configuration file') return # noinspection PyBroadException try: with open(str(conf_file), 'wt') as f: header = '# ClearML SDK configuration file\n' \ 'api {\n' \ ' # Notice: \'host\' is the api server (default port 8008), not the web server.\n' \ ' api_server: %s\n' \ ' web_server: %s\n' \ ' files_server: %s\n' \ ' # Credentials are generated using the webapp, %s/settings\n' \ ' # Override with os environment: CLEARML_API_ACCESS_KEY / CLEARML_API_SECRET_KEY\n' \ ' credentials {"access_key": "%s", "secret_key": "%s"}\n' \ '}\n' \ 'sdk ' % (api_host, web_host, files_host, web_host, credentials['access_key'], credentials['secret_key']) f.write(header) f.write(default_sdk) except Exception: print('Error! Could not write configuration file at: {}'.format( str(conf_file))) return print('\nNew configuration stored in {}'.format(str(conf_file))) print('ClearML setup completed successfully.')