class TermiusApp(App): """Class for CLI application.""" def __init__(self): """Construct new CLI application.""" super(TermiusApp, self).__init__( description='Termius app', version=__version__, command_manager=CommandManager('termius.handlers'), ) self.configure_signals() self.directory_path = Path(expanduser('~/.{}/'.format(self.NAME))) if not self.directory_path.is_dir(): self.directory_path.mkdir(parents=True) def configure_logging(self): """Change logging level for request package.""" super(TermiusApp, self).configure_logging() logging.getLogger('requests').setLevel(logging.WARNING) return # pylint: disable=no-self-use def configure_signals(self): """Bind subscribers to signals.""" post_create_instance.connect(store_ssh_key, sender=SshKey) post_update_instance.connect(store_ssh_key, sender=SshKey) post_delete_instance.connect(delete_ssh_key, sender=SshKey) post_logout.connect(clean_data)
def process(self, path): path = Path(path) if path.is_dir(): self.process_files_in(path) else: self.process_one_file(path)
def get_file(path): result = Path('web') / path if result.is_file(): return str(result) if result.is_dir() and (result / 'index.html').is_file(): return str(result / 'index.html') # File was not found. return None
class TermiusClient(object): def __init__(self): self.app_directory = Path('~/.termius/').expanduser() self.command_mock = Mock(**{'app.directory_path': self.app_directory}) self.prepare() def create_identity(self, label, is_visible): return self._create_instance( Identity, label=label, is_visible=is_visible ) def create_host(self, address, label): return self._create_instance(Host, label=label, address=address) def create_pfrule(self, host_id, local_port, label): return self._create_instance( PFRule, label=label, pftype='L', local_port=local_port, remote_port=22, hostname='localhost' ) def create_group(self, label): return self._create_instance(Group, label=label) def _create_instance(self, model, **kwargs): instance = model(**kwargs) with self.storage: return self.storage.save(instance) def prepare(self): self.clean() if not self.app_directory.is_dir(): self.app_directory.mkdir() self.storage = ApplicationStorage(self.command_mock) def clean(self): if self.app_directory.is_dir(): self._clean_dir(self.app_directory) def _clean_dir(self, dir_path): [self._clean_dir(i) for i in dir_path.iterdir() if i.is_dir()] [i.unlink() for i in dir_path.iterdir() if i.is_file()] dir_path.rmdir()
def is_resource(package, name): """True if name is a resource inside package. Directories are *not* resources. """ package = _get_package(package) _normalize_path(name) try: package_contents = set(contents(package)) except OSError as error: if error.errno not in (errno.ENOENT, errno.ENOTDIR): # We won't hit this in the Python 2 tests, so it'll appear # uncovered. We could mock os.listdir() to return a non-ENOENT or # ENOTDIR, but then we'd have to depend on another external # library since Python 2 doesn't have unittest.mock. It's not # worth it. raise # pragma: nocover return False if name not in package_contents: return False # Just because the given file_name lives as an entry in the package's # contents doesn't necessarily mean it's a resource. Directories are not # resources, so let's try to find out if it's a directory or not. path = Path(package.__file__).parent / name if path.is_file(): return True if path.is_dir(): return False # If it's not a file and it's not a directory, what is it? Well, this # means the file doesn't exist on the file system, so it probably lives # inside a zip file. We have to crack open the zip, look at its table of # contents, and make sure that this entry doesn't have sub-entries. archive_path = package.__loader__.archive # type: ignore package_directory = Path(package.__file__).parent with ZipFile(archive_path) as zf: toc = zf.namelist() relpath = package_directory.relative_to(archive_path) candidate_path = relpath / name for entry in toc: # pragma: nobranch try: relative_to_candidate = Path(entry).relative_to(candidate_path) except ValueError: # The two paths aren't relative to each other so we can ignore it. continue # Since directories aren't explicitly listed in the zip file, we must # infer their 'directory-ness' by looking at the number of path # components in the path relative to the package resource we're # looking up. If there are zero additional parts, it's a file, i.e. a # resource. If there are more than zero it's a directory, i.e. not a # resource. It has to be one of these two cases. return len(relative_to_candidate.parts) == 0 # I think it's impossible to get here. It would mean that we are looking # for a resource in a zip file, there's an entry matching it in the return # value of contents(), but we never actually found it in the zip's table of # contents. raise AssertionError('Impossible situation')
def clone_repository_cached(session, execution, destination): # type: (Session, ExecutionInfo, Path) -> Tuple[VCS, RepoInfo] """ Clone a remote repository. :param execution: execution info :param destination: directory to clone to (in which a directory for the repository will be created) :param session: program session :return: repository information :raises: CommandFailedError if git/hg is not installed """ repo_url = execution.repository # type: str parsed_url = furl(repo_url) no_password_url = parsed_url.copy().remove(password=True).url clone_folder_name = Path(str(furl(repo_url).path)).name # type: str clone_folder = Path(destination) / clone_folder_name cached_repo_path = ( Path(session.config["agent.vcs_cache.path"]).expanduser() / "{}.{}".format(clone_folder_name, md5(ensure_binary(repo_url)).hexdigest()) / clone_folder_name) # type: Path vcs = VcsFactory.create(session, execution_info=execution, location=cached_repo_path) if not find_executable(vcs.executable_name): raise CommandFailedError(vcs.executable_not_found_error_help()) if session.config["agent.vcs_cache.enabled"] and cached_repo_path.exists(): print('Using cached repository in "{}"'.format(cached_repo_path)) else: print("cloning: {}".format(no_password_url)) rm_tree(cached_repo_path) # We clone the entire repository, not a specific branch vcs.clone() # branch=execution.branch) vcs.pull() rm_tree(destination) shutil.copytree(Text(cached_repo_path), Text(clone_folder)) if not clone_folder.is_dir(): raise CommandFailedError( "copying of repository failed: from {} to {}".format( cached_repo_path, clone_folder)) # checkout in the newly copy destination vcs.location = Text(clone_folder) vcs.checkout() repo_info = vcs.get_repository_copy_info(clone_folder) # make sure we have no user/pass in the returned repository structure repo_info = attr.evolve(repo_info, url=no_password_url) return vcs, repo_info
def __call__(self, parser, namespace, path, option_string=None): install_dir = Path(path) if not install_dir.is_dir(): raise argparse.ArgumentTypeError("{0} is not a valid directory".format(path)) binary = Path(install_dir, "sbin/syslog-ng") if not binary.exists(): raise argparse.ArgumentTypeError("{0} not exist".format(binary)) setattr(namespace, self.dest, path)
def updateRepos(): # pylint: disable=invalid-name """Update Mercurial and Git repositories located in ~ and ~/trees .""" home_dir = Path.home() trees = [home_dir, home_dir / "trees"] for tree in trees: for name in sorted(os.listdir(str(tree))): name_path = Path(tree) / name if name_path.is_dir() and (name in REPOS or (name.startswith("funfuzz") and "-" in name)): logger.info("Updating %s ...", name) updateRepo(name_path)
def init(libs_dir="libs"): logging.info("init libs: " + libs_dir) libs_path = Path(libs_dir) if not libs_path.is_dir(): raise Exception("libs dir not exists, " + libs_dir) all_files = fsutils.get_all_files(Path(libs_dir)) for f in all_files: lib = get_lib_info(str(f)) lib.path = f all_libs.append(lib) print("found a lib, name: {}, requires: {}, path: {}".format( lib.name, lib.requires, lib.path))
def mkdatadir(self, original_datadir=None): """Create a temporary directory for this factory's scope. """ # special condition if the datadir is specified as None, which # automatically gets the path that matches the basename of the # module we are in if original_datadir is None: original_datadir = Path( os.path.splitext(self.request.module.__file__)[0]) # get the path to the shared data dir original_path = Path(self.request.fspath.dirname) / original_datadir # make sure that the path exists and it is a directory exists = True if not original_path.exists(): # raise the flag that it doesn't exist so we can generate # a directory for it instead of copying exists = False # make sure the path is a directory if it exists elif not original_path.is_dir(): raise ValueError("datadir path is not a directory") # generate a base temporary directory and receive the path to it temp_path = self.tmp_path_factory.mktemp(TMPDIR_NAME) # in order to use the shutil.copytree util the target directory # must not exist so we specify a dir in the generated tempdir for it temp_data_path = temp_path / DATADIR_DIRNAME # windows-ify the paths original_path = Path(_win32_longpath(original_path)) temp_data_path = Path(_win32_longpath(str(temp_data_path))) # copy or create empty directory depending on whether the # original one exists if exists: # copy all the files in the original data dir to the temp # dir shutil.copytree(original_path, temp_data_path) else: # otherwise just give them a fallback tmpdir temp_data_path.mkdir() return temp_data_path
def data_path(relative_path, relative_to=None): """Returns data path to test file.""" if relative_to is None: # Use BASE_DIR as default. relative_to = BASE_DIR elif not isinstance(relative_to, Path): # Ensure relative_to is a Path. relative_to = Path(relative_to) # If relative_to is not a path, move up one level. if not relative_to.is_dir(): relative_to = relative_to.parent return relative_to / 'data' / relative_path
def get_config(config_file_name=str(find_config())): """Create config parser with defaults and read in the config file.""" default_config_dir = _get_default_config_dir() parser = make_config_parser({'datadir': str(default_config_dir.joinpath('data'))}) parser.add_section('general') parser.read([config_file_name]) env_datadir = os.getenv('REENTRY_DATADIR') if env_datadir: env_datadir_path = Path(env_datadir) if env_datadir_path.exists() and not env_datadir_path.is_dir(): # pylint: disable=no-member raise ValueError('environment variable $REENTRY_DATADIR={} exists, but is not a directory'.format(env_datadir)) parser.set('general', 'datadir', str(env_datadir_path)) return parser
def date_is_cached(self, when): # TODO -Swift Object Storage Checking file_path = Path(self.outputs["readings"]['path']) if not file_path.is_dir(): os.makedirs(file_path) ok = Path(self.netcdf_name_for_date(when)).is_file() logger.debug("\n--> Checking for existence of NetCDF @ %s for %s: %s" % (file_path, when.strftime("%d %m %Y"), ok)) # TODO -if OK put the file into Swift Storage return ok
def inner(shellFilename, hgHash): # pylint: disable=invalid-name,missing-docstring,missing-return-doc # pylint: disable=missing-return-type-doc # pylint: disable=invalid-name conditionArgs = conditionArgPrefix + [str(shellFilename)] + options.runtime_params temp_dir = Path(tempfile.mkdtemp(prefix="abExtTestAndLabel-" + hgHash)) temp_prefix = temp_dir / "t" if hasattr(conditionScript, "init"): # Since we're changing the js shell name, call init() again! conditionScript.init(conditionArgs) if conditionScript.interesting(conditionArgs, str(temp_prefix)): innerResult = ("bad", "interesting") # pylint: disable=invalid-name else: innerResult = ("good", "not interesting") # pylint: disable=invalid-name if temp_dir.is_dir(): sps.rm_tree_incl_readonly(str(temp_dir)) return innerResult
def find_spec(self, name, path, target=None): path = Path('./' + name.replace('.', r'/')) if path.with_suffix('.c').exists(): return ModuleSpec(name, CintegrateLoader(path, type='c'), is_package=True) if path.with_suffix('.cpp').exists(): return ModuleSpec(name, CintegrateLoader(path, type='c++'), is_package=True) if path.with_suffix('.so').exists(): return ModuleSpec(name, CintegrateLoader(path, type='so'), is_package=True) if path.is_dir(): return ModuleSpec(name, CintegrateLoader(path), is_package=True) return None
def update(self, path): if (type(path) != str): index = path[1] path = path[0] config = Path(path) tknzr = TweetTokenizer() lex, doc = dict(), dict() if (config.is_file()): # If just a file with config.open() as f: sentences = sent_tokenize(f.read()) document = path.split('/')[-1] doc[document] = list(sentences) for i, sentence in enumerate(sentences): token_list = tknzr.tokenize(sentence) for word in token_list: if (word not in self.stopwords and self.is_not_number(word) and len(word) > 1): if (pos_tag([word])[0][1] != 'NNP'): word = word.lower() if (word not in lex): lex[word] = Word(word, document, i) else: lex[word].update(document, i) return (lex, doc) elif (config.is_dir() ): # If directory, recursively read each directory and file if (path[-1] != '/'): path = path + '/' iterdir = list() for d in config.iterdir(): iterdir.append(path + d.name) self.outputs = [0] * len(list(iterdir)) p = Pool(self.maxPoolSize) outputs = p.map(self.update, zip(iterdir, list(range(len(iterdir))))) for (lex, doc) in outputs: for k, v in lex.items(): if (k not in self.lexicon): self.lexicon[k] = Word(k, v.documents) self.lexicon[k].count += v.count self.lexicon[k].documents.update(v.documents) self.documents.update(doc) else: print('{} doesn\'t exist.'.format(path))
def get_converted_data(dataset_task, conf_file): if dataset_task: dataset_upload_task = Dataset.get(dataset_id=dataset_task) else: dataset_upload_task = Dataset.get(dataset_project="Nvidia TLT examples with ClearML", dataset_name="Example data") image_directory_path = ( get_field_from_config(conf_file, "image_directory_path") .strip() .strip('"') .rpartition("/")[0] ) # noinspection PyBroadException try: os.makedirs(image_directory_path) except Exception: pass # download the artifact and open it saved_dataset = dataset_upload_task.get_local_copy() dataset_name = os.listdir(saved_dataset)[0] dataset_path = Path(os.path.join(saved_dataset, dataset_name)) if not dataset_path.is_dir() and dataset_path.suffix in (".zip", ".tgz", ".tar.gz"): dataset_suffix = dataset_path.suffix if dataset_suffix == ".zip": from zipfile import ZipFile ZipFile(dataset_path.as_posix()).extractall(path=image_directory_path) elif dataset_suffix == ".tar.gz": import tarfile with tarfile.open(dataset_path.as_posix()) as file: file.extractall(image_directory_path) elif dataset_suffix == ".tgz": import tarfile with tarfile.open(dataset_path.as_posix(), mode="r:gz") as file: file.extractall(image_directory_path) saved_dataset = str(dataset_path) else: os.system("cp -R {}/* {}".format(saved_dataset, image_directory_path)) print(saved_dataset)
def get_file(path): filepath = Path('site') / path if filepath.is_file(): return filepath for format in PAGE_FORMATS: index_file = filepath / ('index' + format) if filepath.is_dir() and index_file.is_file(): return index_file if filepath.suffix == '.css': style_file = filepath.parent / (filepath.stem + '.styl') if style_file.exists(): return style_file if filepath.suffix == '.js': coffee_file = filepath.parent / (filepath.stem + '.coffee') if coffee_file.exists(): return coffee_file return None
def create_tables(self, sql_folder: str = DatabaseUtils.sql_tables_path) -> None: """Create all tables from their schemas :see DatabaseUtils.sql_tables: """ # reach the sql sources folder folder = Path(sql_folder) if not folder.exists() \ or not folder.is_dir(): raise BadFileFormatException # for each file in the folder for file in folder.iterdir(): # if it is an sql file, execute it if file.suffix != DatabaseUtils.sql_extension: continue self._cursor.execute(file.read_text()) # commit transactions changes self._connection.commit()
def _get_system_wide_trusted_ca_certificates(self): trusted_cas, errors = set([]), [] for p in self.system_wide_trusted_ca_search_paths: cert_path = Path(p) if not cert_path.is_dir(): continue for entry in cert_path.iterdir(): cert_file_path = entry.absolute() try: if entry.suffix not in [".pem", ".crt"]: continue trusted_cas.update( self._get_certificates_from_file(cert_file_path)) except IOError: logger.exception("error updating CA") # This error is shown to the user as warning message during "activate changes". # We keep this message for the moment because we think that it is a helpful # trigger for further checking web.log when a really needed certificate can # not be read. # # We know a permission problem with some files that are created by default on # some distros. We simply ignore these files because we assume that they are # not needed. if cert_file_path == Path("/etc/ssl/certs/localhost.crt"): continue errors.append( "Failed to add certificate '%s' to trusted CA certificates. " "See web.log for details." % cert_file_path) break return list(trusted_cas), errors
def Deepzoom(image_or_directory_path,create_static_cache=False,**kwargs): """ Returns a Deepzoom interface corresponding to the given image. Can accept either a filepath (and will read tiles on the fly) or a directory path (that contains an image with the same name as the directory) that contains/will contain a static DeepZoom image directory. :param image_or_directory_path: String or Pathlib object :param create_static_cache: If True, creates a static DeepZoom image directory directory structure *around* the given image (or in the given directory). This is done lazily, saving each tile as it's requested. :param kwargs: Same as DeepzoomInterface :return: DeepZoom """ p = Path(image_or_directory_path).resolve() img = None if p.is_file(): img = _ImageFactory(p) elif p.is_dir(): fList = list(p.glob('%s.*'%p.name)) if len(fList)==0: raise IOError('Invalid Deepzoom directory (%s). ' 'Must contain and image named (%s) to be valid.' ''%(p,'%s.<EXT>'%p.name)) for f in fList: try: img = _ImageFactory(f) except IOError: pass if img is None: raise IOError('Invalid Deepzoom target (%s). ' 'Not a supported image format.'%(p)) if create_static_cache: # do something to DeepZoomGenerator so that it saves on get_tile() dzGen = _CachedInterface(img,**kwargs) else: dzGen = _DeepzoomInterface(img,**kwargs) return dzGen
def _make_file_info(target: pathlib.Path, arcname: Optional[str] = None, dereference=False) -> Dict[str, Any]: f = {} # type: Dict[str, Any] f['origin'] = target if arcname is not None: f['filename'] = pathlib.Path(arcname).as_posix() else: f['filename'] = target.as_posix() if os.name == 'nt': fstat = target.lstat() if target.is_symlink(): if dereference: fstat = target.stat() if stat.S_ISDIR(fstat.st_mode): f['emptystream'] = True f['attributes'] = fstat.st_file_attributes & FILE_ATTRIBUTE_WINDOWS_MASK # type: ignore # noqa else: f['emptystream'] = False f['attributes'] = stat.FILE_ATTRIBUTE_ARCHIVE # type: ignore # noqa f['uncompressed'] = fstat.st_size else: f['emptystream'] = False f['attributes'] = fstat.st_file_attributes & FILE_ATTRIBUTE_WINDOWS_MASK # type: ignore # noqa # f['attributes'] |= stat.FILE_ATTRIBUTE_REPARSE_POINT # type: ignore # noqa elif target.is_dir(): f['emptystream'] = True f['attributes'] = fstat.st_file_attributes & FILE_ATTRIBUTE_WINDOWS_MASK # type: ignore # noqa elif target.is_file(): f['emptystream'] = False f['attributes'] = stat.FILE_ATTRIBUTE_ARCHIVE # type: ignore # noqa f['uncompressed'] = fstat.st_size else: fstat = target.lstat() if target.is_symlink(): if dereference: fstat = target.stat() if stat.S_ISDIR(fstat.st_mode): f['emptystream'] = True f['attributes'] = stat.FILE_ATTRIBUTE_DIRECTORY # type: ignore # noqa f['attributes'] |= FILE_ATTRIBUTE_UNIX_EXTENSION | ( stat.S_IFDIR << 16) f['attributes'] |= (stat.S_IMODE(fstat.st_mode) << 16) else: f['emptystream'] = False f['attributes'] = stat.FILE_ATTRIBUTE_ARCHIVE # type: ignore # noqa f['attributes'] |= FILE_ATTRIBUTE_UNIX_EXTENSION | ( stat.S_IMODE(fstat.st_mode) << 16) else: f['emptystream'] = False f['attributes'] = stat.FILE_ATTRIBUTE_ARCHIVE | stat.FILE_ATTRIBUTE_REPARSE_POINT # type: ignore # noqa f['attributes'] |= FILE_ATTRIBUTE_UNIX_EXTENSION | ( stat.S_IFLNK << 16) f['attributes'] |= (stat.S_IMODE(fstat.st_mode) << 16) elif target.is_dir(): f['emptystream'] = True f['attributes'] = stat.FILE_ATTRIBUTE_DIRECTORY # type: ignore # noqa f['attributes'] |= FILE_ATTRIBUTE_UNIX_EXTENSION | ( stat.S_IFDIR << 16) f['attributes'] |= (stat.S_IMODE(fstat.st_mode) << 16) elif target.is_file(): f['emptystream'] = False f['uncompressed'] = fstat.st_size f['attributes'] = stat.FILE_ATTRIBUTE_ARCHIVE # type: ignore # noqa f['attributes'] |= FILE_ATTRIBUTE_UNIX_EXTENSION | ( stat.S_IMODE(fstat.st_mode) << 16) f['creationtime'] = fstat.st_ctime f['lastwritetime'] = fstat.st_mtime f['lastaccesstime'] = fstat.st_atime return f
def clone_repository_cached(session, execution, destination): # type: (Session, ExecutionInfo, Path) -> Tuple[VCS, RepoInfo] """ Clone a remote repository. :param execution: execution info :param destination: directory to clone to (in which a directory for the repository will be created) :param session: program session :return: repository information :raises: CommandFailedError if git/hg is not installed """ # mock lock repo_lock = Lock() repo_lock_timeout_sec = 300 repo_url = execution.repository # type: str parsed_url = furl(repo_url) no_password_url = parsed_url.copy().remove(password=True).url clone_folder_name = Path(str(furl(repo_url).path)).name # type: str clone_folder = Path(destination) / clone_folder_name standalone_mode = session.config.get("agent.standalone_mode", False) if standalone_mode: cached_repo_path = clone_folder else: vcs_cache_path = Path( session.config["agent.vcs_cache.path"]).expanduser() repo_hash = md5(ensure_binary(repo_url)).hexdigest() # create lock repo_lock = FileLock(filename=(vcs_cache_path / '{}.lock'.format(repo_hash)).as_posix()) # noinspection PyBroadException try: repo_lock.acquire(timeout=repo_lock_timeout_sec) except BaseException: print( 'Could not lock cache folder "{}" (timeout {} sec), using temp vcs cache.' .format(clone_folder_name, repo_lock_timeout_sec)) repo_hash = '{}_{}'.format(repo_hash, str(random()).replace('.', '')) # use mock lock for the context repo_lock = Lock() # select vcs cache folder cached_repo_path = vcs_cache_path / "{}.{}".format( clone_folder_name, repo_hash) / clone_folder_name with repo_lock: vcs = VcsFactory.create(session, execution_info=execution, location=cached_repo_path) if not find_executable(vcs.executable_name): raise CommandFailedError(vcs.executable_not_found_error_help()) if not standalone_mode: if session.config[ "agent.vcs_cache.enabled"] and cached_repo_path.exists(): print( 'Using cached repository in "{}"'.format(cached_repo_path)) else: print("cloning: {}".format(no_password_url)) rm_tree(cached_repo_path) # We clone the entire repository, not a specific branch vcs.clone() # branch=execution.branch) vcs.pull() rm_tree(destination) shutil.copytree(Text(cached_repo_path), Text(clone_folder)) if not clone_folder.is_dir(): raise CommandFailedError( "copying of repository failed: from {} to {}".format( cached_repo_path, clone_folder)) # checkout in the newly copy destination vcs.location = Text(clone_folder) vcs.checkout() repo_info = vcs.get_repository_copy_info(clone_folder) # make sure we have no user/pass in the returned repository structure repo_info = attr.evolve(repo_info, url=no_password_url) return vcs, repo_info
def upload_artifact(self, name, artifact_object=None, metadata=None, preview=None, delete_after_upload=False, auto_pickle=True): # type: (str, Optional[object], Optional[dict], Optional[str], bool, bool) -> bool if not Session.check_min_api_version('2.3'): LoggerRoot.get_base_logger().warning( 'Artifacts not supported by your TRAINS-server version, ' 'please upgrade to the latest server version') return False if name in self._artifacts_container: raise ValueError( "Artifact by the name of {} is already registered, use register_artifact" .format(name)) # cast preview to string if preview: preview = str(preview) # convert string to object if try is a file/folder (dont try to serialize long texts if isinstance(artifact_object, six.string_types) and len(artifact_object) < 2048: # noinspection PyBroadException try: artifact_path = Path(artifact_object) if artifact_path.exists(): artifact_object = artifact_path elif '*' in artifact_object or '?' in artifact_object: # hackish, detect wildcard in tr files folder = Path('').joinpath(*artifact_path.parts[:-1]) if folder.is_dir() and folder.parts: wildcard = artifact_path.parts[-1] if list(Path(folder).rglob(wildcard)): artifact_object = artifact_path except Exception: pass artifact_type_data = tasks.ArtifactTypeData() artifact_type_data.preview = '' override_filename_in_uri = None override_filename_ext_in_uri = None uri = None if np and isinstance(artifact_object, np.ndarray): artifact_type = 'numpy' artifact_type_data.content_type = 'application/numpy' artifact_type_data.preview = preview or str( artifact_object.__repr__()) override_filename_ext_in_uri = '.npz' override_filename_in_uri = name + override_filename_ext_in_uri fd, local_filename = mkstemp(prefix=quote(name, safe="") + '.', suffix=override_filename_ext_in_uri) os.close(fd) np.savez_compressed(local_filename, **{name: artifact_object}) delete_after_upload = True elif pd and isinstance(artifact_object, pd.DataFrame): artifact_type = 'pandas' artifact_type_data.content_type = 'text/csv' artifact_type_data.preview = preview or str( artifact_object.__repr__()) override_filename_ext_in_uri = self._save_format override_filename_in_uri = name fd, local_filename = mkstemp(prefix=quote(name, safe="") + '.', suffix=override_filename_ext_in_uri) os.close(fd) artifact_object.to_csv(local_filename, compression=self._compression) delete_after_upload = True elif isinstance(artifact_object, Image.Image): artifact_type = 'image' artifact_type_data.content_type = 'image/png' desc = str(artifact_object.__repr__()) artifact_type_data.preview = preview or desc[1:desc.find(' at ')] override_filename_ext_in_uri = '.png' override_filename_in_uri = name + override_filename_ext_in_uri fd, local_filename = mkstemp(prefix=quote(name, safe="") + '.', suffix=override_filename_ext_in_uri) os.close(fd) artifact_object.save(local_filename) delete_after_upload = True elif isinstance(artifact_object, dict): artifact_type = 'JSON' artifact_type_data.content_type = 'application/json' preview = preview or json.dumps( artifact_object, sort_keys=True, indent=4) override_filename_ext_in_uri = '.json' override_filename_in_uri = name + override_filename_ext_in_uri fd, local_filename = mkstemp(prefix=quote(name, safe="") + '.', suffix=override_filename_ext_in_uri) os.write(fd, bytes(preview.encode())) os.close(fd) if len(preview) < self.max_preview_size_bytes: artifact_type_data.preview = preview else: artifact_type_data.preview = '# full json too large to store, storing first {}kb\n{}'.format( self.max_preview_size_bytes // 1024, preview[:self.max_preview_size_bytes]) delete_after_upload = True elif isinstance(artifact_object, ( Path, pathlib_Path, ) if pathlib_Path is not None else (Path, )): # check if single file artifact_object = Path(artifact_object) artifact_object.expanduser().absolute() # noinspection PyBroadException try: create_zip_file = not artifact_object.is_file() except Exception: # Hack for windows pathlib2 bug, is_file isn't valid. create_zip_file = True else: # We assume that this is not Windows os if artifact_object.is_dir(): # change to wildcard artifact_object /= '*' if create_zip_file: folder = Path('').joinpath(*artifact_object.parts[:-1]) if not folder.is_dir() or not folder.parts: raise ValueError( "Artifact file/folder '{}' could not be found".format( artifact_object.as_posix())) wildcard = artifact_object.parts[-1] files = list(Path(folder).rglob(wildcard)) override_filename_ext_in_uri = '.zip' override_filename_in_uri = folder.parts[ -1] + override_filename_ext_in_uri fd, zip_file = mkstemp( prefix=quote(folder.parts[-1], safe="") + '.', suffix=override_filename_ext_in_uri) try: artifact_type_data.content_type = 'application/zip' archive_preview = 'Archive content {}:\n'.format( artifact_object.as_posix()) with ZipFile(zip_file, 'w', allowZip64=True, compression=ZIP_DEFLATED) as zf: for filename in sorted(files): if filename.is_file(): relative_file_name = filename.relative_to( folder).as_posix() archive_preview += '{} - {}\n'.format( relative_file_name, humanfriendly.format_size( filename.stat().st_size)) zf.write(filename.as_posix(), arcname=relative_file_name) except Exception as e: # failed uploading folder: LoggerRoot.get_base_logger().warning( 'Exception {}\nFailed zipping artifact folder {}'. format(folder, e)) return False finally: os.close(fd) artifact_type_data.preview = preview or archive_preview artifact_object = zip_file artifact_type = 'archive' artifact_type_data.content_type = mimetypes.guess_type( artifact_object)[0] local_filename = artifact_object delete_after_upload = True else: if not artifact_object.is_file(): raise ValueError( "Artifact file '{}' could not be found".format( artifact_object.as_posix())) override_filename_in_uri = artifact_object.parts[-1] artifact_type_data.preview = preview or '{} - {}\n'.format( artifact_object, humanfriendly.format_size(artifact_object.stat().st_size)) artifact_object = artifact_object.as_posix() artifact_type = 'custom' artifact_type_data.content_type = mimetypes.guess_type( artifact_object)[0] local_filename = artifact_object elif (isinstance(artifact_object, six.string_types) and len(artifact_object) < 4096 and urlparse(artifact_object).scheme in remote_driver_schemes): # we should not upload this, just register local_filename = None uri = artifact_object artifact_type = 'custom' artifact_type_data.content_type = mimetypes.guess_type( artifact_object)[0] elif isinstance(artifact_object, six.string_types): # if we got here, we should store it as text file. artifact_type = 'string' artifact_type_data.content_type = 'text/plain' if preview: artifact_type_data.preview = preview elif len(artifact_object) < self.max_preview_size_bytes: artifact_type_data.preview = artifact_object else: artifact_type_data.preview = '# full text too large to store, storing first {}kb\n{}'.format( self.max_preview_size_bytes // 1024, artifact_object[:self.max_preview_size_bytes]) delete_after_upload = True override_filename_ext_in_uri = '.txt' override_filename_in_uri = name + override_filename_ext_in_uri fd, local_filename = mkstemp(prefix=quote(name, safe="") + '.', suffix=override_filename_ext_in_uri) os.close(fd) # noinspection PyBroadException try: with open(local_filename, 'wt') as f: f.write(artifact_object) except Exception: # cleanup and raise exception os.unlink(local_filename) raise elif auto_pickle: # if we are here it means we do not know what to do with the object, so we serialize it with pickle. artifact_type = 'pickle' artifact_type_data.content_type = 'application/pickle' # noinspection PyBroadException try: artifact_type_data.preview = preview or str( artifact_object.__repr__())[:self.max_preview_size_bytes] except Exception: artifact_type_data.preview = preview or '' delete_after_upload = True override_filename_ext_in_uri = '.pkl' override_filename_in_uri = name + override_filename_ext_in_uri fd, local_filename = mkstemp(prefix=quote(name, safe="") + '.', suffix=override_filename_ext_in_uri) os.close(fd) # noinspection PyBroadException try: with open(local_filename, 'wb') as f: pickle.dump(artifact_object, f) except Exception: # cleanup and raise exception os.unlink(local_filename) raise else: raise ValueError("Artifact type {} not supported".format( type(artifact_object))) # remove from existing list, if exists for artifact in self._task_artifact_list: if artifact.key == name: if artifact.type == self._pd_artifact_type: raise ValueError( "Artifact of name {} already registered, " "use register_artifact instead".format(name)) self._task_artifact_list.remove(artifact) break if not local_filename: file_size = None file_hash = None else: # check that the file to upload exists local_filename = Path(local_filename).absolute() if not local_filename.exists() or not local_filename.is_file(): LoggerRoot.get_base_logger().warning( 'Artifact upload failed, cannot find file {}'.format( local_filename.as_posix())) return False file_hash, _ = self.sha256sum(local_filename.as_posix()) file_size = local_filename.stat().st_size uri = self._upload_local_file( local_filename, name, delete_after_upload=delete_after_upload, override_filename=override_filename_in_uri, override_filename_ext=override_filename_ext_in_uri) timestamp = int(time()) artifact = tasks.Artifact( key=name, type=artifact_type, uri=uri, content_size=file_size, hash=file_hash, timestamp=timestamp, type_data=artifact_type_data, display_data=[(str(k), str(v)) for k, v in metadata.items()] if metadata else None) # update task artifacts with self._task_edit_lock: self._task_artifact_list.append(artifact) self._task.set_artifacts(self._task_artifact_list) return True
class ExperimentMain(object): @property def my_rank_dir(self): return self.exp_dir_path / 'rank_{}'.format(self.my_rank) @property def my_rank(self): return get_mpi_rank_or_0() def main(self): self.comm = get_mpi_comm_world() self.rank = self.comm.Get_rank() self.comm.Barrier() self.setup() self.train() def setup(self): if self.rank == SERVER_RANK: params = get_configuration(print_diagnostics=True, inject_parameters_to_gin=True) experiment_id = params['experiment_id'] self.comm.bcast([experiment_id], root=SERVER_RANK) else: data = None data = self.comm.bcast(data, root=SERVER_RANK) experiment_id = data[0] if self.rank == EVALUATOR_RANK else None params = get_configuration(print_diagnostics=True, inject_parameters_to_gin=True) exp_dir_path = "." self.exp_dir_path = Path(exp_dir_path) if self.my_rank == SERVER_RANK: if not self.exp_dir_path.is_dir(): self.exp_dir_path.mkdir(parents=True) if not self.my_rank_dir.is_dir(): self.my_rank_dir.mkdir(parents=True) self.tee_stdout = TeeStdout(str(self.my_rank_dir / 'stdout'), mode='w') self.tee_stderr = TeeStderr(str(self.my_rank_dir / 'stderr'), mode='w') logger.configure(format_strs=['stdout', 'log', 'csv', 'tensorboard']) self.params = params self.run_eval_worker = self.params.get("run_eval_worker", False) def train(self): if self.params.get("tf_seed", None) is not None: tf.random.set_random_seed(self.params["tf_seed"]) if self.rank == SERVER_RANK: sess = U.make_session( num_cpu=self.params.get('server_tf_num_cpu', 2)) else: sess = U.single_threaded_session() sess.__enter__() value, planner, env_init_kwargs = create_agent(sess) sess.run(tf.global_variables_initializer()) if self.rank == SERVER_RANK: sess.run(value.initializers()) value.sync() # sync value weights to root 0 state_shape = value._replay.memory._state_shape game_buffer_size = self.params['game_buffer_size'] serializer = Serializer() serializer.add_variable("number_of_removed_games", shape=(1, 1), dtype=[np.int32]) serializer.add_variable("worker_step_count", shape=(1, 1), dtype=[np.int32]) serializer.add_variable("worker_total_step_count", shape=(1, 1), dtype=[np.int32]) for i in range(game_buffer_size): serializer.add_variable( name=f"game_{i}", shape=( planner.episode_max_steps, # game max len state_shape, # state (1, 1), # value (1, 1), # action ), dtype=[np.int32, np.float32, np.uint8]) serializer.add_variable(name=f"game_steps_{i}", shape=(1, 1), dtype=[np.int32]) serializer.add_variable(name=f"game_solved_{i}", shape=(1, 1), dtype=[np.int32]) serializer.add_variable(name=f"graph_size_{i}", shape=(1, 1), dtype=[np.int32]) serializer.add_variable(name=f"game_ensemble_std_{i}", shape=(1, 1), dtype=[np.float32]) serializer.add_variable(name=f"worker_step_count_{i}", shape=(1, 1), dtype=[np.int32]) serializer.add_variable(name=f"avg_{i}", shape=(1, 1), dtype=[np.float32]) serializer.add_variable(name=f"num_gen_steps_{i}", shape=(1, 1), dtype=[np.int32]) serializer.add_variable(name=f"env_curriculum_{i}", shape=(1, 1), dtype=[np.float32]) serializer.add_variable(name=f"avg_skip_{i}", shape=(1, 1), dtype=[np.float32]) serializer.add_variable(name=f"worker_avg_result_{i}", shape=(1, 1), dtype=[np.float32]) # serializer.add_variable(name=f"wm_next_frame_errors_rate_{i}", shape=(1, 1), dtype=[np.float32]) # serializer.add_variable(name=f"wm_reward_errors_rate_{i}", shape=(1, 1), dtype=[np.float32]) # serializer.add_variable(name=f"wm_missed_done_rate_{i}", shape=(1, 1), dtype=[np.float32]) # serializer.add_variable(name=f"wm_false_done_rate_{i}", shape=(1, 1), dtype=[np.float32]) # serializer.add_variable(name=f"wm_any_false_done_{i}", shape=(1, 1), dtype=[np.float32]) # serializer.add_variable(name=f"wm_any_missed_done_{i}",shape=(1, 1), dtype=[np.float32]) auto_ml_creator = AutoMLCreator() if auto_ml_creator.is_auto_ml_present: serializer.add_variable(f"auto_ml_parameters_{i}", shape=(1, len(auto_ml_creator.dims)), dtype=[np.int32]) curriculum = Curriculum( enabled=self.params.get('curriculum', False), model=planner._model, curriculum_threshold=self.params.get('curriculum_threshold', 0.8), curriculum_smooth_coeff=self.params.get('curriculum_smooth_coeff', 0.95), curriculum_initial_value=self.params.get( 'curriculum_initial_value', 0.0), curriculum_initial_length_random_walk=self.params.get( 'curriculum_initial_length_random_walk', 50), curriculum_maximal_length_random_walk=self.params.get( 'curriculum_maximal_length_random_walk', 300), curriculum_length_random_walk_delta=self.params.get( "curriculum_length_random_walk_delta", 50), curriculum_max_num_gen_steps=self.params.get( 'curriculum_max_num_gen_steps', 27), curriculum_running_average_drop_on_increase=self.params.get( 'curriculum_running_average_drop_on_increase', 0.2), ) if self.rank == SERVER_RANK: server = Server( value=value, serializer=serializer, training_steps=self.params['training_steps'], train_every_num_steps=self.params['train_every_num_steps'], game_buffer_size=game_buffer_size, exit_training_after_num_games=self.params.get( "exit_trainig_after_num_games", np.inf), curriculum=curriculum, eval_worker_exists=self.run_eval_worker, ) server.run() elif self.rank == EVALUATOR_RANK and self.run_eval_worker: from learning_and_planning.mcts.evaluator_worker import \ EvaluatorWorker eval_worker = EvaluatorWorker(value=value, env_kwargs=env_init_kwargs, planner=planner) eval_worker.run() else: worker = Worker( value=value, training_steps=self.params['training_steps'], planner=planner, serializer=serializer, game_buffer_size=game_buffer_size, curriculum=curriculum, ) worker.run()
class TermiusApp(App): """Class for CLI application.""" def __init__(self): """Construct new CLI application.""" super(TermiusApp, self).__init__( description='Termius - crossplatform SSH and Telnet client', version=__version__, command_manager=CommandManager('termius.handlers'), ) self.configure_signals() self.directory_path = Path(expanduser('~/.{}/'.format(self.NAME))) if not self.directory_path.is_dir(): self.directory_path.mkdir(parents=True) self.command_manager.add_command('help', HelpCommand) def configure_logging(self): """Change logging level for request package.""" super(TermiusApp, self).configure_logging() logging.getLogger('requests').setLevel(logging.WARNING) # pylint: disable=no-self-use def configure_signals(self): """Bind subscribers to signals.""" post_create_instance.connect(store_ssh_key, sender=SshKey) post_update_instance.connect(store_ssh_key, sender=SshKey) post_delete_instance.connect(delete_ssh_key, sender=SshKey) post_logout.connect(clean_data) def build_option_parser(self, description, version, argparse_kwargs=None): """Return an argparse option parser for this application. Subclasses may override this method to extend the parser with more global options. :param description: full description of the application :paramtype description: str :param version: version number for the application :paramtype version: str :param argparse_kwargs: extra keyword argument passed to the ArgumentParser constructor :paramtype extra_kwargs: dict """ argparse_kwargs = argparse_kwargs or {} parser = argparse.ArgumentParser(description=description, add_help=False, **argparse_kwargs) parser.add_argument('--version', action='version', version='%(prog)s {0}'.format(version), help='display version information and exit') verbose_group = parser.add_mutually_exclusive_group() verbose_group.add_argument( '-v', '--verbose', action='count', dest='verbose_level', default=self.DEFAULT_VERBOSE_LEVEL, help='provide a detailed output', ) verbose_group.add_argument( '-q', '--quiet', action='store_const', dest='verbose_level', const=0, help='display warnings and errors only', ) parser.add_argument( '--log-file', action='store', default=None, help='record output into a designated file', ) if self.deferred_help: parser.add_argument( '-h', '--help', dest='deferred_help', action='store_true', help="display help message", ) else: parser.add_argument( '-h', '--help', action=HelpAction, nargs=0, default=self, # tricky help="show the help message", ) parser.add_argument( '--debug', default=False, action='store_true', help='enable debugging mode', ) return parser
class GitRepo(object): def __init__(self, path, remote_url=None, branch_name='master'): self.path = Path(path) self.path_str = str(self.path) self.remote_url = remote_url self.branch_name = branch_name db_latest_key = '%s:%s:%s' % (self.path_str, remote_url or '', branch_name) self.db_latest_key = sha256(db_latest_key).hexdigest() def git(self, *args): """Run a git command against the current repo""" curdir = os.getcwd() try: os.chdir(self.path_str) output = check_output((GIT,) + args, stderr=STDOUT) finally: os.chdir(curdir) return output.strip() @property def current_hash(self): """The git revision ID (hash) of the current HEAD or None if no repo""" try: return self.git('rev-parse', 'HEAD') except OSError: return None def diff(self, start_hash, end_hash): """Return a 2 tuple: (modified files, deleted files)""" diff_out = StringIO(self.git('diff', '--name-status', start_hash, end_hash)) modified = set() removed = set() for line in diff_out: parts = line.split() # delete if parts[0] == 'D': removed.add(parts[1]) # rename elif parts[0][0] == 'R': removed.add(parts[1]) modified.add(parts[2]) # everything else else: # some types (like copy) have two file entries for part in parts[1:]: modified.add(part) return modified, removed def clone(self): """Clone the repo specified in the initial arguments""" if not self.remote_url: raise RuntimeError('remote_url required to clone') self.path.mkdir(parents=True, exist_ok=True) self.git('clone', '--depth', '1', '--branch', self.branch_name, self.remote_url, '.') def pull(self): """Update the repo to the latest of the remote and branch Return the previous hash and the new hash.""" old_hash = self.current_hash self.git('fetch', '-f', self.remote_url, self.branch_name) self.git('checkout', '-f', 'FETCH_HEAD') return old_hash, self.current_hash def update(self): """Updates a repo, cloning if necessary. :return a tuple of lists of modified and deleted files if updated, None if cloned """ if self.path.is_dir(): if not self.path.joinpath('.git').is_dir(): rmtree(self.path_str, ignore_errors=True) self.clone() else: return self.pull() else: self.clone() return None, None def reset(self, new_head): self.git('reset', '--hard', new_head) def get_db_latest(self): try: return GitRepoState.objects.get(repo_id=self.db_latest_key).latest_ref except GitRepoState.DoesNotExist: return None def has_changes(self): return self.current_hash != self.get_db_latest() def set_db_latest(self, latest_ref=None): latest_ref = latest_ref or self.current_hash rs, created = GitRepoState.objects.get_or_create(repo_id=self.db_latest_key, defaults={'latest_ref': latest_ref}) if not created: rs.latest_ref = latest_ref rs.save()
--------------------Files-------------------- """ originals = [] filenames = [] # Test existence of files try: if not cfg_img_path.is_absolute(): input_path = Path(cwd, "image_in", cfg_img_path) else: input_path = cfg_img_path # Files if input_path.is_file(): add_new_image_from_file(input_path) # Directories elif input_path.is_dir(): for img in input_path.glob("*"): add_new_image_from_file(img) else: logger.error("Could not resolve path.") raise OSError except OSError: logger.error("Could not load image/s!") sys.exit() try: input_path_local = Path(cwd, "fonts", cfg_font_path) input_path_windows = Path("C:\\windows\\fonts\\", cfg_font_path) if input_path_local.is_file(): font = ImageFont.truetype(str(input_path_local), cfg_font_size)
class GitRepo(object): def __init__(self, path, remote_url=None, remote_name=None, branch_name='master'): self.path = Path(path) self.path_str = str(self.path) self.remote_url = remote_url self.branch_name = branch_name if not remote_name: remote_name = 'bedrock-dev' if settings.DEV else 'bedrock-prod' self.remote_name = remote_name def git(self, *args): """Run a git command against the current repo""" curdir = os.getcwd() try: os.chdir(self.path_str) output = check_output((GIT,) + args, stderr=STDOUT) finally: os.chdir(curdir) return output.strip() @property def full_branch_name(self): """Full branch name with remote (e.g. origin/master)""" return '{}/{}'.format(self.remote_name, self.branch_name) @property def current_hash(self): """The git revision ID (hash) of the current HEAD""" return self.git('rev-parse', 'HEAD') @property def remote_names(self): """Return a list of the remote names in the repo""" return self.git('remote').split() def has_remote(self): """Return True if the repo has a remote by the correct name""" return self.remote_name in self.remote_names def add_remote(self): """Add the remote to the git repo from the init args""" if not self.remote_url: raise RuntimeError('remote_url required to add a remote') self.git('remote', 'add', self.remote_name, self.remote_url) def diff(self, start_hash, end_hash): """Return a 2 tuple: (modified files, deleted files)""" diff_out = StringIO(self.git('diff', '--name-status', start_hash, end_hash)) modified = set() removed = set() for line in diff_out: parts = line.split() # delete if parts[0] == 'D': removed.add(parts[1]) # rename elif parts[0][0] == 'R': removed.add(parts[1]) modified.add(parts[2]) # everything else else: # some types (like copy) have two file entries for part in parts[1:]: modified.add(part) return modified, removed def clone(self): """Clone the repo specified in the initial arguments""" if not self.remote_url: raise RuntimeError('remote_url required to clone') self.path.mkdir(parents=True, exist_ok=True) self.git('clone', '--origin', self.remote_name, '--depth', '1', '--branch', self.branch_name, self.remote_url, '.') def pull(self): """Update the repo to the latest of the remote and branch Return the previous hash and the new hash.""" if not self.has_remote(): self.add_remote() old_hash = self.current_hash self.git('fetch', self.remote_name) self.git('checkout', '-f', self.full_branch_name) return old_hash, self.current_hash def update(self): """Updates a repo, cloning if necessary. :return a tuple of lists of modified and deleted files if updated, None if cloned """ if self.path.is_dir(): if not self.path.joinpath('.git').is_dir(): rmtree(self.path_str, ignore_errors=True) self.clone() else: return self.diff(*self.pull()) else: self.clone() return None, None
class GitRepo: def __init__(self, path, remote_url=None, branch_name='master', name=None): self.path = Path(path) self.path_str = str(self.path) self.remote_url = remote_url self.branch_name = branch_name db_latest_key = '%s:%s:%s' % (self.path_str, remote_url or '', branch_name) self.db_latest_key = sha256(db_latest_key.encode()).hexdigest() self.repo_name = name or self.path.name def git(self, *args): """Run a git command against the current repo""" curdir = os.getcwd() try: os.chdir(self.path_str) output = check_output((GIT,) + args, stderr=STDOUT) finally: os.chdir(curdir) return output.strip() @property def current_hash(self): """The git revision ID (hash) of the current HEAD or None if no repo""" try: return self.git('rev-parse', 'HEAD') except (OSError, CalledProcessError): return None @property def current_commit_timestamp(self): """The UNIX timestamp of the latest commit""" try: return int(self.git('show', '-s', '--format=%ct', 'HEAD')) except (OSError, CalledProcessError, ValueError): return 0 @property def last_updated(self): if self.current_commit_timestamp: latest_datetime = datetime.fromtimestamp(self.current_commit_timestamp) return timeago.format(latest_datetime) return 'unknown' def diff(self, start_hash, end_hash): """Return a 2 tuple: (modified files, deleted files)""" diff_out = StringIO(self.git('diff', '--name-status', start_hash, end_hash)) modified = set() removed = set() for line in diff_out: parts = line.split() # delete if parts[0] == 'D': removed.add(parts[1]) # rename elif parts[0][0] == 'R': removed.add(parts[1]) modified.add(parts[2]) # everything else else: # some types (like copy) have two file entries for part in parts[1:]: modified.add(part) return modified, removed def clone(self): """Clone the repo specified in the initial arguments""" if not self.remote_url: raise RuntimeError('remote_url required to clone') self.path.mkdir(parents=True, exist_ok=True) self.git('clone', '--depth', '1', '--branch', self.branch_name, self.remote_url, '.') def pull(self): """Update the repo to the latest of the remote and branch Return the previous hash and the new hash.""" old_hash = self.current_hash self.git('fetch', '-f', self.remote_url, self.branch_name) self.git('checkout', '-f', 'FETCH_HEAD') return old_hash, self.current_hash def update(self): """Updates a repo, cloning if necessary. :return a tuple of lists of modified and deleted files if updated, None if cloned """ if self.path.is_dir(): if not self.path.joinpath('.git').is_dir(): rmtree(self.path_str, ignore_errors=True) self.clone() else: return self.pull() else: self.clone() return None, None def reset(self, new_head): self.git('reset', '--hard', new_head) def get_db_latest(self): try: return GitRepoState.objects.get(repo_id=self.db_latest_key).latest_ref except GitRepoState.DoesNotExist: return None def has_changes(self): return self.current_hash != self.get_db_latest() @property def clean_remote_url(self): repo_base = self.remote_url if repo_base.endswith('.git'): repo_base = repo_base[:-4] elif repo_base.endswith('/'): repo_base = repo_base[:-1] return repo_base def set_db_latest(self, latest_ref=None): latest_ref = latest_ref or self.current_hash rs, created = GitRepoState.objects.get_or_create( repo_id=self.db_latest_key, defaults={ 'latest_ref': latest_ref, 'repo_name': self.repo_name, 'repo_url': self.clean_remote_url, 'latest_ref_timestamp': self.current_commit_timestamp, }, ) if not created: rs.latest_ref = latest_ref rs.repo_name = self.repo_name rs.repo_url = self.clean_remote_url rs.latest_ref_timestamp = self.current_commit_timestamp rs.save()
def load(directory): global dataSize, db_name log = open('log.txt', 'w+') # file to log progress dirpath = Path(directory) # Check if path is valid if dirpath.exists(): stime = datetime.datetime.now() # start time for operation print(".fits files will be loaded from " + directory + ".\n") filestoadd = [] # list of all valid files to add to db # load files: conn = sqlite3.connect(db_name) c = conn.cursor() print("Finding files...\n") # if path is a directory, walk through it and find all fits files if dirpath.is_dir(): for root, dirs, files in os.walk(directory): for file in files: if file.endswith("dyn.fit") or file.endswith("dyn.fits"): filestoadd.append(os.path.join(root, file)) # otherwise, it is a file elif directory.endswith("dyn.fit") or directory.endswith("dyn.fits"): filestoadd.append(directory) else: print("No valid files found.") print("{0} files found.".format(len(filestoadd))) log.write("{0} files found.\n".format( len(filestoadd))) # port to log.txt success = 0 # counter for successful adds for i in range(0, len(filestoadd)): print("Loading {0} out of {1} files: {2}".format( i + 1, len(filestoadd), filestoadd[i])) log.write("Loading {0} out of {1} files: {2}\n".format( i + 1, len(filestoadd), filestoadd[i])) # port to log.txt try: header, data = readFile(filestoadd[i]) # insert all header information as numerics and binary data as json object command = "INSERT INTO astrodata VALUES('{0}' , '{1}' , '{2}' , '{3}' , '{4}' , '{5}' , ?)".format( header[0], header[1], header[2], header[3], header[4], header[5]) c.execute(command, (json.dumps(data.tolist()), )) success += 1 except Exception as e: print("Error loading file {0}: {1}".format( filestoadd[i], e.message)) log.write("Error loading file {0}: {1}\n".format( filestoadd[i], e.message)) # port to log.txt conn.commit() dataSize += 1 i += 1 print( "\nDone. {0} out of {1} files added successfully. See log.txt for more information.\n" .format(success, len(filestoadd))) log.write("\nDone. {0} out of {1} files added successfully.\n".format( success, len(filestoadd))) # port to log.txt tottime = datetime.datetime.now() - stime # elapsed time of operation print("Total time elapsed for adding: {0}\n".format(tottime)) log.write("Total time elapsed for adding: {0}\n".format( tottime)) # port to log.txt conn.close() elif directory != "exit": print( "Invalid directory. Type 'exit' to exit to the main menu or try again with a new input.\n" ) directory = input() if directory != "exit": load(directory) log.close()
class TermiusApp(App): """Class for CLI application.""" def __init__(self): """Construct new CLI application.""" super(TermiusApp, self).__init__( description='Termius - crossplatform SSH and Telnet client', version=__version__, command_manager=CommandManager('termius.handlers'), ) self.configure_signals() self.directory_path = Path(expanduser('~/.{}/'.format(self.NAME))) if not self.directory_path.is_dir(): self.directory_path.mkdir(parents=True) self.command_manager.add_command('help', HelpCommand) def configure_logging(self): """Change logging level for request package.""" super(TermiusApp, self).configure_logging() logging.getLogger('requests').setLevel(logging.WARNING) return # pylint: disable=no-self-use def configure_signals(self): """Bind subscribers to signals.""" post_create_instance.connect(store_ssh_key, sender=SshKey) post_update_instance.connect(store_ssh_key, sender=SshKey) post_delete_instance.connect(delete_ssh_key, sender=SshKey) post_logout.connect(clean_data) def build_option_parser(self, description, version, argparse_kwargs=None): """Return an argparse option parser for this application. Subclasses may override this method to extend the parser with more global options. :param description: full description of the application :paramtype description: str :param version: version number for the application :paramtype version: str :param argparse_kwargs: extra keyword argument passed to the ArgumentParser constructor :paramtype extra_kwargs: dict """ argparse_kwargs = argparse_kwargs or {} parser = argparse.ArgumentParser( description=description, add_help=False, **argparse_kwargs ) parser.add_argument( '--version', action='version', version='%(prog)s {0}'.format(version), help='display version information and exit' ) verbose_group = parser.add_mutually_exclusive_group() verbose_group.add_argument( '-v', '--verbose', action='count', dest='verbose_level', default=self.DEFAULT_VERBOSE_LEVEL, help='provide a detailed output', ) verbose_group.add_argument( '-q', '--quiet', action='store_const', dest='verbose_level', const=0, help='display warnings and errors only', ) parser.add_argument( '--log-file', action='store', default=None, help='record output into a designated file', ) if self.deferred_help: parser.add_argument( '-h', '--help', dest='deferred_help', action='store_true', help="display help message", ) else: parser.add_argument( '-h', '--help', action=HelpAction, nargs=0, default=self, # tricky help="show the help message", ) parser.add_argument( '--debug', default=False, action='store_true', help='enable debugging mode', ) return parser
class ReIDDataset(Dataset): """ The ReID Dataset module is a custom Dataset module, specific to parsing the Market & Duke Person Re-Identification datasets. Args: data_dir (str): The path where the dataset is located. transform ([list, torchvision.transforms], optional): Pass the list of transforms to transform the input images. Defaults to None. target_transform ([list, torchvision.transforms], optional): Pass the list of transforms to transform the labels. Defaults to None. ret_camid_n_frame (bool, optional): Whether to return camera ids and frames. True will additionally return cam_ids and frame. Defaults to False. Raises: Exception: If directory does not exist! """ def __init__(self, data_dir: str, transform=None, target_transform=None, ret_camid_n_frame: bool = False): super(ReIDDataset, self).__init__() self.data_dir = Path(data_dir) if not self.data_dir.exists(): raise Exception( f"'Path '{self.data_dir.__str__()}' does not exist!") if not self.data_dir.is_dir(): raise Exception( f"Path '{self.data_dir.__str__()}' is not a directory!") self.transform = transform self.target_transform = target_transform self.ret_camid_n_frame = ret_camid_n_frame self._init_data() def _init_data(self): if 'market' in str(self.data_dir).lower(): self.dataset = 'market' elif 'duke' in str(self.data_dir).lower(): self.dataset = 'duke' self.imgs = list(self.data_dir.glob('*.jpg')) # Filter out labels with -1 self.imgs = [img for img in self.imgs if '-1' not in img.stem] self.cam_ids, self.labels, self.frames = get_ids( self.imgs, self.dataset) self.num_cams = len(set(self.cam_ids)) self.classes = tuple(set(self.labels)) # Convert labels to continuous idxs self.class_to_idx = {label: i for i, label in enumerate(self.classes)} self.targets = [self.class_to_idx[label] for label in self.labels] def __len__(self): return len(self.imgs) def __getitem__(self, index): sample = Image.open(str(self.imgs[index])).convert('RGB') target = self.targets[index] if self.transform: sample = self.transform(sample) if self.target_transform: target = self.target_transform(target) if self.ret_camid_n_frame: cam_id = self.cam_ids[index] frame = self.frames[index] return sample, target, cam_id, frame return sample, target
parser.add_argument("-o", "--out_files", action="store", dest="outFiles", required=True, help="output file or directory") args = parser.parse_args() inPath = Path(args.inFiles) outPath = Path(args.outFiles) # Case 1: Convert single file. if (inPath.is_file()): inSuff = inPath.suffix inStem = inPath.stem outDir = outPath.parent outDir.mkdir(parents=True, exist_ok=True) if (inSuff != tsurfSuffix): msg = 'Only tsurf (*.ts) files are allowed as input.' raise ValueError(msg) outPath = outPath.with_suffix(vtkSuffix) convertFile(inPath, outPath) # Case 2: Convert directory. elif (inPath.is_dir()): convertDir(inPath, outPath) # Case 3: Give up. else: msg = 'Unable to find %s.' % inPath raise ValueError(msg)
class ReIDDataModule(pl.LightningDataModule): def __init__(self, data_dir: str, st_distribution: Optional[str] = None, train_subdir: str = 'bounding_box_train', test_subdir: str = 'bounding_box_test', query_subdir: str = 'query', train_batchsize: int = 16, val_batchsize: int = 16, test_batchsize: int = 16, num_workers: int = 4, random_erasing: float = 0.0, color_jitter: bool = False, save_distribution: Union[bool, str] = False): super().__init__() self.data_dir = Path(data_dir) if not self.data_dir.exists(): raise Exception( f"'Path '{self.data_dir.__str__()}' does not exist!") if not self.data_dir.is_dir(): raise Exception( f"Path '{self.data_dir.__str__()}' is not a directory!") self.train_dir = self.data_dir / train_subdir self.test_dir = self.data_dir / test_subdir self.query_dir = self.data_dir / query_subdir self.train_batchsize = train_batchsize self.test_batchsize = test_batchsize self.val_batchsize = val_batchsize self.num_workers = num_workers self.color_jitter = color_jitter self.random_erasing = random_erasing self.st_distribution = st_distribution self.save_distribution = save_distribution self.prepare_data() def prepare_data(self): train_transforms = [ transforms.Resize((384, 192), interpolation=3), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]) ] test_transforms = train_transforms test_transforms.pop(1) if self.random_erasing > 0: train_transforms.append( transforms.RandomErasing(self.random_erasing)) if self.color_jitter: train_transforms.append(transforms.ColorJitter()) train_transforms = transforms.Compose(train_transforms) self.train = ReIDDataset(self.train_dir, train_transforms) self.num_classes = len(self.train.classes) # train_len = int(len(self.train) * 0.8) # test_len = len(self.train) - train_len # self.train, self.test = random_split(self.train, [train_len, test_len]) test_transforms = transforms.Compose(test_transforms) self.test = ReIDDataset(self.test_dir, test_transforms) self.query = ReIDDataset(self.query_dir, test_transforms, ret_camid_n_frame=True) self.gallery = ReIDDataset(self.test_dir, test_transforms, ret_camid_n_frame=True) self._load_st_distribution() if self.save_distribution: self._save_st_distribution() def _load_st_distribution(self): if isinstance(self.st_distribution, str): self.st_distribution = Path(self.st_distribution) if not (self.st_distribution.exists() and self.st_distribution.is_file()): raise FileNotFoundError( f"Location '{str(self.st_distribution)}' \ does not exist or not a file!") if self.st_distribution.suffix != '.pkl': raise ValueError('File must be of type .pkl') print(f'\nLoading Spatial-Temporal Distribution from \ {self.st_distribution}.\n\n') self.st_distribution = joblib.load(str(self.st_distribution)) elif not self.st_distribution: print('\n\nGenerating Spatial-Temporal Distribution.\n\n') num_cams = self.query.num_cams max_hist = 5000 if self.query.dataset == 'market' else 3000 cam_ids = self.query.cam_ids + self.gallery.cam_ids targets = self.query.targets + self.gallery.targets frames = self.query.frames + self.gallery.frames self.st_distribution = smooth_st_distribution( cam_ids, targets, frames, num_cams, max_hist) def _save_st_distribution(self): if isinstance(self.save_distribution, str): if '.pkl' not in self.save_distribution: self.save_distribution += '.pkl' else: self.save_distribution = self.data_dir + 'st_distribution.pkl' print(f'\nSaving distribution at {self.save_distribution}') joblib.dump(self.st_distribution, self.save_distribution) def train_dataloader(self): return DataLoader(self.train, batch_size=self.train_batchsize, shuffle=True, num_workers=self.num_workers, pin_memory=True) def val_dataloader(self): test_loader = DataLoader(self.test, batch_size=self.test_batchsize, shuffle=False, num_workers=self.num_workers, pin_memory=True) query_indices = range(self.test_batchsize) query_loader = DataLoader(Subset(self.query, query_indices), batch_size=self.test_batchsize, shuffle=False, num_workers=self.num_workers, pin_memory=True) evens = list(range(0, len(self.gallery), 3)) gall_loader = DataLoader(Subset(self.gallery, evens), batch_size=self.test_batchsize, shuffle=True, num_workers=self.num_workers, pin_memory=True) return [query_loader, gall_loader, test_loader] def test_dataloader(self): query_loader = DataLoader(self.query, batch_size=self.test_batchsize, shuffle=False, num_workers=self.num_workers, pin_memory=True) gall_loader = DataLoader(self.gallery, batch_size=self.test_batchsize, shuffle=True, num_workers=self.num_workers, pin_memory=True) return [query_loader, gall_loader]
def _extract_to_cache( cls, cached_file, # type: str name, # type: str cache_context=None, # type: Optional[str] target_folder=None, # type: Optional[str] cache_path_encoding=None, # type: Optional[str] force=False, # type: bool ): # type: (...) -> str """ Extract cached file to cache folder :param str cached_file: local copy of archive file :param str name: name of the target file :param str cache_context: cache context id :param str target_folder: specify target path to use for archive extraction :param str cache_path_encoding: specify representation of the local path of the cached files, this will always point to local cache folder, even if we have direct access file. Used for extracting the cached archived based on cache_path_encoding :param bool force: Force archive extraction even if target folder exists :return: cached folder containing the extracted archive content """ if not cached_file: return cached_file cached_file = Path(cached_file) cache_path_encoding = Path( cache_path_encoding) if cache_path_encoding else None # we support zip and tar.gz files auto-extraction suffix = cached_file.suffix.lower() if suffix == '.gz': suffix = ''.join(a.lower() for a in cached_file.suffixes[-2:]) if suffix not in (".zip", ".tgz", ".tar.gz"): return str(cached_file) cache_folder = Path(cache_path_encoding or cached_file).parent archive_suffix = (cache_path_encoding or cached_file).name[:-len(suffix)] name = encode_string_to_filename(name) if name else name if target_folder: target_folder = Path(target_folder) else: target_folder = cache_folder / CacheManager.get_context_folder_lookup( cache_context).format(archive_suffix, name) if target_folder.is_dir() and not force: # noinspection PyBroadException try: target_folder.touch(exist_ok=True) return target_folder.as_posix() except Exception: pass base_logger = LoggerRoot.get_base_logger() try: # if target folder exists, meaning this is forced ao we extract directly into target folder if target_folder.is_dir(): temp_target_folder = target_folder else: temp_target_folder = cache_folder / "{0}_{1}_{2}".format( target_folder.name, time() * 1000, str(random()).replace('.', '')) temp_target_folder.mkdir(parents=True, exist_ok=True) if suffix == ".zip": ZipFile(cached_file.as_posix()).extractall( path=temp_target_folder.as_posix()) elif suffix == ".tar.gz": with tarfile.open(cached_file.as_posix()) as file: file.extractall(temp_target_folder.as_posix()) elif suffix == ".tgz": with tarfile.open(cached_file.as_posix(), mode='r:gz') as file: file.extractall(temp_target_folder.as_posix()) if temp_target_folder != target_folder: # we assume we will have such folder if we already extract the file # noinspection PyBroadException try: # if rename fails, it means that someone else already manged to extract the file, delete the current # folder and return the already existing cached zip folder shutil.move(temp_target_folder.as_posix(), target_folder.as_posix()) except Exception: if target_folder.exists(): target_folder.touch(exist_ok=True) else: base_logger.warning( "Failed renaming {0} to {1}".format( temp_target_folder.as_posix(), target_folder.as_posix())) try: shutil.rmtree(temp_target_folder.as_posix()) except Exception as ex: base_logger.warning( "Exception {}\nFailed deleting folder {}".format( ex, temp_target_folder.as_posix())) except Exception as ex: # failed extracting the file: base_logger.warning( "Exception {}\nFailed extracting zip file {}".format( ex, cached_file.as_posix())) # noinspection PyBroadException try: target_folder.rmdir() except Exception: pass return cached_file.as_posix() return target_folder.as_posix()
async def collect_parameter_data(self, param, when): """ Collects input parameters for the model as determined by the metadata. """ parameter_dataset_name = None # Only collect from the past! if when.date() < dt.datetime.today().date(): param = self.parameters[param] file_path = Path(param['path']) if not file_path.is_dir(): os.makedirs(file_path) parameter_dataset_name = file_path.joinpath(param['prefix'] + "_" + param['dataset']) if parameter_dataset_name.is_file(): return parameter_dataset_name else: data_file = file_path.joinpath(param['prefix'] + "_" + when.strftime("%Y%m%d") + param['suffix']) logger.debug(data_file) archive_file = Path( str(data_file) + param['compression_suffix']) try: if data_file.is_file(): parameter_dataset_name = self.do_conversion( data_file, param, when) elif not data_file.is_file() and archive_file.is_file(): logger.debug('Found an unexpanded archive: %s' % archive_file) await self.do_expansion(archive_file) parameter_dataset_name = self.do_conversion( data_file, param, when) # Remove the archive? elif not data_file.is_file() and not archive_file.is_file( ): date_string = when.strftime("%Y%m%d") resource = date_string + date_string + param['suffix'] + \ param['compression_suffix'] if await self.do_download(param["url"], resource, archive_file): if await self.do_expansion(archive_file): parameter_dataset_name = await self.do_conversion( data_file, param, when) except URLError as e: logger.warning(e) return None except FileNotFoundError as fnf: logger.warning(fnf) return None if Path(parameter_dataset_name).is_file(): return parameter_dataset_name else: logger.warning("No parameters for that date.") return None
def upload_artifact(self, name, artifact_object=None, metadata=None, delete_after_upload=False): if not Session.check_min_api_version('2.3'): LoggerRoot.get_base_logger().warning( 'Artifacts not supported by your TRAINS-server version, ' 'please upgrade to the latest server version') return False if name in self._artifacts_container: raise ValueError( "Artifact by the name of {} is already registered, use register_artifact" .format(name)) artifact_type_data = tasks.ArtifactTypeData() override_filename_in_uri = None override_filename_ext_in_uri = None uri = None if np and isinstance(artifact_object, np.ndarray): artifact_type = 'numpy' artifact_type_data.content_type = 'application/numpy' artifact_type_data.preview = str(artifact_object.__repr__()) override_filename_ext_in_uri = '.npz' override_filename_in_uri = name + override_filename_ext_in_uri fd, local_filename = mkstemp(prefix=quote(name, safe="") + '.', suffix=override_filename_ext_in_uri) os.close(fd) np.savez_compressed(local_filename, **{name: artifact_object}) delete_after_upload = True elif pd and isinstance(artifact_object, pd.DataFrame): artifact_type = 'pandas' artifact_type_data.content_type = 'text/csv' artifact_type_data.preview = str(artifact_object.__repr__()) override_filename_ext_in_uri = self._save_format override_filename_in_uri = name fd, local_filename = mkstemp(prefix=quote(name, safe="") + '.', suffix=override_filename_ext_in_uri) os.close(fd) artifact_object.to_csv(local_filename, compression=self._compression) delete_after_upload = True elif isinstance(artifact_object, Image.Image): artifact_type = 'image' artifact_type_data.content_type = 'image/png' desc = str(artifact_object.__repr__()) artifact_type_data.preview = desc[1:desc.find(' at ')] override_filename_ext_in_uri = '.png' override_filename_in_uri = name + override_filename_ext_in_uri fd, local_filename = mkstemp(prefix=quote(name, safe="") + '.', suffix=override_filename_ext_in_uri) os.close(fd) artifact_object.save(local_filename) delete_after_upload = True elif isinstance(artifact_object, dict): artifact_type = 'JSON' artifact_type_data.content_type = 'application/json' preview = json.dumps(artifact_object, sort_keys=True, indent=4) override_filename_ext_in_uri = '.json' override_filename_in_uri = name + override_filename_ext_in_uri fd, local_filename = mkstemp(prefix=quote(name, safe="") + '.', suffix=override_filename_ext_in_uri) os.write(fd, bytes(preview.encode())) os.close(fd) artifact_type_data.preview = preview delete_after_upload = True elif isinstance(artifact_object, six.string_types) and urlparse( artifact_object).scheme in remote_driver_schemes: # we should not upload this, just register local_filename = None uri = artifact_object artifact_type = 'custom' artifact_type_data.content_type = mimetypes.guess_type( artifact_object)[0] elif isinstance(artifact_object, six.string_types + (Path, )): # check if single file artifact_object = Path(artifact_object) artifact_object.expanduser().absolute() try: create_zip_file = not artifact_object.is_file() except Exception: # Hack for windows pathlib2 bug, is_file isn't valid. create_zip_file = True else: # We assume that this is not Windows os if artifact_object.is_dir(): # change to wildcard artifact_object /= '*' if create_zip_file: folder = Path('').joinpath(*artifact_object.parts[:-1]) if not folder.is_dir() or not folder.parts: raise ValueError( "Artifact file/folder '{}' could not be found".format( artifact_object.as_posix())) wildcard = artifact_object.parts[-1] files = list(Path(folder).rglob(wildcard)) override_filename_ext_in_uri = '.zip' override_filename_in_uri = folder.parts[ -1] + override_filename_ext_in_uri fd, zip_file = mkstemp( prefix=quote(folder.parts[-1], safe="") + '.', suffix=override_filename_ext_in_uri) try: artifact_type_data.content_type = 'application/zip' artifact_type_data.preview = 'Archive content {}:\n'.format( artifact_object.as_posix()) with ZipFile(zip_file, 'w', allowZip64=True, compression=ZIP_DEFLATED) as zf: for filename in sorted(files): if filename.is_file(): relative_file_name = filename.relative_to( folder).as_posix() artifact_type_data.preview += '{} - {}\n'.format( relative_file_name, humanfriendly.format_size( filename.stat().st_size)) zf.write(filename.as_posix(), arcname=relative_file_name) except Exception as e: # failed uploading folder: LoggerRoot.get_base_logger().warning( 'Exception {}\nFailed zipping artifact folder {}'. format(folder, e)) return None finally: os.close(fd) artifact_object = zip_file artifact_type = 'archive' artifact_type_data.content_type = mimetypes.guess_type( artifact_object)[0] local_filename = artifact_object delete_after_upload = True else: if not artifact_object.is_file(): raise ValueError( "Artifact file '{}' could not be found".format( artifact_object.as_posix())) override_filename_in_uri = artifact_object.parts[-1] artifact_object = artifact_object.as_posix() artifact_type = 'custom' artifact_type_data.content_type = mimetypes.guess_type( artifact_object)[0] local_filename = artifact_object else: raise ValueError("Artifact type {} not supported".format( type(artifact_object))) # remove from existing list, if exists for artifact in self._task_artifact_list: if artifact.key == name: if artifact.type == self._pd_artifact_type: raise ValueError( "Artifact of name {} already registered, " "use register_artifact instead".format(name)) self._task_artifact_list.remove(artifact) break if not local_filename: file_size = None file_hash = None else: # check that the file to upload exists local_filename = Path(local_filename).absolute() if not local_filename.exists() or not local_filename.is_file(): LoggerRoot.get_base_logger().warning( 'Artifact upload failed, cannot find file {}'.format( local_filename.as_posix())) return False file_hash, _ = self.sha256sum(local_filename.as_posix()) file_size = local_filename.stat().st_size uri = self._upload_local_file( local_filename, name, delete_after_upload=delete_after_upload, override_filename=override_filename_in_uri, override_filename_ext=override_filename_ext_in_uri) timestamp = int(time()) artifact = tasks.Artifact( key=name, type=artifact_type, uri=uri, content_size=file_size, hash=file_hash, timestamp=timestamp, type_data=artifact_type_data, display_data=[(str(k), str(v)) for k, v in metadata.items()] if metadata else None) # update task artifacts with self._task_edit_lock: self._task_artifact_list.append(artifact) self._task.set_artifacts(self._task_artifact_list) return True
print("Greetings! This script is still in beta so please let me know if there is any issues with the script itself.\nSo far, it seems that things are just barely getting started and this is my first full blown python script so I'm sorry if it looks bad \n") subprocess.call(["ls", "-lah"]) print("Completed ls -lah subprocess call \n") path = os.getcwd() print("The current path of this system is %s \n" % path) print("Checking if /home/steamcmd is available... \n") my_file = Path ("/home/steamcmd") if my_file.is_dir(): os.chdir("/home/steamcmd") print("/home/steamcmd is already on the server, moving on..") else: print("Not finding /home/steamcmd.. making folder") os.mkdir("/home/steamcmd") print("Folder should be created, testing it now \n") os.chdir("/home/steamcmd") <<<<<<< HEAD print("Successfully create /home/steamcmd") ======= print("Successfully create /home/steamcmd") >>>>>>> master