class TermiusApp(App):
"""Class for CLI application."""
def __init__(self):
"""Construct new CLI application."""
super(TermiusApp, self).__init__(
description='Termius app',
version=__version__,
command_manager=CommandManager('termius.handlers'),
)
self.configure_signals()
self.directory_path = Path(expanduser('~/.{}/'.format(self.NAME)))
if not self.directory_path.is_dir():
self.directory_path.mkdir(parents=True)
def configure_logging(self):
"""Change logging level for request package."""
super(TermiusApp, self).configure_logging()
logging.getLogger('requests').setLevel(logging.WARNING)
return
# pylint: disable=no-self-use
def configure_signals(self):
"""Bind subscribers to signals."""
post_create_instance.connect(store_ssh_key, sender=SshKey)
post_update_instance.connect(store_ssh_key, sender=SshKey)
post_delete_instance.connect(delete_ssh_key, sender=SshKey)
post_logout.connect(clean_data)
def process(self, path):
path = Path(path)
if path.is_dir():
self.process_files_in(path)
else:
self.process_one_file(path)
def get_file(path):
result = Path('web') / path
if result.is_file():
return str(result)
if result.is_dir() and (result / 'index.html').is_file():
return str(result / 'index.html')
# File was not found.
return None
class TermiusClient(object):
def __init__(self):
self.app_directory = Path('~/.termius/').expanduser()
self.command_mock = Mock(**{'app.directory_path': self.app_directory})
self.prepare()
def create_identity(self, label, is_visible):
return self._create_instance(
Identity, label=label, is_visible=is_visible
)
def create_host(self, address, label):
return self._create_instance(Host, label=label, address=address)
def create_pfrule(self, host_id, local_port, label):
return self._create_instance(
PFRule, label=label,
pftype='L', local_port=local_port,
remote_port=22, hostname='localhost'
)
def create_group(self, label):
return self._create_instance(Group, label=label)
def _create_instance(self, model, **kwargs):
instance = model(**kwargs)
with self.storage:
return self.storage.save(instance)
def prepare(self):
self.clean()
if not self.app_directory.is_dir():
self.app_directory.mkdir()
self.storage = ApplicationStorage(self.command_mock)
def clean(self):
if self.app_directory.is_dir():
self._clean_dir(self.app_directory)
def _clean_dir(self, dir_path):
[self._clean_dir(i) for i in dir_path.iterdir() if i.is_dir()]
[i.unlink() for i in dir_path.iterdir() if i.is_file()]
dir_path.rmdir()
def is_resource(package, name):
"""True if name is a resource inside package.
Directories are *not* resources.
"""
package = _get_package(package)
_normalize_path(name)
try:
package_contents = set(contents(package))
except OSError as error:
if error.errno not in (errno.ENOENT, errno.ENOTDIR):
# We won't hit this in the Python 2 tests, so it'll appear
# uncovered. We could mock os.listdir() to return a non-ENOENT or
# ENOTDIR, but then we'd have to depend on another external
# library since Python 2 doesn't have unittest.mock. It's not
# worth it.
raise # pragma: nocover
return False
if name not in package_contents:
return False
# Just because the given file_name lives as an entry in the package's
# contents doesn't necessarily mean it's a resource. Directories are not
# resources, so let's try to find out if it's a directory or not.
path = Path(package.__file__).parent / name
if path.is_file():
return True
if path.is_dir():
return False
# If it's not a file and it's not a directory, what is it? Well, this
# means the file doesn't exist on the file system, so it probably lives
# inside a zip file. We have to crack open the zip, look at its table of
# contents, and make sure that this entry doesn't have sub-entries.
archive_path = package.__loader__.archive # type: ignore
package_directory = Path(package.__file__).parent
with ZipFile(archive_path) as zf:
toc = zf.namelist()
relpath = package_directory.relative_to(archive_path)
candidate_path = relpath / name
for entry in toc: # pragma: nobranch
try:
relative_to_candidate = Path(entry).relative_to(candidate_path)
except ValueError:
# The two paths aren't relative to each other so we can ignore it.
continue
# Since directories aren't explicitly listed in the zip file, we must
# infer their 'directory-ness' by looking at the number of path
# components in the path relative to the package resource we're
# looking up. If there are zero additional parts, it's a file, i.e. a
# resource. If there are more than zero it's a directory, i.e. not a
# resource. It has to be one of these two cases.
return len(relative_to_candidate.parts) == 0
# I think it's impossible to get here. It would mean that we are looking
# for a resource in a zip file, there's an entry matching it in the return
# value of contents(), but we never actually found it in the zip's table of
# contents.
raise AssertionError('Impossible situation')
def clone_repository_cached(session, execution, destination):
# type: (Session, ExecutionInfo, Path) -> Tuple[VCS, RepoInfo]
"""
Clone a remote repository.
:param execution: execution info
:param destination: directory to clone to (in which a directory for the repository will be created)
:param session: program session
:return: repository information
:raises: CommandFailedError if git/hg is not installed
"""
repo_url = execution.repository # type: str
parsed_url = furl(repo_url)
no_password_url = parsed_url.copy().remove(password=True).url
clone_folder_name = Path(str(furl(repo_url).path)).name # type: str
clone_folder = Path(destination) / clone_folder_name
cached_repo_path = (
Path(session.config["agent.vcs_cache.path"]).expanduser() /
"{}.{}".format(clone_folder_name,
md5(ensure_binary(repo_url)).hexdigest()) /
clone_folder_name) # type: Path
vcs = VcsFactory.create(session,
execution_info=execution,
location=cached_repo_path)
if not find_executable(vcs.executable_name):
raise CommandFailedError(vcs.executable_not_found_error_help())
if session.config["agent.vcs_cache.enabled"] and cached_repo_path.exists():
print('Using cached repository in "{}"'.format(cached_repo_path))
else:
print("cloning: {}".format(no_password_url))
rm_tree(cached_repo_path)
# We clone the entire repository, not a specific branch
vcs.clone() # branch=execution.branch)
vcs.pull()
rm_tree(destination)
shutil.copytree(Text(cached_repo_path), Text(clone_folder))
if not clone_folder.is_dir():
raise CommandFailedError(
"copying of repository failed: from {} to {}".format(
cached_repo_path, clone_folder))
# checkout in the newly copy destination
vcs.location = Text(clone_folder)
vcs.checkout()
repo_info = vcs.get_repository_copy_info(clone_folder)
# make sure we have no user/pass in the returned repository structure
repo_info = attr.evolve(repo_info, url=no_password_url)
return vcs, repo_info
def __call__(self, parser, namespace, path, option_string=None):
install_dir = Path(path)
if not install_dir.is_dir():
raise argparse.ArgumentTypeError("{0} is not a valid directory".format(path))
binary = Path(install_dir, "sbin/syslog-ng")
if not binary.exists():
raise argparse.ArgumentTypeError("{0} not exist".format(binary))
setattr(namespace, self.dest, path)
def updateRepos(): # pylint: disable=invalid-name
"""Update Mercurial and Git repositories located in ~ and ~/trees ."""
home_dir = Path.home()
trees = [home_dir, home_dir / "trees"]
for tree in trees:
for name in sorted(os.listdir(str(tree))):
name_path = Path(tree) / name
if name_path.is_dir() and (name in REPOS or
(name.startswith("funfuzz")
and "-" in name)):
logger.info("Updating %s ...", name)
updateRepo(name_path)
def init(libs_dir="libs"):
logging.info("init libs: " + libs_dir)
libs_path = Path(libs_dir)
if not libs_path.is_dir():
raise Exception("libs dir not exists, " + libs_dir)
all_files = fsutils.get_all_files(Path(libs_dir))
for f in all_files:
lib = get_lib_info(str(f))
lib.path = f
all_libs.append(lib)
print("found a lib, name: {}, requires: {}, path: {}".format(
lib.name, lib.requires, lib.path))
def mkdatadir(self, original_datadir=None):
"""Create a temporary directory for this factory's scope.
"""
# special condition if the datadir is specified as None, which
# automatically gets the path that matches the basename of the
# module we are in
if original_datadir is None:
original_datadir = Path(
os.path.splitext(self.request.module.__file__)[0])
# get the path to the shared data dir
original_path = Path(self.request.fspath.dirname) / original_datadir
# make sure that the path exists and it is a directory
exists = True
if not original_path.exists():
# raise the flag that it doesn't exist so we can generate
# a directory for it instead of copying
exists = False
# make sure the path is a directory if it exists
elif not original_path.is_dir():
raise ValueError("datadir path is not a directory")
# generate a base temporary directory and receive the path to it
temp_path = self.tmp_path_factory.mktemp(TMPDIR_NAME)
# in order to use the shutil.copytree util the target directory
# must not exist so we specify a dir in the generated tempdir for it
temp_data_path = temp_path / DATADIR_DIRNAME
# windows-ify the paths
original_path = Path(_win32_longpath(original_path))
temp_data_path = Path(_win32_longpath(str(temp_data_path)))
# copy or create empty directory depending on whether the
# original one exists
if exists:
# copy all the files in the original data dir to the temp
# dir
shutil.copytree(original_path, temp_data_path)
else:
# otherwise just give them a fallback tmpdir
temp_data_path.mkdir()
return temp_data_path
def data_path(relative_path, relative_to=None):
"""Returns data path to test file."""
if relative_to is None:
# Use BASE_DIR as default.
relative_to = BASE_DIR
elif not isinstance(relative_to, Path):
# Ensure relative_to is a Path.
relative_to = Path(relative_to)
# If relative_to is not a path, move up one level.
if not relative_to.is_dir():
relative_to = relative_to.parent
return relative_to / 'data' / relative_path
def get_config(config_file_name=str(find_config())):
"""Create config parser with defaults and read in the config file."""
default_config_dir = _get_default_config_dir()
parser = make_config_parser({'datadir': str(default_config_dir.joinpath('data'))})
parser.add_section('general')
parser.read([config_file_name])
env_datadir = os.getenv('REENTRY_DATADIR')
if env_datadir:
env_datadir_path = Path(env_datadir)
if env_datadir_path.exists() and not env_datadir_path.is_dir(): # pylint: disable=no-member
raise ValueError('environment variable $REENTRY_DATADIR={} exists, but is not a directory'.format(env_datadir))
parser.set('general', 'datadir', str(env_datadir_path))
return parser
def data_path(relative_path, relative_to=None):
"""Returns data path to test file."""
if relative_to is None:
# Use BASE_DIR as default.
relative_to = BASE_DIR
elif not isinstance(relative_to, Path):
# Ensure relative_to is a Path.
relative_to = Path(relative_to)
# If relative_to is not a path, move up one level.
if not relative_to.is_dir():
relative_to = relative_to.parent
return relative_to / 'data' / relative_path
def date_is_cached(self, when):
# TODO -Swift Object Storage Checking
file_path = Path(self.outputs["readings"]['path'])
if not file_path.is_dir():
os.makedirs(file_path)
ok = Path(self.netcdf_name_for_date(when)).is_file()
logger.debug("\n--> Checking for existence of NetCDF @ %s for %s: %s" %
(file_path, when.strftime("%d %m %Y"), ok))
# TODO -if OK put the file into Swift Storage
return ok
def inner(shellFilename, hgHash): # pylint: disable=invalid-name,missing-docstring,missing-return-doc
# pylint: disable=missing-return-type-doc
# pylint: disable=invalid-name
conditionArgs = conditionArgPrefix + [str(shellFilename)] + options.runtime_params
temp_dir = Path(tempfile.mkdtemp(prefix="abExtTestAndLabel-" + hgHash))
temp_prefix = temp_dir / "t"
if hasattr(conditionScript, "init"):
# Since we're changing the js shell name, call init() again!
conditionScript.init(conditionArgs)
if conditionScript.interesting(conditionArgs, str(temp_prefix)):
innerResult = ("bad", "interesting") # pylint: disable=invalid-name
else:
innerResult = ("good", "not interesting") # pylint: disable=invalid-name
if temp_dir.is_dir():
sps.rm_tree_incl_readonly(str(temp_dir))
return innerResult
def find_spec(self, name, path, target=None):
path = Path('./' + name.replace('.', r'/'))
if path.with_suffix('.c').exists():
return ModuleSpec(name,
CintegrateLoader(path, type='c'),
is_package=True)
if path.with_suffix('.cpp').exists():
return ModuleSpec(name,
CintegrateLoader(path, type='c++'),
is_package=True)
if path.with_suffix('.so').exists():
return ModuleSpec(name,
CintegrateLoader(path, type='so'),
is_package=True)
if path.is_dir():
return ModuleSpec(name, CintegrateLoader(path), is_package=True)
return None
def update(self, path):
if (type(path) != str):
index = path[1]
path = path[0]
config = Path(path)
tknzr = TweetTokenizer()
lex, doc = dict(), dict()
if (config.is_file()): # If just a file
with config.open() as f:
sentences = sent_tokenize(f.read())
document = path.split('/')[-1]
doc[document] = list(sentences)
for i, sentence in enumerate(sentences):
token_list = tknzr.tokenize(sentence)
for word in token_list:
if (word not in self.stopwords
and self.is_not_number(word)
and len(word) > 1):
if (pos_tag([word])[0][1] != 'NNP'):
word = word.lower()
if (word not in lex):
lex[word] = Word(word, document, i)
else:
lex[word].update(document, i)
return (lex, doc)
elif (config.is_dir()
): # If directory, recursively read each directory and file
if (path[-1] != '/'): path = path + '/'
iterdir = list()
for d in config.iterdir():
iterdir.append(path + d.name)
self.outputs = [0] * len(list(iterdir))
p = Pool(self.maxPoolSize)
outputs = p.map(self.update, zip(iterdir,
list(range(len(iterdir)))))
for (lex, doc) in outputs:
for k, v in lex.items():
if (k not in self.lexicon):
self.lexicon[k] = Word(k, v.documents)
self.lexicon[k].count += v.count
self.lexicon[k].documents.update(v.documents)
self.documents.update(doc)
else:
print('{} doesn\'t exist.'.format(path))
def get_converted_data(dataset_task, conf_file):
if dataset_task:
dataset_upload_task = Dataset.get(dataset_id=dataset_task)
else:
dataset_upload_task = Dataset.get(dataset_project="Nvidia TLT examples with ClearML",
dataset_name="Example data")
image_directory_path = (
get_field_from_config(conf_file, "image_directory_path")
.strip()
.strip('"')
.rpartition("/")[0]
)
# noinspection PyBroadException
try:
os.makedirs(image_directory_path)
except Exception:
pass
# download the artifact and open it
saved_dataset = dataset_upload_task.get_local_copy()
dataset_name = os.listdir(saved_dataset)[0]
dataset_path = Path(os.path.join(saved_dataset, dataset_name))
if not dataset_path.is_dir() and dataset_path.suffix in (".zip", ".tgz", ".tar.gz"):
dataset_suffix = dataset_path.suffix
if dataset_suffix == ".zip":
from zipfile import ZipFile
ZipFile(dataset_path.as_posix()).extractall(path=image_directory_path)
elif dataset_suffix == ".tar.gz":
import tarfile
with tarfile.open(dataset_path.as_posix()) as file:
file.extractall(image_directory_path)
elif dataset_suffix == ".tgz":
import tarfile
with tarfile.open(dataset_path.as_posix(), mode="r:gz") as file:
file.extractall(image_directory_path)
saved_dataset = str(dataset_path)
else:
os.system("cp -R {}/* {}".format(saved_dataset, image_directory_path))
print(saved_dataset)
def get_file(path):
filepath = Path('site') / path
if filepath.is_file():
return filepath
for format in PAGE_FORMATS:
index_file = filepath / ('index' + format)
if filepath.is_dir() and index_file.is_file():
return index_file
if filepath.suffix == '.css':
style_file = filepath.parent / (filepath.stem + '.styl')
if style_file.exists():
return style_file
if filepath.suffix == '.js':
coffee_file = filepath.parent / (filepath.stem + '.coffee')
if coffee_file.exists():
return coffee_file
return None
def create_tables(self,
sql_folder: str = DatabaseUtils.sql_tables_path) -> None:
"""Create all tables from their schemas
:see DatabaseUtils.sql_tables:
"""
# reach the sql sources folder
folder = Path(sql_folder)
if not folder.exists() \
or not folder.is_dir():
raise BadFileFormatException
# for each file in the folder
for file in folder.iterdir():
# if it is an sql file, execute it
if file.suffix != DatabaseUtils.sql_extension:
continue
self._cursor.execute(file.read_text())
# commit transactions changes
self._connection.commit()
def _get_system_wide_trusted_ca_certificates(self):
trusted_cas, errors = set([]), []
for p in self.system_wide_trusted_ca_search_paths:
cert_path = Path(p)
if not cert_path.is_dir():
continue
for entry in cert_path.iterdir():
cert_file_path = entry.absolute()
try:
if entry.suffix not in [".pem", ".crt"]:
continue
trusted_cas.update(
self._get_certificates_from_file(cert_file_path))
except IOError:
logger.exception("error updating CA")
# This error is shown to the user as warning message during "activate changes".
# We keep this message for the moment because we think that it is a helpful
# trigger for further checking web.log when a really needed certificate can
# not be read.
#
# We know a permission problem with some files that are created by default on
# some distros. We simply ignore these files because we assume that they are
# not needed.
if cert_file_path == Path("/etc/ssl/certs/localhost.crt"):
continue
errors.append(
"Failed to add certificate '%s' to trusted CA certificates. "
"See web.log for details." % cert_file_path)
break
return list(trusted_cas), errors
def Deepzoom(image_or_directory_path,create_static_cache=False,**kwargs):
"""
Returns a Deepzoom interface corresponding to the given image. Can accept
either a filepath (and will read tiles on the fly) or a directory path (that
contains an image with the same name as the directory) that contains/will
contain a static DeepZoom image directory.
:param image_or_directory_path: String or Pathlib object
:param create_static_cache: If True, creates a static DeepZoom image
directory directory structure *around* the given image (or in the given
directory). This is done lazily, saving each tile as it's requested.
:param kwargs: Same as DeepzoomInterface
:return: DeepZoom
"""
p = Path(image_or_directory_path).resolve()
img = None
if p.is_file():
img = _ImageFactory(p)
elif p.is_dir():
fList = list(p.glob('%s.*'%p.name))
if len(fList)==0:
raise IOError('Invalid Deepzoom directory (%s). '
'Must contain and image named (%s) to be valid.'
''%(p,'%s.<EXT>'%p.name))
for f in fList:
try: img = _ImageFactory(f)
except IOError: pass
if img is None:
raise IOError('Invalid Deepzoom target (%s). '
'Not a supported image format.'%(p))
if create_static_cache:
# do something to DeepZoomGenerator so that it saves on get_tile()
dzGen = _CachedInterface(img,**kwargs)
else:
dzGen = _DeepzoomInterface(img,**kwargs)
return dzGen
def _make_file_info(target: pathlib.Path,
arcname: Optional[str] = None,
dereference=False) -> Dict[str, Any]:
f = {} # type: Dict[str, Any]
f['origin'] = target
if arcname is not None:
f['filename'] = pathlib.Path(arcname).as_posix()
else:
f['filename'] = target.as_posix()
if os.name == 'nt':
fstat = target.lstat()
if target.is_symlink():
if dereference:
fstat = target.stat()
if stat.S_ISDIR(fstat.st_mode):
f['emptystream'] = True
f['attributes'] = fstat.st_file_attributes & FILE_ATTRIBUTE_WINDOWS_MASK # type: ignore # noqa
else:
f['emptystream'] = False
f['attributes'] = stat.FILE_ATTRIBUTE_ARCHIVE # type: ignore # noqa
f['uncompressed'] = fstat.st_size
else:
f['emptystream'] = False
f['attributes'] = fstat.st_file_attributes & FILE_ATTRIBUTE_WINDOWS_MASK # type: ignore # noqa
# f['attributes'] |= stat.FILE_ATTRIBUTE_REPARSE_POINT # type: ignore # noqa
elif target.is_dir():
f['emptystream'] = True
f['attributes'] = fstat.st_file_attributes & FILE_ATTRIBUTE_WINDOWS_MASK # type: ignore # noqa
elif target.is_file():
f['emptystream'] = False
f['attributes'] = stat.FILE_ATTRIBUTE_ARCHIVE # type: ignore # noqa
f['uncompressed'] = fstat.st_size
else:
fstat = target.lstat()
if target.is_symlink():
if dereference:
fstat = target.stat()
if stat.S_ISDIR(fstat.st_mode):
f['emptystream'] = True
f['attributes'] = stat.FILE_ATTRIBUTE_DIRECTORY # type: ignore # noqa
f['attributes'] |= FILE_ATTRIBUTE_UNIX_EXTENSION | (
stat.S_IFDIR << 16)
f['attributes'] |= (stat.S_IMODE(fstat.st_mode) << 16)
else:
f['emptystream'] = False
f['attributes'] = stat.FILE_ATTRIBUTE_ARCHIVE # type: ignore # noqa
f['attributes'] |= FILE_ATTRIBUTE_UNIX_EXTENSION | (
stat.S_IMODE(fstat.st_mode) << 16)
else:
f['emptystream'] = False
f['attributes'] = stat.FILE_ATTRIBUTE_ARCHIVE | stat.FILE_ATTRIBUTE_REPARSE_POINT # type: ignore # noqa
f['attributes'] |= FILE_ATTRIBUTE_UNIX_EXTENSION | (
stat.S_IFLNK << 16)
f['attributes'] |= (stat.S_IMODE(fstat.st_mode) << 16)
elif target.is_dir():
f['emptystream'] = True
f['attributes'] = stat.FILE_ATTRIBUTE_DIRECTORY # type: ignore # noqa
f['attributes'] |= FILE_ATTRIBUTE_UNIX_EXTENSION | (
stat.S_IFDIR << 16)
f['attributes'] |= (stat.S_IMODE(fstat.st_mode) << 16)
elif target.is_file():
f['emptystream'] = False
f['uncompressed'] = fstat.st_size
f['attributes'] = stat.FILE_ATTRIBUTE_ARCHIVE # type: ignore # noqa
f['attributes'] |= FILE_ATTRIBUTE_UNIX_EXTENSION | (
stat.S_IMODE(fstat.st_mode) << 16)
f['creationtime'] = fstat.st_ctime
f['lastwritetime'] = fstat.st_mtime
f['lastaccesstime'] = fstat.st_atime
return f
def clone_repository_cached(session, execution, destination):
# type: (Session, ExecutionInfo, Path) -> Tuple[VCS, RepoInfo]
"""
Clone a remote repository.
:param execution: execution info
:param destination: directory to clone to (in which a directory for the repository will be created)
:param session: program session
:return: repository information
:raises: CommandFailedError if git/hg is not installed
"""
# mock lock
repo_lock = Lock()
repo_lock_timeout_sec = 300
repo_url = execution.repository # type: str
parsed_url = furl(repo_url)
no_password_url = parsed_url.copy().remove(password=True).url
clone_folder_name = Path(str(furl(repo_url).path)).name # type: str
clone_folder = Path(destination) / clone_folder_name
standalone_mode = session.config.get("agent.standalone_mode", False)
if standalone_mode:
cached_repo_path = clone_folder
else:
vcs_cache_path = Path(
session.config["agent.vcs_cache.path"]).expanduser()
repo_hash = md5(ensure_binary(repo_url)).hexdigest()
# create lock
repo_lock = FileLock(filename=(vcs_cache_path /
'{}.lock'.format(repo_hash)).as_posix())
# noinspection PyBroadException
try:
repo_lock.acquire(timeout=repo_lock_timeout_sec)
except BaseException:
print(
'Could not lock cache folder "{}" (timeout {} sec), using temp vcs cache.'
.format(clone_folder_name, repo_lock_timeout_sec))
repo_hash = '{}_{}'.format(repo_hash,
str(random()).replace('.', ''))
# use mock lock for the context
repo_lock = Lock()
# select vcs cache folder
cached_repo_path = vcs_cache_path / "{}.{}".format(
clone_folder_name, repo_hash) / clone_folder_name
with repo_lock:
vcs = VcsFactory.create(session,
execution_info=execution,
location=cached_repo_path)
if not find_executable(vcs.executable_name):
raise CommandFailedError(vcs.executable_not_found_error_help())
if not standalone_mode:
if session.config[
"agent.vcs_cache.enabled"] and cached_repo_path.exists():
print(
'Using cached repository in "{}"'.format(cached_repo_path))
else:
print("cloning: {}".format(no_password_url))
rm_tree(cached_repo_path)
# We clone the entire repository, not a specific branch
vcs.clone() # branch=execution.branch)
vcs.pull()
rm_tree(destination)
shutil.copytree(Text(cached_repo_path), Text(clone_folder))
if not clone_folder.is_dir():
raise CommandFailedError(
"copying of repository failed: from {} to {}".format(
cached_repo_path, clone_folder))
# checkout in the newly copy destination
vcs.location = Text(clone_folder)
vcs.checkout()
repo_info = vcs.get_repository_copy_info(clone_folder)
# make sure we have no user/pass in the returned repository structure
repo_info = attr.evolve(repo_info, url=no_password_url)
return vcs, repo_info
def upload_artifact(self,
name,
artifact_object=None,
metadata=None,
preview=None,
delete_after_upload=False,
auto_pickle=True):
# type: (str, Optional[object], Optional[dict], Optional[str], bool, bool) -> bool
if not Session.check_min_api_version('2.3'):
LoggerRoot.get_base_logger().warning(
'Artifacts not supported by your TRAINS-server version, '
'please upgrade to the latest server version')
return False
if name in self._artifacts_container:
raise ValueError(
"Artifact by the name of {} is already registered, use register_artifact"
.format(name))
# cast preview to string
if preview:
preview = str(preview)
# convert string to object if try is a file/folder (dont try to serialize long texts
if isinstance(artifact_object,
six.string_types) and len(artifact_object) < 2048:
# noinspection PyBroadException
try:
artifact_path = Path(artifact_object)
if artifact_path.exists():
artifact_object = artifact_path
elif '*' in artifact_object or '?' in artifact_object:
# hackish, detect wildcard in tr files
folder = Path('').joinpath(*artifact_path.parts[:-1])
if folder.is_dir() and folder.parts:
wildcard = artifact_path.parts[-1]
if list(Path(folder).rglob(wildcard)):
artifact_object = artifact_path
except Exception:
pass
artifact_type_data = tasks.ArtifactTypeData()
artifact_type_data.preview = ''
override_filename_in_uri = None
override_filename_ext_in_uri = None
uri = None
if np and isinstance(artifact_object, np.ndarray):
artifact_type = 'numpy'
artifact_type_data.content_type = 'application/numpy'
artifact_type_data.preview = preview or str(
artifact_object.__repr__())
override_filename_ext_in_uri = '.npz'
override_filename_in_uri = name + override_filename_ext_in_uri
fd, local_filename = mkstemp(prefix=quote(name, safe="") + '.',
suffix=override_filename_ext_in_uri)
os.close(fd)
np.savez_compressed(local_filename, **{name: artifact_object})
delete_after_upload = True
elif pd and isinstance(artifact_object, pd.DataFrame):
artifact_type = 'pandas'
artifact_type_data.content_type = 'text/csv'
artifact_type_data.preview = preview or str(
artifact_object.__repr__())
override_filename_ext_in_uri = self._save_format
override_filename_in_uri = name
fd, local_filename = mkstemp(prefix=quote(name, safe="") + '.',
suffix=override_filename_ext_in_uri)
os.close(fd)
artifact_object.to_csv(local_filename,
compression=self._compression)
delete_after_upload = True
elif isinstance(artifact_object, Image.Image):
artifact_type = 'image'
artifact_type_data.content_type = 'image/png'
desc = str(artifact_object.__repr__())
artifact_type_data.preview = preview or desc[1:desc.find(' at ')]
override_filename_ext_in_uri = '.png'
override_filename_in_uri = name + override_filename_ext_in_uri
fd, local_filename = mkstemp(prefix=quote(name, safe="") + '.',
suffix=override_filename_ext_in_uri)
os.close(fd)
artifact_object.save(local_filename)
delete_after_upload = True
elif isinstance(artifact_object, dict):
artifact_type = 'JSON'
artifact_type_data.content_type = 'application/json'
preview = preview or json.dumps(
artifact_object, sort_keys=True, indent=4)
override_filename_ext_in_uri = '.json'
override_filename_in_uri = name + override_filename_ext_in_uri
fd, local_filename = mkstemp(prefix=quote(name, safe="") + '.',
suffix=override_filename_ext_in_uri)
os.write(fd, bytes(preview.encode()))
os.close(fd)
if len(preview) < self.max_preview_size_bytes:
artifact_type_data.preview = preview
else:
artifact_type_data.preview = '# full json too large to store, storing first {}kb\n{}'.format(
self.max_preview_size_bytes // 1024,
preview[:self.max_preview_size_bytes])
delete_after_upload = True
elif isinstance(artifact_object, (
Path,
pathlib_Path,
) if pathlib_Path is not None else (Path, )):
# check if single file
artifact_object = Path(artifact_object)
artifact_object.expanduser().absolute()
# noinspection PyBroadException
try:
create_zip_file = not artifact_object.is_file()
except Exception: # Hack for windows pathlib2 bug, is_file isn't valid.
create_zip_file = True
else: # We assume that this is not Windows os
if artifact_object.is_dir():
# change to wildcard
artifact_object /= '*'
if create_zip_file:
folder = Path('').joinpath(*artifact_object.parts[:-1])
if not folder.is_dir() or not folder.parts:
raise ValueError(
"Artifact file/folder '{}' could not be found".format(
artifact_object.as_posix()))
wildcard = artifact_object.parts[-1]
files = list(Path(folder).rglob(wildcard))
override_filename_ext_in_uri = '.zip'
override_filename_in_uri = folder.parts[
-1] + override_filename_ext_in_uri
fd, zip_file = mkstemp(
prefix=quote(folder.parts[-1], safe="") + '.',
suffix=override_filename_ext_in_uri)
try:
artifact_type_data.content_type = 'application/zip'
archive_preview = 'Archive content {}:\n'.format(
artifact_object.as_posix())
with ZipFile(zip_file,
'w',
allowZip64=True,
compression=ZIP_DEFLATED) as zf:
for filename in sorted(files):
if filename.is_file():
relative_file_name = filename.relative_to(
folder).as_posix()
archive_preview += '{} - {}\n'.format(
relative_file_name,
humanfriendly.format_size(
filename.stat().st_size))
zf.write(filename.as_posix(),
arcname=relative_file_name)
except Exception as e:
# failed uploading folder:
LoggerRoot.get_base_logger().warning(
'Exception {}\nFailed zipping artifact folder {}'.
format(folder, e))
return False
finally:
os.close(fd)
artifact_type_data.preview = preview or archive_preview
artifact_object = zip_file
artifact_type = 'archive'
artifact_type_data.content_type = mimetypes.guess_type(
artifact_object)[0]
local_filename = artifact_object
delete_after_upload = True
else:
if not artifact_object.is_file():
raise ValueError(
"Artifact file '{}' could not be found".format(
artifact_object.as_posix()))
override_filename_in_uri = artifact_object.parts[-1]
artifact_type_data.preview = preview or '{} - {}\n'.format(
artifact_object,
humanfriendly.format_size(artifact_object.stat().st_size))
artifact_object = artifact_object.as_posix()
artifact_type = 'custom'
artifact_type_data.content_type = mimetypes.guess_type(
artifact_object)[0]
local_filename = artifact_object
elif (isinstance(artifact_object, six.string_types)
and len(artifact_object) < 4096
and urlparse(artifact_object).scheme in remote_driver_schemes):
# we should not upload this, just register
local_filename = None
uri = artifact_object
artifact_type = 'custom'
artifact_type_data.content_type = mimetypes.guess_type(
artifact_object)[0]
elif isinstance(artifact_object, six.string_types):
# if we got here, we should store it as text file.
artifact_type = 'string'
artifact_type_data.content_type = 'text/plain'
if preview:
artifact_type_data.preview = preview
elif len(artifact_object) < self.max_preview_size_bytes:
artifact_type_data.preview = artifact_object
else:
artifact_type_data.preview = '# full text too large to store, storing first {}kb\n{}'.format(
self.max_preview_size_bytes // 1024,
artifact_object[:self.max_preview_size_bytes])
delete_after_upload = True
override_filename_ext_in_uri = '.txt'
override_filename_in_uri = name + override_filename_ext_in_uri
fd, local_filename = mkstemp(prefix=quote(name, safe="") + '.',
suffix=override_filename_ext_in_uri)
os.close(fd)
# noinspection PyBroadException
try:
with open(local_filename, 'wt') as f:
f.write(artifact_object)
except Exception:
# cleanup and raise exception
os.unlink(local_filename)
raise
elif auto_pickle:
# if we are here it means we do not know what to do with the object, so we serialize it with pickle.
artifact_type = 'pickle'
artifact_type_data.content_type = 'application/pickle'
# noinspection PyBroadException
try:
artifact_type_data.preview = preview or str(
artifact_object.__repr__())[:self.max_preview_size_bytes]
except Exception:
artifact_type_data.preview = preview or ''
delete_after_upload = True
override_filename_ext_in_uri = '.pkl'
override_filename_in_uri = name + override_filename_ext_in_uri
fd, local_filename = mkstemp(prefix=quote(name, safe="") + '.',
suffix=override_filename_ext_in_uri)
os.close(fd)
# noinspection PyBroadException
try:
with open(local_filename, 'wb') as f:
pickle.dump(artifact_object, f)
except Exception:
# cleanup and raise exception
os.unlink(local_filename)
raise
else:
raise ValueError("Artifact type {} not supported".format(
type(artifact_object)))
# remove from existing list, if exists
for artifact in self._task_artifact_list:
if artifact.key == name:
if artifact.type == self._pd_artifact_type:
raise ValueError(
"Artifact of name {} already registered, "
"use register_artifact instead".format(name))
self._task_artifact_list.remove(artifact)
break
if not local_filename:
file_size = None
file_hash = None
else:
# check that the file to upload exists
local_filename = Path(local_filename).absolute()
if not local_filename.exists() or not local_filename.is_file():
LoggerRoot.get_base_logger().warning(
'Artifact upload failed, cannot find file {}'.format(
local_filename.as_posix()))
return False
file_hash, _ = self.sha256sum(local_filename.as_posix())
file_size = local_filename.stat().st_size
uri = self._upload_local_file(
local_filename,
name,
delete_after_upload=delete_after_upload,
override_filename=override_filename_in_uri,
override_filename_ext=override_filename_ext_in_uri)
timestamp = int(time())
artifact = tasks.Artifact(
key=name,
type=artifact_type,
uri=uri,
content_size=file_size,
hash=file_hash,
timestamp=timestamp,
type_data=artifact_type_data,
display_data=[(str(k), str(v))
for k, v in metadata.items()] if metadata else None)
# update task artifacts
with self._task_edit_lock:
self._task_artifact_list.append(artifact)
self._task.set_artifacts(self._task_artifact_list)
return True
class ExperimentMain(object):
@property
def my_rank_dir(self):
return self.exp_dir_path / 'rank_{}'.format(self.my_rank)
@property
def my_rank(self):
return get_mpi_rank_or_0()
def main(self):
self.comm = get_mpi_comm_world()
self.rank = self.comm.Get_rank()
self.comm.Barrier()
self.setup()
self.train()
def setup(self):
if self.rank == SERVER_RANK:
params = get_configuration(print_diagnostics=True,
inject_parameters_to_gin=True)
experiment_id = params['experiment_id']
self.comm.bcast([experiment_id], root=SERVER_RANK)
else:
data = None
data = self.comm.bcast(data, root=SERVER_RANK)
experiment_id = data[0] if self.rank == EVALUATOR_RANK else None
params = get_configuration(print_diagnostics=True,
inject_parameters_to_gin=True)
exp_dir_path = "."
self.exp_dir_path = Path(exp_dir_path)
if self.my_rank == SERVER_RANK:
if not self.exp_dir_path.is_dir():
self.exp_dir_path.mkdir(parents=True)
if not self.my_rank_dir.is_dir():
self.my_rank_dir.mkdir(parents=True)
self.tee_stdout = TeeStdout(str(self.my_rank_dir / 'stdout'), mode='w')
self.tee_stderr = TeeStderr(str(self.my_rank_dir / 'stderr'), mode='w')
logger.configure(format_strs=['stdout', 'log', 'csv', 'tensorboard'])
self.params = params
self.run_eval_worker = self.params.get("run_eval_worker", False)
def train(self):
if self.params.get("tf_seed", None) is not None:
tf.random.set_random_seed(self.params["tf_seed"])
if self.rank == SERVER_RANK:
sess = U.make_session(
num_cpu=self.params.get('server_tf_num_cpu', 2))
else:
sess = U.single_threaded_session()
sess.__enter__()
value, planner, env_init_kwargs = create_agent(sess)
sess.run(tf.global_variables_initializer())
if self.rank == SERVER_RANK:
sess.run(value.initializers())
value.sync() # sync value weights to root 0
state_shape = value._replay.memory._state_shape
game_buffer_size = self.params['game_buffer_size']
serializer = Serializer()
serializer.add_variable("number_of_removed_games",
shape=(1, 1),
dtype=[np.int32])
serializer.add_variable("worker_step_count",
shape=(1, 1),
dtype=[np.int32])
serializer.add_variable("worker_total_step_count",
shape=(1, 1),
dtype=[np.int32])
for i in range(game_buffer_size):
serializer.add_variable(
name=f"game_{i}",
shape=(
planner.episode_max_steps, # game max len
state_shape, # state
(1, 1), # value
(1, 1), # action
),
dtype=[np.int32, np.float32, np.uint8])
serializer.add_variable(name=f"game_steps_{i}",
shape=(1, 1),
dtype=[np.int32])
serializer.add_variable(name=f"game_solved_{i}",
shape=(1, 1),
dtype=[np.int32])
serializer.add_variable(name=f"graph_size_{i}",
shape=(1, 1),
dtype=[np.int32])
serializer.add_variable(name=f"game_ensemble_std_{i}",
shape=(1, 1),
dtype=[np.float32])
serializer.add_variable(name=f"worker_step_count_{i}",
shape=(1, 1),
dtype=[np.int32])
serializer.add_variable(name=f"avg_{i}",
shape=(1, 1),
dtype=[np.float32])
serializer.add_variable(name=f"num_gen_steps_{i}",
shape=(1, 1),
dtype=[np.int32])
serializer.add_variable(name=f"env_curriculum_{i}",
shape=(1, 1),
dtype=[np.float32])
serializer.add_variable(name=f"avg_skip_{i}",
shape=(1, 1),
dtype=[np.float32])
serializer.add_variable(name=f"worker_avg_result_{i}",
shape=(1, 1),
dtype=[np.float32])
# serializer.add_variable(name=f"wm_next_frame_errors_rate_{i}", shape=(1, 1), dtype=[np.float32])
# serializer.add_variable(name=f"wm_reward_errors_rate_{i}", shape=(1, 1), dtype=[np.float32])
# serializer.add_variable(name=f"wm_missed_done_rate_{i}", shape=(1, 1), dtype=[np.float32])
# serializer.add_variable(name=f"wm_false_done_rate_{i}", shape=(1, 1), dtype=[np.float32])
# serializer.add_variable(name=f"wm_any_false_done_{i}", shape=(1, 1), dtype=[np.float32])
# serializer.add_variable(name=f"wm_any_missed_done_{i}",shape=(1, 1), dtype=[np.float32])
auto_ml_creator = AutoMLCreator()
if auto_ml_creator.is_auto_ml_present:
serializer.add_variable(f"auto_ml_parameters_{i}",
shape=(1, len(auto_ml_creator.dims)),
dtype=[np.int32])
curriculum = Curriculum(
enabled=self.params.get('curriculum', False),
model=planner._model,
curriculum_threshold=self.params.get('curriculum_threshold', 0.8),
curriculum_smooth_coeff=self.params.get('curriculum_smooth_coeff',
0.95),
curriculum_initial_value=self.params.get(
'curriculum_initial_value', 0.0),
curriculum_initial_length_random_walk=self.params.get(
'curriculum_initial_length_random_walk', 50),
curriculum_maximal_length_random_walk=self.params.get(
'curriculum_maximal_length_random_walk', 300),
curriculum_length_random_walk_delta=self.params.get(
"curriculum_length_random_walk_delta", 50),
curriculum_max_num_gen_steps=self.params.get(
'curriculum_max_num_gen_steps', 27),
curriculum_running_average_drop_on_increase=self.params.get(
'curriculum_running_average_drop_on_increase', 0.2),
)
if self.rank == SERVER_RANK:
server = Server(
value=value,
serializer=serializer,
training_steps=self.params['training_steps'],
train_every_num_steps=self.params['train_every_num_steps'],
game_buffer_size=game_buffer_size,
exit_training_after_num_games=self.params.get(
"exit_trainig_after_num_games", np.inf),
curriculum=curriculum,
eval_worker_exists=self.run_eval_worker,
)
server.run()
elif self.rank == EVALUATOR_RANK and self.run_eval_worker:
from learning_and_planning.mcts.evaluator_worker import \
EvaluatorWorker
eval_worker = EvaluatorWorker(value=value,
env_kwargs=env_init_kwargs,
planner=planner)
eval_worker.run()
else:
worker = Worker(
value=value,
training_steps=self.params['training_steps'],
planner=planner,
serializer=serializer,
game_buffer_size=game_buffer_size,
curriculum=curriculum,
)
worker.run()
class TermiusApp(App):
"""Class for CLI application."""
def __init__(self):
"""Construct new CLI application."""
super(TermiusApp, self).__init__(
description='Termius - crossplatform SSH and Telnet client',
version=__version__,
command_manager=CommandManager('termius.handlers'),
)
self.configure_signals()
self.directory_path = Path(expanduser('~/.{}/'.format(self.NAME)))
if not self.directory_path.is_dir():
self.directory_path.mkdir(parents=True)
self.command_manager.add_command('help', HelpCommand)
def configure_logging(self):
"""Change logging level for request package."""
super(TermiusApp, self).configure_logging()
logging.getLogger('requests').setLevel(logging.WARNING)
# pylint: disable=no-self-use
def configure_signals(self):
"""Bind subscribers to signals."""
post_create_instance.connect(store_ssh_key, sender=SshKey)
post_update_instance.connect(store_ssh_key, sender=SshKey)
post_delete_instance.connect(delete_ssh_key, sender=SshKey)
post_logout.connect(clean_data)
def build_option_parser(self, description, version, argparse_kwargs=None):
"""Return an argparse option parser for this application.
Subclasses may override this method to extend
the parser with more global options.
:param description: full description of the application
:paramtype description: str
:param version: version number for the application
:paramtype version: str
:param argparse_kwargs: extra keyword argument passed to the
ArgumentParser constructor
:paramtype extra_kwargs: dict
"""
argparse_kwargs = argparse_kwargs or {}
parser = argparse.ArgumentParser(description=description,
add_help=False,
**argparse_kwargs)
parser.add_argument('--version',
action='version',
version='%(prog)s {0}'.format(version),
help='display version information and exit')
verbose_group = parser.add_mutually_exclusive_group()
verbose_group.add_argument(
'-v',
'--verbose',
action='count',
dest='verbose_level',
default=self.DEFAULT_VERBOSE_LEVEL,
help='provide a detailed output',
)
verbose_group.add_argument(
'-q',
'--quiet',
action='store_const',
dest='verbose_level',
const=0,
help='display warnings and errors only',
)
parser.add_argument(
'--log-file',
action='store',
default=None,
help='record output into a designated file',
)
if self.deferred_help:
parser.add_argument(
'-h',
'--help',
dest='deferred_help',
action='store_true',
help="display help message",
)
else:
parser.add_argument(
'-h',
'--help',
action=HelpAction,
nargs=0,
default=self, # tricky
help="show the help message",
)
parser.add_argument(
'--debug',
default=False,
action='store_true',
help='enable debugging mode',
)
return parser
class GitRepo(object):
def __init__(self, path, remote_url=None, branch_name='master'):
self.path = Path(path)
self.path_str = str(self.path)
self.remote_url = remote_url
self.branch_name = branch_name
db_latest_key = '%s:%s:%s' % (self.path_str, remote_url or '',
branch_name)
self.db_latest_key = sha256(db_latest_key).hexdigest()
def git(self, *args):
"""Run a git command against the current repo"""
curdir = os.getcwd()
try:
os.chdir(self.path_str)
output = check_output((GIT,) + args, stderr=STDOUT)
finally:
os.chdir(curdir)
return output.strip()
@property
def current_hash(self):
"""The git revision ID (hash) of the current HEAD or None if no repo"""
try:
return self.git('rev-parse', 'HEAD')
except OSError:
return None
def diff(self, start_hash, end_hash):
"""Return a 2 tuple: (modified files, deleted files)"""
diff_out = StringIO(self.git('diff', '--name-status', start_hash, end_hash))
modified = set()
removed = set()
for line in diff_out:
parts = line.split()
# delete
if parts[0] == 'D':
removed.add(parts[1])
# rename
elif parts[0][0] == 'R':
removed.add(parts[1])
modified.add(parts[2])
# everything else
else:
# some types (like copy) have two file entries
for part in parts[1:]:
modified.add(part)
return modified, removed
def clone(self):
"""Clone the repo specified in the initial arguments"""
if not self.remote_url:
raise RuntimeError('remote_url required to clone')
self.path.mkdir(parents=True, exist_ok=True)
self.git('clone', '--depth', '1', '--branch',
self.branch_name, self.remote_url, '.')
def pull(self):
"""Update the repo to the latest of the remote and branch
Return the previous hash and the new hash."""
old_hash = self.current_hash
self.git('fetch', '-f', self.remote_url, self.branch_name)
self.git('checkout', '-f', 'FETCH_HEAD')
return old_hash, self.current_hash
def update(self):
"""Updates a repo, cloning if necessary.
:return a tuple of lists of modified and deleted files if updated, None if cloned
"""
if self.path.is_dir():
if not self.path.joinpath('.git').is_dir():
rmtree(self.path_str, ignore_errors=True)
self.clone()
else:
return self.pull()
else:
self.clone()
return None, None
def reset(self, new_head):
self.git('reset', '--hard', new_head)
def get_db_latest(self):
try:
return GitRepoState.objects.get(repo_id=self.db_latest_key).latest_ref
except GitRepoState.DoesNotExist:
return None
def has_changes(self):
return self.current_hash != self.get_db_latest()
def set_db_latest(self, latest_ref=None):
latest_ref = latest_ref or self.current_hash
rs, created = GitRepoState.objects.get_or_create(repo_id=self.db_latest_key,
defaults={'latest_ref': latest_ref})
if not created:
rs.latest_ref = latest_ref
rs.save()
--------------------Files--------------------
"""
originals = []
filenames = []
# Test existence of files
try:
if not cfg_img_path.is_absolute():
input_path = Path(cwd, "image_in", cfg_img_path)
else:
input_path = cfg_img_path
# Files
if input_path.is_file():
add_new_image_from_file(input_path)
# Directories
elif input_path.is_dir():
for img in input_path.glob("*"):
add_new_image_from_file(img)
else:
logger.error("Could not resolve path.")
raise OSError
except OSError:
logger.error("Could not load image/s!")
sys.exit()
try:
input_path_local = Path(cwd, "fonts", cfg_font_path)
input_path_windows = Path("C:\\windows\\fonts\\", cfg_font_path)
if input_path_local.is_file():
font = ImageFont.truetype(str(input_path_local), cfg_font_size)
class GitRepo(object):
def __init__(self, path, remote_url=None, remote_name=None, branch_name='master'):
self.path = Path(path)
self.path_str = str(self.path)
self.remote_url = remote_url
self.branch_name = branch_name
if not remote_name:
remote_name = 'bedrock-dev' if settings.DEV else 'bedrock-prod'
self.remote_name = remote_name
def git(self, *args):
"""Run a git command against the current repo"""
curdir = os.getcwd()
try:
os.chdir(self.path_str)
output = check_output((GIT,) + args, stderr=STDOUT)
finally:
os.chdir(curdir)
return output.strip()
@property
def full_branch_name(self):
"""Full branch name with remote (e.g. origin/master)"""
return '{}/{}'.format(self.remote_name, self.branch_name)
@property
def current_hash(self):
"""The git revision ID (hash) of the current HEAD"""
return self.git('rev-parse', 'HEAD')
@property
def remote_names(self):
"""Return a list of the remote names in the repo"""
return self.git('remote').split()
def has_remote(self):
"""Return True if the repo has a remote by the correct name"""
return self.remote_name in self.remote_names
def add_remote(self):
"""Add the remote to the git repo from the init args"""
if not self.remote_url:
raise RuntimeError('remote_url required to add a remote')
self.git('remote', 'add', self.remote_name, self.remote_url)
def diff(self, start_hash, end_hash):
"""Return a 2 tuple: (modified files, deleted files)"""
diff_out = StringIO(self.git('diff', '--name-status', start_hash, end_hash))
modified = set()
removed = set()
for line in diff_out:
parts = line.split()
# delete
if parts[0] == 'D':
removed.add(parts[1])
# rename
elif parts[0][0] == 'R':
removed.add(parts[1])
modified.add(parts[2])
# everything else
else:
# some types (like copy) have two file entries
for part in parts[1:]:
modified.add(part)
return modified, removed
def clone(self):
"""Clone the repo specified in the initial arguments"""
if not self.remote_url:
raise RuntimeError('remote_url required to clone')
self.path.mkdir(parents=True, exist_ok=True)
self.git('clone', '--origin', self.remote_name, '--depth', '1',
'--branch', self.branch_name, self.remote_url, '.')
def pull(self):
"""Update the repo to the latest of the remote and branch
Return the previous hash and the new hash."""
if not self.has_remote():
self.add_remote()
old_hash = self.current_hash
self.git('fetch', self.remote_name)
self.git('checkout', '-f', self.full_branch_name)
return old_hash, self.current_hash
def update(self):
"""Updates a repo, cloning if necessary.
:return a tuple of lists of modified and deleted files if updated, None if cloned
"""
if self.path.is_dir():
if not self.path.joinpath('.git').is_dir():
rmtree(self.path_str, ignore_errors=True)
self.clone()
else:
return self.diff(*self.pull())
else:
self.clone()
return None, None
class GitRepo:
def __init__(self, path, remote_url=None, branch_name='master', name=None):
self.path = Path(path)
self.path_str = str(self.path)
self.remote_url = remote_url
self.branch_name = branch_name
db_latest_key = '%s:%s:%s' % (self.path_str, remote_url or '', branch_name)
self.db_latest_key = sha256(db_latest_key.encode()).hexdigest()
self.repo_name = name or self.path.name
def git(self, *args):
"""Run a git command against the current repo"""
curdir = os.getcwd()
try:
os.chdir(self.path_str)
output = check_output((GIT,) + args, stderr=STDOUT)
finally:
os.chdir(curdir)
return output.strip()
@property
def current_hash(self):
"""The git revision ID (hash) of the current HEAD or None if no repo"""
try:
return self.git('rev-parse', 'HEAD')
except (OSError, CalledProcessError):
return None
@property
def current_commit_timestamp(self):
"""The UNIX timestamp of the latest commit"""
try:
return int(self.git('show', '-s', '--format=%ct', 'HEAD'))
except (OSError, CalledProcessError, ValueError):
return 0
@property
def last_updated(self):
if self.current_commit_timestamp:
latest_datetime = datetime.fromtimestamp(self.current_commit_timestamp)
return timeago.format(latest_datetime)
return 'unknown'
def diff(self, start_hash, end_hash):
"""Return a 2 tuple: (modified files, deleted files)"""
diff_out = StringIO(self.git('diff', '--name-status', start_hash, end_hash))
modified = set()
removed = set()
for line in diff_out:
parts = line.split()
# delete
if parts[0] == 'D':
removed.add(parts[1])
# rename
elif parts[0][0] == 'R':
removed.add(parts[1])
modified.add(parts[2])
# everything else
else:
# some types (like copy) have two file entries
for part in parts[1:]:
modified.add(part)
return modified, removed
def clone(self):
"""Clone the repo specified in the initial arguments"""
if not self.remote_url:
raise RuntimeError('remote_url required to clone')
self.path.mkdir(parents=True, exist_ok=True)
self.git('clone', '--depth', '1', '--branch',
self.branch_name, self.remote_url, '.')
def pull(self):
"""Update the repo to the latest of the remote and branch
Return the previous hash and the new hash."""
old_hash = self.current_hash
self.git('fetch', '-f', self.remote_url, self.branch_name)
self.git('checkout', '-f', 'FETCH_HEAD')
return old_hash, self.current_hash
def update(self):
"""Updates a repo, cloning if necessary.
:return a tuple of lists of modified and deleted files if updated, None if cloned
"""
if self.path.is_dir():
if not self.path.joinpath('.git').is_dir():
rmtree(self.path_str, ignore_errors=True)
self.clone()
else:
return self.pull()
else:
self.clone()
return None, None
def reset(self, new_head):
self.git('reset', '--hard', new_head)
def get_db_latest(self):
try:
return GitRepoState.objects.get(repo_id=self.db_latest_key).latest_ref
except GitRepoState.DoesNotExist:
return None
def has_changes(self):
return self.current_hash != self.get_db_latest()
@property
def clean_remote_url(self):
repo_base = self.remote_url
if repo_base.endswith('.git'):
repo_base = repo_base[:-4]
elif repo_base.endswith('/'):
repo_base = repo_base[:-1]
return repo_base
def set_db_latest(self, latest_ref=None):
latest_ref = latest_ref or self.current_hash
rs, created = GitRepoState.objects.get_or_create(
repo_id=self.db_latest_key,
defaults={
'latest_ref': latest_ref,
'repo_name': self.repo_name,
'repo_url': self.clean_remote_url,
'latest_ref_timestamp': self.current_commit_timestamp,
},
)
if not created:
rs.latest_ref = latest_ref
rs.repo_name = self.repo_name
rs.repo_url = self.clean_remote_url
rs.latest_ref_timestamp = self.current_commit_timestamp
rs.save()
def load(directory):
global dataSize, db_name
log = open('log.txt', 'w+') # file to log progress
dirpath = Path(directory)
# Check if path is valid
if dirpath.exists():
stime = datetime.datetime.now() # start time for operation
print(".fits files will be loaded from " + directory + ".\n")
filestoadd = [] # list of all valid files to add to db
# load files:
conn = sqlite3.connect(db_name)
c = conn.cursor()
print("Finding files...\n")
# if path is a directory, walk through it and find all fits files
if dirpath.is_dir():
for root, dirs, files in os.walk(directory):
for file in files:
if file.endswith("dyn.fit") or file.endswith("dyn.fits"):
filestoadd.append(os.path.join(root, file))
# otherwise, it is a file
elif directory.endswith("dyn.fit") or directory.endswith("dyn.fits"):
filestoadd.append(directory)
else:
print("No valid files found.")
print("{0} files found.".format(len(filestoadd)))
log.write("{0} files found.\n".format(
len(filestoadd))) # port to log.txt
success = 0 # counter for successful adds
for i in range(0, len(filestoadd)):
print("Loading {0} out of {1} files: {2}".format(
i + 1, len(filestoadd), filestoadd[i]))
log.write("Loading {0} out of {1} files: {2}\n".format(
i + 1, len(filestoadd), filestoadd[i])) # port to log.txt
try:
header, data = readFile(filestoadd[i])
# insert all header information as numerics and binary data as json object
command = "INSERT INTO astrodata VALUES('{0}' , '{1}' , '{2}' , '{3}' , '{4}' , '{5}' , ?)".format(
header[0], header[1], header[2], header[3], header[4],
header[5])
c.execute(command, (json.dumps(data.tolist()), ))
success += 1
except Exception as e:
print("Error loading file {0}: {1}".format(
filestoadd[i], e.message))
log.write("Error loading file {0}: {1}\n".format(
filestoadd[i], e.message)) # port to log.txt
conn.commit()
dataSize += 1
i += 1
print(
"\nDone. {0} out of {1} files added successfully. See log.txt for more information.\n"
.format(success, len(filestoadd)))
log.write("\nDone. {0} out of {1} files added successfully.\n".format(
success, len(filestoadd))) # port to log.txt
tottime = datetime.datetime.now() - stime # elapsed time of operation
print("Total time elapsed for adding: {0}\n".format(tottime))
log.write("Total time elapsed for adding: {0}\n".format(
tottime)) # port to log.txt
conn.close()
elif directory != "exit":
print(
"Invalid directory. Type 'exit' to exit to the main menu or try again with a new input.\n"
)
directory = input()
if directory != "exit":
load(directory)
log.close()
class TermiusApp(App):
"""Class for CLI application."""
def __init__(self):
"""Construct new CLI application."""
super(TermiusApp, self).__init__(
description='Termius - crossplatform SSH and Telnet client',
version=__version__,
command_manager=CommandManager('termius.handlers'),
)
self.configure_signals()
self.directory_path = Path(expanduser('~/.{}/'.format(self.NAME)))
if not self.directory_path.is_dir():
self.directory_path.mkdir(parents=True)
self.command_manager.add_command('help', HelpCommand)
def configure_logging(self):
"""Change logging level for request package."""
super(TermiusApp, self).configure_logging()
logging.getLogger('requests').setLevel(logging.WARNING)
return
# pylint: disable=no-self-use
def configure_signals(self):
"""Bind subscribers to signals."""
post_create_instance.connect(store_ssh_key, sender=SshKey)
post_update_instance.connect(store_ssh_key, sender=SshKey)
post_delete_instance.connect(delete_ssh_key, sender=SshKey)
post_logout.connect(clean_data)
def build_option_parser(self, description, version,
argparse_kwargs=None):
"""Return an argparse option parser for this application.
Subclasses may override this method to extend
the parser with more global options.
:param description: full description of the application
:paramtype description: str
:param version: version number for the application
:paramtype version: str
:param argparse_kwargs: extra keyword argument passed to the
ArgumentParser constructor
:paramtype extra_kwargs: dict
"""
argparse_kwargs = argparse_kwargs or {}
parser = argparse.ArgumentParser(
description=description,
add_help=False,
**argparse_kwargs
)
parser.add_argument(
'--version',
action='version',
version='%(prog)s {0}'.format(version),
help='display version information and exit'
)
verbose_group = parser.add_mutually_exclusive_group()
verbose_group.add_argument(
'-v', '--verbose',
action='count',
dest='verbose_level',
default=self.DEFAULT_VERBOSE_LEVEL,
help='provide a detailed output',
)
verbose_group.add_argument(
'-q', '--quiet',
action='store_const',
dest='verbose_level',
const=0,
help='display warnings and errors only',
)
parser.add_argument(
'--log-file',
action='store',
default=None,
help='record output into a designated file',
)
if self.deferred_help:
parser.add_argument(
'-h', '--help',
dest='deferred_help',
action='store_true',
help="display help message",
)
else:
parser.add_argument(
'-h', '--help',
action=HelpAction,
nargs=0,
default=self, # tricky
help="show the help message",
)
parser.add_argument(
'--debug',
default=False,
action='store_true',
help='enable debugging mode',
)
return parser
class ReIDDataset(Dataset):
"""
The ReID Dataset module is a custom Dataset module, specific to parsing
the Market & Duke Person Re-Identification datasets.
Args:
data_dir (str): The path where the dataset is located.
transform ([list, torchvision.transforms], optional): Pass the list of
transforms to transform the input images. Defaults to None.
target_transform ([list, torchvision.transforms], optional): Pass the
list of transforms to transform the labels. Defaults to None.
ret_camid_n_frame (bool, optional): Whether to return camera ids and
frames. True will additionally return cam_ids and frame.
Defaults to False.
Raises:
Exception: If directory does not exist!
"""
def __init__(self,
data_dir: str,
transform=None,
target_transform=None,
ret_camid_n_frame: bool = False):
super(ReIDDataset, self).__init__()
self.data_dir = Path(data_dir)
if not self.data_dir.exists():
raise Exception(
f"'Path '{self.data_dir.__str__()}' does not exist!")
if not self.data_dir.is_dir():
raise Exception(
f"Path '{self.data_dir.__str__()}' is not a directory!")
self.transform = transform
self.target_transform = target_transform
self.ret_camid_n_frame = ret_camid_n_frame
self._init_data()
def _init_data(self):
if 'market' in str(self.data_dir).lower():
self.dataset = 'market'
elif 'duke' in str(self.data_dir).lower():
self.dataset = 'duke'
self.imgs = list(self.data_dir.glob('*.jpg'))
# Filter out labels with -1
self.imgs = [img for img in self.imgs if '-1' not in img.stem]
self.cam_ids, self.labels, self.frames = get_ids(
self.imgs, self.dataset)
self.num_cams = len(set(self.cam_ids))
self.classes = tuple(set(self.labels))
# Convert labels to continuous idxs
self.class_to_idx = {label: i for i, label in enumerate(self.classes)}
self.targets = [self.class_to_idx[label] for label in self.labels]
def __len__(self):
return len(self.imgs)
def __getitem__(self, index):
sample = Image.open(str(self.imgs[index])).convert('RGB')
target = self.targets[index]
if self.transform:
sample = self.transform(sample)
if self.target_transform:
target = self.target_transform(target)
if self.ret_camid_n_frame:
cam_id = self.cam_ids[index]
frame = self.frames[index]
return sample, target, cam_id, frame
return sample, target
parser.add_argument("-o",
"--out_files",
action="store",
dest="outFiles",
required=True,
help="output file or directory")
args = parser.parse_args()
inPath = Path(args.inFiles)
outPath = Path(args.outFiles)
# Case 1: Convert single file.
if (inPath.is_file()):
inSuff = inPath.suffix
inStem = inPath.stem
outDir = outPath.parent
outDir.mkdir(parents=True, exist_ok=True)
if (inSuff != tsurfSuffix):
msg = 'Only tsurf (*.ts) files are allowed as input.'
raise ValueError(msg)
outPath = outPath.with_suffix(vtkSuffix)
convertFile(inPath, outPath)
# Case 2: Convert directory.
elif (inPath.is_dir()):
convertDir(inPath, outPath)
# Case 3: Give up.
else:
msg = 'Unable to find %s.' % inPath
raise ValueError(msg)
class ReIDDataModule(pl.LightningDataModule):
def __init__(self,
data_dir: str,
st_distribution: Optional[str] = None,
train_subdir: str = 'bounding_box_train',
test_subdir: str = 'bounding_box_test',
query_subdir: str = 'query',
train_batchsize: int = 16,
val_batchsize: int = 16,
test_batchsize: int = 16,
num_workers: int = 4,
random_erasing: float = 0.0,
color_jitter: bool = False,
save_distribution: Union[bool, str] = False):
super().__init__()
self.data_dir = Path(data_dir)
if not self.data_dir.exists():
raise Exception(
f"'Path '{self.data_dir.__str__()}' does not exist!")
if not self.data_dir.is_dir():
raise Exception(
f"Path '{self.data_dir.__str__()}' is not a directory!")
self.train_dir = self.data_dir / train_subdir
self.test_dir = self.data_dir / test_subdir
self.query_dir = self.data_dir / query_subdir
self.train_batchsize = train_batchsize
self.test_batchsize = test_batchsize
self.val_batchsize = val_batchsize
self.num_workers = num_workers
self.color_jitter = color_jitter
self.random_erasing = random_erasing
self.st_distribution = st_distribution
self.save_distribution = save_distribution
self.prepare_data()
def prepare_data(self):
train_transforms = [
transforms.Resize((384, 192), interpolation=3),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
]
test_transforms = train_transforms
test_transforms.pop(1)
if self.random_erasing > 0:
train_transforms.append(
transforms.RandomErasing(self.random_erasing))
if self.color_jitter:
train_transforms.append(transforms.ColorJitter())
train_transforms = transforms.Compose(train_transforms)
self.train = ReIDDataset(self.train_dir, train_transforms)
self.num_classes = len(self.train.classes)
# train_len = int(len(self.train) * 0.8)
# test_len = len(self.train) - train_len
# self.train, self.test = random_split(self.train, [train_len, test_len])
test_transforms = transforms.Compose(test_transforms)
self.test = ReIDDataset(self.test_dir, test_transforms)
self.query = ReIDDataset(self.query_dir,
test_transforms,
ret_camid_n_frame=True)
self.gallery = ReIDDataset(self.test_dir,
test_transforms,
ret_camid_n_frame=True)
self._load_st_distribution()
if self.save_distribution:
self._save_st_distribution()
def _load_st_distribution(self):
if isinstance(self.st_distribution, str):
self.st_distribution = Path(self.st_distribution)
if not (self.st_distribution.exists()
and self.st_distribution.is_file()):
raise FileNotFoundError(
f"Location '{str(self.st_distribution)}' \
does not exist or not a file!")
if self.st_distribution.suffix != '.pkl':
raise ValueError('File must be of type .pkl')
print(f'\nLoading Spatial-Temporal Distribution from \
{self.st_distribution}.\n\n')
self.st_distribution = joblib.load(str(self.st_distribution))
elif not self.st_distribution:
print('\n\nGenerating Spatial-Temporal Distribution.\n\n')
num_cams = self.query.num_cams
max_hist = 5000 if self.query.dataset == 'market' else 3000
cam_ids = self.query.cam_ids + self.gallery.cam_ids
targets = self.query.targets + self.gallery.targets
frames = self.query.frames + self.gallery.frames
self.st_distribution = smooth_st_distribution(
cam_ids, targets, frames, num_cams, max_hist)
def _save_st_distribution(self):
if isinstance(self.save_distribution, str):
if '.pkl' not in self.save_distribution:
self.save_distribution += '.pkl'
else:
self.save_distribution = self.data_dir + 'st_distribution.pkl'
print(f'\nSaving distribution at {self.save_distribution}')
joblib.dump(self.st_distribution, self.save_distribution)
def train_dataloader(self):
return DataLoader(self.train,
batch_size=self.train_batchsize,
shuffle=True,
num_workers=self.num_workers,
pin_memory=True)
def val_dataloader(self):
test_loader = DataLoader(self.test,
batch_size=self.test_batchsize,
shuffle=False,
num_workers=self.num_workers,
pin_memory=True)
query_indices = range(self.test_batchsize)
query_loader = DataLoader(Subset(self.query, query_indices),
batch_size=self.test_batchsize,
shuffle=False,
num_workers=self.num_workers,
pin_memory=True)
evens = list(range(0, len(self.gallery), 3))
gall_loader = DataLoader(Subset(self.gallery, evens),
batch_size=self.test_batchsize,
shuffle=True,
num_workers=self.num_workers,
pin_memory=True)
return [query_loader, gall_loader, test_loader]
def test_dataloader(self):
query_loader = DataLoader(self.query,
batch_size=self.test_batchsize,
shuffle=False,
num_workers=self.num_workers,
pin_memory=True)
gall_loader = DataLoader(self.gallery,
batch_size=self.test_batchsize,
shuffle=True,
num_workers=self.num_workers,
pin_memory=True)
return [query_loader, gall_loader]
def _extract_to_cache(
cls,
cached_file, # type: str
name, # type: str
cache_context=None, # type: Optional[str]
target_folder=None, # type: Optional[str]
cache_path_encoding=None, # type: Optional[str]
force=False, # type: bool
):
# type: (...) -> str
"""
Extract cached file to cache folder
:param str cached_file: local copy of archive file
:param str name: name of the target file
:param str cache_context: cache context id
:param str target_folder: specify target path to use for archive extraction
:param str cache_path_encoding: specify representation of the local path of the cached files,
this will always point to local cache folder, even if we have direct access file.
Used for extracting the cached archived based on cache_path_encoding
:param bool force: Force archive extraction even if target folder exists
:return: cached folder containing the extracted archive content
"""
if not cached_file:
return cached_file
cached_file = Path(cached_file)
cache_path_encoding = Path(
cache_path_encoding) if cache_path_encoding else None
# we support zip and tar.gz files auto-extraction
suffix = cached_file.suffix.lower()
if suffix == '.gz':
suffix = ''.join(a.lower() for a in cached_file.suffixes[-2:])
if suffix not in (".zip", ".tgz", ".tar.gz"):
return str(cached_file)
cache_folder = Path(cache_path_encoding or cached_file).parent
archive_suffix = (cache_path_encoding
or cached_file).name[:-len(suffix)]
name = encode_string_to_filename(name) if name else name
if target_folder:
target_folder = Path(target_folder)
else:
target_folder = cache_folder / CacheManager.get_context_folder_lookup(
cache_context).format(archive_suffix, name)
if target_folder.is_dir() and not force:
# noinspection PyBroadException
try:
target_folder.touch(exist_ok=True)
return target_folder.as_posix()
except Exception:
pass
base_logger = LoggerRoot.get_base_logger()
try:
# if target folder exists, meaning this is forced ao we extract directly into target folder
if target_folder.is_dir():
temp_target_folder = target_folder
else:
temp_target_folder = cache_folder / "{0}_{1}_{2}".format(
target_folder.name,
time() * 1000,
str(random()).replace('.', ''))
temp_target_folder.mkdir(parents=True, exist_ok=True)
if suffix == ".zip":
ZipFile(cached_file.as_posix()).extractall(
path=temp_target_folder.as_posix())
elif suffix == ".tar.gz":
with tarfile.open(cached_file.as_posix()) as file:
file.extractall(temp_target_folder.as_posix())
elif suffix == ".tgz":
with tarfile.open(cached_file.as_posix(), mode='r:gz') as file:
file.extractall(temp_target_folder.as_posix())
if temp_target_folder != target_folder:
# we assume we will have such folder if we already extract the file
# noinspection PyBroadException
try:
# if rename fails, it means that someone else already manged to extract the file, delete the current
# folder and return the already existing cached zip folder
shutil.move(temp_target_folder.as_posix(),
target_folder.as_posix())
except Exception:
if target_folder.exists():
target_folder.touch(exist_ok=True)
else:
base_logger.warning(
"Failed renaming {0} to {1}".format(
temp_target_folder.as_posix(),
target_folder.as_posix()))
try:
shutil.rmtree(temp_target_folder.as_posix())
except Exception as ex:
base_logger.warning(
"Exception {}\nFailed deleting folder {}".format(
ex, temp_target_folder.as_posix()))
except Exception as ex:
# failed extracting the file:
base_logger.warning(
"Exception {}\nFailed extracting zip file {}".format(
ex, cached_file.as_posix()))
# noinspection PyBroadException
try:
target_folder.rmdir()
except Exception:
pass
return cached_file.as_posix()
return target_folder.as_posix()
async def collect_parameter_data(self, param, when):
""" Collects input parameters for the model as determined by the metadata. """
parameter_dataset_name = None
# Only collect from the past!
if when.date() < dt.datetime.today().date():
param = self.parameters[param]
file_path = Path(param['path'])
if not file_path.is_dir():
os.makedirs(file_path)
parameter_dataset_name = file_path.joinpath(param['prefix'] + "_" +
param['dataset'])
if parameter_dataset_name.is_file():
return parameter_dataset_name
else:
data_file = file_path.joinpath(param['prefix'] + "_" +
when.strftime("%Y%m%d") +
param['suffix'])
logger.debug(data_file)
archive_file = Path(
str(data_file) + param['compression_suffix'])
try:
if data_file.is_file():
parameter_dataset_name = self.do_conversion(
data_file, param, when)
elif not data_file.is_file() and archive_file.is_file():
logger.debug('Found an unexpanded archive: %s' %
archive_file)
await self.do_expansion(archive_file)
parameter_dataset_name = self.do_conversion(
data_file, param, when)
# Remove the archive?
elif not data_file.is_file() and not archive_file.is_file(
):
date_string = when.strftime("%Y%m%d")
resource = date_string + date_string + param['suffix'] + \
param['compression_suffix']
if await self.do_download(param["url"], resource,
archive_file):
if await self.do_expansion(archive_file):
parameter_dataset_name = await self.do_conversion(
data_file, param, when)
except URLError as e:
logger.warning(e)
return None
except FileNotFoundError as fnf:
logger.warning(fnf)
return None
if Path(parameter_dataset_name).is_file():
return parameter_dataset_name
else:
logger.warning("No parameters for that date.")
return None
def upload_artifact(self,
name,
artifact_object=None,
metadata=None,
delete_after_upload=False):
if not Session.check_min_api_version('2.3'):
LoggerRoot.get_base_logger().warning(
'Artifacts not supported by your TRAINS-server version, '
'please upgrade to the latest server version')
return False
if name in self._artifacts_container:
raise ValueError(
"Artifact by the name of {} is already registered, use register_artifact"
.format(name))
artifact_type_data = tasks.ArtifactTypeData()
override_filename_in_uri = None
override_filename_ext_in_uri = None
uri = None
if np and isinstance(artifact_object, np.ndarray):
artifact_type = 'numpy'
artifact_type_data.content_type = 'application/numpy'
artifact_type_data.preview = str(artifact_object.__repr__())
override_filename_ext_in_uri = '.npz'
override_filename_in_uri = name + override_filename_ext_in_uri
fd, local_filename = mkstemp(prefix=quote(name, safe="") + '.',
suffix=override_filename_ext_in_uri)
os.close(fd)
np.savez_compressed(local_filename, **{name: artifact_object})
delete_after_upload = True
elif pd and isinstance(artifact_object, pd.DataFrame):
artifact_type = 'pandas'
artifact_type_data.content_type = 'text/csv'
artifact_type_data.preview = str(artifact_object.__repr__())
override_filename_ext_in_uri = self._save_format
override_filename_in_uri = name
fd, local_filename = mkstemp(prefix=quote(name, safe="") + '.',
suffix=override_filename_ext_in_uri)
os.close(fd)
artifact_object.to_csv(local_filename,
compression=self._compression)
delete_after_upload = True
elif isinstance(artifact_object, Image.Image):
artifact_type = 'image'
artifact_type_data.content_type = 'image/png'
desc = str(artifact_object.__repr__())
artifact_type_data.preview = desc[1:desc.find(' at ')]
override_filename_ext_in_uri = '.png'
override_filename_in_uri = name + override_filename_ext_in_uri
fd, local_filename = mkstemp(prefix=quote(name, safe="") + '.',
suffix=override_filename_ext_in_uri)
os.close(fd)
artifact_object.save(local_filename)
delete_after_upload = True
elif isinstance(artifact_object, dict):
artifact_type = 'JSON'
artifact_type_data.content_type = 'application/json'
preview = json.dumps(artifact_object, sort_keys=True, indent=4)
override_filename_ext_in_uri = '.json'
override_filename_in_uri = name + override_filename_ext_in_uri
fd, local_filename = mkstemp(prefix=quote(name, safe="") + '.',
suffix=override_filename_ext_in_uri)
os.write(fd, bytes(preview.encode()))
os.close(fd)
artifact_type_data.preview = preview
delete_after_upload = True
elif isinstance(artifact_object, six.string_types) and urlparse(
artifact_object).scheme in remote_driver_schemes:
# we should not upload this, just register
local_filename = None
uri = artifact_object
artifact_type = 'custom'
artifact_type_data.content_type = mimetypes.guess_type(
artifact_object)[0]
elif isinstance(artifact_object, six.string_types + (Path, )):
# check if single file
artifact_object = Path(artifact_object)
artifact_object.expanduser().absolute()
try:
create_zip_file = not artifact_object.is_file()
except Exception: # Hack for windows pathlib2 bug, is_file isn't valid.
create_zip_file = True
else: # We assume that this is not Windows os
if artifact_object.is_dir():
# change to wildcard
artifact_object /= '*'
if create_zip_file:
folder = Path('').joinpath(*artifact_object.parts[:-1])
if not folder.is_dir() or not folder.parts:
raise ValueError(
"Artifact file/folder '{}' could not be found".format(
artifact_object.as_posix()))
wildcard = artifact_object.parts[-1]
files = list(Path(folder).rglob(wildcard))
override_filename_ext_in_uri = '.zip'
override_filename_in_uri = folder.parts[
-1] + override_filename_ext_in_uri
fd, zip_file = mkstemp(
prefix=quote(folder.parts[-1], safe="") + '.',
suffix=override_filename_ext_in_uri)
try:
artifact_type_data.content_type = 'application/zip'
artifact_type_data.preview = 'Archive content {}:\n'.format(
artifact_object.as_posix())
with ZipFile(zip_file,
'w',
allowZip64=True,
compression=ZIP_DEFLATED) as zf:
for filename in sorted(files):
if filename.is_file():
relative_file_name = filename.relative_to(
folder).as_posix()
artifact_type_data.preview += '{} - {}\n'.format(
relative_file_name,
humanfriendly.format_size(
filename.stat().st_size))
zf.write(filename.as_posix(),
arcname=relative_file_name)
except Exception as e:
# failed uploading folder:
LoggerRoot.get_base_logger().warning(
'Exception {}\nFailed zipping artifact folder {}'.
format(folder, e))
return None
finally:
os.close(fd)
artifact_object = zip_file
artifact_type = 'archive'
artifact_type_data.content_type = mimetypes.guess_type(
artifact_object)[0]
local_filename = artifact_object
delete_after_upload = True
else:
if not artifact_object.is_file():
raise ValueError(
"Artifact file '{}' could not be found".format(
artifact_object.as_posix()))
override_filename_in_uri = artifact_object.parts[-1]
artifact_object = artifact_object.as_posix()
artifact_type = 'custom'
artifact_type_data.content_type = mimetypes.guess_type(
artifact_object)[0]
local_filename = artifact_object
else:
raise ValueError("Artifact type {} not supported".format(
type(artifact_object)))
# remove from existing list, if exists
for artifact in self._task_artifact_list:
if artifact.key == name:
if artifact.type == self._pd_artifact_type:
raise ValueError(
"Artifact of name {} already registered, "
"use register_artifact instead".format(name))
self._task_artifact_list.remove(artifact)
break
if not local_filename:
file_size = None
file_hash = None
else:
# check that the file to upload exists
local_filename = Path(local_filename).absolute()
if not local_filename.exists() or not local_filename.is_file():
LoggerRoot.get_base_logger().warning(
'Artifact upload failed, cannot find file {}'.format(
local_filename.as_posix()))
return False
file_hash, _ = self.sha256sum(local_filename.as_posix())
file_size = local_filename.stat().st_size
uri = self._upload_local_file(
local_filename,
name,
delete_after_upload=delete_after_upload,
override_filename=override_filename_in_uri,
override_filename_ext=override_filename_ext_in_uri)
timestamp = int(time())
artifact = tasks.Artifact(
key=name,
type=artifact_type,
uri=uri,
content_size=file_size,
hash=file_hash,
timestamp=timestamp,
type_data=artifact_type_data,
display_data=[(str(k), str(v))
for k, v in metadata.items()] if metadata else None)
# update task artifacts
with self._task_edit_lock:
self._task_artifact_list.append(artifact)
self._task.set_artifacts(self._task_artifact_list)
return True
print("Greetings! This script is still in beta so please let me know if there is any issues with the script itself.\nSo far, it seems that things are just barely getting started and this is my first full blown python script so I'm sorry if it looks bad \n")
subprocess.call(["ls", "-lah"])
print("Completed ls -lah subprocess call \n")
path = os.getcwd()
print("The current path of this system is %s \n" % path)
print("Checking if /home/steamcmd is available... \n")
my_file = Path ("/home/steamcmd")
if my_file.is_dir():
os.chdir("/home/steamcmd")
print("/home/steamcmd is already on the server, moving on..")
else:
print("Not finding /home/steamcmd.. making folder")
os.mkdir("/home/steamcmd")
print("Folder should be created, testing it now \n")
os.chdir("/home/steamcmd")
<<<<<<< HEAD
print("Successfully create /home/steamcmd")
=======
print("Successfully create /home/steamcmd")
>>>>>>> master
