def put(remote_file, local_file, mode="wct", perm=None):
"""Copy the local_file from the execute machine to the submit machine, naming it remote_file.
The optional perm argument describes the file access permissions in a Unix format.
The optional mode argument is one or more of the following characters describing the remote_file file:
w, open for writing; a, force all writes to append; t, truncate before use;
c, create the file, if it does not exist; x, fail if c is given and the file already exists.
Args:
remote_file (str, optional): File on submit machine.
local_file (str, optional): File on execute machine.
mode (str, optional): File open modes (one or more of 'watcx'). Defaults to 'wct'.
w, open for writing;
a, force all writes to append;
t, truncate before use;
c, create the file, if it does not exist;
x, fail if c is given and the file already exists. Defaults to None.
perm (str, optional): Describes the file access permissions in a Unix format. Defaults to None.
"""
opt_params = {}
if mode:
# Add "w" to along with the following characters to reproduce condor_chirp behavior
for c in "act":
if c in mode:
mode += "w"
break
opt_params["flags"] = mode
if perm:
opt_params["mode"] = perm
with htchirp.HTChirp() as chirp:
chirp.put(remote_file, local_file, mode, perm)
def phase(phasestring):
"""Tell HTCondor that the job is changing phases.
Args:
phasestring (str): New phase.
"""
with htchirp.HTChirp() as chirp:
chirp.phase(phasestring)
def ulog(text):
"""Appends Message to the job event log.
Args:
text (str): Message to log.
"""
with htchirp.HTChirp() as chirp:
chirp.ulog(text)
def whoami():
"""Get the user's current identity.
Returns:
str: The user's identity.
"""
with htchirp.HTChirp() as chirp:
return chirp.whoami()
def remove(remote_file):
"""Remove the remote_file file from the submit machine.
Args:
remote_file (str, optional): File on submit machine.
"""
with htchirp.HTChirp() as chirp:
chirp.remove(remote_file)
def rmdir(remotepath, r=False):
"""Delete the directory specified by RemotePath. If the optional -r is specified, recursively delete the entire directory.
Args:
remotepath (str): Path to directory on the submit machine.
r (bool, optional): Recursively delete remotepath. Defaults to False.
"""
with htchirp.HTChirp() as chirp:
chirp.rmdir(remotepath, r)
def truncate(remotepath, length):
"""Truncates the file at remotepath to length bytes.
Args:
remotepath (str): File on the submit machine.
length (int): Truncated length.
"""
with htchirp.HTChirp() as chirp:
chirp.truncate(remotepath, length)
def set_job_attr(job_attribute, attribute_value):
"""Sets the named job ClassAd attribute with the given attribute value.
Args:
job_attribute (str): Job ClassAd attribute.
attribute_value (str): Job ClassAd value.
"""
with htchirp.HTChirp() as chirp:
chirp.set_job_attr(job_attribute, attribute_value)
def chmod(remotepath, mode):
"""Change the permissions of remotepath to mode. mode describes the file access
permissions in a Unix format; 660 is an example Unix format.
Args:
remotepath (str): Target path on the submit machine.
mode (str): Permission mode to set
"""
with htchirp.HTChirp() as chirp:
chirp.chmod(remotepath, int(mode, 8))
def lchown(remotepath, uid, gid):
"""Change the ownership of remotepath to uid and gid. Changes the link, if remotepath is a symbolic link.
Args:
remotepath (str): File on the submit machine.
uid (int): User's UID.
gid (int): User's GID.
"""
with htchirp.HTChirp() as chirp:
chirp.lchown(remotepath, uid, gid)
def access(remotepath, mode):
"""Check access permissions for RemotePath. Mode is one or more of the characters
r, w, x, or f, representing read, write, execute, and existence, respectively.
Args:
remotepath (str): Path to examine on the submit machine.
mode (str): Mode to check (one or more of 'rwxf').
"""
with htchirp.HTChirp() as chirp:
chirp.access(remotepath, mode)
def link(oldpath, newpath, s=False):
"""Create a hard link from OldRemotePath to NewRemotePath.
Args:
oldpath (str): File path to link from on the submit machine.
newpath (str): File path to link to on the submit machine.
s (bool, optional): Create a symbolic link instead. Defaults to False.
"""
with htchirp.HTChirp() as chirp:
chirp.link(oldpath, newpath, s)
def utime(remotepath, actime, mtime):
"""Change the access to actime and modification time to mtime of remotepath.
Args:
remotepath (str): Target path on the submit machine.
actime (int): Access time, in seconds (Unix epoch).
mtime (int): Modification time, in seconds (Unix epoch).
"""
with htchirp.HTChirp() as chirp:
chirp.utime(remotepath, actime, mtime)
def readlink(remotepath):
"""Read the contents of the file defined by the symbolic link remotepath.
Args:
remotepath (str): File path to link on the submit machine.
Returns:
str: Contents of the link.
"""
with htchirp.HTChirp() as chirp:
return chirp.readlink(remotepath[0]).decode()
def whoareyou(remotepath):
"""Get the identity of RemoteHost.
Args:
remotepath (str): Remote host
Returns:
str: The server's identity
"""
with htchirp.HTChirp() as chirp:
return chirp.whoareyou(remotepath)
def get_job_attr(job_attribute):
"""Prints the named job ClassAd attribute to standard output.
Args:
job_attribute (str, optional): Job ClassAd attribute.
Returns:
str: The value of the job attribute as a string.
"""
with htchirp.HTChirp() as chirp:
return chirp.get_job_attr(job_attribute)
def statfs(remotepath):
"""Get file system metadata for remotepath.
Args:
remotepath (str): File path to link on the submit machine.
Returns:
dict: Dict of filesystem metadata.
"""
with htchirp.HTChirp() as chirp:
return chirp.statfs(remotepath)
def get_job_attr_delayed(job_attribute):
"""Prints the named job ClassAd attribute to standard output, potentially reading the cached value
from a recent set_job_attr_delayed.
Args:
job_attribute (str, optional): Job ClassAd attribute.
Returns:
str: The value of the job attribute as a string.
"""
with htchirp.HTChirp() as chirp:
return chirp.get_job_attr_delayed(job_attribute)
def fetch(remote_file, local_file):
"""Copy the remote_file from the submit machine to the execute machine, naming it local_file.
Args:
remote_file (str, optional): File on submit machine.
local_file (str, optional): File on execute machine.
Returns:
int: Bytes written
"""
with htchirp.HTChirp() as chirp:
return chirp.fetch(remote_file, local_file)
def set_job_attr_delayed(job_attribute, attribute_value):
"""Sets the named job ClassAd attribute with the given attribute value, but does not immediately
synchronize the value with the submit side. It can take 15 minutes before the synchronization occurs.
This has much less overhead than the non delayed version. With this option, jobs do not need ClassAd
attribute WantIOProxy set. With this option, job attribute names are restricted to begin with the case
sensitive substring Chirp.
Args:
job_attribute (str): Job ClassAd attribute.
attribute_value (str): Job ClassAd value.
"""
with htchirp.HTChirp() as chirp:
chirp.set_job_attr_delayed(job_attribute, attribute_value)
def read(remote_file, length, offset=None, stride=(None, None)):
"""Read length bytes from remote_file. Optionally, implement a stride by starting the read at offset
and reading stride(length) bytes with a stride of stride(skip) bytes.
Args:
remote_file (str): File on the submit machine.
length (int): Number of bytes to read.
offset (int, optional): Number of bytes to offset from beginning of file. Defaults to None.
stride (tuple, optional): Number of bytes to read followed by number of bytes to skip per stride. Defaults to (None, None).
Returns:
str: Data read from file
"""
with htchirp.HTChirp() as chirp:
return chirp.read(remote_file, length, offset, stride[0], stride[1])
def lstat(remotepath):
"""Get metadata for remotepath. Examines the file, if it is a symbolic link.
Args:
remotepath (str): File path to link on the submit machine.
Returns:
dict: Dict of file metadata.
"""
with htchirp.HTChirp() as chirp:
out = chirp.lstat(remotepath)
for key in ["atime", "mtime", "ctime"]:
out[key] = datetime.fromtimestamp(out[key])
return out
def getdir(remotepath, l=False):
"""List the contents of the directory specified by RemotePath.
Args:
remotepath (str): Path to directory on the submit machine.
l (bool, optional): Returns a dict of file metadata. Defaults to False.
Returns:
list: List of files, when l is False.
dict: Dictionary of files with their metadata, when l is True.
"""
with htchirp.HTChirp() as chirp:
out = chirp.getdir(remotepath, l)
for item in out:
for key in ["atime", "mtime", "ctime"]:
out[item][key] = datetime.fromtimestamp(out[item][key])
return out
def write(remote_file, local_file, length, offset=None, stride=(None, None)):
"""Write the contents of local_file to remote_file. Optionally, start writing to the remote file at offset
and write stride(length) bytes with a stride of stride(skip) bytes. If the optional length follows
local_file, then the write will halt after length input bytes have been written. Otherwise, the entire
contents of local_file will be written.
Args:
remote_file (str): File on the submit machine.
local_file (str): File on execute machine.
length (int): Number of bytes to write.
offset (int, optional): Number of bytes to offset from beginning of file. Defaults to None.
stride (tuple, optional): Number of bytes to read followed by number of bytes to skip per stride. Defaults to (None, None).
"""
data = open(local_file).read()
with htchirp.HTChirp() as chirp:
chirp.write(data,
remote_file,
length=length,
offset=offset,
stride_length=stride[0],
stride_skip=stride[1])
