def __del__(self):
if self.log_on_del:
self.stop_block()
from oarphpy.util import create_log
log = create_log()
log.info('\n' + str(self) + '\n')
def maybe_log_progress(self, every_n=-1):
if every_n >= 0:
self.__log_freq = every_n
if self.n >= self.__last_log + self.__log_freq:
from oarphpy.util import log
log.info("Progress for \n" + str(self))
self.__last_log = self.n
# Track last log because `n` may increase inconsistently
if every_n == -1 and (self.n >= (1.7 * self.__log_freq)):
self.__log_freq = int(1.7 * self.__log_freq)
def get_infos(only_visible=True):
from oarphpy.util import log
# Much safer than pycuda and Tensorflow, which can both segfault if the
# nvidia driver is absent :P
try:
cmd = "nvidia-smi --query-gpu=index,name,utilization.memory,name,memory.total,memory.free,memory.used --format=csv,nounits"
out = run_cmd(cmd, collect=True)
except Exception as e:
log.info("No GPUs found")
return []
# NB: nvidia doesn't actually return *valid* csv.
# Why would they? They make hardware, not software!
out = out.decode('utf-8')
out = out.replace(', ', ',')
import csv
rows = list(csv.DictReader(out.split('\n')))
infos = [GPUInfo.from_nvidia_smi(row) for row in rows]
log.info("Found GPUs: %s" % ([str(info) for info in infos], ))
if only_visible:
if 'CUDA_VISIBLE_DEVICES' in os.environ:
allowed_gpus = set(
int(g)
for g in os.environ['CUDA_VISIBLE_DEVICES'].split(',')
if g)
log.info("... restricting to GPUs %s ..." % (allowed_gpus, ))
infos = [info for info in infos if info.index in allowed_gpus]
return infos
def download(uri, dest, try_expand=True):
"""Fetch `uri`, which is a file or archive, and put in `dest`, which
is either a destination file path or destination directory."""
import math
from oarphpy.util import log
from oarphpy.util.thruput_observer import ThruputObserver
# Import urllib
try:
import urllib.error as urlliberror
import urllib.request
HTTPError = urlliberror.HTTPError
URLError = urlliberror.URLError
except ImportError:
import urllib2 as urllib
HTTPError = urllib.HTTPError
URLError = urllib.URLError
import urllib.request
import patoolib
if os.path.exists(dest):
return
fname = os.path.split(uri)[-1]
tempdest = tempfile.NamedTemporaryFile(suffix='_' + fname)
try:
log.info("Fetching %s ..." % uri)
response = urllib.request.urlopen(uri)
size = int(response.info().get('Content-Length').strip())
log.info("... downloading %s MB ..." % (float(size) * 1e-6))
chunk_size = min(size, 8192)
t = ThruputObserver(name=uri,
log_freq=10000,
n_total=math.ceil(size / chunk_size))
while True:
with t.observe(n=1, num_bytes=chunk_size):
data = response.read(chunk_size)
if not data:
break
tempdest.write(data)
t.maybe_log_progress()
sys.stdout.write("")
sys.stdout.flush()
log.info("... fetched!")
except HTTPError as e:
raise Exception("[HTTP Error] {code}: {reason}.".format(
code=e.code, reason=e.reason))
except URLError as e:
raise Exception("[URL Error] {reason}.".format(reason=e.reason))
tempdest.flush()
if try_expand:
try:
# Is it an archive? expand!
mkdir(dest)
patoolib.extract_archive(tempdest.name, outdir=dest)
log.info("Extracted archive.")
except Exception:
# Just move the file
shutil.move(tempdest.name, dest)
tempdest.delete = False
else:
shutil.move(tempdest.name, dest)
tempdest.delete = False
log.info("Downloaded to %s" % dest)