def time_func(func, *args, **kw):
''' Run the supplied function and arguments.
Return a the elapsed time in seconds and the function's own return value.
'''
from cs.logutils import LogTime
with LogTime(repr(func)) as L:
ret = func(*args, **kw)
return L.elapsed, ret
def canonicalize(self):
from cs.chunkyString import ChunkyString
cs = ChunkyString()
cs.write(' ') # HACK: work around bug in ZSI.wstools.c14n
with LogTime("BigElementProxy.canonicalize()"):
ZSI.wstools.Utility.Canonicalize(self.node, output=cs)
cs = str(cs)
##if isdebug: print >>sys.stderr, "BigElementProxy.canonicalize: XML=[%s]" % cs
return cs
def handle(self, *a, **kw):
''' Wrapper for FUSE handler methods.
'''
syscall = method.__name__
if syscall == 'write':
fh, offset, bs = a
arg_desc = [
str(a[0]),
str(a[1]),
"%d bytes:%r..." % (len(bs), bytes(bs[:16]))
]
else:
arg_desc = [(("<%s>" % (type(arg).__name__, )) if isinstance(
arg, llfuse.RequestContext) else repr(arg)) for arg in a]
arg_desc.extend("%s=%r" % (kw_name, kw_value)
for kw_name, kw_value in kw.items())
arg_desc = ','.join(arg_desc)
with Pfx("%s.%s(%s)", type(self).__name__, syscall, arg_desc):
trace = syscall in (
##'getxattr',
##'setxattr',
##'statfs',
)
if trace:
X("CALL %s(%s)", syscall, arg_desc)
fs = self._vtfs
try:
with stackattrs(defaults, fs=fs):
with fs.S:
with LogTime("SLOW SYSCALL", threshold=5.0):
result = method(self, *a, **kw)
if trace:
if isinstance(result, bytes):
X("CALL %s result => %d bytes, %r...", syscall,
len(result), result[:16])
else:
X("CALL %s result => %s", syscall, result)
return result
##except FuseOSError as e:
## warning("=> FuseOSError %s", e, exc_info=False)
## raise
except OSError as e:
##warning("=> OSError %s => FuseOSError", e, exc_info=False)
raise FuseOSError(e.errno) from e
except MissingHashcodeError as e:
error("raising IOError from missing hashcode: %s", e)
raise FuseOSError(errno.EIO) from e
except Exception as e:
exception("unexpected exception, raising EINVAL %s:%s",
type(e), e)
raise FuseOSError(errno.EINVAL) from e
except BaseException as e:
error("UNCAUGHT EXCEPTION: %s", e)
raise RuntimeError("UNCAUGHT EXCEPTION") from e
except:
error("=> EXCEPTION %r", sys.exc_info())
def callSOAP(url, action, xml, retAction, retTypecode, onerror=None):
''' Call the specified web services URL with an action and SOAP XML string.
Return the parsed response, which should have the action retAction
and be of type retTypecode.
'''
if onerror is not None:
ret = None
try:
ret = callSOAP(url,
action,
xml,
retAction,
retTypecode,
onerror=None)
except urllib2.HTTPError as e:
onerror(action, xml, e)
except urllib2.URLError as e:
onerror(action, xml, e)
except AssertionError as e:
onerror(action, xml, e)
return ret
rq = urllib2.Request(url, xml)
rq.add_header('Accept-Encoding', 'identity')
rq.add_header('Soapaction', '"%s"' % action)
rq.add_header('Content-Type', 'text/xml; charset="utf-8"')
with LogTime('callSOAP(%s): call %s with %d bytes of XML', action, url,
len(xml)):
U = urllib2.urlopen(rq)
I = U.info()
assert I.type in ('text/xml', 'application/soap+xml'), \
"%s: expected text/xml, got \"%s\" from %s %s" % (cs.pfx.cmd, I.type, action, url)
retxml = ''.join(U.readlines())
with LogTime('callSOAP(%s): decode %d bytes of %s response', action,
len(retxml), retAction):
ret = xml2pyobj(retxml, retTypecode)
return ret
def run(self):
''' Run the download.
'''
url = self.url
upd = self.upd
proxy = self.proxy = upd.insert(1)
proxy.prefix = url + ' '
with proxy:
try:
ydl_opts = {
'progress_hooks': [self.update_progress],
'format': DEFAULT_OUTPUT_FORMAT,
'logger': logging.getLogger(),
'outtmpl': DEFAULT_OUTPUT_FILENAME_TEMPLATE,
##'skip_download': True,
'writeinfojson': False,
'updatetime': False,
'process_info': [self.process_info]
}
if self.kw_opts:
ydl_opts.update(self.kw_opts)
ydl = self.ydl = YoutubeDL(ydl_opts)
proxy('await run slot...')
with self.sem:
proxy('extract_info...')
self.tick()
ie_result = ydl.extract_info(url, download=False, process=True)
output_path = ydl.prepare_filename(ie_result)
proxy.prefix = (ie_result.get('title') or output_path) + ' '
proxy('download...')
self.tick()
with LogTime("%s.download(%r)", type(ydl).__name__, url) as LT:
with ydl:
ydl.download([url])
proxy("elapsed %ds, saving metadata ...", LT.elapsed)
self.tick()
tagged_path = self.fstags[output_path]
for key, value in ie_result.items():
tag_name = FSTAGS_PREFIX + '.' + key
tagged_path.set(tag_name, value)
self.fstags.sync()
print(output_path)
except DownloadError as e:
error("download fails: %s", e)
def adjust_delta(self, delta):
''' Adjust capacity by `delta` by calling release() or acquire()
an appropriate number of times.
If `delta` lowers the semaphore capacity then adjust() may block
until the overcapacity is released.
'''
newvalue = self.__value + delta
with self.__lock:
if delta > 0:
while delta > 0:
self.__sem.release()
delta -= 1
else:
while delta < 0:
with LogTime("AdjustableSemaphore(%s): acquire excess capacity",
self.__name):
self.__sem.acquire(True)
delta += 1
self.__value = newvalue
def rinse(soap, tc):
''' Turn SOAP into a python object.
'''
with LogTime("parse SOAP into %s object", tc):
parse = ParsedSoap(soap).Parse(tc)
return parse
def __enter__(self):
with LogTime("%s(%d).__enter__: acquire", self.__name, self.__value):
self.acquire()