def download(self, basefile=None, reporter=None): if basefile: # expect a basefile on the form "subrepoalias:basefile" or # just "subrepoalias:" subrepoalias, basefile = basefile.split(":") else: subrepoalias = None if not basefile: basefile = None # ie convert '' => None found = False for cls in self.subrepos: if (subrepoalias is None or cls.alias == subrepoalias): found = True inst = self.get_instance(cls) basefiles = [] try: ret = inst.download(basefile, reporter=basefiles.append) except Exception as e: loc = util.location_exception(e) self.log.error("download for %s failed: %s (%s)" % (cls.alias, e, loc)) ret = False finally: for b in basefiles: util.link_or_copy(inst.store.documententry_path(b), self.store.documententry_path(b)) # msbfs/entries/.root.json -> myndfs/entries/msbfs.json util.link_or_copy(inst.store.documententry_path(".root"), self.store.documententry_path(inst.alias)) if not found: self.log.error("Couldn't find any subrepo with alias %s" % subrepoalias)
def download(self, basefile=None): for c in self.subrepos: inst = self.get_instance(c) # make sure that our store has access to our now # initialized subrepo objects if c not in self.store.docrepo_instances: self.store.docrepo_instances[c] = inst try: # temporarily re-set the logging level so that the # subrepos INFO messages get reported (see note in # get_instance). loglevel_workaround = False if (self.log.getEffectiveLevel() == logging.INFO and inst.log.getEffectiveLevel() == logging.INFO + 1): loglevel_workaround = True inst.log.setLevel(self.log.getEffectiveLevel()) ret = inst.download(basefile) if loglevel_workaround: inst.log.setLevel(self.log.getEffectiveLevel() + 1) except Exception as e: # be resilient loc = util.location_exception(e) self.log.error("download for %s failed: %s (%s)" % (c.alias, e, loc)) ret = False if basefile and ret: # we got the doc we want, we're done! return
def download(self, basefile=None): for c in self.subrepos: inst = self.get_instance(c) # make sure that our store has access to our now # initialized subrepo objects if c not in self.store.docrepo_instances: self.store.docrepo_instances[c] = inst try: ret = inst.download(basefile) except Exception as e: # be resilient loc = util.location_exception(e) self.log.error("download for %s failed: %s (%s)" % (c.alias, e, loc)) ret = False if basefile and ret: # we got the doc we want, we're done! return
def download(self, basefile=None): for c in self.subrepos: inst = self.get_instance(c) # make sure that our store has access to our now # initialized subrepo objects if c not in self.store.docrepo_instances: self.store.docrepo_instances[c] = inst try: ret = inst.download(basefile) except Exception as e: # be resilient loc = util.location_exception(e) self.log.error("download for %s failed: %s (%s)" % (c.alias, e, loc)) ret = False if basefile and ret: # we got the doc we want, we're done! return
def download(self, basefile=None, reporter=None): if basefile: # expect a basefile on the form "subrepoalias:basefile" or # just "subrepoalias:" subrepoalias, basefile = basefile.split(":") else: subrepoalias = None if not basefile: basefile = None # ie convert '' => None found = False for cls in self.subrepos: if (subrepoalias is None or cls.alias == subrepoalias): found = True inst = self.get_instance(cls) # the feature where subrepos has a slighly higher # loglevel to avoid creating almost-duplicate "OK" log # messages is not useful for downloading. So we work # around it here. subrepo_loglevel = inst.log.getEffectiveLevel() if subrepo_loglevel == self.log.getEffectiveLevel() + 1: inst.log.setLevel(self.log.getEffectiveLevel()) basefiles = [] try: ret = inst.download(basefile, reporter=basefiles.append) except Exception as e: loc = util.location_exception(e) self.log.error("download for %s failed: %s (%s)" % (cls.alias, e, loc)) ret = False finally: inst.log.setLevel(subrepo_loglevel) for b in basefiles: util.link_or_copy(inst.store.documententry_path(b), self.store.documententry_path(b)) # msbfs/entries/.root.json -> myndfs/entries/msbfs.json util.link_or_copy( inst.store.documententry_path(".root"), self.store.documententry_path(inst.alias)) if not found: self.log.error("Couldn't find any subrepo with alias %s" % subrepoalias)
def download(self, basefile=None, reporter=None): if basefile: # expect a basefile on the form "subrepoalias:basefile" or # just "subrepoalias:" subrepoalias, basefile = basefile.split(":") else: subrepoalias = None if not basefile: basefile = None # ie convert '' => None found = False for cls in self.subrepos: if (subrepoalias is None or cls.alias == subrepoalias): found = True inst = self.get_instance(cls) # the feature where subrepos has a slighly higher # loglevel to avoid creating almost-duplicate "OK" log # messages is not useful for downloading. So we work # around it here. subrepo_loglevel = inst.log.getEffectiveLevel() if subrepo_loglevel == self.log.getEffectiveLevel() + 1: inst.log.setLevel(self.log.getEffectiveLevel()) basefiles = [] try: ret = inst.download(basefile, reporter=basefiles.append) except Exception as e: loc = util.location_exception(e) self.log.error("download for %s failed: %s (%s)" % (cls.alias, e, loc)) ret = False finally: inst.log.setLevel(subrepo_loglevel) for b in basefiles: util.link_or_copy(inst.store.documententry_path(b), self.store.documententry_path(b)) # msbfs/entries/.root.json -> myndfs/entries/msbfs.json util.link_or_copy(inst.store.documententry_path(".root"), self.store.documententry_path(inst.alias)) if not found: self.log.error("Couldn't find any subrepo with alias %s" % subrepoalias)
def updateentry(f, section, entrypath, *args, **kwargs): """runs the provided function with the provided arguments, captures any logged events emitted, catches any errors, and records the result in the entry file under the provided section. The basefile is assumed to be the first element in args. """ def clear(key, d): if key in d: del d[key] logstream = StringIO() handler = logging.StreamHandler(logstream) # FIXME: Think about which format is optimal for storing in # docentry. Do we need eg name and levelname? Should we log # date as well as time? fmt = "%(asctime)s %(name)s %(levelname)s %(message)s" formatter = logging.Formatter(fmt, datefmt="%H:%M:%S") handler.setFormatter(formatter) handler.setLevel(logging.WARNING) rootlog = logging.getLogger() rootlog.addHandler(handler) start = datetime.datetime.now() try: ret = f(*args, **kwargs) success = True except DocumentRemovedError as e: success = "removed" raise except Exception as e: success = False errortype, errorval, errortb = sys.exc_info() raise except KeyboardInterrupt as e: success = None raise else: return ret finally: rootlog.removeHandler(handler) if success is not None: warnings = logstream.getvalue() entry = DocumentEntry(entrypath) if section not in entry.status: entry.status[section] = {} entry.status[section]['success'] = success entry.status[section]['date'] = start delta = datetime.datetime.now() - start try: duration = delta.total_seconds() except AttributeError: # probably on py26, wich lack total_seconds() duration = delta.seconds + (delta.microseconds / 1000000.0) entry.status[section]['duration'] = duration if warnings: entry.status[section]['warnings'] = warnings else: clear('warnings', entry.status[section]) if not success: entry.status[section]['traceback'] = "".join( format_tb(errortb)) entry.status[section]['error'] = "%s: %s (%s)" % ( errorval.__class__.__name__, errorval, util.location_exception(errorval)) else: clear('traceback', entry.status[section]) clear('error', entry.status[section]) entry.save()
def updateentry(f, section, entrypath, entrypath_arg, *args, **kwargs): """runs the provided function with the provided arguments, captures any logged events emitted, catches any errors, and records the result in the entry file under the provided section. Entrypath should be a function that takes a basefile string and returns the full path to the entry file for that basefile. """ def clear(key, d): if key in d: del d[key] logstream = StringIO() handler = logging.StreamHandler(logstream) # FIXME: Think about which format is optimal for storing in # docentry. Do we need eg name and levelname? Should we log # date as well as time? fmt = "%(asctime)s %(name)s %(levelname)s %(message)s" formatter = logging.Formatter(fmt, datefmt="%H:%M:%S") handler.setFormatter(formatter) handler.setLevel(logging.WARNING) rootlog = logging.getLogger() rootlog.addHandler(handler) start = datetime.datetime.now() try: ret = f(*args, **kwargs) success = True except DocumentRemovedError as e: success = "removed" raise except DocumentRenamedError as e: entrypath_arg = e.newbasefile success = True return e.returnvalue except Exception as e: success = False errortype, errorval, errortb = sys.exc_info() raise except KeyboardInterrupt as e: success = None raise else: return ret finally: rootlog.removeHandler(handler) if success is not None: warnings = logstream.getvalue() entry = DocumentEntry(entrypath(entrypath_arg)) if section not in entry.status: entry.status[section] = {} entry.status[section]['success'] = success entry.status[section]['date'] = start delta = datetime.datetime.now()-start try: duration = delta.total_seconds() except AttributeError: # probably on py26, wich lack total_seconds() duration = delta.seconds + (delta.microseconds / 1000000.0) entry.status[section]['duration'] = duration if warnings: entry.status[section]['warnings'] = warnings else: clear('warnings', entry.status[section]) if not success: entry.status[section]['traceback'] = "".join(format_tb(errortb)) entry.status[section]['error'] = "%s: %s (%s)" % (errorval.__class__.__name__, errorval, util.location_exception(errorval)) else: clear('traceback', entry.status[section]) clear('error', entry.status[section]) entry.save()