def appsave(): '''Save a MD doc given its WOPI context, and return a JSON-formatted message. The actual save is asynchronous.''' # fetch metadata from request try: meta = urlparse.unquote(flask.request.headers['X-EFSS-Metadata']) wopisrc = meta[:meta.index('?t=')] acctok = meta[meta.index('?t=') + 3:] isclose = flask.request.args.get('close') == 'true' docid = flask.request.args.get('id') WB.log.info( 'msg="Save: requested action" isclose="%s" docid="%s" wopisrc="%s" token="%s"' % (isclose, docid, wopisrc, acctok[-20:])) except (KeyError, ValueError) as e: WB.log.error( 'msg="Save: malformed or missing metadata" client="%s" headers="%s" exception="%s" error="%s"' % (flask.request.remote_addr, flask.request.headers, type(e), e)) return codimd.jsonify( 'Malformed or missing metadata, could not save. %s' % RECOVER_MSG), http.client.INTERNAL_SERVER_ERROR # decide whether to notify the save thread donotify = isclose or wopisrc not in WB.openfiles or WB.openfiles[wopisrc][ 'lastsave'] < time.time() - WB.saveinterval # enqueue the request, it will be processed asynchronously with WB.savecv: if wopisrc in WB.openfiles: WB.openfiles[wopisrc]['tosave'] = True WB.openfiles[wopisrc]['toclose'][acctok[-20:]] = isclose else: WB.log.info( 'msg="Save: repopulating missing metadata" wopisrc="%s" token="%s"' % (wopisrc, acctok[-20:])) WB.openfiles[wopisrc] = { 'acctok': acctok, 'tosave': True, 'lastsave': int(time.time() - WB.saveinterval), 'toclose': { acctok[-20:]: isclose }, 'docid': docid, } # if it's the first time we heard about this wopisrc, remove any potential stale response try: del WB.saveresponses[wopisrc] except KeyError: pass if donotify: # note that the save thread stays locked until we release the context, after return! WB.savecv.notify() # return latest known state for this document if wopisrc in WB.saveresponses: resp = WB.saveresponses[wopisrc] WB.log.info( 'msg="Save: returned response" response="%s" token="%s"' % (resp, acctok[-20:])) del WB.saveresponses[wopisrc] return resp WB.log.info('msg="Save: enqueued action" immediate="%s" token="%s"' % (donotify, acctok[-20:])) return '{}', http.client.ACCEPTED
def savedirty(self, openfile, wopisrc): '''save documents that are dirty for more than `saveinterval` or that are being closed''' wopilock = None if openfile['tosave'] and ( _intersection(openfile['toclose']) or (openfile['lastsave'] < time.time() - WB.saveinterval)): try: wopilock = wopi.getlock(wopisrc, openfile['acctok']) except wopi.InvalidLock: WB.log.info( 'msg="SaveThread: attempting to relock file" token="%s" docid="%s"' % (openfile['acctok'][-20:], openfile['docid'])) try: wopilock = WB.saveresponses[wopisrc] = wopi.relock( wopisrc, openfile['acctok'], openfile['docid'], _intersection(openfile['toclose'])) except wopi.InvalidLock as ile: # even this attempt failed, give up # TODO here we should save the file on a local storage to help later recovery WB.saveresponses[wopisrc] = codimd.jsonify( str(ile)), http.client.INTERNAL_SERVER_ERROR # set some 'fake' metadata, will be automatically cleaned up later openfile['lastsave'] = int(time.time()) openfile['tosave'] = False openfile['toclose'] = {'invalid-lock': True} return None WB.log.info('msg="SaveThread: saving file" token="%s" docid="%s"' % (openfile['acctok'][-20:], openfile['docid'])) WB.saveresponses[wopisrc] = codimd.savetostorage( wopisrc, openfile['acctok'], _intersection(openfile['toclose']), wopilock) openfile['lastsave'] = int(time.time()) openfile['tosave'] = False return wopilock
def handleexception(ex): '''Generic method to log any uncaught exception''' if isinstance(ex, (Flask_NotFound, Flask_MethodNotAllowed)): return ex ex_type, ex_value, ex_traceback = sys.exc_info() WB.log.error( 'msg="Unexpected exception caught" exception="%s" type="%s" traceback="%s"' % (ex, ex_type, traceback.format_exception(ex_type, ex_value, ex_traceback))) return codimd.jsonify('Internal error, please contact support. %s' % RECOVER_MSG), http.client.INTERNAL_SERVER_ERROR
def appsave(): '''Save a MD doc given its WOPI context, and return a JSON-formatted message. The actual save is asynchronous.''' # fetch metadata from request try: meta = urllib.parse.unquote(flask.request.headers['X-EFSS-Metadata']) wopisrc = meta[:meta.index('?t=')] acctok = meta[meta.index('?t=') + 3:] isclose = 'close' in flask.request.args and flask.request.args[ 'close'] == 'true' except (KeyError, ValueError) as e: WB.log.error( 'msg="Save: malformed or missing metadata" client="%s" headers="%s" exception="%s" error="%s"' % (flask.request.remote_addr, flask.request.headers, type(e), e)) return codimd.jsonify( 'Malformed or missing metadata, could not save. %s' % codimd.RECOVER_MSG), http.client.BAD_REQUEST # decide whether to notify the save thread donotify = isclose or wopisrc not in WB.openfiles or WB.openfiles[wopisrc][ 'lastsave'] < time.time() - WB.saveinterval # enqueue the request, it will be processed asynchronously with WB.savecv: if wopisrc in WB.openfiles: WB.openfiles[wopisrc]['tosave'] = True WB.openfiles[wopisrc]['toclose'][acctok[-20:]] = isclose else: WB.log.debug( 'msg="Save: repopulating missing metadata" token="%s"' % acctok[-20:]) WB.openfiles[wopisrc] = { 'acctok': acctok, 'tosave': True, 'lastsave': int(time.time() - WB.saveinterval), 'toclose': { acctok[-20:]: isclose }, } if donotify: # note that the save thread stays locked until we release the context, after return! WB.savecv.notify() # return latest known state for this document if wopisrc in WB.saveresponses: resp = WB.saveresponses[wopisrc] WB.log.info( 'msg="Save: returned response" response="%s" isclose="%s" token="%s"' % (resp, isclose, acctok[-20:])) del WB.saveresponses[wopisrc] return resp WB.log.info('msg="Save: enqueued action" isclose="%s" token="%s"' % (isclose, acctok[-20:])) return '{}', http.client.ACCEPTED
def savethread_do(): '''Perform the pending save to storage operations''' WB.log.info('msg="Savethread starting"') while WB.active: with WB.savecv: # sleep for one minute or until awaken WB.savecv.wait(60) if not WB.active: break # execute a round of sync to storage; list is needed as we may delete entries from the dict for wopisrc, openfile in list(WB.openfiles.items()): try: wopilock = None # save documents that are dirty for more than `saveinterval` or that are being closed if openfile['tosave'] and ( _intersection(openfile['toclose']) or (openfile['lastsave'] < time.time() - WB.saveinterval)): wopilock = wopi.getlock(wopisrc, openfile['acctok']) WB.saveresponses[wopisrc] = codimd.codimdtostorage( wopisrc, openfile['acctok'], _intersection(openfile['toclose']), wopilock) openfile['lastsave'] = int(time.time()) openfile['tosave'] = False # refresh locks of open idle documents every 30 minutes if openfile['lastsave'] < time.time() - (1800 + WB.saveinterval): wopilock = wopi.getlock( wopisrc, openfile['acctok'], raiseifmissing=False) if not wopilock else wopilock if not wopilock: # not a problem here, just forget this document (may have been closed by another wopibridge) WB.log.debug( 'msg="Savethread: cleaning up metadata" url="%s"' % wopisrc) del WB.openfiles[wopisrc] continue wopilock = wopi.refreshlock(wopisrc, openfile['acctok'], wopilock) # in case we get soon a save callback, we want to honor it immediately openfile['lastsave'] = int( time.time()) - WB.saveinterval # remove state for closed documents after some time if _union(openfile['toclose']) and not openfile['tosave']: # check lock wopilock = wopi.getlock( wopisrc, openfile['acctok'], raiseifmissing=False) if not wopilock else wopilock if not wopilock: # not a problem here, just forget this document like above WB.log.debug( 'msg="Savethread: cleaning up metadata" url="%s"' % wopisrc) del WB.openfiles[wopisrc] continue # refresh state openfile['toclose'] = { t: wopilock['toclose'][t] or t not in openfile['toclose'] or openfile['toclose'][t] for t in wopilock['toclose'] } if _intersection(openfile['toclose'] ) and openfile['lastsave'] <= int( time.time()) - WB.saveinterval: # nobody is still on this document and some time has passed, unlock res = wopi.request(wopisrc, openfile['acctok'], 'POST', headers={ 'X-WOPI-Lock': json.dumps(wopilock), 'X-Wopi-Override': 'UNLOCK' }) if res.status_code != http.client.OK: WB.log.warning( 'msg="Savethread: calling WOPI Unlock failed" lastsavetime="%s" token="%s" response="%s"' % (openfile['lastsave'], openfile['acctok'][-20:], res.status_code)) else: WB.log.info( 'msg="Savethread: unlocked document" lastsavetime="%s" token="%s"' % (openfile['lastsave'], openfile['acctok'][-20:])) del WB.openfiles[wopisrc] else: # some user still on it or last operation happened not long ago, just refresh lock wopi.refreshlock(wopisrc, openfile['acctok'], wopilock, toclose=openfile['toclose']) except wopi.InvalidLock as e: # WOPI lock got lost, this is fatal WB.saveresponses[wopisrc] = codimd.jsonify('Missing or malformed lock when saving the file. %s' % codimd.RECOVER_MSG), \ http.client.NOT_FOUND del WB.openfiles[wopisrc] except Exception as e: # pylint: disable=broad-except ex_type, ex_value, ex_traceback = sys.exc_info() WB.log.error( 'msg="Savethread: unexpected exception caught" exception="%s" type="%s" traceback="%s"' % (e, ex_type, traceback.format_exception(ex_type, ex_value, ex_traceback))) WB.log.info('msg="Savethread terminated, shutting down"')