def savedirty(self, openfile, wopisrc):
'''save documents that are dirty for more than `saveinterval` or that are being closed'''
wopilock = None
if openfile['tosave'] and (
_intersection(openfile['toclose']) or
(openfile['lastsave'] < time.time() - WB.saveinterval)):
try:
wopilock = wopi.getlock(wopisrc, openfile['acctok'])
except wopi.InvalidLock:
WB.log.info(
'msg="SaveThread: attempting to relock file" token="%s" docid="%s"'
% (openfile['acctok'][-20:], openfile['docid']))
try:
wopilock = WB.saveresponses[wopisrc] = wopi.relock(
wopisrc, openfile['acctok'], openfile['docid'],
_intersection(openfile['toclose']))
except wopi.InvalidLock as ile:
# even this attempt failed, give up
# TODO here we should save the file on a local storage to help later recovery
WB.saveresponses[wopisrc] = codimd.jsonify(
str(ile)), http.client.INTERNAL_SERVER_ERROR
# set some 'fake' metadata, will be automatically cleaned up later
openfile['lastsave'] = int(time.time())
openfile['tosave'] = False
openfile['toclose'] = {'invalid-lock': True}
return None
WB.log.info('msg="SaveThread: saving file" token="%s" docid="%s"' %
(openfile['acctok'][-20:], openfile['docid']))
WB.saveresponses[wopisrc] = codimd.savetostorage(
wopisrc, openfile['acctok'],
_intersection(openfile['toclose']), wopilock)
openfile['lastsave'] = int(time.time())
openfile['tosave'] = False
return wopilock
def cleanup(self, openfile, wopisrc, wopilock):
'''remove state for closed documents after some time'''
if _union(openfile['toclose']) and not openfile['tosave']:
# check lock
try:
wopilock = wopi.getlock(
wopisrc, openfile['acctok']) if not wopilock else wopilock
except wopi.InvalidLock:
# nothing to do here, this document may have been closed by another wopibridge
if openfile['lastsave'] < time.time() - WB.unlockinterval:
# yet cleanup only after the unlockinterval time, cf. the InvalidLock handling in savedirty()
WB.log.info(
'msg="SaveThread: cleaning up metadata, file already unlocked" url="%s"'
% wopisrc)
del WB.openfiles[wopisrc]
return
# reconcile list of toclose tokens
openfile['toclose'] = {
t: wopilock['toclose'][t]
or (t in openfile['toclose'] and openfile['toclose'][t])
for t in wopilock['toclose']
}
if _intersection(openfile['toclose']):
if openfile['lastsave'] < int(time.time()) - WB.unlockinterval:
# nobody is still on this document and some time has passed, unlock
res = wopi.request(wopisrc,
openfile['acctok'],
'POST',
headers={
'X-WOPI-Lock': json.dumps(wopilock),
'X-Wopi-Override': 'UNLOCK'
})
if res.status_code != http.client.OK:
WB.log.warning(
'msg="SaveThread: failed to unlock" lastsavetime="%s" token="%s" response="%s"'
% (openfile['lastsave'], openfile['acctok'][-20:],
res.status_code))
else:
WB.log.info(
'msg="SaveThread: unlocked document" lastsavetime="%s" token="%s"'
% (openfile['lastsave'], openfile['acctok'][-20:]))
del WB.openfiles[wopisrc]
elif openfile['toclose'] != wopilock['toclose']:
# some user still on it, refresh lock if the toclose part has changed
wopi.refreshlock(wopisrc,
openfile['acctok'],
wopilock,
toclose=openfile['toclose'])
def closewhenidle(self, openfile, wopisrc, wopilock):
'''close and unlock documents tha are idle for more than 60 minutes.
They will transparently be relocked when/if the session resumes, but we seem to miss some close notifications,
therefore this also works as a cleanup step'''
if openfile['lastsave'] < int(time.time()) - 3600:
try:
wopilock = wopi.getlock(
wopisrc, openfile['acctok']) if not wopilock else wopilock
# this will force a close in the cleanup step
openfile['toclose'] = {t: True for t in openfile['toclose']}
WB.log.info(
'msg="SaveThread: force-closing document" lastsavetime="%s" toclosetokens="%s"'
% (openfile['lastsave'], openfile['toclose']))
except wopi.InvalidLock:
# lock is gone, just cleanup our metadata
WB.log.warning(
'msg="SaveThread: cleaning up metadata, detected missed close event" url="%s"'
% wopisrc)
del WB.openfiles[wopisrc]
return wopilock
def savethread_do():
'''Perform the pending save to storage operations'''
WB.log.info('msg="Savethread starting"')
while WB.active:
with WB.savecv:
# sleep for one minute or until awaken
WB.savecv.wait(60)
if not WB.active:
break
# execute a round of sync to storage; list is needed as we may delete entries from the dict
for wopisrc, openfile in list(WB.openfiles.items()):
try:
wopilock = None
# save documents that are dirty for more than `saveinterval` or that are being closed
if openfile['tosave'] and (
_intersection(openfile['toclose']) or
(openfile['lastsave'] <
time.time() - WB.saveinterval)):
wopilock = wopi.getlock(wopisrc, openfile['acctok'])
WB.saveresponses[wopisrc] = codimd.codimdtostorage(
wopisrc, openfile['acctok'],
_intersection(openfile['toclose']), wopilock)
openfile['lastsave'] = int(time.time())
openfile['tosave'] = False
# refresh locks of open idle documents every 30 minutes
if openfile['lastsave'] < time.time() - (1800 +
WB.saveinterval):
wopilock = wopi.getlock(
wopisrc, openfile['acctok'],
raiseifmissing=False) if not wopilock else wopilock
if not wopilock:
# not a problem here, just forget this document (may have been closed by another wopibridge)
WB.log.debug(
'msg="Savethread: cleaning up metadata" url="%s"'
% wopisrc)
del WB.openfiles[wopisrc]
continue
wopilock = wopi.refreshlock(wopisrc,
openfile['acctok'],
wopilock)
# in case we get soon a save callback, we want to honor it immediately
openfile['lastsave'] = int(
time.time()) - WB.saveinterval
# remove state for closed documents after some time
if _union(openfile['toclose']) and not openfile['tosave']:
# check lock
wopilock = wopi.getlock(
wopisrc, openfile['acctok'],
raiseifmissing=False) if not wopilock else wopilock
if not wopilock:
# not a problem here, just forget this document like above
WB.log.debug(
'msg="Savethread: cleaning up metadata" url="%s"'
% wopisrc)
del WB.openfiles[wopisrc]
continue
# refresh state
openfile['toclose'] = {
t: wopilock['toclose'][t]
or t not in openfile['toclose']
or openfile['toclose'][t]
for t in wopilock['toclose']
}
if _intersection(openfile['toclose']
) and openfile['lastsave'] <= int(
time.time()) - WB.saveinterval:
# nobody is still on this document and some time has passed, unlock
res = wopi.request(wopisrc,
openfile['acctok'],
'POST',
headers={
'X-WOPI-Lock':
json.dumps(wopilock),
'X-Wopi-Override':
'UNLOCK'
})
if res.status_code != http.client.OK:
WB.log.warning(
'msg="Savethread: calling WOPI Unlock failed" lastsavetime="%s" token="%s" response="%s"'
% (openfile['lastsave'],
openfile['acctok'][-20:],
res.status_code))
else:
WB.log.info(
'msg="Savethread: unlocked document" lastsavetime="%s" token="%s"'
% (openfile['lastsave'],
openfile['acctok'][-20:]))
del WB.openfiles[wopisrc]
else:
# some user still on it or last operation happened not long ago, just refresh lock
wopi.refreshlock(wopisrc,
openfile['acctok'],
wopilock,
toclose=openfile['toclose'])
except wopi.InvalidLock as e:
# WOPI lock got lost, this is fatal
WB.saveresponses[wopisrc] = codimd.jsonify('Missing or malformed lock when saving the file. %s' % codimd.RECOVER_MSG), \
http.client.NOT_FOUND
del WB.openfiles[wopisrc]
except Exception as e: # pylint: disable=broad-except
ex_type, ex_value, ex_traceback = sys.exc_info()
WB.log.error(
'msg="Savethread: unexpected exception caught" exception="%s" type="%s" traceback="%s"'
% (e, ex_type,
traceback.format_exception(ex_type, ex_value,
ex_traceback)))
WB.log.info('msg="Savethread terminated, shutting down"')
def appopen():
'''Open a MD doc by contacting the provided WOPISrc with the given access_token'''
try:
wopisrc = urllib.parse.unquote(flask.request.args['WOPISrc'])
acctok = flask.request.args['access_token']
WB.log.info('msg="Open called" client="%s" token="%s"' %
(flask.request.remote_addr, acctok[-20:]))
except KeyError as e:
WB.log.error(
'msg="Open: unable to open the file, missing WOPI context" error="%s"'
% e)
return _guireturn('Missing arguments'), http.client.BAD_REQUEST
# WOPI GetFileInfo
try:
res = wopi.request(wopisrc, acctok, 'GET')
filemd = res.json()
except json.decoder.JSONDecodeError as e:
WB.log.warning(
'msg="Malformed JSON from WOPI" error="%s" response="%d"' %
(e, res.status_code))
return _guireturn('Invalid WOPI context'), http.client.NOT_FOUND
try:
# use the 'UserCanWrite' attribute to decide whether the file is to be opened in read-only mode
if filemd['UserCanWrite']:
try:
wopilock = wopi.getlock(wopisrc, acctok, raiseifmissing=False)
if not wopilock:
# first user opening this file, fetch it
wopilock = codimd.storagetocodimd(filemd, wopisrc, acctok)
else:
WB.log.info('msg="Lock already held" lock="%s"' % wopilock)
# add this token to the list, if not already in
if acctok[-20:] not in wopilock['toclose']:
wopilock = wopi.refreshlock(wopisrc, acctok, wopilock)
except wopi.InvalidLock:
# lock is invalid/corrupted: force read-only mode
WB.log.info(
'msg="Missing or invalid lock, forcing read-only mode" lock="%s" token="%s"'
% (wopilock, acctok[-20:]))
filemd['UserCanWrite'] = False
# and fetch the file from storage
wopilock = codimd.storagetocodimd(filemd, wopisrc, acctok)
# WOPI Lock
res = wopi.request(wopisrc,
acctok,
'POST',
headers={
'X-WOPI-Lock': json.dumps(wopilock),
'X-Wopi-Override': 'LOCK'
})
if res.status_code != http.client.OK:
# Failed to lock the file: open in read-only mode
WB.log.warning(
'msg="Failed to lock the file" response="%d" token="%s"' %
(res.status_code, acctok[-20:]))
filemd['UserCanWrite'] = False
else:
# user has no write privileges, just fetch document and push it to CodiMD
wopilock = codimd.storagetocodimd(filemd, wopisrc, acctok)
if filemd['UserCanWrite']:
# keep track of this open document for the save thread and for statistical purposes
if wopisrc in WB.openfiles:
# use the new acctok and the new/current wopilock content
WB.openfiles[wopisrc]['acctok'] = acctok
WB.openfiles[wopisrc]['toclose'] = wopilock['toclose']
else:
WB.openfiles[wopisrc] = {
'acctok': acctok,
'tosave': False,
'lastsave': int(time.time()) - WB.saveinterval,
'toclose': {
acctok[-20:]: False
},
}
# also clear any potential stale response for this document
try:
del WB.saveresponses[wopisrc]
except KeyError:
pass
# create the external redirect URL to be returned to the client:
# metadata will be used for autosave (this is an extended feature of CodiMD)
redirecturl = codimd.codimdexturl + wopilock['docid'] + '?metadata=' + \
urllib.parse.quote_plus('%s?t=%s' % (wopisrc, acctok)) + '&'
else:
# read-only mode: in this case redirect to publish mode or normal view
# to quickly jump in slide mode depending on the content
redirecturl = codimd.codimdexturl + wopilock['docid'] + \
('/publish?' if wopilock['app'] != 'slide' else '?')
# append displayName (again this is an extended feature of CodiMD)
redirecturl += 'displayName=' + urllib.parse.quote_plus(
filemd['UserFriendlyName'])
WB.log.info('msg="Redirecting client to CodiMD" redirecturl="%s"' %
redirecturl)
return flask.redirect(redirecturl)
except codimd.CodiMDFailure:
# this can be risen by storagetocodimd
return _guireturn('Unable to contact CodiMD, please try again later'
), http.client.INTERNAL_SERVER_ERROR
def appopen():
'''Open a MD doc by contacting the provided WOPISrc with the given access_token'''
try:
wopisrc = urlparse.unquote(flask.request.args['WOPISrc'])
acctok = flask.request.args['access_token']
WB.log.info(
'msg="Open called" client="%s" user-agent="%s" token="%s"' %
(flask.request.remote_addr, flask.request.user_agent,
acctok[-20:]))
except KeyError as e:
WB.log.error(
'msg="Open: unable to open the file, missing WOPI context" error="%s"'
% e)
return _guireturn('Missing arguments'), http.client.BAD_REQUEST
# WOPI GetFileInfo
res = wopi.request(wopisrc, acctok, 'GET')
if res.status_code != http.client.OK:
WB.log.warning(
'msg="Open: unable to fetch file WOPI metadata" response="%d"' %
res.status_code)
return _guireturn('Invalid WOPI context'), http.client.NOT_FOUND
filemd = res.json()
try:
# use the 'UserCanWrite' attribute to decide whether the file is to be opened in read-only mode
if filemd['UserCanWrite']:
try:
# was it already being worked on?
wopilock = wopi.getlock(wopisrc, acctok)
WB.log.info('msg="Lock already held" lock="%s" token="%s"' %
(wopilock, acctok[-20:]))
# add this token to the list, if not already in
if acctok[-20:] not in wopilock['toclose']:
wopilock = wopi.refreshlock(wopisrc, acctok, wopilock)
except wopi.InvalidLock as e:
if str(e) != str(int(http.client.NOT_FOUND)):
# lock is invalid/corrupted: force read-only mode
WB.log.info(
'msg="Invalid lock, forcing read-only mode" error="%s" token="%s"'
% (e, acctok[-20:]))
filemd['UserCanWrite'] = False
# otherwise, this is the first user opening the file; in both cases, fetch it
wopilock = codimd.loadfromstorage(filemd, wopisrc, acctok)
# and WOPI Lock it
res = wopi.request(wopisrc,
acctok,
'POST',
headers={
'X-WOPI-Lock': json.dumps(wopilock),
'X-Wopi-Override': 'LOCK'
})
if res.status_code != http.client.OK:
# failed to lock the file: open in read-only mode
WB.log.warning(
'msg="Failed to lock the file" response="%d" token="%s"'
% (res.status_code, acctok[-20:]))
filemd['UserCanWrite'] = False
else:
# user has no write privileges, just fetch document and push it to CodiMD
wopilock = codimd.loadfromstorage(filemd, wopisrc, acctok)
except codimd.CodiMDFailure:
# this can be raised by loadfromstorage
return _guireturn(
'Unable to connect to CodiMD, please try again later or contact support'
), http.client.INTERNAL_SERVER_ERROR
# here we append the user browser to the displayName
# TODO need to review this for production usage, it should actually come from WOPI if configured accordingly
redirecturl = _redirecttoapp(filemd['UserCanWrite'], wopisrc, acctok, wopilock) + \
'displayName=' + urlparse.quote_plus(filemd['UserFriendlyName'] + \
'@' + _renderagent(flask.request.user_agent))
WB.log.info('msg="Redirecting client to CodiMD" redirecturl="%s"' %
redirecturl)
return flask.redirect(redirecturl)