def savetostorage(wopisrc, acctok, isclose, wopilock):
'''Copy document from CodiMD back to storage'''
# get document from CodiMD
try:
log.info(
'msg="Fetching file from CodiMD" isclose="%s" codimdurl="%s" token="%s"'
% (isclose, codimdurl + wopilock['docid'], acctok[-20:]))
mddoc = _fetchfromcodimd(wopilock, acctok)
except CodiMDFailure:
return jsonify(
'Could not save file, failed to fetch document from CodiMD'
), http.client.INTERNAL_SERVER_ERROR
if wopilock['digest'] != 'dirty':
# so far the file was not touched: before forcing a put let's validate the contents
h = hashlib.sha1()
h.update(mddoc)
if h.hexdigest() == wopilock['digest']:
log.info('msg="File unchanged, skipping save" token="%s"' %
acctok[-20:])
return '{}', http.client.ACCEPTED
# check if we have attachments
wasbundle = os.path.splitext(wopilock['filename'])[1] == '.zmd'
bundlefile, attresponse = _getattachments(
mddoc.decode(), wopilock['filename'].replace('.zmd', '.md'),
(wasbundle and not isclose))
# WOPI PutFile for the file or the bundle if it already existed
if (wasbundle ^ (not bundlefile)) or not isclose:
res = wopi.request(wopisrc,
acctok,
'POST',
headers={'X-WOPI-Lock': json.dumps(wopilock)},
contents=(bundlefile if wasbundle else mddoc))
reply = _dealwithputfile('PutFile', wopisrc, res)
if reply:
return reply
wopi.refreshlock(wopisrc, acctok, wopilock, isdirty=True)
log.info('msg="Save completed" filename="%s" isclose="%s" token="%s"' %
(wopilock['filename'], isclose, acctok[-20:]))
# combine the responses
return attresponse if attresponse else (
jsonify('File saved successfully'), http.client.OK)
# on close, use saveas for either the new bundle, if this is the first time we have attachments,
# or the single md file, if there are no more attachments.
return _saveas(
wopisrc, acctok, wopilock,
os.path.splitext(wopilock['filename'])[0] +
('.zmd' if bundlefile else '.md'), bundlefile if bundlefile else mddoc)
def cleanup(self, openfile, wopisrc, wopilock):
'''remove state for closed documents after some time'''
if _union(openfile['toclose']) and not openfile['tosave']:
# check lock
try:
wopilock = wopi.getlock(
wopisrc, openfile['acctok']) if not wopilock else wopilock
except wopi.InvalidLock:
# nothing to do here, this document may have been closed by another wopibridge
if openfile['lastsave'] < time.time() - WB.unlockinterval:
# yet cleanup only after the unlockinterval time, cf. the InvalidLock handling in savedirty()
WB.log.info(
'msg="SaveThread: cleaning up metadata, file already unlocked" url="%s"'
% wopisrc)
del WB.openfiles[wopisrc]
return
# reconcile list of toclose tokens
openfile['toclose'] = {
t: wopilock['toclose'][t]
or (t in openfile['toclose'] and openfile['toclose'][t])
for t in wopilock['toclose']
}
if _intersection(openfile['toclose']):
if openfile['lastsave'] < int(time.time()) - WB.unlockinterval:
# nobody is still on this document and some time has passed, unlock
res = wopi.request(wopisrc,
openfile['acctok'],
'POST',
headers={
'X-WOPI-Lock': json.dumps(wopilock),
'X-Wopi-Override': 'UNLOCK'
})
if res.status_code != http.client.OK:
WB.log.warning(
'msg="SaveThread: failed to unlock" lastsavetime="%s" token="%s" response="%s"'
% (openfile['lastsave'], openfile['acctok'][-20:],
res.status_code))
else:
WB.log.info(
'msg="SaveThread: unlocked document" lastsavetime="%s" token="%s"'
% (openfile['lastsave'], openfile['acctok'][-20:]))
del WB.openfiles[wopisrc]
elif openfile['toclose'] != wopilock['toclose']:
# some user still on it, refresh lock if the toclose part has changed
wopi.refreshlock(wopisrc,
openfile['acctok'],
wopilock,
toclose=openfile['toclose'])
def savethread_do():
'''Perform the pending save to storage operations'''
WB.log.info('msg="Savethread starting"')
while WB.active:
with WB.savecv:
# sleep for one minute or until awaken
WB.savecv.wait(60)
if not WB.active:
break
# execute a round of sync to storage; list is needed as we may delete entries from the dict
for wopisrc, openfile in list(WB.openfiles.items()):
try:
wopilock = None
# save documents that are dirty for more than `saveinterval` or that are being closed
if openfile['tosave'] and (
_intersection(openfile['toclose']) or
(openfile['lastsave'] <
time.time() - WB.saveinterval)):
wopilock = wopi.getlock(wopisrc, openfile['acctok'])
WB.saveresponses[wopisrc] = codimd.codimdtostorage(
wopisrc, openfile['acctok'],
_intersection(openfile['toclose']), wopilock)
openfile['lastsave'] = int(time.time())
openfile['tosave'] = False
# refresh locks of open idle documents every 30 minutes
if openfile['lastsave'] < time.time() - (1800 +
WB.saveinterval):
wopilock = wopi.getlock(
wopisrc, openfile['acctok'],
raiseifmissing=False) if not wopilock else wopilock
if not wopilock:
# not a problem here, just forget this document (may have been closed by another wopibridge)
WB.log.debug(
'msg="Savethread: cleaning up metadata" url="%s"'
% wopisrc)
del WB.openfiles[wopisrc]
continue
wopilock = wopi.refreshlock(wopisrc,
openfile['acctok'],
wopilock)
# in case we get soon a save callback, we want to honor it immediately
openfile['lastsave'] = int(
time.time()) - WB.saveinterval
# remove state for closed documents after some time
if _union(openfile['toclose']) and not openfile['tosave']:
# check lock
wopilock = wopi.getlock(
wopisrc, openfile['acctok'],
raiseifmissing=False) if not wopilock else wopilock
if not wopilock:
# not a problem here, just forget this document like above
WB.log.debug(
'msg="Savethread: cleaning up metadata" url="%s"'
% wopisrc)
del WB.openfiles[wopisrc]
continue
# refresh state
openfile['toclose'] = {
t: wopilock['toclose'][t]
or t not in openfile['toclose']
or openfile['toclose'][t]
for t in wopilock['toclose']
}
if _intersection(openfile['toclose']
) and openfile['lastsave'] <= int(
time.time()) - WB.saveinterval:
# nobody is still on this document and some time has passed, unlock
res = wopi.request(wopisrc,
openfile['acctok'],
'POST',
headers={
'X-WOPI-Lock':
json.dumps(wopilock),
'X-Wopi-Override':
'UNLOCK'
})
if res.status_code != http.client.OK:
WB.log.warning(
'msg="Savethread: calling WOPI Unlock failed" lastsavetime="%s" token="%s" response="%s"'
% (openfile['lastsave'],
openfile['acctok'][-20:],
res.status_code))
else:
WB.log.info(
'msg="Savethread: unlocked document" lastsavetime="%s" token="%s"'
% (openfile['lastsave'],
openfile['acctok'][-20:]))
del WB.openfiles[wopisrc]
else:
# some user still on it or last operation happened not long ago, just refresh lock
wopi.refreshlock(wopisrc,
openfile['acctok'],
wopilock,
toclose=openfile['toclose'])
except wopi.InvalidLock as e:
# WOPI lock got lost, this is fatal
WB.saveresponses[wopisrc] = codimd.jsonify('Missing or malformed lock when saving the file. %s' % codimd.RECOVER_MSG), \
http.client.NOT_FOUND
del WB.openfiles[wopisrc]
except Exception as e: # pylint: disable=broad-except
ex_type, ex_value, ex_traceback = sys.exc_info()
WB.log.error(
'msg="Savethread: unexpected exception caught" exception="%s" type="%s" traceback="%s"'
% (e, ex_type,
traceback.format_exception(ex_type, ex_value,
ex_traceback)))
WB.log.info('msg="Savethread terminated, shutting down"')
def appopen():
'''Open a MD doc by contacting the provided WOPISrc with the given access_token'''
try:
wopisrc = urllib.parse.unquote(flask.request.args['WOPISrc'])
acctok = flask.request.args['access_token']
WB.log.info('msg="Open called" client="%s" token="%s"' %
(flask.request.remote_addr, acctok[-20:]))
except KeyError as e:
WB.log.error(
'msg="Open: unable to open the file, missing WOPI context" error="%s"'
% e)
return _guireturn('Missing arguments'), http.client.BAD_REQUEST
# WOPI GetFileInfo
try:
res = wopi.request(wopisrc, acctok, 'GET')
filemd = res.json()
except json.decoder.JSONDecodeError as e:
WB.log.warning(
'msg="Malformed JSON from WOPI" error="%s" response="%d"' %
(e, res.status_code))
return _guireturn('Invalid WOPI context'), http.client.NOT_FOUND
try:
# use the 'UserCanWrite' attribute to decide whether the file is to be opened in read-only mode
if filemd['UserCanWrite']:
try:
wopilock = wopi.getlock(wopisrc, acctok, raiseifmissing=False)
if not wopilock:
# first user opening this file, fetch it
wopilock = codimd.storagetocodimd(filemd, wopisrc, acctok)
else:
WB.log.info('msg="Lock already held" lock="%s"' % wopilock)
# add this token to the list, if not already in
if acctok[-20:] not in wopilock['toclose']:
wopilock = wopi.refreshlock(wopisrc, acctok, wopilock)
except wopi.InvalidLock:
# lock is invalid/corrupted: force read-only mode
WB.log.info(
'msg="Missing or invalid lock, forcing read-only mode" lock="%s" token="%s"'
% (wopilock, acctok[-20:]))
filemd['UserCanWrite'] = False
# and fetch the file from storage
wopilock = codimd.storagetocodimd(filemd, wopisrc, acctok)
# WOPI Lock
res = wopi.request(wopisrc,
acctok,
'POST',
headers={
'X-WOPI-Lock': json.dumps(wopilock),
'X-Wopi-Override': 'LOCK'
})
if res.status_code != http.client.OK:
# Failed to lock the file: open in read-only mode
WB.log.warning(
'msg="Failed to lock the file" response="%d" token="%s"' %
(res.status_code, acctok[-20:]))
filemd['UserCanWrite'] = False
else:
# user has no write privileges, just fetch document and push it to CodiMD
wopilock = codimd.storagetocodimd(filemd, wopisrc, acctok)
if filemd['UserCanWrite']:
# keep track of this open document for the save thread and for statistical purposes
if wopisrc in WB.openfiles:
# use the new acctok and the new/current wopilock content
WB.openfiles[wopisrc]['acctok'] = acctok
WB.openfiles[wopisrc]['toclose'] = wopilock['toclose']
else:
WB.openfiles[wopisrc] = {
'acctok': acctok,
'tosave': False,
'lastsave': int(time.time()) - WB.saveinterval,
'toclose': {
acctok[-20:]: False
},
}
# also clear any potential stale response for this document
try:
del WB.saveresponses[wopisrc]
except KeyError:
pass
# create the external redirect URL to be returned to the client:
# metadata will be used for autosave (this is an extended feature of CodiMD)
redirecturl = codimd.codimdexturl + wopilock['docid'] + '?metadata=' + \
urllib.parse.quote_plus('%s?t=%s' % (wopisrc, acctok)) + '&'
else:
# read-only mode: in this case redirect to publish mode or normal view
# to quickly jump in slide mode depending on the content
redirecturl = codimd.codimdexturl + wopilock['docid'] + \
('/publish?' if wopilock['app'] != 'slide' else '?')
# append displayName (again this is an extended feature of CodiMD)
redirecturl += 'displayName=' + urllib.parse.quote_plus(
filemd['UserFriendlyName'])
WB.log.info('msg="Redirecting client to CodiMD" redirecturl="%s"' %
redirecturl)
return flask.redirect(redirecturl)
except codimd.CodiMDFailure:
# this can be risen by storagetocodimd
return _guireturn('Unable to contact CodiMD, please try again later'
), http.client.INTERNAL_SERVER_ERROR
def appopen():
'''Open a MD doc by contacting the provided WOPISrc with the given access_token'''
try:
wopisrc = urlparse.unquote(flask.request.args['WOPISrc'])
acctok = flask.request.args['access_token']
WB.log.info(
'msg="Open called" client="%s" user-agent="%s" token="%s"' %
(flask.request.remote_addr, flask.request.user_agent,
acctok[-20:]))
except KeyError as e:
WB.log.error(
'msg="Open: unable to open the file, missing WOPI context" error="%s"'
% e)
return _guireturn('Missing arguments'), http.client.BAD_REQUEST
# WOPI GetFileInfo
res = wopi.request(wopisrc, acctok, 'GET')
if res.status_code != http.client.OK:
WB.log.warning(
'msg="Open: unable to fetch file WOPI metadata" response="%d"' %
res.status_code)
return _guireturn('Invalid WOPI context'), http.client.NOT_FOUND
filemd = res.json()
try:
# use the 'UserCanWrite' attribute to decide whether the file is to be opened in read-only mode
if filemd['UserCanWrite']:
try:
# was it already being worked on?
wopilock = wopi.getlock(wopisrc, acctok)
WB.log.info('msg="Lock already held" lock="%s" token="%s"' %
(wopilock, acctok[-20:]))
# add this token to the list, if not already in
if acctok[-20:] not in wopilock['toclose']:
wopilock = wopi.refreshlock(wopisrc, acctok, wopilock)
except wopi.InvalidLock as e:
if str(e) != str(int(http.client.NOT_FOUND)):
# lock is invalid/corrupted: force read-only mode
WB.log.info(
'msg="Invalid lock, forcing read-only mode" error="%s" token="%s"'
% (e, acctok[-20:]))
filemd['UserCanWrite'] = False
# otherwise, this is the first user opening the file; in both cases, fetch it
wopilock = codimd.loadfromstorage(filemd, wopisrc, acctok)
# and WOPI Lock it
res = wopi.request(wopisrc,
acctok,
'POST',
headers={
'X-WOPI-Lock': json.dumps(wopilock),
'X-Wopi-Override': 'LOCK'
})
if res.status_code != http.client.OK:
# failed to lock the file: open in read-only mode
WB.log.warning(
'msg="Failed to lock the file" response="%d" token="%s"'
% (res.status_code, acctok[-20:]))
filemd['UserCanWrite'] = False
else:
# user has no write privileges, just fetch document and push it to CodiMD
wopilock = codimd.loadfromstorage(filemd, wopisrc, acctok)
except codimd.CodiMDFailure:
# this can be raised by loadfromstorage
return _guireturn(
'Unable to connect to CodiMD, please try again later or contact support'
), http.client.INTERNAL_SERVER_ERROR
# here we append the user browser to the displayName
# TODO need to review this for production usage, it should actually come from WOPI if configured accordingly
redirecturl = _redirecttoapp(filemd['UserCanWrite'], wopisrc, acctok, wopilock) + \
'displayName=' + urlparse.quote_plus(filemd['UserFriendlyName'] + \
'@' + _renderagent(flask.request.user_agent))
WB.log.info('msg="Redirecting client to CodiMD" redirecturl="%s"' %
redirecturl)
return flask.redirect(redirecturl)
def codimdtostorage(wopisrc, acctok, isclose, wopilock):
'''Copy document from CodiMD back to storage'''
# get document from CodiMD
log.info('msg="Fetching file from CodiMD" isclose="%s" codimdurl="%s" token="%s"' %
(isclose, codimdurl + wopilock['docid'], acctok[-20:]))
res = requests.get(
codimdurl + wopilock['docid'] + '/download', verify=not skipsslverify)
if res.status_code != http.client.OK:
return jsonify('Failed to fetch document from CodiMD: got HTTP %d' % res.status_code), res.status_code
mddoc = res.content
if isclose and wopilock['digest'] != 'dirty':
# so far the file was not touched and we are about to close: before forcing a put let's validate the contents
h = hashlib.sha1()
h.update(mddoc)
if h.hexdigest() == wopilock['digest']:
log.info('msg="File unchanged, skipping save" token="%s"' % acctok[-20:])
return '{}', http.client.ACCEPTED
# check if we have attachments
wasbundle = os.path.splitext(wopilock['filename'])[1] == '.zmd'
bundlefile = _getattachments(mddoc.decode(), wopilock['filename'].replace(
'.zmd', '.md'), (wasbundle and not isclose))
log.debug('msg="Before Put/PutRelative" notbundlefile="%s" wasbundle="%s" isclose="%s"' %
(not bundlefile, wasbundle, isclose))
# WOPI PutFile for the file or the bundle if it already existed
if (wasbundle ^ (not bundlefile)) or not isclose:
res = wopi.request(wopisrc, acctok, 'POST', headers={'X-WOPI-Lock': json.dumps(wopilock)},
contents=(bundlefile if wasbundle else mddoc))
if res.status_code != http.client.OK:
log.error('msg="Calling WOPI PutFile failed" url="%s" response="%s"' % (
wopisrc, res.status_code))
# in case of conflict do not show the "recover" message as a conflict file has been saved anyway
details = '. %s' % res.content.decode() if res.status_code == http.client.CONFLICT \
else ' (%s). %s' % (res.content.decode(), RECOVER_MSG)
return jsonify('Error saving the file' + details), res.status_code
# and refresh the WOPI lock
wopi.refreshlock(wopisrc, acctok, wopilock, isdirty=True)
log.info('msg="Save completed" filename="%s" token="%s"' % (wopilock['filename'], acctok[-20:]))
return jsonify('File saved successfully'), http.client.OK
# On close, use WOPI PutRelative for either the new bundle, if this is the first time we have attachments,
# or the single md file, if there are no more attachments.
putrelheaders = {'X-WOPI-Lock': json.dumps(wopilock),
'X-WOPI-Override': 'PUT_RELATIVE',
# SuggestedTarget to not overwrite a possibly existing file
'X-WOPI-SuggestedTarget': os.path.splitext(wopilock['filename'])[0] + ('.zmd' if bundlefile else '.md')
}
res = wopi.request(wopisrc, acctok, 'POST', headers=putrelheaders, contents=(
bundlefile if bundlefile else mddoc))
if res.status_code != http.client.OK:
log.error('msg="Calling WOPI PutRelative failed" url="%s" response="%s"' % (
wopisrc, res.status_code))
return jsonify('Error saving the file: %s. %s' % (res.content.decode(), RECOVER_MSG)), res.status_code
# use the new file's metadata from PutRelative to remove the previous file: we can do that only on close
# because we need to keep using the current wopisrc/acctok until the session is alive in CodiMD
newname = res.json()['Name']
# unlock and delete original file
res = wopi.request(wopisrc, acctok, 'POST', headers={'X-WOPI-Lock': json.dumps(wopilock), 'X-Wopi-Override': 'UNLOCK'})
if res.status_code != http.client.OK:
log.warning('msg="Failed to unlock the previous file" token="%s" response="%d"' % (
acctok[-20:], res.status_code))
else:
res = wopi.request(wopisrc, acctok, 'POST', headers={'X-Wopi-Override': 'DELETE'})
if res.status_code != http.client.OK:
log.warning('msg="Failed to delete the previous file" token="%s" response="%d"' % (
acctok[-20:], res.status_code))
else:
log.info('msg="Previous file unlocked and removed successfully" token="%s"' % acctok[-20:])
# update our metadata: note we already hold the condition variable as we're called within the save thread
#WB.openfiles[newwopisrc] = {'acctok': newacctok, 'tosave': False,
# 'lastsave': int(time.time()),
# 'toclose': {newacctok[-20:]: True},
# }
#del WB.openfiles[wopisrc]
log.info('msg="Final save completed" filename"%s" token="%s"' % (newname, acctok[-20:]))
return jsonify('File saved successfully'), http.client.OK