def init_routes(self): wr_api_spec.set_curr_tag('Bug Reporting') @self.app.post('/api/v1/report/dnlr') def report_issues(): useragent = request.headers.get('User-Agent') data = request.json or {} self.do_report(data, useragent) return {} @self.app.post('/api/v1/report/ui') def report_issues(): useragent = request.headers.get('User-Agent') data = request.json or {} data['state'] = 'ui-report' self.do_report(data, useragent) return {}
def init_routes(self): wr_api_spec.set_curr_tag('Collections') @self.app.post('/api/v1/collections') @self.api(query=['user'], req=['title', 'url', 'public', 'public_index'], resp='collection') def create_collection(): user = self.get_user(api=True, redir_check=False) data = request.json or {} title = data.get('title', '') coll_name = self.sanitize_title(title) #if not title: # title._raise_error(400, 'please enter a title to record') url = data.get('url', '') #if not url: # self._raise_error(400, 'please enter a URL to record') if not coll_name: self._raise_error(400, 'invalid_coll_name') doi = data.get('doi', '') # TODO: generate doi here # #TODO: add redis object, key: jahr.monat, value: counter is_public = data.get('public', False) is_public_index = data.get('public_index', False) is_external = data.get('external', False) is_anon = self.access.is_anon(user) creatorList = data.get('creatorList', '') subjectHeaderList = data.get('subjectHeaderList', '') personHeaderList = data.get('personHeaderList', '') publisher = data.get('publisher', '') #if not publisher: # self._raise_error(400, 'please enter the publisher of the resource') personHeadingText = data.get('personHeadingText', '') collTitle = data.get('collTitle', '') #if not collTitle: # self._raise_error(400, 'please enter the authership information of the resource') noteToDachs = data.get('noteToDachs', '') publisherOriginal = data.get('publisherOriginal', '') pubTitleOriginal = data.get('pubTitleOriginal', '') collYear = data.get('collYear', '') copTitle = data.get('copTitle', '') subjectHeadingText = data.get('subjectHeadingText', '') surName = data.get('surName', '') persName = data.get('persName', '') usermail = data.get('usermail', '') #if not usermail: # self._raise_error(400, 'invalid email adress') selectedGroupName = data.get('selectedGroupName', '') projektcode = data.get('projektcode', '') publishYear = data.get('publishYear', '') listID = data.get('listID', 0) ticketState = data.get('ticketState') isCollLoaded = data.get('isCollLoaded', True) recordingUrl = data.get('recordingUrl', '') recordingTimestamp = data.get('recordingTimestamp', '') if is_external: if not self.allow_external: self._raise_error(403, 'external_not_allowed') #if not is_anon: # self._raise_error(400, 'not_valid_for_external') elif is_anon: if coll_name != 'temp': self._raise_error(400, 'invalid_temp_coll_name') if user.has_collection(coll_name): self._raise_error(400, 'duplicate_name') try: collection = user.create_collection( coll_name, title=title, url=url, creatorList=creatorList, noteToDachs=noteToDachs, subjectHeaderList=subjectHeaderList, personHeaderList=personHeaderList, publisher=publisher, collTitle=collTitle, publisherOriginal=publisherOriginal, pubTitleOriginal=pubTitleOriginal, personHeadingText=personHeadingText, collYear=collYear, copTitle=copTitle, subjectHeadingText=subjectHeadingText, surName=surName, persName=persName, usermail=usermail, selectedGroupName=selectedGroupName, projektcode=projektcode, publishYear=publishYear, listID=listID, desc='', public=is_public, public_index=is_public_index, ticketState=ticketState, isCollLoaded=isCollLoaded, recordingUrl=recordingUrl, recordingTimestamp=recordingTimestamp, doi=doi) if is_external: collection.set_external(True) user.mark_updated() self.flash_message( 'Created collection <b>{0}</b>!'.format( collection.get_prop('title')), 'success') resp = {'collection': collection.serialize()} except DupeNameException as de: self._raise_error(400, 'duplicate_name') except Exception as ve: print(ve) self.flash_message(str(ve)) self._raise_error(400, 'duplicate_name') return resp @self.app.post('/api/v1/collectionsduplicate') @self.api(query=['user'], req=['title'], resp='collection') def create_collection_with_Warc(): user = self.get_user(api=True, redir_check=False) data = request.json or {} title = data.get('title', '') coll_name = self.sanitize_title(title) resp = None collection = None collections = user.get_collections() for _col in collections: print(_col.get('title')) if _col.get('title') == coll_name: coll_name += "_duplicate" title = coll_name #if not title: # title._raise_error(400, 'please enter a title to record') url = _col.get('url', '') #if not url: # self._raise_error(400, 'please enter a URL to record') coll_name = title if not coll_name: self._raise_error(400, 'invalid_coll_name') doi = _col.get('doi', '') # TODO: generate doi here # #TODO: add redis object, key: jahr.monat, value: counter is_public = _col.get('public', False) is_public_index = _col.get('public_index', False) is_external = _col.get('external', False) is_anon = self.access.is_anon(user) creatorList = _col.get('creatorList', '') subjectHeaderList = _col.get('subjectHeaderList', '') personHeaderList = _col.get('personHeaderList', '') publisher = _col.get('publisher', '') #if not publisher: # self._raise_error(400, 'please enter the publisher of the resource') personHeadingText = _col.get('personHeadingText', '') collTitle = _col.get('collTitle', '') #if not collTitle: # self._raise_error(400, 'please enter the authership information of the resource') noteToDachs = _col.get('noteToDachs', '') publisherOriginal = _col.get('publisherOriginal', '') pubTitleOriginal = _col.get('pubTitleOriginal', '') collYear = _col.get('collYear', '') copTitle = _col.get('copTitle', '') subjectHeadingText = _col.get('subjectHeadingText', '') surName = _col.get('surName', '') persName = _col.get('persName', '') usermail = _col.get('usermail', '') #if not usermail: # self._raise_error(400, 'invalid email adress') selectedGroupName = _col.get('selectedGroupName', '') projektcode = _col.get('projektcode', '') publishYear = _col.get('publishYear', '') listID = _col.get('listID', 0) ticketState = _col.get('ticketState') isCollLoaded = _col.get('isCollLoaded', True) recordingUrl = _col.get('recordingUrl', '') recordingTimestamp = _col.get('recordingTimestamp', '') if is_external: if not self.allow_external: self._raise_error(403, 'external_not_allowed') #if not is_anon: # self._raise_error(400, 'not_valid_for_external') elif is_anon: if coll_name != 'temp': self._raise_error(400, 'invalid_temp_coll_name') if user.has_collection(coll_name): self._raise_error(400, 'duplicate_name') try: collection = user.create_collection( coll_name, title=title, url=url, creatorList=creatorList, noteToDachs=noteToDachs, subjectHeaderList=subjectHeaderList, personHeaderList=personHeaderList, publisher=publisher, collTitle=collTitle, publisherOriginal=publisherOriginal, pubTitleOriginal=pubTitleOriginal, personHeadingText=personHeadingText, collYear=collYear, copTitle=copTitle, subjectHeadingText=subjectHeadingText, surName=surName, persName=persName, usermail=usermail, selectedGroupName=selectedGroupName, projektcode=projektcode, publishYear=publishYear, listID=listID, desc='', public=is_public, public_index=is_public_index, ticketState=ticketState, isCollLoaded=isCollLoaded, recordingUrl=recordingUrl, recordingTimestamp=recordingTimestamp, doi=doi) if is_external: collection.set_external(True) user.mark_updated() self.flash_message( 'Created collection <b>{0}</b>!'.format( collection.get_prop('title')), 'success') resp = {'collection': collection.serialize()} except DupeNameException as de: self._raise_error(400, 'duplicate_name') except Exception as ve: print(ve) self.flash_message(str(ve)) self._raise_error(400, 'duplicate_name') for recording in _col.get_recordings(load=True): _col.copy_recording(recording, collection) #for n, warc_path in recording.iter_all_files(): # print(warc_path) return resp self._raise_error(400, 'object to duplicate not found') @self.app.get('/api/v1/collections') @self.api(query=[ 'user', 'include_recordings', 'include_lists', 'include_pages' ], resp='collections') def get_collections(): user = self.get_user(api=True, redir_check=False) kwargs = { 'include_recordings': get_bool(request.query.get('include_recordings')), 'include_lists': get_bool(request.query.get('include_lists')), 'include_pages': get_bool(request.query.get('include_pages')), } collections = user.get_collections() return { 'collections': [coll.serialize(**kwargs) for coll in collections] } @self.app.get('/api/v1/collection/<coll_name>') @self.api(query=['user'], resp='collection') def get_collection(coll_name): user = self.get_user(api=True, redir_check=False) return self.get_collection_info(coll_name, user=user) @self.app.delete('/api/v1/collection/<coll_name>') @self.api(query=['user'], resp='deleted') def delete_collection(coll_name): user, collection = self.load_user_coll(coll_name=coll_name) errs = user.remove_collection(collection, delete=True) if errs.get('error'): return self._raise_error(400, errs['error']) else: return {'deleted_id': coll_name} @self.app.put('/api/v1/collection/<coll_name>/warc') def add_external_warc(coll_name): if not self.allow_external: self._raise_error(403, 'external_not_allowed') user, collection = self.load_user_coll(coll_name=coll_name) self.access.assert_can_admin_coll(collection) if not collection.is_external(): self._raise_error(400, 'external_only') num_added = collection.add_warcs(request.json.get('warcs', {})) return {'success': num_added} @self.app.put('/api/v1/collection/<coll_name>/cdx') def add_external_cdxj(coll_name): if not self.allow_external: self._raise_error(403, 'external_not_allowed') user, collection = self.load_user_coll(coll_name=coll_name) self.access.assert_can_admin_coll(collection) if not collection.is_external(): self._raise_error(400, 'external_only') num_added = collection.add_cdxj(request.body.read()) return {'success': num_added} @self.app.post('/api/v1/collection/<coll_name>') @self.api(query=['user'], req=['title'], resp='collection') def update_collection(coll_name): user, collection = self.load_user_coll(coll_name=coll_name) self.access.assert_can_admin_coll(collection) ticketStateChanged = False data = request.json or {} if 'title' in data: new_coll_title = data['title'] new_coll_name = self.sanitize_title(new_coll_title) if not new_coll_name: self._raise_error(400, 'invalid_coll_name') try: new_coll_name = user.colls.rename(collection, new_coll_name, allow_dupe=False) except DupeNameException as de: self._raise_error(400, 'duplicate_name') collection['title'] = new_coll_title if 'creatorList' in data: collection['creatorList'] = data['creatorList'] #if 'doi' in data: # collection['doi'] = data['doi'] if 'subjectHeaderList' in data: collection['subjectHeaderList'] = data['subjectHeaderList'] if 'personHeaderList' in data: collection['personHeaderList'] = data['personHeaderList'] if 'publisherOriginal' in data: collection['publisherOriginal'] = data['publisherOriginal'] if 'publisher' in data: collection['publisher'] = data['publisher'] if 'collTitle' in data: collection['collTitle'] = data['collTitle'] if 'collYear' in data: collection['collYear'] = data['collYear'] if 'copTitle' in data: collection['copTitle'] = data['copTitle'] if 'noteToDachs' in data: collection['noteToDachs'] = data['noteToDachs'] if 'surName' in data: collection['surName'] = data['surName'] if 'persName' in data: collection['persName'] = data['persName'] if 'personHeadingText' in data: collection['personHeadingText'] = data['personHeadingText'] if 'pubTitleOriginal' in data: collection['pubTitleOriginal'] = data['pubTitleOriginal'] if 'subjectHeadingText' in data: collection['subjectHeadingText'] = data['subjectHeadingText'] if 'usermail' in data: collection['usermail'] = data['usermail'] if 'selectedGroupName' in data: collection['selectedGroupName'] = data['selectedGroupName'] if 'ticketState' in data and data[ 'ticketState'] == "approved" and 'projektcode' in data and data[ 'projektcode'] != "" and collection['doi'] is None: collection['projektcode'] = data['projektcode'] today = datetime.utcnow() possibleDOIBase = "10.25354/" + data['projektcode'] + "." + str( today.year) + "." + str(today.month) possibleDOI = "10.25354/" + data['projektcode'] + "." + str( today.year) + "." + str(today.month) tempInc = 1 while self.redis.sismember('doimodel', possibleDOI) == 1: possibleDOI = possibleDOIBase + "-" + str(tempInc) tempInc += 1 self.redis.sadd('doimodel', possibleDOI) collection['doi'] = possibleDOI print("reviewControllerpost" + possibleDOI) else: print(collection['doi']) if 'publishYear' in data: collection['publishYear'] = data['publishYear'] if 'listID' in data: collection['listID'] = data['listID'] if 'isCollLoaded' in data: collection.set_bool_prop('isCollLoaded', data['isCollLoaded']) if 'recordingUrl' in data: collection['recordingUrl'] = data['recordingUrl'] if 'recordingTimestamp' in data: collection['recordingTimestamp'] = data['recordingTimestamp'] if 'desc' in data: collection['desc'] = data['desc'] if 'ticketState' in data: if collection['ticketState'] != data['ticketState']: prevState = collection['ticketState'] newState = data['ticketState'] ticketStateChanged = True print("Ticket State changed from {} to {}".format( collection['title'], newState)) collection['ticketState'] = data['ticketState'] if 'url' in data: collection['url'] = data['url'] if ticketStateChanged: if data['ticketState'] == 'complete': reviewerMailText = template( 'webrecorder/templates/complete_mail.html', coll_name=coll_name, coll_doi=collection['doi']) mail = MIMEMultipart() mail['FROM'] = '*****@*****.**' mail['TO'] = collection['usermail'] mail[ 'subject'] = 'Webrecorder: DOI creation has been completed!' host = "relays.uni-heidelberg.de" mailServer = smtplib.SMTP(host) mail.attach(MIMEText(reviewerMailText, "html")) msgBody = mail.as_string() mailServer.sendmail('*****@*****.**', collection['usermail'], msgBody) mailServer.quit() elif data['ticketState'] == 'denied': reviewerMailText = template( 'webrecorder/templates/deny_mail.html', coll_name=coll_name) mail = MIMEMultipart() mail['FROM'] = '*****@*****.**' mail['TO'] = collection['usermail'] mail[ 'subject'] = 'Webrecorder: Your archive request has been reviewed and denied!' host = "relays.uni-heidelberg.de" mailServer = smtplib.SMTP(host) mail.attach(MIMEText(reviewerMailText, "html")) msgBody = mail.as_string() mailServer.sendmail('*****@*****.**', collection['usermail'], msgBody) mailServer.quit() elif data['ticketState'] == 'pending': reviewerMailText = template( 'webrecorder/templates/pending_mail.html', coll_name=coll_name) mail = MIMEMultipart() mail['FROM'] = '*****@*****.**' mail['TO'] = collection['usermail'] mail[ 'subject'] = 'Webrecorder: New collection awaiting review!' host = "relays.uni-heidelberg.de" mailServer = smtplib.SMTP(host) MSG = "Your archives state has been changesd from {} to {}. We will inform you with further updates as soon as possible.".format( prevState, newState) mail.attach(MIMEText(reviewerMailText, "html")) msgBody = mail.as_string() #part1 = MIMEText(MSG, 'plain') #part2 = MIMEText(html, 'html') #mail.attach(part1) #mail.attach(part2) mailServer.sendmail('*****@*****.**', collection['usermail'], msgBody) mailServer.quit() #self.cork.mailer = Mailer('*****@*****.**', 'smtp://relays.uni-heidelberg.de:25') # TODO: notify the user if this is a request from the admin panel if 'public' in data: #if self.access.is_superuser() and data.get('notify'): # pass collection.set_public(data['public']) if 'public_index' in data: collection.set_bool_prop('public_index', data['public_index']) collection.mark_updated() return {'collection': collection.serialize()} @self.app.get('/api/v1/collection/<coll_name>/page_bookmarks') @self.api(query=['user'], resp='bookmarks') def get_page_bookmarks(coll_name): user, collection = self.load_user_coll(coll_name=coll_name) rec = request.query.get('rec') if rec: recording = collection.get_recording(rec) if not recording: return {'page_bookmarks': {}} rec_pages = collection.list_rec_pages(recording) else: rec_pages = None return { 'page_bookmarks': collection.get_all_page_bookmarks(rec_pages) } # DAT @self.app.post('/api/v1/collection/<coll_name>/dat/share') def dat_do_share(coll_name): user, collection = self.load_user_coll(coll_name=coll_name) print(user) # BETA only self.require_admin_beta_access(collection) try: data = request.json or {} print(data) result = DatShare.dat_share.share( collection, data.get('always_update', False)) except Exception as e: result = {'error': 'api_error', 'details': str(e)} if 'error' in result: self._raise_error(400, result['error']) return result @self.app.post('/api/v1/collection/<coll_name>/dat/unshare') def dat_do_unshare(coll_name): user, collection = self.load_user_coll(coll_name=coll_name) # BETA only self.require_admin_beta_access(collection) try: result = DatShare.dat_share.unshare(collection) except Exception as e: result = {'error': 'api_error', 'details': str(e)} if 'error' in result: self._raise_error(400, result['error']) return result @self.app.post('/api/v1/collection/<coll_name>/sendmeta') @self.api(query=['user'], resp='reviewed') def send_meta(coll_name): user, collection = self.load_user_coll(coll_name=coll_name) # Serializing json json_object = json.dumps(collection, indent=4) print(json_object) # Writing to sample.json with open("sample.json", "w") as outfile: outfile.write(json_object) @self.app.post('/api/v1/collection/<coll_name>/commit') def commit_file(coll_name): user, collection = self.load_user_coll(coll_name=coll_name) self.access.assert_can_admin_coll(collection) data = request.json or {} res = collection.commit_all(data.get('commit_id')) if not res: return {'success': True} else: return {'commit_id': res} # LEGACY ENDPOINTS (to remove) # Collection view (all recordings) @self.app.get(['/<user>/<coll_name>', '/<user>/<coll_name>/']) @self.jinja2_view('collection_info.html') def coll_info(user, coll_name): return self.get_collection_info_for_view(user, coll_name) @self.app.get([ '/<user>/<coll_name>/<rec_list:re:([\w,-]+)>', '/<user>/<coll_name>/<rec_list:re:([\w,-]+)>/' ]) @self.jinja2_view('collection_info.html') def coll_info(user, coll_name, rec_list): #rec_list = [self.sanitize_title(title) for title in rec_list.split(',')] return self.get_collection_info_for_view(user, coll_name) wr_api_spec.set_curr_tag(None)
def init_routes(self): wr_api_spec.set_curr_tag('External Archives') @self.app.get('/api/v1/client_archives') def get_client_archives(): return self.client_archives wr_api_spec.set_curr_tag('Browsers') @self.app.get('/api/v1/create_remote_browser') def create_browser(): """ Api to launch remote browser instances """ sesh = self.get_session() if sesh.is_new() and self.is_content_request(): self._raise_error(403, 'invalid_browser_request') browser_id = request.query['browser'] Stats(self.redis).incr_browser(browser_id) user = self.get_user(redir_check=False) data = request.query coll_name = data.getunicode('coll', '') rec = data.get('rec', '') mode = data.get('mode', '') url = data.getunicode('url', '') timestamp = data.get('timestamp', '') sources = '' inv_sources = '' patch_rec = '' collection = user.get_collection_by_name(coll_name) recording = collection.get_recording(rec) if not collection: self._raise_error(404, 'no_such_collection') if mode == 'extract': # Extract from All, Patch from None sources = '*' inv_sources = '*' elif mode.startswith('extract:'): # Extract from One, Patch from all but one sources = mode.split(':', 1)[1] inv_sources = sources # load patch recording also #patch_recording = collection.get_recording(recording['patch_rec']) if recording: patch_rec = recording.get_prop('patch_rec') mode = 'extract' elif mode.startswith('extract_only:'): # Extract from one only, no patching sources = mode.split(':', 1)[1] inv_sources = '*' mode = 'extract' if mode in self.MODIFY_MODES: if not recording: return self._raise_error(404, 'no_such_recording') #rec = recording.my_id elif mode in ('replay', 'replay-coll'): rec = '*' else: return self._raise_error(400, 'invalid_mode') browser_can_write = '1' if self.access.can_write_coll(collection) else '0' remote_ip = self._get_remote_ip() # build kwargs kwargs = dict(user=user.name, id=sesh.get_id(), coll=collection.my_id, rec=rec, coll_name=quote(coll_name), #rec_name=quote(rec_name, safe='/*'), type=mode, sources=sources, inv_sources=inv_sources, patch_rec=patch_rec, remote_ip=remote_ip, ip=remote_ip, browser=browser_id, url=url, request_ts=timestamp, browser_can_write=browser_can_write) data = self.browser_mgr.request_new_browser(kwargs) if 'error_message' in data: self._raise_error(400, data['error_message']) return data # UPDATE REMOTE BROWSER CONFIG @self.app.get('/api/v1/update_remote_browser/<reqid>') def update_remote_browser(reqid): user, collection = self.load_user_coll(api=True) timestamp = request.query.getunicode('timestamp') type_ = request.query.getunicode('type') # if switching mode, need to have write access # for timestamp, only read access if type_: self.access.assert_can_write_coll(collection) else: self.access.assert_can_read_coll(collection) return self.browser_mgr.update_remote_browser(reqid, type_=type_, timestamp=timestamp) # REDIRECTS @self.app.route('/record/<wb_url:path>', method='ANY') def redir_new_temp_rec(wb_url): coll_name = 'temp' rec_title = self.DEF_REC_NAME wb_url = self.add_query(wb_url) return self.do_create_new_and_redir(coll_name, rec_title, wb_url, 'record') @self.app.route('/$record/<coll_name>/<rec_title>/<wb_url:path>', method='ANY') def redir_new_record(coll_name, rec_title, wb_url): wb_url = self.add_query(wb_url) return self.do_create_new_and_redir(coll_name, rec_title, wb_url, 'record') # API NEW wr_api_spec.set_curr_tag('Recordings') @self.app.post('/api/v1/new') def api_create_new(): self.redir_host() url = request.json.get('url') coll = request.json.get('coll') mode = request.json.get('mode') desc = request.json.get('desc', '') browser = request.json.get('browser') is_content = request.json.get('is_content') and not browser timestamp = request.json.get('timestamp') wb_url = self.construct_wburl(url, timestamp, browser, is_content) host = self.content_host if is_content else self.app_host if not host: host = request.urlparts.netloc full_url = request.environ['wsgi.url_scheme'] + '://' + host url, rec, patch_rec = self.do_create_new(coll, '', wb_url, mode, desc=desc) full_url += url return {'url': full_url, 'rec_name': rec, 'patch_rec_name': patch_rec } # COOKIES wr_api_spec.set_curr_tag('Cookies') @self.app.post('/api/v1/auth/cookie') def add_cookie(): user, collection = self.load_user_coll() data = request.json or {} rec_name = data.get('rec', '*') recording = collection.get_recording(rec_name) name = data.get('name') value = data.get('value') domain = data.get('domain') if not domain: return self._raise_error(400, 'domain_missing') self.add_cookie(user, collection, recording, name, value, domain) return {'success': domain} # PROXY @self.app.route('/_proxy/<url:path>', method='ANY') def do_proxy(url): return self.do_proxy(url) # PROXY with CORS @self.app.route('/proxy-fetch/<url:path>', method='GET') def do_proxy_fetch_cors(url): res = self.do_proxy(url) if 'HTTP_ORIGIN' in request.environ: self.set_options_headers(None, None, res) return res # LIVE DEBUG #@self.app.route('/live/<wb_url:path>', method='ANY') def live(wb_url): request.path_shift(1) return self.handle_routing(wb_url, user='******', coll='temp', rec='', type='live') # EMDED @self.app.route('/_embed/<user>/<coll>/<wb_url:path>', method='ANY') def embed_replay(user, coll, wb_url): request.path_shift(3) #return self.do_replay_coll_or_rec(user, coll, wb_url, is_embed=True) return self.handle_routing(wb_url, user, coll, '*', type='replay-coll', is_embed=True) # DISPLAY @self.app.route('/_embed_noborder/<user>/<coll>/<wb_url:path>', method='ANY') def embed_replay(user, coll, wb_url): request.path_shift(3) #return self.do_replay_coll_or_rec(user, coll, wb_url, is_embed=True, # is_display=True) return self.handle_routing(wb_url, user, coll, '*', type='replay-coll', is_embed=True, is_display=True) # CONTENT ROUTES # Record @self.app.route('/<user>/<coll>/<rec>/record/<wb_url:path>', method='ANY') def do_record(user, coll, rec, wb_url): request.path_shift(4) return self.handle_routing(wb_url, user, coll, rec, type='record', redir_route='record') # Patch @self.app.route('/<user>/<coll>/<rec>/patch/<wb_url:path>', method='ANY') def do_patch(user, coll, rec, wb_url): request.path_shift(4) return self.handle_routing(wb_url, user, coll, rec, type='patch', redir_route='patch') # Extract @self.app.route('/<user>/<coll>/<rec>/extract\:<archive>/<wb_url:path>', method='ANY') def do_extract_patch_archive(user, coll, rec, wb_url, archive): request.path_shift(4) return self.handle_routing(wb_url, user, coll, rec, type='extract', sources=archive, inv_sources=archive, redir_route='extract:' + archive) @self.app.route('/<user>/<coll>/<rec>/extract_only\:<archive>/<wb_url:path>', method='ANY') def do_extract_only_archive(user, coll, rec, wb_url, archive): request.path_shift(4) return self.handle_routing(wb_url, user, coll, rec, type='extract', sources=archive, inv_sources='*', redir_route='extract_only:' + archive) @self.app.route('/<user>/<coll>/<rec>/extract/<wb_url:path>', method='ANY') def do_extract_all(user, coll, rec, wb_url): request.path_shift(4) return self.handle_routing(wb_url, user, coll, rec, type='extract', sources='*', inv_sources='*', redir_route='extract') # REPLAY # Replay List @self.app.route('/<user>/<coll>/list/<list_id>/<bk_id>/<wb_url:path>', method='ANY') def do_replay_rec(user, coll, list_id, bk_id, wb_url): request.path_shift(5) return self.handle_routing(wb_url, user, coll, '*', type='replay-coll') # Replay Recording @self.app.route('/<user>/<coll>/<rec>/replay/<wb_url:path>', method='ANY') def do_replay_rec(user, coll, rec, wb_url): request.path_shift(4) return self.handle_routing(wb_url, user, coll, rec, type='replay') # Replay Coll @self.app.route('/<user>/<coll>/<wb_url:path>', method='ANY') def do_replay_coll(user, coll, wb_url): request.path_shift(2) return self.handle_routing(wb_url, user, coll, '*', type='replay-coll') # Session redir @self.app.get(['/_set_session']) def set_sesh(): sesh = self.get_session() if self.is_content_request(): cookie = request.query.getunicode('cookie') sesh.set_id_from_cookie(cookie) return self.redirect(request.query.getunicode('path')) else: url = request.environ['wsgi.url_scheme'] + '://' + self.content_host self.set_options_headers(self.content_host, self.app_host) response.headers['Cache-Control'] = 'no-cache' cookie = request.query.getunicode('webrec.sesh_cookie') # otherwise, check if content cookie provided # already have same session, just redirect back # likely a real 404 not found if sesh.is_same_session(request.query.getunicode('content_cookie')): redirect(url + request.query.getunicode('path')) # if anon, ensure session is persisted before setting content session # generate cookie to pass if not cookie: self.access.init_session_user(persist=True) cookie = sesh.get_cookie() cookie = quote(cookie) url += '/_set_session?{0}&cookie={1}'.format(request.environ['QUERY_STRING'], cookie) redirect(url) # OPTIONS @self.app.route('/_set_session', method='OPTIONS') def set_sesh_options(): self.set_options_headers(self.content_host, self.app_host) return '' @self.app.route('/_clear_session', method='OPTIONS') def set_clear_options(): self.set_options_headers(self.app_host, self.content_host) return '' # CLEAR CONTENT SESSION @self.app.get(['/_clear_session']) def clear_sesh(): self.set_options_headers(self.app_host, self.content_host) response.headers['Cache-Control'] = 'no-cache' if not self.is_content_request(): self._raise_error(400, 'invalid_request') try: # delete session (will updated cookie) self.get_session().delete() return {'success': 'logged_out'} except Exception as e: self._raise_error(400, 'invalid_request')
def init_routes(self): wr_api_spec.set_curr_tag('Auth') # USER CHECKS @self.app.get('/api/v1/auth/check_username/<username>') def test_username(username): """async precheck username availability on signup form""" if self.user_manager.is_username_available(username): return {'success': True} else: return self._raise_error(400, 'username_not_available') # GET CURRENT USER @self.app.get('/api/v1/auth/curr_user') def load_user(): return self.load_user() # AUTH NEW SESSION @self.app.post('/api/v1/auth/anon_user') def new_auth(): return self.new_auth() # REGISTRATION @self.app.post('/api/v1/auth/register') def api_register_user(): data = request.json or {} msg, redir_extra = self.user_manager.register_user( data, self.get_host()) if 'success' in msg: return msg response.status = 400 return {'errors': msg} @self.app.post('/api/v1/auth/validate') def api_validate_reg_user(): data = request.json or {} reg = data.get('reg') cookie = request.environ.get('webrec.request_cookie', '') username = request.query.getunicode('username') result = self.user_manager.validate_registration( reg, cookie, username) if 'error' in result or 'errors' in result: response.status = 400 return result # LOGIN @self.app.post('/api/v1/auth/login') def login(): """Authenticate users""" if not self.access.is_anon(): return self._raise_error(403, 'already_logged_in') include_colls = get_bool(request.query.get('include_colls', False)) result = self.user_manager.login_user(request.json or {}) if 'success' in result: data = { 'user': self.access.session_user.serialize(include_colls) } if result.get('new_coll_name'): data['new_coll_name'] = result['new_coll_name'] return data #self._raise_error(401, result.get('error', '')) response.status = 401 return result @self.app.post('/api/v1/auth/logout') def logout(): self.get_user_or_raise() self.user_manager.logout() data = {'success': 'logged_out'} return data @self.app.post('/api/v1/auth/ensure_login') def ensure_login(): return self.ensure_login() # PASSWORD @self.app.post('/api/v1/auth/password/reset_request') def request_reset_password(): data = request.json or {} email = data.get('email', '') username = data.get('username', '') host = self.get_host() try: self.user_manager.cork.send_password_reset_email( username=username, email_addr=email, subject='Conifer password reset confirmation', email_template='webrecorder/templates/emailreset.html', host=host) return {'success': True} except Exception as e: import traceback traceback.print_exc() self._raise_error(404, 'no_such_user') @self.app.post('/api/v1/auth/password/reset') def reset_password(): #self.get_user_or_raise() if not self.access.is_anon(): return self._raise_error(403, 'already_logged_in') data = request.json or {} password = data.get('newPass', '') confirm_password = data.get('newPass2', '') reset_code = data.get('resetCode', '') try: self.user_manager.reset_password(password, confirm_password, reset_code) return {'success': True} except ValidationException as ve: self._raise_error(403, str(ve)) @self.app.post('/api/v1/auth/password/update') def update_password(): self.get_user_or_raise() data = request.json or {} curr_password = data.get('currPass', '') password = data.get('newPass', '') confirm_password = data.get('newPass2', '') try: self.user_manager.update_password(curr_password, password, confirm_password) return {'success': True} except ValidationException as ve: return self._raise_error(403, str(ve)) # Skip POST request recording @self.app.post('/api/v1/auth/skipreq') def skip_req(): data = request.json or {} url = data.get('url', '') self.access.session_user.mark_skip_url(url) return {'success': True} # USER API wr_api_spec.set_curr_tag('Users') @self.app.get('/api/v1/user/<username>') def api_get_user(username): """API enpoint to return user info""" return self.load_user(username) @self.app.delete('/api/v1/user/<username>') def api_delete_user(username): """API enpoint to delete a user""" self.get_user_or_raise(username, 404, 'not_found') # TODO? add validation #self.validate_csrf() try: assert (self.user_manager.delete_user(username)) request.environ['webrec.delete_all_cookies'] = 'all' except: #return {'error_message': 'Could not delete user: '******'error_deleting') data = {'deleted_user': username} return data @self.app.post('/api/v1/user/<username>') def update_user(username): user = self.get_user(user=username) data = request.json or {} if 'desc' in data: user['desc'] = data['desc'] if 'full_name' in data: user['full_name'] = data['full_name'][:150] if 'display_url' in data: user['display_url'] = data['display_url'][:500] return {'success': True} # OLD VIEWS BELOW # ==================================================================== @self.app.get(['/<username>', '/<username>/']) @self.jinja2_view('user.html') def user_info(username): self.redir_host() user = self.get_user(user=username) if self.access.is_anon(user): self.redirect('/' + user.my_id + '/temp') result = { 'user': user.name, 'user_info': user.serialize(), 'collections': [coll.serialize() for coll in user.get_collections()], } if not result['user_info'].get('desc'): result['user_info']['desc'] = self.default_user_desc.format( user) return result wr_api_spec.set_curr_tag(None)
def init_routes(self): wr_api_spec.set_curr_tag('Recordings') @self.app.post('/api/v1/recordings') @self.api(query=['user', 'coll'], req=['title', 'desc'], resp='recording') def create_recording(): user, collection = self.load_user_coll() data = request.json or {} title = data.get('title', '') desc = data.get('desc', '') recording = collection.create_recording(title=title, desc=desc) collection.mark_updated() return {'recording': recording.serialize()} @self.app.get('/api/v1/recordings') @self.api(query=['user', 'coll'], resp='recordings') def get_recordings(): user, collection = self.load_user_coll() recs = collection.get_recordings() return {'recordings': [rec.serialize() for rec in recs]} @self.app.get('/api/v1/recording/<rec>') @self.api(query=['user', 'coll'], resp='recording') def get_recording(rec): user, collection, recording = self.load_recording(rec) if recording: return {'recording': recording.serialize()} else: self._raise_error(404, 'no_such_recording') @self.app.post('/api/v1/recording/<rec>') @self.api(query=['user', 'coll'], req=['desc'], resp='recording') def update_rec_desc(rec): user, collection, recording = self.load_recording(rec) user.access.assert_can_write_coll(collection) data = request.json or {} desc = data.get('desc', '') recording['desc'] = desc recording.mark_updated() return {'recording': recording.serialize()} @self.app.delete('/api/v1/recording/<rec>') @self.api(query=['user', 'coll'], resp='deleted') def delete_recording(rec): user, collection, recording = self.load_recording(rec) errs = collection.remove_recording(recording, delete=True) if errs.get('error'): return self._raise_error(400, errs['error']) else: return {'deleted_id': rec} @self.app.post('/api/v1/recording/<rec>/move/<new_coll_name>') @self.api(query=['user', 'coll'], resp='rec_move') def move_recording(rec, new_coll_name): user, collection, recording = self.load_recording(rec) new_collection = user.get_collection_by_name(new_coll_name) if not new_collection: self._raise_error(400, 'no_such_collection') user.access.assert_can_admin_coll(new_collection) new_rec = collection.move_recording(recording, new_collection) if new_rec: collection.mark_updated() new_collection.mark_updated() return {'coll_id': new_coll_name, 'rec_id': new_rec} else: self._raise_error(400, 'move_error') @self.app.post('/api/v1/recording/<rec>/copy/<new_coll_name>') @self.api(query=['user', 'coll'], resp='recording') def copy_recording(rec, new_coll_name): user, collection, recording = self.load_recording(rec) new_collection = user.get_collection_by_name(new_coll_name) if not new_collection: return self._raise_error(400, 'no_such_collection') user.access.assert_can_write_coll(collection) user.access.assert_can_admin_coll(new_collection) new_rec = new_collection.create_recording() if new_rec.copy_data_from_recording(recording): new_rec.mark_updated() return {'recording': new_rec.serialize()} else: return self._raise_error(400, 'copy_error') @self.app.post('/api/v1/recording/<rec>/pages') @self.api(query=['user', 'coll'], req=['url', 'timestamp', 'title', 'browser'], resp='page_id') def add_page(rec): user, collection, recording = self.load_recording(rec) page_data = request.json or {} page_id = collection.add_page(page_data, recording) recording.mark_updated() return {'page_id': page_id} @self.app.get('/api/v1/recording/<rec>/pages') @self.api(query=['user', 'coll'], resp='pages') def list_pages(rec): user, collection, recording = self.load_recording(rec) pages = collection.list_rec_pages(recording) return {'pages': pages} @self.app.get('/api/v1/recording/<rec>/num_pages') @self.api(query=['user', 'coll'], resp='count_pages') def get_num_pages(rec): user, collection, recording = self.load_recording(rec) return {'count': recording.count_pages() } @self.app.delete('/api/v1/recording/<rec>/pages') @self.api(query=['user', 'coll'], resp='deleted') def delete_page(rec): user, collection, recording = self.load_recording(rec) url = request.json.get('url') ts = request.json.get('timestamp') return recording.delete_page(url, ts)
def init_routes(self): wr_api_spec.set_curr_tag('Recordings') @self.app.post('/api/v1/recordings') @self.api(query=['user', 'coll'], req=['title', 'desc'], resp='recording') def create_recording(): user, collection = self.load_user_coll() data = request.json or {} title = data.get('title', '') desc = data.get('desc', '') recording = collection.create_recording(title=title, desc=desc) collection.mark_updated() return {'recording': recording.serialize()} @self.app.get('/api/v1/recordings') @self.api(query=['user', 'coll'], resp='recordings') def get_recordings(): user, collection = self.load_user_coll() recs = collection.get_recordings() return {'recordings': [rec.serialize() for rec in recs]} @self.app.get('/api/v1/recording/<rec>') @self.api(query=['user', 'coll'], resp='recording') def get_recording(rec): user, collection, recording = self.load_recording(rec) if recording: return {'recording': recording.serialize()} else: self._raise_error(404, 'no_such_recording') @self.app.post('/api/v1/recording/<rec>') @self.api(query=['user', 'coll'], req=['desc'], resp='recording') def update_rec_desc(rec): user, collection, recording = self.load_recording(rec) user.access.assert_can_write_coll(collection) data = request.json or {} desc = data.get('desc', '') recording['desc'] = desc recording.mark_updated() return {'recording': recording.serialize()} @self.app.delete('/api/v1/recording/<rec>') @self.api(query=['user', 'coll'], resp='deleted') def delete_recording(rec): user, collection, recording = self.load_recording(rec) errs = collection.remove_recording(recording, delete=True) if errs.get('error'): return self._raise_error(400, errs['error']) else: return {'deleted_id': rec} @self.app.post('/api/v1/recording/<rec>/move/<new_coll_name>') @self.api(query=['user', 'coll'], resp='rec_move') def move_recording(rec, new_coll_name): user, collection, recording = self.load_recording(rec) new_collection = user.get_collection_by_name(new_coll_name) if not new_collection: self._raise_error(400, 'no_such_collection') user.access.assert_can_admin_coll(new_collection) new_rec = collection.move_recording(recording, new_collection) if new_rec: collection.mark_updated() new_collection.mark_updated() return {'coll_id': new_coll_name, 'rec_id': new_rec} else: self._raise_error(400, 'move_error') @self.app.post('/api/v1/recording/<rec>/copy/<new_coll_name>') @self.api(query=['user', 'coll'], resp='recording') def copy_recording(rec, new_coll_name): user, collection, recording = self.load_recording(rec) new_collection = user.get_collection_by_name(new_coll_name) if not new_collection: return self._raise_error(400, 'no_such_collection') user.access.assert_can_write_coll(collection) user.access.assert_can_admin_coll(new_collection) new_rec = new_collection.create_recording() if new_rec.copy_data_from_recording(recording): new_rec.mark_updated() return {'recording': new_rec.serialize()} else: return self._raise_error(400, 'copy_error') @self.app.post('/api/v1/recording/<rec>/pages') @self.api(query=['user', 'coll'], req=['url', 'timestamp', 'title', 'browser'], resp='page_id') def add_page(rec): user, collection, recording = self.load_recording(rec) page_data = request.json or {} page_id = collection.add_page(page_data, recording) recording.mark_updated() return {'page_id': page_id} @self.app.get('/api/v1/recording/<rec>/pages') @self.api(query=['user', 'coll'], resp='pages') def list_pages(rec): user, collection, recording = self.load_recording(rec) pages = collection.list_rec_pages(recording) return {'pages': pages} @self.app.get('/api/v1/recording/<rec>/num_pages') @self.api(query=['user', 'coll'], resp='count_pages') def get_num_pages(rec): user, collection, recording = self.load_recording(rec) return {'count': recording.count_pages()} @self.app.delete('/api/v1/recording/<rec>/pages') @self.api(query=['user', 'coll'], resp='deleted') def delete_page(rec): user, collection, recording = self.load_recording(rec) url = request.json.get('url') ts = request.json.get('timestamp') return recording.delete_page(url, ts)
def init_routes(self): wr_api_spec.set_curr_tag('Collections') @self.app.post('/api/v1/collections') @self.api(query=['user'], req=['title', 'public', 'public_index'], resp='collection') def create_collection(): user = self.get_user(api=True, redir_check=False) data = request.json or {} title = data.get('title', '') coll_name = self.sanitize_title(title) if not coll_name: self._raise_error(400, 'invalid_coll_name') is_public = data.get('public', False) is_public_index = data.get('public_index', False) is_external = data.get('external', False) is_anon = self.access.is_anon(user) if is_external: if not self.allow_external: self._raise_error(403, 'external_not_allowed') if not is_anon: self._raise_error(400, 'not_valid_for_external') elif is_anon: if coll_name != 'temp': self._raise_error(400, 'invalid_temp_coll_name') if user.has_collection(coll_name): self._raise_error(400, 'duplicate_name') try: collection = user.create_collection(coll_name, title=title, desc='', public=is_public, public_index=is_public_index) if is_external: collection.set_external(True) user.mark_updated() self.flash_message('Created collection <b>{0}</b>!'.format(collection.get_prop('title')), 'success') resp = {'collection': collection.serialize()} except DupeNameException as de: self._raise_error(400, 'duplicate_name') except Exception as ve: print(ve) self.flash_message(str(ve)) self._raise_error(400, 'duplicate_name') return resp @self.app.get('/api/v1/collections') @self.api(query=['user', 'include_recordings', 'include_lists', 'include_pages'], resp='collections') def get_collections(): user = self.get_user(api=True, redir_check=False) kwargs = {'include_recordings': get_bool(request.query.get('include_recordings')), 'include_lists': get_bool(request.query.get('include_lists')), 'include_pages': get_bool(request.query.get('include_pages')), } collections = user.get_collections() return {'collections': [coll.serialize(**kwargs) for coll in collections]} @self.app.get('/api/v1/collection/<coll_name>') @self.api(query=['user'], resp='collection') def get_collection(coll_name): user = self.get_user(api=True, redir_check=False) return self.get_collection_info(coll_name, user=user) @self.app.delete('/api/v1/collection/<coll_name>') @self.api(query=['user'], resp='deleted') def delete_collection(coll_name): user, collection = self.load_user_coll(coll_name=coll_name) errs = user.remove_collection(collection, delete=True) if errs.get('error'): return self._raise_error(400, errs['error']) else: return {'deleted_id': coll_name} @self.app.put('/api/v1/collection/<coll_name>/warc') def add_external_warc(coll_name): if not self.allow_external: self._raise_error(403, 'external_not_allowed') user, collection = self.load_user_coll(coll_name=coll_name) self.access.assert_can_admin_coll(collection) if not collection.is_external(): self._raise_error(400, 'external_only') num_added = collection.add_warcs(request.json.get('warcs', {})) return {'success': num_added} @self.app.put('/api/v1/collection/<coll_name>/cdx') def add_external_cdxj(coll_name): if not self.allow_external: self._raise_error(403, 'external_not_allowed') user, collection = self.load_user_coll(coll_name=coll_name) self.access.assert_can_admin_coll(collection) if not collection.is_external(): self._raise_error(400, 'external_only') num_added = collection.add_cdxj(request.body.read()) return {'success': num_added} @self.app.post('/api/v1/collection/<coll_name>') @self.api(query=['user'], req=['title', 'desc', 'public', 'public_index'], resp='collection') def update_collection(coll_name): user, collection = self.load_user_coll(coll_name=coll_name) self.access.assert_can_admin_coll(collection) data = request.json or {} if 'title' in data: new_coll_title = data['title'] new_coll_name = self.sanitize_title(new_coll_title) if not new_coll_name: self._raise_error(400, 'invalid_coll_name') try: new_coll_name = user.colls.rename(collection, new_coll_name, allow_dupe=False) except DupeNameException as de: self._raise_error(400, 'duplicate_name') collection['title'] = new_coll_title if 'desc' in data: collection['desc'] = data['desc'] # TODO: notify the user if this is a request from the admin panel if 'public' in data: #if self.access.is_superuser() and data.get('notify'): # pass collection.set_public(data['public']) if 'public_index' in data: collection.set_bool_prop('public_index', data['public_index']) collection.mark_updated() return {'collection': collection.serialize()} @self.app.get('/api/v1/collection/<coll_name>/page_bookmarks') @self.api(query=['user'], resp='bookmarks') def get_page_bookmarks(coll_name): user, collection = self.load_user_coll(coll_name=coll_name) rec = request.query.get('rec') if rec: recording = collection.get_recording(rec) if not recording: return {'page_bookmarks': {}} rec_pages = collection.list_rec_pages(recording) else: rec_pages = None return {'page_bookmarks': collection.get_all_page_bookmarks(rec_pages)} # DAT @self.app.post('/api/v1/collection/<coll_name>/dat/share') def dat_do_share(coll_name): user, collection = self.load_user_coll(coll_name=coll_name) self.access.assert_can_admin_coll(collection) # BETA only try: self.cork.require(role='beta-archivist') except: self._raise_error(400, 'not_allowed') try: data = request.json or {} result = DatShare.dat_share.share(collection, data.get('always_update', False)) except Exception as e: result = {'error': 'api_error', 'details': str(e)} if 'error' in result: self._raise_error(400, result['error']) return result @self.app.post('/api/v1/collection/<coll_name>/dat/unshare') def dat_do_unshare(coll_name): user, collection = self.load_user_coll(coll_name=coll_name) self.access.assert_can_admin_coll(collection) # BETA only try: self.cork.require(role='beta-archivist') except: self._raise_error(400, 'not_allowed') try: result = DatShare.dat_share.unshare(collection) except Exception as e: result = {'error': 'api_error', 'details': str(e)} if 'error' in result: self._raise_error(400, result['error']) return result @self.app.post('/api/v1/collection/<coll_name>/commit') def commit_file(coll_name): user, collection = self.load_user_coll(coll_name=coll_name) self.access.assert_can_admin_coll(collection) data = request.json or {} res = collection.commit_all(data.get('commit_id')) if not res: return {'success': True} else: return {'commit_id': res} # LEGACY ENDPOINTS (to remove) # Collection view (all recordings) @self.app.get(['/<user>/<coll_name>', '/<user>/<coll_name>/']) @self.jinja2_view('collection_info.html') def coll_info(user, coll_name): return self.get_collection_info_for_view(user, coll_name) @self.app.get(['/<user>/<coll_name>/<rec_list:re:([\w,-]+)>', '/<user>/<coll_name>/<rec_list:re:([\w,-]+)>/']) @self.jinja2_view('collection_info.html') def coll_info(user, coll_name, rec_list): #rec_list = [self.sanitize_title(title) for title in rec_list.split(',')] return self.get_collection_info_for_view(user, coll_name)
def init_routes(self): wr_api_spec.set_curr_tag('Admin') @self.app.get('/api/v1/admin/defaults') @self.admin_view def get_defaults(): data = self.redis.hgetall('h:defaults') data['max_size'] = int(data['max_size']) data['max_anon_size'] = int(data['max_anon_size']) return {'defaults': data} @self.app.put('/api/v1/admin/defaults') def update_defaults(): data = request.json if 'max_size' in data: try: self.redis.hset('h:defaults', 'max_size', int(data['max_size'])) except Exception as e: return {'error': 'error setting max_size'} if 'max_anon_size' in data: try: self.redis.hset('h:defaults', 'max_anon_size', int(data['max_anon_size'])) except Exception as e: return {'error': 'error setting max_anon_size'} data = self.redis.hgetall('h:defaults') data['max_size'] = int(data['max_size']) data['max_anon_size'] = int(data['max_anon_size']) return {'defaults': data} @self.app.get('/api/v1/admin/user_roles') @self.admin_view def api_get_user_roles(): return {"roles": self.user_manager.get_roles()} @self.app.get('/api/v1/admin/dashboard') @self.admin_view def api_dashboard(): cache_key = self.cache_template.format('dashboard') expiry = 5 * 60 # 5 min cache = self.redis.get(cache_key) if cache: return json.loads(cache) users = self.user_manager.all_users temp = self.redis.hgetall(self.temp_usage_key) user = self.redis.hgetall(self.user_usage_key) temp = [(k, int(v)) for k, v in temp.items()] user = [(k, int(v)) for k, v in user.items()] all_collections = [] for username in users: u = self.get_user(user=username) all_collections.extend( [c.serialize() for c in u.get_collections()] ) data = { 'user_count': len(users), 'collections': all_collections, 'temp_usage': sorted(temp, key=itemgetter(0)), 'user_usage': sorted(user, key=itemgetter(0)), } self.redis.setex(cache_key, expiry, json.dumps(data, cls=CustomJSONEncoder)) return data @self.app.get('/api/v1/admin/users') @self.admin_view def api_users(): """Full admin API resource of all users. Containing user info and public collections - Provides basic (1 dimension) RESTful sorting - TODO: Pagination """ sorting = request.query.getunicode('sort', None) sort_key = sub(r'^-{1}?', '', sorting) if sorting is not None else None reverse = sorting.startswith('-') if sorting is not None else False def dt(d): return datetime.strptime(d, '%Y-%m-%d %H:%M:%S.%f') # sortable fields, with optional key unpacking functions filters = { 'created': {'key': lambda obj: dt(obj[1]['creation_date'])}, 'email': {'key': lambda obj: obj[1]['email_addr']}, 'last_login': {'key': lambda obj: dt(obj[1]['last_login'])}, 'name': {'key': lambda obj: json.loads(obj[1]['desc'] or '{}')['name']}, 'username': {}, } if sorting is not None and sort_key not in filters: raise HTTPError(400, 'Bad Request') sort_by = filters[sort_key] if sorting is not None else None users = sorted(self.user_manager.all_users, key=sort_by, reverse=reverse) return {'users': [self.user_manager.all_users[user].serialize() for user in users]} @self.app.get('/api/v1/admin/temp-users') @self.admin_view def temp_users(): """ Resource returning active temp users """ temp_user_keys = list(self.redis.scan_iter(self.temp_user_search)) temp_user_data = [] for user_key in temp_user_keys: username = user_key.split(':')[1] user = self.user_manager.all_users[username] if not user or not user.get_prop('created_at'): continue temp_user_data.append(user.serialize()) return {'users': temp_user_data} @self.app.post('/api/v1/admin/users') @self.admin_view def api_create_user(): """API enpoint to create a user""" data = request.json errs, res = self.user_manager.create_user_as_admin( email=data['email'], username=data['username'], role=data['role'], passwd=data['password'], passwd2=data['password'], name=data.get('full_name', '')) # validate if errs: return {'errors': errs} user, first_coll = res return {'user': user.name, 'first_coll': first_coll.name if first_coll else ''} @self.app.put('/api/v1/admin/user/<username>') @self.admin_view def api_update_user(username): """API enpoint to update user info (full access) """ user = self.get_user(user=username) errs = self.user_manager.update_user_as_admin(user, request.json) if errs: return {'errors': errs} return {'user': user.serialize()} # Grafana Stats APIs wr_api_spec.set_curr_tag('Stats') @self.app.get('/api/v1/stats/') @self.admin_view def stats_ping(): return {} @self.app.post('/api/v1/stats/search') @self.admin_view def stats_search(): stats = sorted(list(self.all_stats.keys())) response.content_type = 'application/json' return json.dumps(stats) @self.app.post('/api/v1/stats/query') @self.admin_view def stats_query(): stats = self.grafana_time_stats(request.json) response.content_type = 'application/json' return json.dumps(stats) @self.app.post('/api/v1/stats/annotations') @self.admin_view def stats_annotations(): return []
def init_routes(self): wr_api_spec.set_curr_tag('External Archives') @self.app.get('/api/v1/client_archives') def get_client_archives(): return self.client_archives wr_api_spec.set_curr_tag('Browsers') @self.app.get('/api/v1/create_remote_browser') def create_browser(): """ Api to launch remote browser instances """ sesh = self.get_session() if sesh.is_new() and self.is_content_request(): self._raise_error(403, 'invalid_browser_request') browser_id = request.query['browser'] Stats(self.redis).incr_browser(browser_id) user = self.get_user(redir_check=False) data = request.query coll_name = data.getunicode('coll', '') rec = data.get('rec', '') mode = data.get('mode', '') url = data.getunicode('url', '') timestamp = data.get('timestamp', '') sources = '' inv_sources = '' patch_rec = '' collection = user.get_collection_by_name(coll_name) recording = collection.get_recording(rec) if not collection: self._raise_error(404, 'no_such_collection') if mode == 'extract': # Extract from All, Patch from None sources = '*' inv_sources = '*' elif mode.startswith('extract:'): # Extract from One, Patch from all but one sources = mode.split(':', 1)[1] inv_sources = sources # load patch recording also #patch_recording = collection.get_recording(recording['patch_rec']) if recording: patch_rec = recording.get_prop('patch_rec') mode = 'extract' elif mode.startswith('extract_only:'): # Extract from one only, no patching sources = mode.split(':', 1)[1] inv_sources = '*' mode = 'extract' if mode in self.MODIFY_MODES: if not recording: return self._raise_error(404, 'no_such_recording') #rec = recording.my_id elif mode in ('replay', 'replay-coll'): rec = '*' else: return self._raise_error(400, 'invalid_mode') browser_can_write = '1' if self.access.can_write_coll(collection) else '0' remote_ip = self._get_remote_ip() # build kwargs kwargs = dict(user=user.name, id=sesh.get_id(), coll=collection.my_id, rec=rec, coll_name=quote(coll_name), #rec_name=quote(rec_name, safe='/*'), type=mode, sources=sources, inv_sources=inv_sources, patch_rec=patch_rec, remote_ip=remote_ip, ip=remote_ip, browser=browser_id, url=url, request_ts=timestamp, browser_can_write=browser_can_write) data = self.browser_mgr.request_new_browser(kwargs) if 'error_message' in data: self._raise_error(400, data['error_message']) return data # UPDATE REMOTE BROWSER CONFIG @self.app.get('/api/v1/update_remote_browser/<reqid>') def update_remote_browser(reqid): user, collection = self.load_user_coll(api=True) timestamp = request.query.getunicode('timestamp') type_ = request.query.getunicode('type') # if switching mode, need to have write access # for timestamp, only read access if type_: self.access.assert_can_write_coll(collection) else: self.access.assert_can_read_coll(collection) return self.browser_mgr.update_remote_browser(reqid, type_=type_, timestamp=timestamp) # REDIRECTS @self.app.route('/record/<wb_url:path>', method='ANY') def redir_new_temp_rec(wb_url): coll_name = 'temp' rec_title = self.DEF_REC_NAME wb_url = self.add_query(wb_url) return self.do_create_new_and_redir(coll_name, rec_title, wb_url, 'record') @self.app.route('/$record/<coll_name>/<rec_title>/<wb_url:path>', method='ANY') def redir_new_record(coll_name, rec_title, wb_url): wb_url = self.add_query(wb_url) return self.do_create_new_and_redir(coll_name, rec_title, wb_url, 'record') # API NEW wr_api_spec.set_curr_tag('Recordings') @self.app.post('/api/v1/new') def api_create_new(): self.redir_host() url = request.json.get('url') coll = request.json.get('coll') mode = request.json.get('mode') desc = request.json.get('desc', '') browser = request.json.get('browser') is_content = request.json.get('is_content') and not browser timestamp = request.json.get('timestamp') wb_url = self.construct_wburl(url, timestamp, browser, is_content) host = self.content_host if is_content else self.app_host if not host: host = request.urlparts.netloc full_url = request.environ['wsgi.url_scheme'] + '://' + host url, rec, patch_rec = self.do_create_new(coll, '', wb_url, mode, desc=desc) full_url += url return {'url': full_url, 'user': self.access.session_user.name, 'rec_name': rec, 'patch_rec_name': patch_rec } # COOKIES wr_api_spec.set_curr_tag('Cookies') @self.app.post('/api/v1/auth/cookie') def add_cookie(): user, collection = self.load_user_coll() data = request.json or {} rec_name = data.get('rec', '*') recording = collection.get_recording(rec_name) name = data.get('name') value = data.get('value') domain = data.get('domain') if not domain: return self._raise_error(400, 'domain_missing') self.add_cookie(user, collection, recording, name, value, domain) return {'success': domain} # PROXY @self.app.route('/_proxy/<url:path>', method='ANY') def do_proxy(url): return self.do_proxy(url) # PROXY with CORS @self.app.route('/proxy-fetch/<url:path>', method='GET') def do_proxy_fetch_cors(url): res = self.do_proxy(url) if 'HTTP_ORIGIN' in request.environ: self.set_options_headers(None, None, res) return res @self.app.route('/api/v1/remote/put-record', method='PUT') def do_put_record(): return self.do_put_record() # LIVE DEBUG #@self.app.route('/live/<wb_url:path>', method='ANY') def live(wb_url): request.path_shift(1) return self.handle_routing(wb_url, user='******', coll='temp', rec='', type='live') # EMDED @self.app.route('/_embed/<user>/<coll>/<wb_url:path>', method='ANY') def embed_replay(user, coll, wb_url): request.path_shift(3) #return self.do_replay_coll_or_rec(user, coll, wb_url, is_embed=True) return self.handle_routing(wb_url, user, coll, '*', type='replay-coll', is_embed=True) # DISPLAY @self.app.route('/_embed_noborder/<user>/<coll>/<wb_url:path>', method='ANY') def embed_replay(user, coll, wb_url): request.path_shift(3) #return self.do_replay_coll_or_rec(user, coll, wb_url, is_embed=True, # is_display=True) return self.handle_routing(wb_url, user, coll, '*', type='replay-coll', is_embed=True, is_display=True) # CONTENT ROUTES # Record @self.app.route('/<user>/<coll>/<rec>/record/<wb_url:path>', method='ANY') def do_record(user, coll, rec, wb_url): request.path_shift(4) return self.handle_routing(wb_url, user, coll, rec, type='record', redir_route='record') # Patch @self.app.route('/<user>/<coll>/<rec>/patch/<wb_url:path>', method='ANY') def do_patch(user, coll, rec, wb_url): request.path_shift(4) return self.handle_routing(wb_url, user, coll, rec, type='patch', redir_route='patch') # Extract @self.app.route('/<user>/<coll>/<rec>/extract\:<archive>/<wb_url:path>', method='ANY') def do_extract_patch_archive(user, coll, rec, wb_url, archive): request.path_shift(4) return self.handle_routing(wb_url, user, coll, rec, type='extract', sources=archive, inv_sources=archive, redir_route='extract:' + archive) @self.app.route('/<user>/<coll>/<rec>/extract_only\:<archive>/<wb_url:path>', method='ANY') def do_extract_only_archive(user, coll, rec, wb_url, archive): request.path_shift(4) return self.handle_routing(wb_url, user, coll, rec, type='extract', sources=archive, inv_sources='*', redir_route='extract_only:' + archive) @self.app.route('/<user>/<coll>/<rec>/extract/<wb_url:path>', method='ANY') def do_extract_all(user, coll, rec, wb_url): request.path_shift(4) return self.handle_routing(wb_url, user, coll, rec, type='extract', sources='*', inv_sources='*', redir_route='extract') # REPLAY # Replay List @self.app.route('/<user>/<coll>/list/<list_id>/<bk_id>/<wb_url:path>', method='ANY') def do_replay_rec(user, coll, list_id, bk_id, wb_url): request.path_shift(5) return self.handle_routing(wb_url, user, coll, '*', type='replay-coll') # Replay Recording @self.app.route('/<user>/<coll>/<rec>/replay/<wb_url:path>', method='ANY') def do_replay_rec(user, coll, rec, wb_url): request.path_shift(4) return self.handle_routing(wb_url, user, coll, rec, type='replay') # Replay Coll @self.app.route('/<user>/<coll>/<wb_url:path>', method='ANY') def do_replay_coll(user, coll, wb_url): request.path_shift(2) # BEGIN PERMA CUSTOMIZATION # see self.refuse_playback for more information if any(url in wb_url for url in self.refuse_playback): self._raise_error(404, 'no_such_recording') # END PERMA CUSTOMIZATION return self.handle_routing(wb_url, user, coll, '*', type='replay-coll') # Session redir @self.app.get(['/_set_session']) def set_sesh(): sesh = self.get_session() if self.is_content_request(): cookie = request.query.getunicode('cookie') sesh.set_id_from_cookie(cookie) return self.redirect(request.query.getunicode('path')) else: url = request.environ['wsgi.url_scheme'] + '://' + self.content_host self.set_options_headers(self.content_host, self.app_host) response.headers['Cache-Control'] = 'no-cache' cookie = request.query.getunicode('webrec.sesh_cookie') # otherwise, check if content cookie provided # already have same session, just redirect back # likely a real 404 not found if sesh.is_same_session(request.query.getunicode('content_cookie')): redirect(url + request.query.getunicode('path')) # if anon, ensure session is persisted before setting content session # generate cookie to pass if not cookie: self.access.init_session_user(persist=True) cookie = sesh.get_cookie() cookie = quote(cookie) url += '/_set_session?{0}&cookie={1}'.format(request.environ['QUERY_STRING'], cookie) redirect(url) # OPTIONS @self.app.route('/_set_session', method='OPTIONS') def set_sesh_options(): self.set_options_headers(self.content_host, self.app_host) return '' @self.app.route('/_clear_session', method='OPTIONS') def set_clear_options(): self.set_options_headers(self.app_host, self.content_host) return '' # CLEAR CONTENT SESSION @self.app.get(['/_clear_session']) def clear_sesh(): self.set_options_headers(self.app_host, self.content_host) response.headers['Cache-Control'] = 'no-cache' if not self.is_content_request(): self._raise_error(400, 'invalid_request') try: # delete session (will updated cookie) self.get_session().delete() return {'success': 'logged_out'} except Exception as e: self._raise_error(400, 'invalid_request')
def init_routes(self): # LISTS wr_api_spec.set_curr_tag('Lists') @self.app.get('/api/v1/lists') @self.api(query=['user', 'coll', 'include_bookmarks'], resp='lists') def get_lists(): user, collection = self.load_user_coll() include_bookmarks = request.query.getunicode('include_bookmarks') or 'all' lists = collection.get_lists() return { 'lists': [blist.serialize(include_bookmarks=include_bookmarks) for blist in lists] } @self.app.post('/api/v1/lists') @self.api(query=['user', 'coll', 'include_bookmarks'], req_desc='List properties', req=['title', 'desc', 'public', 'before_id'], resp='list') def add_list(): user, collection = self.load_user_coll() blist = collection.create_bookmark_list(request.json) collection.mark_updated() return {'list': blist.serialize()} @self.app.get('/api/v1/list/<list_id>') @self.api(query=['user', 'coll', 'include_bookmarks'], resp='list') def get_list(list_id): user, collection, blist = self.load_user_coll_list(list_id) self.access.assert_can_read_list(blist) include_bookmarks = request.query.getunicode('include_bookmarks') or 'all' return {'list': blist.serialize(check_slug=list_id, include_bookmarks=include_bookmarks)} @self.app.post('/api/v1/list/<list_id>') @self.api(query=['user', 'coll'], req_desc='Update List properties', req=['title', 'desc', 'public'], resp='list') def update_list(list_id): user, collection, blist = self.load_user_coll_list(list_id) blist.update(request.json) blist.mark_updated() return {'list': blist.serialize()} @self.app.delete('/api/v1/list/<list_id>') @self.api(query=['user', 'coll'], resp='deleted') def delete_list(list_id): user, collection, blist = self.load_user_coll_list(list_id) if collection.remove_list(blist): collection.mark_updated() return {'deleted_id': list_id} else: self._raise_error(400, 'error_deleting') @self.app.post('/api/v1/list/<list_id>/move') @self.api(query=['user', 'coll'], req=['before_id'], resp='success') def move_list_before(list_id): user, collection, blist = self.load_user_coll_list(list_id) before_id = request.json.get('before_id') if before_id: before = self.load_list(collection, before_id) else: before = None blist.mark_updated() collection.move_list_before(blist, before) return {'success': True} @self.app.post('/api/v1/lists/reorder') @self.api(query=['user', 'coll'], req_desc='An array of existing list ids in a new order', req=['order'], resp='success') def reorder_lists(): user, collection = self.load_user_coll() new_order = request.json.get('order', []) if collection.lists.reorder_objects(new_order): collection.mark_updated() return {'success': True} else: return self._raise_error(400, 'invalid_order') #BOOKMARKS wr_api_spec.set_curr_tag('Bookmarks') @self.app.post('/api/v1/list/<list_id>/bookmarks') @self.api(query=['user', 'coll'], req_desc='Bookmark properties', req=['title', 'url', 'timestamp', 'browser', 'desc', 'page_id', 'before_id'], resp='bookmark') def create_bookmark(list_id): user, collection, blist = self.load_user_coll_list(list_id) bookmark = blist.create_bookmark(request.json) if bookmark: blist.mark_updated() return {'bookmark': bookmark} else: return self._raise_error(400, 'invalid_page') @self.app.post('/api/v1/list/<list_id>/bulk_bookmarks') @self.api(query=['user', 'coll'], req_desc='List of Bookmarks', req={'type': 'array', 'item_type': ['title', 'url', 'timestamp', 'browser', 'desc', 'page_id', 'before_id']}, resp='list') def create_bookmarks(list_id): user, collection, blist = self.load_user_coll_list(list_id) bookmark_list = request.json blist.add_bookmarks(bookmark_list) return {'success': True} @self.app.get('/api/v1/list/<list_id>/bookmarks') @self.api(query=['user', 'coll'], resp='bookmarks') def get_bookmarks(list_id): user, collection, blist = self.load_user_coll_list(list_id) bookmarks = blist.get_bookmarks() return {'bookmarks': bookmarks} @self.app.get('/api/v1/bookmark/<bid>') @self.api(query=['user', 'coll', 'list'], resp='bookmark') def get_bookmark(bid): user, collection, blist = self.load_user_coll_list() bookmark = blist.get_bookmark(bid) return {'bookmark': bookmark} @self.app.post('/api/v1/bookmark/<bid>') @self.api(query=['user', 'coll', 'list'], req_desc='Bookmark properties', req=['title', 'url', 'timestamp', 'browser', 'desc', 'page_id', 'before_id'], resp='bookmark') def update_bookmark(bid): user, collection, blist = self.load_user_coll_list() bookmark = blist.update_bookmark(bid, request.json) blist.mark_updated() return {'bookmark': bookmark} @self.app.delete('/api/v1/bookmark/<bid>') @self.api(query=['user', 'coll', 'list'], resp='deleted') def delete_bookmark(bid): user, collection, blist = self.load_user_coll_list() if blist.remove_bookmark(bid): blist.mark_updated() return {'deleted_id': bid} else: self._raise_error(404, 'no_such_bookmark') @self.app.post('/api/v1/list/<list_id>/bookmarks/reorder') @self.api(query=['user', 'coll'], req_desc='An array of existing bookmark ids in a new order', req=['order'], resp='success') def reorder_bookmarks(list_id): user, collection, blist = self.load_user_coll_list(list_id) new_order = request.json.get('order', []) if blist.reorder_bookmarks(new_order): blist.mark_updated() return {'success': True} else: self._raise_error(400, 'invalid_order')
def init_routes(self): wr_api_spec.set_curr_tag('Collections') @self.app.post('/api/v1/collections') @self.api(query=['user'], req=['title', 'public', 'public_index'], resp='collection') def create_collection(): user = self.get_user(api=True, redir_check=False) data = request.json or {} title = data.get('title', '') coll_name = self.sanitize_title(title) if not coll_name: self._raise_error(400, 'invalid_coll_name') is_public = data.get('public', False) is_public_index = data.get('public_index', False) is_external = data.get('external', False) is_anon = self.access.is_anon(user) if is_external: if not self.allow_external: self._raise_error(403, 'external_not_allowed') #if not is_anon: # self._raise_error(400, 'not_valid_for_external') elif is_anon: if coll_name != 'temp': self._raise_error(400, 'invalid_temp_coll_name') if user.has_collection(coll_name): self._raise_error(400, 'duplicate_name') try: collection = user.create_collection(coll_name, title=title, desc='', public=is_public, public_index=is_public_index) if is_external: collection.set_external(True) # if auto-indexing is on, mark new collections as auto-indexed to distinguish from prev collections if self.is_search_auto: collection.set_bool_prop('autoindexed', True) user.mark_updated() self.flash_message('Created collection <b>{0}</b>!'.format(collection.get_prop('title')), 'success') resp = {'collection': collection.serialize()} except DupeNameException as de: self._raise_error(400, 'duplicate_name') except Exception as ve: print(ve) self.flash_message(str(ve)) self._raise_error(400, 'duplicate_name') return resp @self.app.get('/api/v1/collections') @self.api(query=['user', 'include_recordings', 'include_lists', 'include_pages'], resp='collections') def get_collections(): user = self.get_user(api=True, redir_check=False) kwargs = {'include_recordings': get_bool(request.query.get('include_recordings')), 'include_lists': get_bool(request.query.get('include_lists')), 'include_pages': get_bool(request.query.get('include_pages')), } collections = user.get_collections() return {'collections': [coll.serialize(**kwargs) for coll in collections]} @self.app.get('/api/v1/collection/<coll_name>') @self.api(query=['user'], resp='collection') def get_collection(coll_name): user = self.get_user(api=True, redir_check=False) return self.get_collection_info(coll_name, user=user) @self.app.delete('/api/v1/collection/<coll_name>') @self.api(query=['user'], resp='deleted') def delete_collection(coll_name): user, collection = self.load_user_coll(coll_name=coll_name) errs = user.remove_collection(collection, delete=True) if errs.get('error'): return self._raise_error(400, errs['error']) else: return {'deleted_id': coll_name} @self.app.put('/api/v1/collection/<coll_name>/warc') def add_external_warc(coll_name): if not self.allow_external: self._raise_error(403, 'external_not_allowed') user, collection = self.load_user_coll(coll_name=coll_name) self.access.assert_can_admin_coll(collection) if not collection.is_external(): self._raise_error(400, 'external_only') num_added = collection.add_warcs(request.json.get('warcs', {})) return {'success': num_added} @self.app.put('/api/v1/collection/<coll_name>/cdx') def add_external_cdxj(coll_name): if not self.allow_external: self._raise_error(403, 'external_not_allowed') user, collection = self.load_user_coll(coll_name=coll_name) self.access.assert_can_admin_coll(collection) if not collection.is_external(): self._raise_error(400, 'external_only') num_added = collection.add_cdxj(request.body.read()) return {'success': num_added} @self.app.post('/api/v1/collection/<coll_name>') @self.api(query=['user'], req=['title', 'desc', 'public', 'public_index'], resp='collection') def update_collection(coll_name): user, collection = self.load_user_coll(coll_name=coll_name) self.access.assert_can_admin_coll(collection) data = request.json or {} if 'title' in data: new_coll_title = data['title'] new_coll_name = self.sanitize_title(new_coll_title) if not new_coll_name: self._raise_error(400, 'invalid_coll_name') try: new_coll_name = user.colls.rename(collection, new_coll_name, allow_dupe=False) except DupeNameException as de: self._raise_error(400, 'duplicate_name') collection['title'] = new_coll_title if 'desc' in data: collection['desc'] = data['desc'] # TODO: notify the user if this is a request from the admin panel if 'public' in data: #if self.access.is_superuser() and data.get('notify'): # pass collection.set_public(data['public']) if 'public_index' in data: collection.set_bool_prop('public_index', data['public_index']) collection.mark_updated() return {'collection': collection.serialize()} @self.app.get('/api/v1/collection/<coll_name>/page_bookmarks') @self.api(query=['user'], resp='bookmarks') def get_page_bookmarks(coll_name): user, collection = self.load_user_coll(coll_name=coll_name) rec = request.query.get('rec') if rec: recording = collection.get_recording(rec) if not recording: return {'page_bookmarks': {}} rec_pages = collection.list_rec_pages(recording) else: rec_pages = None return {'page_bookmarks': collection.get_all_page_bookmarks(rec_pages)} @self.app.get('/api/v1/url_search') def do_url_search(): user, collection = self.load_user_coll() results = [] search = request.query.getunicode('search', '').lower() url_query = request.query.getunicode('url', '').lower() has_query = search or url_query ts_from = request.query.getunicode('from') ts_to = request.query.getunicode('to') date_filter = ts_from and ts_to if date_filter: try: ts_from = int(ts_from) ts_to = int(ts_to) except ValueError: date_filter = False session = request.query.getunicode('session') # remove trailing comma, mimes = request.query.getunicode('mime', '').rstrip(',') mimes = mimes.split(',') if mimes else [] # search pages or default to page search if no mime supplied if 'text/html' in mimes or len(mimes) == 0: try: mimes.remove('text/html') except ValueError: pass # shortcut empty search if not has_query and not date_filter and not session: results = collection.list_pages() else: for page in collection.list_pages(): # check for legacy hidden flag if page.get('hidden', False): continue if date_filter: try: # trim seconds ts = int(page['timestamp'][:12]) except ValueError: continue if ts < ts_from or ts > ts_to: continue if session and page['rec'] != session: continue if search and search not in page.get('title', '').lower(): continue if url_query and url_query not in page['url'].lower(): continue results.append(page) # search non-page cdx if len(mimes): for line, _ in collection.get_cdxj_iter(): cdxj = CDXObject(line.encode('utf-8')) if date_filter: try: # trim seconds ts = int(cdxj['timestamp'][:12]) except ValueError: continue if ts < ts_from or ts > ts_to: continue if search and search not in cdxj['url'].lower(): continue if url_query and url_query not in cdxj['url'].lower(): continue if mimes and not any(cdxj['mime'].startswith(mime) for mime in mimes): continue results.append({'url': cdxj['url'], 'timestamp': cdxj['timestamp'], 'mime': cdxj['mime']}) return {'results': results} @self.app.get('/api/v1/text_search') def do_text_search(): if not self.solr_mgr: self._raise_error(400, 'not_supported') user, collection = self.load_user_coll() return self.solr_mgr.query_solr(collection.my_id, request.query) # DAT @self.app.post('/api/v1/collection/<coll_name>/dat/share') def dat_do_share(coll_name): user, collection = self.load_user_coll(coll_name=coll_name) # BETA only self.require_admin_beta_access(collection) try: data = request.json or {} result = DatShare.dat_share.share(collection, data.get('always_update', False)) except Exception as e: result = {'error': 'api_error', 'details': str(e)} if 'error' in result: self._raise_error(400, result['error']) return result @self.app.post('/api/v1/collection/<coll_name>/dat/unshare') def dat_do_unshare(coll_name): user, collection = self.load_user_coll(coll_name=coll_name) # BETA only self.require_admin_beta_access(collection) try: result = DatShare.dat_share.unshare(collection) except Exception as e: result = {'error': 'api_error', 'details': str(e)} if 'error' in result: self._raise_error(400, result['error']) return result @self.app.post('/api/v1/collection/<coll_name>/commit') def commit_file(coll_name): user, collection = self.load_user_coll(coll_name=coll_name) self.access.assert_can_admin_coll(collection) data = request.json or {} res = collection.commit_all(data.get('commit_id')) if not res: return {'success': True} else: return {'commit_id': res} @self.app.post('/api/v1/collection/<coll_name>/generate_derivs') def generate_derivs(coll_name): user, collection = self.load_user_coll(coll_name=coll_name) self.access.assert_can_admin_coll(collection) if not self.is_search_auto: self._raise_error(400, 'not_supported') title = 'Derivates Regenerated on ' + datetime.datetime.now().isoformat() derivs_recording = collection.create_recording(title=title, rec_type='derivs') res = collection.requeue_pages_for_derivs(derivs_recording.my_id, get_bool(request.query.get('include_existing'))) if res > 0: collection.set_bool_prop('autoindexed', True) return {'queued': res} # LEGACY ENDPOINTS (to remove) # Collection view (all recordings) @self.app.get(['/<user>/<coll_name>', '/<user>/<coll_name>/']) @self.jinja2_view('collection_info.html') def coll_info(user, coll_name): return self.get_collection_info_for_view(user, coll_name) @self.app.get(['/<user>/<coll_name>/<rec_list:re:([\w,-]+)>', '/<user>/<coll_name>/<rec_list:re:([\w,-]+)>/']) @self.jinja2_view('collection_info.html') def coll_info(user, coll_name, rec_list): #rec_list = [self.sanitize_title(title) for title in rec_list.split(',')] return self.get_collection_info_for_view(user, coll_name) wr_api_spec.set_curr_tag(None)
def init_routes(self): wr_api_spec.set_curr_tag('Auth') # USER CHECKS @self.app.get('/api/v1/auth/check_username/<username>') def test_username(username): """async precheck username availability on signup form""" if self.user_manager.is_username_available(username): return {'success': True} else: return self._raise_error(400, 'username_not_available') # GET CURRENT USER @self.app.get('/api/v1/auth/curr_user') def load_user(): return self.load_user() # AUTH NEW SESSION @self.app.post('/api/v1/auth/anon_user') def new_auth(): return self.new_auth() # REGISTRATION @self.app.post('/api/v1/auth/register') def api_register_user(): data = request.json or {} msg, redir_extra = self.user_manager.register_user(data, self.get_host()) if 'success' in msg: return msg response.status = 400 return {'errors': msg} @self.app.post('/api/v1/auth/validate') def api_validate_reg_user(): data = request.json or {} reg = data.get('reg') cookie = request.environ.get('webrec.request_cookie', '') username = request.query.getunicode('username') result = self.user_manager.validate_registration(reg, cookie, username) if 'error' in result or 'errors' in result: response.status = 400 return result # LOGIN @self.app.post('/api/v1/auth/login') def login(): """Authenticate users""" if not self.access.is_anon(): return self._raise_error(403, 'already_logged_in') include_colls = get_bool(request.query.get('include_colls', False)) result = self.user_manager.login_user(request.json or {}) if 'success' in result: data = {'user': self.access.session_user.serialize(include_colls)} if result.get('new_coll_name'): data['new_coll_name'] = result['new_coll_name'] return data #self._raise_error(401, result.get('error', '')) response.status = 401 return result @self.app.post('/api/v1/auth/logout') def logout(): self.get_user_or_raise() self.user_manager.logout() data = {'success': 'logged_out'} return data @self.app.post('/api/v1/auth/ensure_login') def ensure_login(): return self.ensure_login() # PASSWORD @self.app.post('/api/v1/auth/password/reset_request') def request_reset_password(): data = request.json or {} email = data.get('email', '') username = data.get('username', '') host = self.get_host() try: self.user_manager.cork.send_password_reset_email( username=username, email_addr=email, subject='webrecorder.io password reset confirmation', email_template='webrecorder/templates/emailreset.html', host=host) return {'success': True} except Exception as e: import traceback traceback.print_exc() self._raise_error(404, 'no_such_user') @self.app.post('/api/v1/auth/password/reset') def reset_password(): #self.get_user_or_raise() if not self.access.is_anon(): return self._raise_error(403, 'already_logged_in') data = request.json or {} password = data.get('newPass', '') confirm_password = data.get('newPass2', '') reset_code = data.get('resetCode', '') try: self.user_manager.reset_password(password, confirm_password, reset_code) return {'success': True} except ValidationException as ve: self._raise_error(403, str(ve)) @self.app.post('/api/v1/auth/password/update') def update_password(): self.get_user_or_raise() data = request.json or {} curr_password = data.get('currPass', '') password = data.get('newPass', '') confirm_password = data.get('newPass2', '') try: self.user_manager.update_password(curr_password, password, confirm_password) return {'success': True} except ValidationException as ve: return self._raise_error(403, str(ve)) # Skip POST request recording @self.app.post('/api/v1/auth/skipreq') def skip_req(): data = request.json or {} url = data.get('url', '') self.access.session_user.mark_skip_url(url) return {'success': True} # USER API wr_api_spec.set_curr_tag('Users') @self.app.get('/api/v1/user/<username>') def api_get_user(username): """API enpoint to return user info""" return self.load_user(username) @self.app.delete('/api/v1/user/<username>') def api_delete_user(username): """API enpoint to delete a user""" self.get_user_or_raise(username, 404, 'not_found') # TODO? add validation #self.validate_csrf() try: assert(self.user_manager.delete_user(username)) request.environ['webrec.delete_all_cookies'] = 'all' except: #return {'error_message': 'Could not delete user: '******'error_deleting') data = {'deleted_user': username} return data @self.app.post('/api/v1/user/<username>') def update_user(username): user = self.get_user(user=username) data = request.json or {} if 'desc' in data: user['desc'] = data['desc'] if 'full_name' in data: user['full_name'] = data['full_name'][:150] if 'display_url' in data: user['display_url'] = data['display_url'][:500] return {'success': True} # OLD VIEWS BELOW # ==================================================================== @self.app.get(['/<username>', '/<username>/']) @self.jinja2_view('user.html') def user_info(username): self.redir_host() user = self.get_user(user=username) if self.access.is_anon(user): self.redirect('/' + user.my_id + '/temp') result = { 'user': user.name, 'user_info': user.serialize(), 'collections': [coll.serialize() for coll in user.get_collections()], } if not result['user_info'].get('desc'): result['user_info']['desc'] = self.default_user_desc.format(user) return result
def init_routes(self): # LISTS wr_api_spec.set_curr_tag('Lists') @self.app.get('/api/v1/lists') @self.api(query=['user', 'coll', 'include_bookmarks'], resp='lists') def get_lists(): user, collection = self.load_user_coll() include_bookmarks = request.query.getunicode( 'include_bookmarks') or 'all' lists = collection.get_lists() return { 'lists': [ blist.serialize(include_bookmarks=include_bookmarks) for blist in lists ] } @self.app.post('/api/v1/lists') @self.api(query=['user', 'coll', 'include_bookmarks'], req_desc='List properties', req=['title', 'desc', 'public', 'before_id'], resp='list') def add_list(): user, collection = self.load_user_coll() blist = collection.create_bookmark_list(request.json) collection.mark_updated() return {'list': blist.serialize()} @self.app.get('/api/v1/list/<list_id>') @self.api(query=['user', 'coll', 'include_bookmarks'], resp='list') def get_list(list_id): user, collection, blist = self.load_user_coll_list(list_id) self.access.assert_can_read_list(blist) include_bookmarks = request.query.getunicode( 'include_bookmarks') or 'all' return { 'list': blist.serialize(check_slug=list_id, include_bookmarks=include_bookmarks) } @self.app.post('/api/v1/list/<list_id>') @self.api(query=['user', 'coll'], req_desc='Update List properties', req=['title', 'desc', 'public'], resp='list') def update_list(list_id): user, collection, blist = self.load_user_coll_list(list_id) blist.update(request.json) blist.mark_updated() return {'list': blist.serialize()} @self.app.delete('/api/v1/list/<list_id>') @self.api(query=['user', 'coll'], resp='deleted') def delete_list(list_id): user, collection, blist = self.load_user_coll_list(list_id) if collection.remove_list(blist): collection.mark_updated() return {'deleted_id': list_id} else: self._raise_error(400, 'error_deleting') @self.app.post('/api/v1/list/<list_id>/move') @self.api(query=['user', 'coll'], req=['before_id'], resp='success') def move_list_before(list_id): user, collection, blist = self.load_user_coll_list(list_id) before_id = request.json.get('before_id') if before_id: before = self.load_list(collection, before_id) else: before = None blist.mark_updated() collection.move_list_before(blist, before) return {'success': True} @self.app.post('/api/v1/lists/reorder') @self.api(query=['user', 'coll'], req_desc='An array of existing list ids in a new order', req=['order'], resp='success') def reorder_lists(): user, collection = self.load_user_coll() new_order = request.json.get('order', []) if collection.lists.reorder_objects(new_order): collection.mark_updated() return {'success': True} else: return self._raise_error(400, 'invalid_order') #BOOKMARKS wr_api_spec.set_curr_tag('Bookmarks') @self.app.post('/api/v1/list/<list_id>/bookmarks') @self.api(query=['user', 'coll'], req_desc='Bookmark properties', req=[ 'title', 'url', 'timestamp', 'browser', 'desc', 'page_id', 'before_id' ], resp='bookmark') def create_bookmark(list_id): user, collection, blist = self.load_user_coll_list(list_id) bookmark = blist.create_bookmark(request.json) if bookmark: blist.mark_updated() return {'bookmark': bookmark} else: return self._raise_error(400, 'invalid_page') @self.app.post('/api/v1/list/<list_id>/bulk_bookmarks') @self.api(query=['user', 'coll'], req_desc='List of Bookmarks', req={ 'type': 'array', 'item_type': [ 'title', 'url', 'timestamp', 'browser', 'desc', 'page_id', 'before_id' ] }, resp='list') def create_bookmarks(list_id): user, collection, blist = self.load_user_coll_list(list_id) bookmark_list = request.json blist.add_bookmarks(bookmark_list) return {'success': True} @self.app.get('/api/v1/list/<list_id>/bookmarks') @self.api(query=['user', 'coll'], resp='bookmarks') def get_bookmarks(list_id): user, collection, blist = self.load_user_coll_list(list_id) bookmarks = blist.get_bookmarks() return {'bookmarks': bookmarks} @self.app.get('/api/v1/bookmark/<bid>') @self.api(query=['user', 'coll', 'list'], resp='bookmark') def get_bookmark(bid): user, collection, blist = self.load_user_coll_list() bookmark = blist.get_bookmark(bid) return {'bookmark': bookmark} @self.app.post('/api/v1/bookmark/<bid>') @self.api(query=['user', 'coll', 'list'], req_desc='Bookmark properties', req=[ 'title', 'url', 'timestamp', 'browser', 'desc', 'page_id', 'before_id' ], resp='bookmark') def update_bookmark(bid): user, collection, blist = self.load_user_coll_list() bookmark = blist.update_bookmark(bid, request.json) blist.mark_updated() return {'bookmark': bookmark} @self.app.delete('/api/v1/bookmark/<bid>') @self.api(query=['user', 'coll', 'list'], resp='deleted') def delete_bookmark(bid): user, collection, blist = self.load_user_coll_list() if blist.remove_bookmark(bid): blist.mark_updated() return {'deleted_id': bid} else: self._raise_error(404, 'no_such_bookmark') @self.app.post('/api/v1/list/<list_id>/bookmarks/reorder') @self.api(query=['user', 'coll'], req_desc='An array of existing bookmark ids in a new order', req=['order'], resp='success') def reorder_bookmarks(list_id): user, collection, blist = self.load_user_coll_list(list_id) new_order = request.json.get('order', []) if blist.reorder_bookmarks(new_order): blist.mark_updated() return {'success': True} else: self._raise_error(400, 'invalid_order') wr_api_spec.set_curr_tag(None)
def init_routes(self): wr_api_spec.set_curr_tag('Collections') @self.app.post('/api/v1/collections') @self.api(query=['user'], req=['title', 'public', 'public_index'], resp='collection') def create_collection(): user = self.get_user(api=True, redir_check=False) data = request.json or {} title = data.get('title', '') coll_name = self.sanitize_title(title) if not coll_name: self._raise_error(400, 'invalid_coll_name') is_public = data.get('public', False) is_public_index = data.get('public_index', False) is_external = data.get('external', False) is_anon = self.access.is_anon(user) if is_external: if not self.allow_external: self._raise_error(403, 'external_not_allowed') #if not is_anon: # self._raise_error(400, 'not_valid_for_external') elif is_anon: if coll_name != 'temp': self._raise_error(400, 'invalid_temp_coll_name') if user.has_collection(coll_name): self._raise_error(400, 'duplicate_name') try: collection = user.create_collection(coll_name, title=title, desc='', public=is_public, public_index=is_public_index) if is_external: collection.set_external(True) user.mark_updated() self.flash_message('Created collection <b>{0}</b>!'.format(collection.get_prop('title')), 'success') resp = {'collection': collection.serialize()} except DupeNameException as de: self._raise_error(400, 'duplicate_name') except Exception as ve: print(ve) self.flash_message(str(ve)) self._raise_error(400, 'duplicate_name') return resp @self.app.get('/api/v1/collections') @self.api(query=['user', 'include_recordings', 'include_lists', 'include_pages'], resp='collections') def get_collections(): user = self.get_user(api=True, redir_check=False) kwargs = {'include_recordings': get_bool(request.query.get('include_recordings')), 'include_lists': get_bool(request.query.get('include_lists')), 'include_pages': get_bool(request.query.get('include_pages')), } collections = user.get_collections() return {'collections': [coll.serialize(**kwargs) for coll in collections]} @self.app.get('/api/v1/collection/<coll_name>') @self.api(query=['user'], resp='collection') def get_collection(coll_name): user = self.get_user(api=True, redir_check=False) return self.get_collection_info(coll_name, user=user) @self.app.delete('/api/v1/collection/<coll_name>') @self.api(query=['user'], resp='deleted') def delete_collection(coll_name): user, collection = self.load_user_coll(coll_name=coll_name) errs = user.remove_collection(collection, delete=True) if errs.get('error'): return self._raise_error(400, errs['error']) else: return {'deleted_id': coll_name} @self.app.put('/api/v1/collection/<coll_name>/warc') def add_external_warc(coll_name): if not self.allow_external: self._raise_error(403, 'external_not_allowed') user, collection = self.load_user_coll(coll_name=coll_name) self.access.assert_can_admin_coll(collection) if not collection.is_external(): self._raise_error(400, 'external_only') num_added = collection.add_warcs(request.json.get('warcs', {})) return {'success': num_added} @self.app.put('/api/v1/collection/<coll_name>/cdx') def add_external_cdxj(coll_name): if not self.allow_external: self._raise_error(403, 'external_not_allowed') user, collection = self.load_user_coll(coll_name=coll_name) self.access.assert_can_admin_coll(collection) if not collection.is_external(): self._raise_error(400, 'external_only') num_added = collection.add_cdxj(request.body.read()) return {'success': num_added} @self.app.post('/api/v1/collection/<coll_name>') @self.api(query=['user'], req=['title', 'desc', 'public', 'public_index'], resp='collection') def update_collection(coll_name): user, collection = self.load_user_coll(coll_name=coll_name) self.access.assert_can_admin_coll(collection) data = request.json or {} if 'title' in data: new_coll_title = data['title'] new_coll_name = self.sanitize_title(new_coll_title) if not new_coll_name: self._raise_error(400, 'invalid_coll_name') try: new_coll_name = user.colls.rename(collection, new_coll_name, allow_dupe=False) except DupeNameException as de: self._raise_error(400, 'duplicate_name') collection['title'] = new_coll_title if 'desc' in data: collection['desc'] = data['desc'] # TODO: notify the user if this is a request from the admin panel if 'public' in data: #if self.access.is_superuser() and data.get('notify'): # pass collection.set_public(data['public']) if 'public_index' in data: collection.set_bool_prop('public_index', data['public_index']) collection.mark_updated() return {'collection': collection.serialize()} @self.app.get('/api/v1/collection/<coll_name>/page_bookmarks') @self.api(query=['user'], resp='bookmarks') def get_page_bookmarks(coll_name): user, collection = self.load_user_coll(coll_name=coll_name) rec = request.query.get('rec') if rec: recording = collection.get_recording(rec) if not recording: return {'page_bookmarks': {}} rec_pages = collection.list_rec_pages(recording) else: rec_pages = None return {'page_bookmarks': collection.get_all_page_bookmarks(rec_pages)} # DAT @self.app.post('/api/v1/collection/<coll_name>/dat/share') def dat_do_share(coll_name): user, collection = self.load_user_coll(coll_name=coll_name) # BETA only self.require_admin_beta_access(collection) try: data = request.json or {} result = DatShare.dat_share.share(collection, data.get('always_update', False)) except Exception as e: result = {'error': 'api_error', 'details': str(e)} if 'error' in result: self._raise_error(400, result['error']) return result @self.app.post('/api/v1/collection/<coll_name>/dat/unshare') def dat_do_unshare(coll_name): user, collection = self.load_user_coll(coll_name=coll_name) # BETA only self.require_admin_beta_access(collection) try: result = DatShare.dat_share.unshare(collection) except Exception as e: result = {'error': 'api_error', 'details': str(e)} if 'error' in result: self._raise_error(400, result['error']) return result @self.app.post('/api/v1/collection/<coll_name>/commit') def commit_file(coll_name): user, collection = self.load_user_coll(coll_name=coll_name) self.access.assert_can_admin_coll(collection) data = request.json or {} res = collection.commit_all(data.get('commit_id')) if not res: return {'success': True} else: return {'commit_id': res} # LEGACY ENDPOINTS (to remove) # Collection view (all recordings) @self.app.get(['/<user>/<coll_name>', '/<user>/<coll_name>/']) @self.jinja2_view('collection_info.html') def coll_info(user, coll_name): return self.get_collection_info_for_view(user, coll_name) @self.app.get(['/<user>/<coll_name>/<rec_list:re:([\w,-]+)>', '/<user>/<coll_name>/<rec_list:re:([\w,-]+)>/']) @self.jinja2_view('collection_info.html') def coll_info(user, coll_name, rec_list): #rec_list = [self.sanitize_title(title) for title in rec_list.split(',')] return self.get_collection_info_for_view(user, coll_name)