def get(self): """ Given a long URL, returns a short URL. """ long_url = self.get_argument('longUrl', None) # Is decoded by Tornado. domain = self.get_argument('domain', self.settings['default_domain']) # Normalize and validate long_url. try: long_url = utils.normalize_url(long_url) assert utils.validate_url(long_url) == True except: logging.info('Wrong URL', exc_info=1) return self.finish({'status_code': 500, 'status_txt': 'INVALID_URI', 'data': []}) # Validate domain. if not utils.validate_url('http://' + domain): return self.finish({'status_code': 500, 'status_txt': 'INVALID_ARG_DOMAIN', 'data': []}) # Generate a unique hash, assemble short url and store result in Redis. url_hash = utils.get_hash_from_map(long_url) #check exist hash if do some customize is_exist = self.load_url(url_hash) if is_exist: url_hash = utils.generate_hash(self.application.redis, self.settings['redis_namespace'], self.settings['hash_salt']) short_url = 'http://' + domain + '/' + url_hash self.store_url(url_hash, long_url) # Return success response. data = {'long_url': long_url, 'url': short_url, 'hash': url_hash, 'global_hash': url_hash} self.finish({'status_code': 200, 'status_txt': 'OK', 'data': data})
def get(self): """ Given a long URL, returns a short URL. """ long_url = self.get_argument('longUrl', None) # decoded by Tornado. android_url = self.get_argument('androidUrl', None) # decoded by Tornado. android_fallback_url = self.get_argument('androidFallbackUrl', None) # decoded by Tornado. ios_url = self.get_argument('iosUrl', None) # decoded by Tornado. ios_fallback_url = self.get_argument('iosFallbackUrl', None) # decoded by Tornado. domain = self.get_argument('domain', self.settings['default_domain']) # Normalize and validate long_url. try: long_url = utils.normalize_url(long_url) assert utils.validate_url(long_url) is True if android_url: # TODO: Validate and normalize! pass # android_url = utils.normalize_url(android_url) # assert utils.validate_url(android_url) is True if android_fallback_url: android_fallback_url = utils.normalize_url(android_fallback_url) assert utils.validate_url(android_fallback_url) is True if ios_url: # TODO: Validate and normalize! pass # ios_url = utils.normalize_url(ios_url) # assert utils.validate_url(ios_url) is True if ios_fallback_url: ios_fallback_url = utils.normalize_url(ios_fallback_url) assert utils.validate_url(ios_fallback_url) is True except: logging.info('Wrong URL', exc_info=1) return self.finish({'status_code': 500, 'status_txt': 'INVALID_URI', 'data': []}) # Validate domain. if not utils.validate_url('http://' + domain): return self.finish( {'status_code': 500, 'status_txt': 'INVALID_ARG_DOMAIN', 'data': []}) # Generate a unique hash, assemble short url and store result in Redis. url_hash = utils.generate_hash(self.application.redis, self.settings['redis_namespace'], self.settings['hash_salt']) short_url = 'http://' + domain + '/' + url_hash self.store_url(url_hash, long_url, android_url, android_fallback_url, ios_url, ios_fallback_url) # Return success response. data = { 'long_url': long_url, 'android_url': android_url, 'android_fallback_url': android_fallback_url, 'ios_url': ios_url, 'ios_fallback_url': ios_fallback_url, 'url': short_url, 'hash': url_hash, 'global_hash': url_hash } self.finish({'status_code': 200, 'status_txt': 'OK', 'data': data})
def get(self): long_url = self.get_argument('longUrl', None) url_hash = self.get_argument('hash', None) try: long_url = utils.normalize_url(long_url) assert utils.validate_url(long_url) == True except: logging.info('Wrong URL', exc_info=1) return self.finish({ 'status_code': 500, 'status_txt': 'INVALID_URI', 'data': [] }) domain = self.get_argument('domain', self.settings['default_domain']) short_url = 'http://' + domain + '/' + url_hash self.store_url(url_hash, long_url) print 'xxxxx' data = { 'long_url': long_url, 'url': short_url, 'hash': url_hash, 'global_hash': url_hash } self.finish({'status_code': 200, 'status_txt': 'OK', 'data': data})
def test_ampersand_splitting(self): correct_result = 'https://www.youtube.com/watch?v=1Lfv5tUGsn8' self.assertEqual( validate_url('https://www.youtube.com/watch?v=1Lfv5tUGsn8' '&list=TLPQMTMwNDIwMjHlkFY9B3CuTQ&index=1'), correct_result) self.assertEqual( validate_url('https://www.youtube.com/watch?v=1Lfv5tUGsn8&'), correct_result) self.assertEqual( validate_url('https://www.youtube.com/watch?v=1Lfv5tUGsn8'), correct_result)
def get(self): """ Given a long URL, returns a short URL. """ long_url = self.get_argument('longUrl', None) # Is decoded by Tornado. domain = self.get_argument('domain', self.settings['default_domain']) # Normalize and validate long_url. try: long_url = utils.normalize_url(long_url) assert utils.validate_url(long_url) == True except: logging.info('Wrong URL', exc_info=1) return self.finish({ 'status_code': 500, 'status_txt': 'INVALID_URI', 'data': [] }) # Validate domain. if not utils.validate_url('http://' + domain): return self.finish({ 'status_code': 500, 'status_txt': 'INVALID_ARG_DOMAIN', 'data': [] }) # Generate a unique hash, assemble short url and store result in Redis. url_hash = utils.get_hash_from_map(long_url) #check exist hash if do some customize is_exist = self.load_url(url_hash) if is_exist: url_hash = utils.generate_hash(self.application.redis, self.settings['redis_namespace'], self.settings['hash_salt']) short_url = 'http://' + domain + '/' + url_hash self.store_url(url_hash, long_url) # Return success response. data = { 'long_url': long_url, 'url': short_url, 'hash': url_hash, 'global_hash': url_hash } self.finish({'status_code': 200, 'status_txt': 'OK', 'data': data})
def work_with_paths(paths): for path in paths: if validate_url(path): title, body = parse_article_from_url(path) text = title + '\n' + body else: with open(path) as f: text = f.read() people = person_extractor.extract_people(text) print(people)
def shorten_url(): long_url = long_url_from_request(request) validated_url = validate_url(long_url) if not validated_url: Submmit.add_bad_redirect(db) return jsonify('Not a valid URL') code = unique_code(db, validate_url) Submmit.long_short(db, code, validated_url) return jsonify('http://localhost:5000/' + code)
def write(path: str, longurl: str) -> None: """ Write an entry into Azure Table Storage that maps a URL path to a longer url (link shortening). """ # validation & setup utils.validate_url(longurl) azure_full_url = utils.create_azure_url("AZURE_WRITESHORTURL_URL", path, longurl=longurl) # GET request try: r = requests.get(azure_full_url) except: print(emoji.emojize(":x: ", use_aliases=True), end="", flush=True) typer.echo("Something went wrong when submitting GET request!") raise typer.Exit(-1) # write result utils.print_response_status(r)
def _create_bs4_instance(self, source): is_url = validate_url(source) if is_url == True and self.source == None: content = self._get_content_from_web(source) source = content.text else: source = self.source if source == None: raise AttributeError("url or source required") self.html = BeautifulSoup(source, "html.parser")
def scrap_site(): try: url = request.args.get('url', '') if request.method == "GET" else request.get_json().get('url', '') valid_url = utils.validate_url(url) if not valid_url: return jsonify({ "status": 401, "data": "", "errors": ["Invalid URL"] }) site = requests.get(url) soup = bs(site.text, 'html.parser') results = miniseo.collect(soup) return jsonify({ "status": 200, "data": results, "errors": [] }) except Exception as e: print(e) return jsonify({ "status": 500, "data": "", "errors": ["Server error"] })
def get(self): long_url = self.get_argument('longUrl', None) url_hash = self.get_argument('hash', None) try: long_url = utils.normalize_url(long_url) assert utils.validate_url(long_url) == True except: logging.info('Wrong URL', exc_info=1) return self.finish({'status_code': 500, 'status_txt': 'INVALID_URI', 'data': []}) domain = self.get_argument('domain', self.settings['default_domain']) short_url = 'http://' + domain + '/' + url_hash self.store_url(url_hash, long_url) print 'xxxxx' data = {'long_url': long_url, 'url': short_url, 'hash': url_hash, 'global_hash': url_hash} self.finish({'status_code': 200, 'status_txt': 'OK', 'data': data})
def add_bookmark(db): """Add bookmark to database.""" category = request.forms.get("category") title = request.forms.get("title") url = request.forms.get("url") if title != "" and url != "": url = validate_url(url) if url: if category == "": category = "uncategorized" new_bookmark = Bookmark(category, title, url) db.add(new_bookmark) db.commit() message = "Added!" else: message = "Error: Url seems invalid." else: message = "Error: Title and Url are required." bookmarks = db.query(Bookmark.category).all() return template("index", categories=bookmarks, alert_message=message)
async def url_shortener(request): """ View to that recieves an url and returns a smaller version of it """ if request.method == 'POST': # get data from post data = await request.json() # validate structure of the json contained by the request if 'url' not in data.keys(): raise HTTPBadRequest(reason='url not in body') url = data['url'] # validate the provide url if not validate_url(url): raise HTTPBadRequest(reason='wrong url format') _url = get_or_create_url(url) # generate response return web.json_response( data={ 'status': 201, 'url': f'http://localhost:8000/sht/{_url.hash}' })
def handle_add_event(): """ Validate and add event, flash and redirect to frontpage """ # TODO Aikaperusteinen hidaste tapahtumien lisäyksen spämmäämiseen if request.method == 'GET': flash(u'Käytä lomaketta') return redirect(url_for('handle_list_events')) # Get data from form add_short_name = strip_tags(request.form['short_name'].strip()) add_name = strip_tags(request.form['name'].strip()) add_url = request.form['url'].strip().lower() add_start_time = strip_tags(request.form['start_time'].strip()) add_end_time = strip_tags(request.form['end_time'].strip()) ayah_session_secret = strip_tags(request.form['session_secret'].strip()) # Setup Are You A Human check ayah.configure(app.config['ARE_YOU_HUMAN_PUBLISHER_KEY'], app.config['ARE_YOU_HUMAN_SCORING_KEY']) # Validate short name if is_valid_short_name(add_short_name) != True: flash(u'Tapahtuman lyhytnimi ei ole kelvollinen. Tarkista tiedot ja yritä uudelleen.') return redirect(url_for('handle_list_events')) # Parse times and validate them add_start_time = parse_date(add_start_time) add_end_time = parse_date(add_end_time) if add_start_time == False: flash(u'Tapahtuman alkuaika ei ole kelvollinen. Tarkista tiedot ja yritä uudelleen.') return redirect(url_for('handle_list_events')) if add_end_time == False: flash(u'Tapahtuman päättymisaika ei ole kelvollinen. Tarkista tiedot ja yritä uudelleen.') return redirect(url_for('handle_list_events')) # Check that end time is later than start time if add_start_time >= add_end_time: flash(u'Tapahtuma ei voi päättyä ennen alkamistaan. Tarkista tiedot ja yritä uudelleen.') return redirect(url_for('handle_list_events')) # Check if event already exists with short name if event_exists(add_short_name): flash(u'Tapahtuman lyhytnimi on jo käytössä. Valitse toinen nimi tai poista olemassa oleva tapahtuma.') return redirect(url_for('handle_list_events')) # Call Are You A Human scoring service if ayah.score_result(ayah_session_secret) == False: flash(u'Spämmitarkistus epäonnistui. Tarkista että et ole spämmirobotti ja/tai yritä uudelleen.') return redirect(url_for('handle_list_events')) # Request URL and check it's valid add_url = validate_url(add_url) if add_url == False: flash(u'Tapahtuman URL-osoite ei ole kelvollinen. Tarkista tiedot ja yritä uudelleen.') return redirect(url_for('handle_list_events')) # TODO Spämmitarkistuksia lyhyt nimi, nimi ja URL. Domain/sanalistan mukaan pisteytys. # Add event to system if add_event(short_name=add_short_name, name=add_name, url=add_url, start_time=add_start_time, end_time=add_end_time) == False: flash( u'Tapahtuman lisäämisessä tapahtui mystinen virhe. Tarkista tiedot ja yritä uudelleen. Ongelman jatkuessa ota yhteyttä.') return redirect(url_for('handle_list_events')) else: secret_key = get_event_secret_key(add_short_name) flash( u'Tapahtuma lisättiin onnistuneesti. Tapahtuman salainen avain on %s. Pidä se tallessa esimerkiksi poistoa varten.' % secret_key) return redirect(url_for('handle_list_events'))
def get(self): """ Given a long URL, returns a short URL. """ long_url = self.get_argument('longUrl', None) # decoded by Tornado. android_url = self.get_argument('androidUrl', None) # decoded by Tornado. android_fallback_url = self.get_argument('androidFallbackUrl', None) # decoded by Tornado. ios_url = self.get_argument('iosUrl', None) # decoded by Tornado. ios_fallback_url = self.get_argument('iosFallbackUrl', None) # decoded by Tornado. domain = self.get_argument('domain', self.settings['default_domain']) # Normalize and validate long_url. try: long_url = utils.normalize_url(long_url) assert utils.validate_url(long_url) is True if android_url: # TODO: Validate and normalize! pass # android_url = utils.normalize_url(android_url) # assert utils.validate_url(android_url) is True if android_fallback_url: android_fallback_url = utils.normalize_url( android_fallback_url) assert utils.validate_url(android_fallback_url) is True if ios_url: # TODO: Validate and normalize! pass # ios_url = utils.normalize_url(ios_url) # assert utils.validate_url(ios_url) is True if ios_fallback_url: ios_fallback_url = utils.normalize_url(ios_fallback_url) assert utils.validate_url(ios_fallback_url) is True except: logging.info('Wrong URL', exc_info=1) return self.finish({ 'status_code': 500, 'status_txt': 'INVALID_URI', 'data': [] }) # Validate domain. if not utils.validate_url('http://' + domain): return self.finish({ 'status_code': 500, 'status_txt': 'INVALID_ARG_DOMAIN', 'data': [] }) # Generate a unique hash, assemble short url and store result in Redis. url_hash = utils.generate_hash(self.application.redis, self.settings['redis_namespace'], self.settings['hash_salt']) short_url = 'http://' + domain + '/' + url_hash self.store_url(url_hash, long_url, android_url, android_fallback_url, ios_url, ios_fallback_url) # Return success response. data = { 'long_url': long_url, 'android_url': android_url, 'android_fallback_url': android_fallback_url, 'ios_url': ios_url, 'ios_fallback_url': ios_fallback_url, 'url': short_url, 'hash': url_hash, 'global_hash': url_hash } self.finish({'status_code': 200, 'status_txt': 'OK', 'data': data})
def youtube_lookup(url): return YouTube( utils.validate_url(url) ).streams.filter(**VIDEO_SETTINGS).first().url
def clean_github(self): data = self.cleaned_data['github'] validate_url(data, 'github.com') return data
def clean_devpost(self): data = self.cleaned_data['devpost'] validate_url(data, 'devpost.com') return data
def prepare_asset(request): data = request.POST or request.FORM or {} if 'model' in data: data = json.loads(data['model']) def get(key): val = data.get(key, '') return val.strip() if isinstance(val, basestring) else val if all([get('name'), get('uri') or (request.files.file_upload != ""), get('mimetype')]): asset = { 'name': get('name'), 'mimetype': get('mimetype'), 'asset_id': get('asset_id'), 'is_enabled': get('is_enabled'), 'nocache': get('nocache'), } uri = get('uri') or False if not asset['asset_id']: asset['asset_id'] = uuid.uuid4().hex try: file_upload = request.files.file_upload filename = file_upload.filename except AttributeError: file_upload = None filename = None if filename and 'web' in asset['mimetype']: raise Exception("Invalid combination. Can't upload a web resource.") if uri and filename: raise Exception("Invalid combination. Can't select both URI and a file.") if uri and not uri.startswith('/'): if not validate_url(uri): raise Exception("Invalid URL. Failed to add asset.") else: asset['uri'] = uri else: asset['uri'] = uri if filename: asset['uri'] = path.join(settings['assetdir'], asset['asset_id']) with open(asset['uri'], 'w') as f: while True: chunk = file_upload.file.read(1024) if not chunk: break f.write(chunk) if "video" in asset['mimetype']: video_duration = get_video_duration(asset['uri']) if video_duration: asset['duration'] = int(video_duration.total_seconds()) else: asset['duration'] = 'N/A' else: # Crashes if it's not an int. We want that. asset['duration'] = int(get('duration')) if get('start_date'): asset['start_date'] = datetime.strptime(get('start_date').split(".")[0], "%Y-%m-%dT%H:%M:%S") else: asset['start_date'] = "" if get('end_date'): asset['end_date'] = datetime.strptime(get('end_date').split(".")[0], "%Y-%m-%dT%H:%M:%S") else: asset['end_date'] = "" if not asset['asset_id']: raise Exception if not asset['uri']: raise Exception return asset else: raise Exception("Not enough information provided. Please specify 'name', 'uri', and 'mimetype'.")
def prepare_asset(request): req = Request(request.environ) data = None data = json.loads(req.form['model']) if 'model' in req.form else req.form def get(key): val = data.get(key, '') if isinstance(val, unicode): return val.strip() elif isinstance(val, basestring): return val.strip().decode('utf-8') else: return val if all([get('name'), get('uri') or req.files.get('file_upload'), get('mimetype')]): asset = { 'name': get('name'), 'mimetype': get('mimetype'), 'asset_id': get('asset_id'), 'is_enabled': get('is_enabled'), 'nocache': get('nocache'), } uri = get('uri') or False if not asset['asset_id']: asset['asset_id'] = uuid.uuid4().hex try: file_upload = req.files.get('file_upload') filename = file_upload.filename except AttributeError: file_upload = None filename = None if filename and 'web' in asset['mimetype']: raise Exception("Invalid combination. Can't upload a web resource.") if uri and filename: raise Exception("Invalid combination. Can't select both URI and a file.") if uri and not uri.startswith('/'): if not validate_url(uri): raise Exception("Invalid URL. Failed to add asset.") else: asset['uri'] = uri else: asset['uri'] = uri if filename: asset['uri'] = path.join(settings['assetdir'], asset['asset_id']) file_upload.save(asset['uri']) if "video" in asset['mimetype']: video_duration = get_video_duration(asset['uri']) if video_duration: asset['duration'] = int(video_duration.total_seconds()) else: asset['duration'] = 'N/A' else: # Crashes if it's not an int. We want that. asset['duration'] = int(get('duration')) if get('start_date'): asset['start_date'] = datetime.strptime(get('start_date').split(".")[0], "%Y-%m-%dT%H:%M:%S") else: asset['start_date'] = "" if get('end_date'): asset['end_date'] = datetime.strptime(get('end_date').split(".")[0], "%Y-%m-%dT%H:%M:%S") else: asset['end_date'] = "" if not asset['asset_id']: raise Exception if not asset['uri']: raise Exception return asset else: raise Exception("Not enough information provided. Please specify 'name', 'uri', and 'mimetype'.")
def prepare_asset(request): data = request.POST or request.FORM or {} if "model" in data: data = json.loads(data["model"]) def get(key): val = data.get(key, "") return val.strip() if isinstance(val, basestring) else val if all([get("name"), get("uri") or (request.files.file_upload != ""), get("mimetype")]): asset = { "name": get("name"), "mimetype": get("mimetype"), "asset_id": get("asset_id"), "is_enabled": get("is_enabled"), "nocache": get("nocache"), } uri = get("uri") or False if not asset["asset_id"]: asset["asset_id"] = uuid.uuid4().hex try: file_upload = request.files.file_upload filename = file_upload.filename except AttributeError: file_upload = None filename = None if filename and "web" in asset["mimetype"]: raise Exception("Invalid combination. Can't upload a web resource.") if uri and filename: raise Exception("Invalid combination. Can't select both URI and a file.") if uri and not uri.startswith("/"): if not validate_url(uri): raise Exception("Invalid URL. Failed to add asset.") else: asset["uri"] = uri else: asset["uri"] = uri if filename: asset["uri"] = path.join(settings["assetdir"], asset["asset_id"]) with open(asset["uri"], "w") as f: while True: chunk = file_upload.file.read(1024) if not chunk: break f.write(chunk) if "video" in asset["mimetype"]: video_duration = get_video_duration(asset["uri"]) if video_duration: asset["duration"] = int(video_duration.total_seconds()) else: asset["duration"] = "N/A" else: # Crashes if it's not an int. We want that. asset["duration"] = int(get("duration")) if get("start_date"): asset["start_date"] = datetime.strptime(get("start_date").split(".")[0], "%Y-%m-%dT%H:%M:%S") else: asset["start_date"] = "" if get("end_date"): asset["end_date"] = datetime.strptime(get("end_date").split(".")[0], "%Y-%m-%dT%H:%M:%S") else: asset["end_date"] = "" if not asset["asset_id"]: raise Exception if not asset["uri"]: raise Exception return asset else: raise Exception("Not enough information provided. Please specify 'name', 'uri', and 'mimetype'.")
def prepare_asset(request): data = request.POST or request.FORM or {} if 'model' in data: data = json.loads(data['model']) def get(key): val = data.get(key, '') if isinstance(val, unicode): return val.strip() elif isinstance(val, basestring): return val.strip().decode('utf-8') else: return val if all([ get('name'), get('uri') or (request.files.file_upload != ""), get('mimetype') ]): asset = { 'name': get('name'), 'mimetype': get('mimetype'), 'asset_id': get('asset_id'), 'is_enabled': get('is_enabled'), 'nocache': get('nocache'), } uri = get('uri') or False if not asset['asset_id']: asset['asset_id'] = uuid.uuid4().hex try: file_upload = request.files.file_upload filename = file_upload.filename except AttributeError: file_upload = None filename = None if filename and 'web' in asset['mimetype']: raise Exception( "Invalid combination. Can't upload a web resource.") if uri and filename: raise Exception( "Invalid combination. Can't select both URI and a file.") if uri and not uri.startswith('/'): if not validate_url(uri): raise Exception("Invalid URL. Failed to add asset.") else: asset['uri'] = uri else: asset['uri'] = uri if filename: asset['uri'] = path.join(settings['assetdir'], asset['asset_id']) with open(asset['uri'], 'w') as f: while True: chunk = file_upload.file.read(1024) if not chunk: break f.write(chunk) if "video" in asset['mimetype']: video_duration = get_video_duration(asset['uri']) if video_duration: asset['duration'] = int(video_duration.total_seconds()) else: asset['duration'] = 'N/A' else: # Crashes if it's not an int. We want that. asset['duration'] = int(get('duration')) if get('start_date'): asset['start_date'] = datetime.strptime( get('start_date').split(".")[0], "%Y-%m-%dT%H:%M:%S") else: asset['start_date'] = "" if get('end_date'): asset['end_date'] = datetime.strptime( get('end_date').split(".")[0], "%Y-%m-%dT%H:%M:%S") else: asset['end_date'] = "" if not asset['asset_id']: raise Exception if not asset['uri']: raise Exception return asset else: raise Exception( "Not enough information provided. Please specify 'name', 'uri', and 'mimetype'." )
# Infinite loop. logging.debug('Entering infinite loop.') while True: asset = scheduler.get_next_asset() logging.debug('got asset' + str(asset)) check_update() if asset is None: # The playlist is empty, go to sleep. logging.info('Playlist is empty. Going to sleep.') sleep(5) elif not url_fails(asset['uri']): logging.info('Showing asset %s.' % asset["name"]) watchdog() if "image" in asset["mimetype"]: img_uri = asset['uri'] if asset['nocache'] == 1 and validate_url(img_uri): img_uri += '?_nocache=' + str(time()) view_image(img_uri, asset["asset_id"], asset["duration"]) elif "video" in asset["mimetype"]: view_video(asset["uri"]) elif "web" in asset["mimetype"]: view_web(asset["uri"], asset["duration"]) else: print "Unknown MimeType, or MimeType missing" else: logging.info('Asset {0} is not available, skipping.'.format(asset['uri']))