def test_get_client_ip(self): request = self.factory.get('/app/') ip = Url.get_client_ip(request) self.assertEqual(ip, '127.0.0.1') request = self.factory.get('/app/', HTTP_X_FORWARDED_FOR="8.8.8.8") ip = Url.get_client_ip(request) self.assertEqual(ip, '8.8.8.8')
def index(): form = URLConverterForm() if form.validate_on_submit(): url = Url.query.filter_by(long_url=form.url.data).first() if url is None: url = Url(long_url=form.url.data) db.session.add(url) db.session.flush() url.short_url = encode_id(url.id) db.session.add(url) db.session.commit() form.url.data = request.base_url + url.short_url return render_template('index.html', form=form)
def handle(self): """ Handle TCP datagrams """ data = loads(self.request.recv(1024).strip()) print "Recebido de '%s': %s \n" % (self.client_address[0], data) try: obj = Url.objects.filter(title=data['hostname']).update( ip=data['ip'], urls=data['urls'], slug=data['customer']) if obj: self.request.sendall(dumps({'return': 'Successfully updated'})) else: obj = Url(title=data['hostname'], ip=data['ip'], urls=data['urls'], slug=data['customer']) obj.save() self.request.sendall(dumps({'return': 'Successfully created'})) except Url.DoesNotExist: obj = Url(title=data['hostname'], ip=data['ip'], urls=data['urls'], slug=data['customer']) obj.save() self.request.sendall(dumps({'return': 'Successfully created'}))
def dispatch_request(self) -> jsonify: url = request.values.get('url') if not url_validator(url): return jsonify({'error': 'Not a valid URL.'}), 400 # Bad request. status = 200 # OK. start = 0 while True: code = generate_code(url, start) obj = db.session.query(Url).filter_by(code=code).first() data = {'url': url, 'code': code} if not obj: obj = Url(**data) db.session.add(obj) db.session.commit() status = 201 # Created. if obj and obj.url != url: # Edge case scenario; # In order to avoid conflicts, increment starting index and try again. start += 1 continue break return jsonify(data), status
def add_url(): form = UrlForm() if form.validate_on_submit(): url = Url(original_url=form.original_url.data) db.session.add(url) db.session.commit() return redirect('/index') return render_template('add_url.html', title='Add Url', form=form)
def post(self): args = parser.parse_args() destination = args['url'] if destination is None: return { 'Error': 'url is a required parameter' }, 400 # The odds of two urls getting the same random hash are kinda low, given that # there are 62 ^ 5 possibilities. If it happens, let's just try again. while True: try: url = Url(destination) db.session.add(url) db.session.commit() return jsonify(url.json()) except exc.IntegrityError: db.session.rollback() except: return { 'Error': 'URL Creation failed' }
def test_expand_shortened_link(self): link = Url(self.key, self.test_url) db.session.add(link) db.session.commit() # Get /<key> r = self.client.get(self.key) self.assertEqual(r.status_code, 302) # Redirect to expanded link assert self.test_url and "redirected" in r.data
def shortened_link(url): if url_exists(url): link = Url.query.filter_by(url=url).first() # if the link is same as short link if link is not None: return current_app.config['BASE_LINK'] + link.random_code random_hash = random_hash_key(url) shortened = Url(random_hash, url) db.session.add(shortened) db.session.commit() return current_app.config['BASE_LINK'] + random_hash return None
def generate_short_link(): form = UrlForm() username = current_user if current_user.is_authenticated: us_link = Url(url_link=form.url_link.data, user_id=username.id) us_link.create_short_link(form.url_link.data) db.session.add(us_link) db.session.commit() else: us_link = Url(url_link=form.url_link.data) us_link.create_short_link(form.url_link.data) db.session.add(us_link) db.session.commit()
def add_node(request): ctx = {} provider = request.POST.get('new_provider') parent_id = request.POST.get('parent_id') parent_provider = request.POST.get('parent_provider') relation = request.POST.get('relation') relation_back = request.POST.get('relation_back') if request.method == 'POST' and relation != '': if provider == 'text': node = Node(user=request.user, text=request.POST.get('text')) node.save() make_relation(request.user, parent_id, parent_provider, node.id, provider, relation, relation_back) elif provider == 'url': url_text = request.POST.get('url') import hashlib m = hashlib.md5() m.update(url_text.encode('utf-8')) url_hash = m.hexdigest() try: title, image = get_url_info(url_text, url_hash) url = Url(user=request.user, url=url_text, name=title, image=image) url.url_hash = url_hash url.save() except: url = Url.objects.get(url_hash=url_hash) make_relation(request.user, parent_id, parent_provider, url_hash, provider, relation, relation_back) elif provider == 'file': # todo - add files new_path = os.path.join(parent_id, request.POST.get('name')) if os.path.isdir(parent_id) and os.path.isdir(new_path): os.mkdir(new_path) make_relation(request.user, parent_id, parent_provider, new_path, provider, relation, relation_back) return HttpResponseRedirect(request.META['HTTP_REFERER'])
def detail_url(short_url): url = Url.query.filter_by(short_url=short_url).first() form = EditURLForm(obj=url) if form.validate_on_submit(): new_short_url = form.short_url.data if Url.short_url_exists(new_short_url) and short_url != new_short_url: flash("Short url is already in use.") return redirect(url_for("main.detail_url", short_url=url.short_url)) url.short_url = new_short_url url.element_text = form.element_text.data return redirect(url_for("main.detail_url", short_url=url.short_url)) return render_template("detail_url.html", url=url, form=form)
def create_short_url(): json_data = request.get_json() if not json_data: return {"message": "No input data provided"}, 400 schema = UrlSchema() try: data = schema.load(json_data) letters = string.ascii_lowercase result_str = ''.join(random.choice(letters) for i in range(6)) link = Url(long_url=data['long_url'], short_url=result_str) db.session.add(link) db.session.commit() return jsonify( short_link="http://localhost:5000/{}".format(result_str)), 201 except ValidationError as err: return err.messages, 422
def test_set_info(self): url = 'www.google.com' request = self.factory.get('/app/') url_obj = Url() url_obj.set_info(request, url) url_obj.save() self.assertEqual(url_obj.long_url, url) self.assertEqual(url_obj.ip, '127.0.0.1') self.assertIsNotNone(url_obj.time) self.assertIsNotNone(url_obj.shortened_url)
def count_words_at_url(url, to_store_in_db=1): errors = [] resp = requests.get(url) count = len(resp.text.split()) try: result = Url(url=url, count=count) if (to_store_in_db): db.session.add(result) db.session.commit() print(result.id) return result.id return result except: errors.append("Unable to add url to database.") return {"error": errors}
def test_redirecting_speed_without_redis(self): for i in range(1000): url_obj = Url() url_obj.long_url = f'www.google.com/{i}' url_obj.time = datetime.now(pytz.utc) url_obj.shortened_url = url_obj.generate_id(6) url_obj.save() url_obj_500 = Url.objects.filter(long_url='www.google.com/500').first() start = process_time() for i in range(1000): response = self.client.get(f'/app/detail_without_redis/{url_obj_500.shortened_url}/') self.assertEqual(response.status_code, 302) self.assertEqual(response.url, 'https://www.google.com/500') end = process_time() print(f'Process time without Redis: {end - start}')
def post(self, name, url): """ POST request method for class GetUrlContents URL: '/url/<name>/<url>' :param name: Name used for identifying content in the system :param url: URL which content will be saved in the system :return: Task ID of the started task """ if not url.startswith("http"): url = "https://" + url new_url_object = Url(name=name.encode("utf-8"), url=url.encode("utf-8")) db.session.add(new_url_object) commit_to_database() url_schema = UrlSchema() result = get_data_from_url.delay(url_schema.dump(new_url_object)) return result.task_id, 200
def index(): form = UrlForm() link = Url() if form.validate_on_submit(): if form.submit.data: tmp = form.url_link.data if re.match( r'http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+', tmp): generate_short_link() else: flash('Invalid url') return render_template('index.html', form=form, link=link) elif form.submit2.data: pass link = Url.query.filter_by(url_link=form.url_link.data).all() return render_template( 'index.html', form=form, link=link, )
def index(): urls = Url.query.all() form = URLForm() if form.validate_on_submit(): full_url = form.full_url.data short_url = form.short_url.data if Url.short_url_exists(short_url): flash("Short url is already exist. Try another one.") return redirect(url_for("main.index")) new_url = Url(full_url=full_url, clicks=0, created=datetime.today()) element_text = new_url.check_url() if not element_text: flash("Check url!!! Failed request %s" % new_url.full_url) return redirect(url_for("main.index")) new_url.element_text = element_text db.session.add(new_url) db.session.commit() new_url.store_short_url(short_url) flash("Your url has been created successfully.") return redirect(url_for("main.index")) return render_template("index.html", urls=urls, form=form)
def handle(self): """ Handle TCP datagrams """ data = loads(self.request.recv(1024).strip()) print "Recebido de '%s': %s \n" % (self.client_address[0], data) try: obj = Url.objects.filter(title=data['hostname']).update(ip=data['ip'],urls=data['urls'],slug=data['customer']) if obj: self.request.sendall(dumps({'return':'Successfully updated'})) else: obj = Url(title=data['hostname'],ip=data['ip'],urls=data['urls'],slug=data['customer']) obj.save() self.request.sendall(dumps({'return':'Successfully created'})) except Url.DoesNotExist: obj = Url(title=data['hostname'],ip=data['ip'],urls=data['urls'],slug=data['customer']) obj.save() self.request.sendall(dumps({'return':'Successfully created'}))
def test_get_shorted_url(self): url_obj = Url() url_obj.shortened_url = 'AF45Dd' shortened_url = url_obj.get_shorted_url() self.assertEqual(shortened_url, f'localhost:8000/app/AF45Dd')
def new_url_deets(): url = url = Url(url="https://www.google.com/", count=313) return url
parser = argparse.ArgumentParser() parser.add_argument('-t', '--generate-test-data', action='store_true', help='if present, generate test data for db') parser.add_argument('-d', '--drop-tables', action='store_true', help='drop all exited tables') parser.add_argument('-v', '--verbose', action='store_true', help='turn on debug for flask app') args = parser.parse_args() if args.drop_tables: db.drop_all() db.create_all() if args.generate_test_data: habr_urls = [ 'https://habrahabr.ru/all/page{}/'.format(i) for i in range(1, 100) ] for url in habr_urls: url = Url(long_url=url) db.session.add(url) db.session.flush() url.short_url = encode_id(url.id) db.session.add(url) db.session.commit() app.run(debug=args.verbose)
def insert(short_id, origin_url, remote_addr): url = Url(short_link=short_id, long_link=origin_url, ip=remote_addr) db.session.add(url) db.session.commit()
def test_generate_id(self): size = 6 id = Url.generate_id(size) self.assertEqual(len(id), size)
def write_database(post_id, user_id, text, created_at, created_at_dt, reply_to_user_id, reply_to_scrname, reply_to_status_id, retweet_count, favorite_count, is_retweet, original_tweet_id, original_tweet_retweets, original_text, original_tweet_created_at, original_tweet_likes, original_author_id, original_author_scrname, polarity, polarity_val, tag_list, url_list, user_scrname, user_name, user_location, user_created, user_followers, user_friends, user_statuses, query): #print("starting db entry for postid = {}".format(post_id)) if query[4:7] == 'Sen': district = 'sen' district_name = query[2:7] dist_type = 2 else: district = query[4:6] district_name = query[2:6] dist_type = 1 #POST TABLE: #If tweet ID not already in database, add to Post table if db.session.query(Post).filter(Post.post_id == post_id).count() == 0: #print('adding post') #USER table #If User already in User table, update dynamic elements, associate with this post this_user = db.session.query(User).filter( User.user_id == user_id).first() if this_user != None: this_user.user_location = user_location this_user.user_followers = user_followers this_user.user_friends = user_friends this_user.user_statuses = user_statuses db.session.add(this_user) #Otherise, add User to user table, associate with this post else: this_user = User(user_id, user_scrname, user_name, user_location,\ user_created, user_followers, user_friends, user_statuses) db.session.add(this_user) #POST table new_post = Post(post_id, user_id, text, created_at, created_at_dt, reply_to_user_id, reply_to_scrname, reply_to_status_id, retweet_count, favorite_count, is_retweet, original_tweet_id, original_tweet_retweets, original_text, original_tweet_created_at, original_tweet_likes, original_author_id, original_author_scrname, polarity, polarity_val) db.session.add(new_post) #If original tweet is in database, update its retweeted count. If not, do nothing if original_tweet_id != None: orig_tweet = db.session.query(Post).\ filter(Post.post_id == original_tweet_id).first() if orig_tweet != None: orig_tweet.retweet_count = original_tweet_retweets db.session.add(orig_tweet) #HASHTAG TABLE # If tweet is being added, iterate through tag/url list, and create a # Hashtag/Url table row for each tag for item in tag_list: #If hashtag is not already in Hashtag table, create new row hash_search = db.session.query(Hashtag).\ filter(Hashtag.hashtag == item).first() if hash_search == None: new_hashtag = Hashtag(item) db.session.add(new_hashtag) else: new_hashtag = hash_search #Add association to posthash_assoc_table new_post.hashtags.append(new_hashtag) #db.session.add(posthash_assoc.hashtag) #add one row to Post_extended per hashtag # NOTE: this means number of rows per post_id = cartesian product # of hashtags times districts (if picked up in search for every dist) new_row = Post_extended( post_id, user_id, created_at, created_at_dt, retweet_count, is_retweet, original_tweet_id, original_text, original_tweet_created_at, original_author_id, original_author_scrname, polarity, polarity_val, item, district_name, dist_type, user_scrname) db.session.add(new_row) #print("added newrow for hash {}".format(item)) #DISTRICT TABLE #capture District_id from 1st query term: state = query[2:4].lower() #Handle Senate districts differently than congressional # if query[4:7] == 'Sen': # district = 'sen' # district_name = query[2:7] # dist_type = 2 # else: # district = query[4:6] # district_name = query[2:6] # dist_type = 1 #Check if district is in DB, add if not district_search = db.session.query(District).\ filter(District.district_name == district_name).first() if district_search == None: new_district = District(state, district, district_name, dist_type) db.session.add(new_district) else: new_district = district_search #Add association to postdist_assoc_table new_post.districts.append(new_district) #URL TABLE #if URLS exist, add to db if len(url_list) > 0: for item in url_list: url_search = db.session.query(Url).filter( Url.url == item).first() if url_search == None: new_url = Url(item) db.session.add(new_url) else: new_url = url_search #Add association to postDistAssoc_table new_post.urls.append(new_url) #associate user with post this_user.user_posts.append(new_post) #If tweet ID in db (from another dist query), add new association to Post table else: #print('ID there, trying plan B') district_check = db.session.query(District.district_name).\ join(Post.districts).\ filter(Post.post_id==post_id).all() check = 0 for result in district_check: #iterate through associated dists if result[0] == district_name: check = 1 #print("already there") #if find match, check = 1, do nothing if check == 0: #print("adding newdist") # if no match, add to postdist_assoc sql_command = '''INSERT INTO postdist_assoc (post_id, district_name) VALUES (post_id, district_name);''' conn = db.engine.connect() conn.execute(sql_command) conn.close()