def test_get_client_ip(self):
request = self.factory.get('/app/')
ip = Url.get_client_ip(request)
self.assertEqual(ip, '127.0.0.1')
request = self.factory.get('/app/', HTTP_X_FORWARDED_FOR="8.8.8.8")
ip = Url.get_client_ip(request)
self.assertEqual(ip, '8.8.8.8')
def index():
form = URLConverterForm()
if form.validate_on_submit():
url = Url.query.filter_by(long_url=form.url.data).first()
if url is None:
url = Url(long_url=form.url.data)
db.session.add(url)
db.session.flush()
url.short_url = encode_id(url.id)
db.session.add(url)
db.session.commit()
form.url.data = request.base_url + url.short_url
return render_template('index.html', form=form)
def handle(self):
"""
Handle TCP datagrams
"""
data = loads(self.request.recv(1024).strip())
print "Recebido de '%s': %s \n" % (self.client_address[0], data)
try:
obj = Url.objects.filter(title=data['hostname']).update(
ip=data['ip'], urls=data['urls'], slug=data['customer'])
if obj:
self.request.sendall(dumps({'return': 'Successfully updated'}))
else:
obj = Url(title=data['hostname'],
ip=data['ip'],
urls=data['urls'],
slug=data['customer'])
obj.save()
self.request.sendall(dumps({'return': 'Successfully created'}))
except Url.DoesNotExist:
obj = Url(title=data['hostname'],
ip=data['ip'],
urls=data['urls'],
slug=data['customer'])
obj.save()
self.request.sendall(dumps({'return': 'Successfully created'}))
def dispatch_request(self) -> jsonify:
url = request.values.get('url')
if not url_validator(url):
return jsonify({'error': 'Not a valid URL.'}), 400 # Bad request.
status = 200 # OK.
start = 0
while True:
code = generate_code(url, start)
obj = db.session.query(Url).filter_by(code=code).first()
data = {'url': url, 'code': code}
if not obj:
obj = Url(**data)
db.session.add(obj)
db.session.commit()
status = 201 # Created.
if obj and obj.url != url:
# Edge case scenario;
# In order to avoid conflicts, increment starting index and try again.
start += 1
continue
break
return jsonify(data), status
def add_url():
form = UrlForm()
if form.validate_on_submit():
url = Url(original_url=form.original_url.data)
db.session.add(url)
db.session.commit()
return redirect('/index')
return render_template('add_url.html', title='Add Url', form=form)
def post(self):
args = parser.parse_args()
destination = args['url']
if destination is None:
return { 'Error': 'url is a required parameter' }, 400
# The odds of two urls getting the same random hash are kinda low, given that
# there are 62 ^ 5 possibilities. If it happens, let's just try again.
while True:
try:
url = Url(destination)
db.session.add(url)
db.session.commit()
return jsonify(url.json())
except exc.IntegrityError:
db.session.rollback()
except:
return { 'Error': 'URL Creation failed' }
def test_expand_shortened_link(self):
link = Url(self.key, self.test_url)
db.session.add(link)
db.session.commit()
# Get /<key>
r = self.client.get(self.key)
self.assertEqual(r.status_code, 302)
# Redirect to expanded link
assert self.test_url and "redirected" in r.data
def shortened_link(url):
if url_exists(url):
link = Url.query.filter_by(url=url).first()
# if the link is same as short link
if link is not None:
return current_app.config['BASE_LINK'] + link.random_code
random_hash = random_hash_key(url)
shortened = Url(random_hash, url)
db.session.add(shortened)
db.session.commit()
return current_app.config['BASE_LINK'] + random_hash
return None
def generate_short_link():
form = UrlForm()
username = current_user
if current_user.is_authenticated:
us_link = Url(url_link=form.url_link.data, user_id=username.id)
us_link.create_short_link(form.url_link.data)
db.session.add(us_link)
db.session.commit()
else:
us_link = Url(url_link=form.url_link.data)
us_link.create_short_link(form.url_link.data)
db.session.add(us_link)
db.session.commit()
def add_node(request):
ctx = {}
provider = request.POST.get('new_provider')
parent_id = request.POST.get('parent_id')
parent_provider = request.POST.get('parent_provider')
relation = request.POST.get('relation')
relation_back = request.POST.get('relation_back')
if request.method == 'POST' and relation != '':
if provider == 'text':
node = Node(user=request.user, text=request.POST.get('text'))
node.save()
make_relation(request.user, parent_id, parent_provider, node.id,
provider, relation, relation_back)
elif provider == 'url':
url_text = request.POST.get('url')
import hashlib
m = hashlib.md5()
m.update(url_text.encode('utf-8'))
url_hash = m.hexdigest()
try:
title, image = get_url_info(url_text, url_hash)
url = Url(user=request.user,
url=url_text,
name=title,
image=image)
url.url_hash = url_hash
url.save()
except:
url = Url.objects.get(url_hash=url_hash)
make_relation(request.user, parent_id, parent_provider, url_hash,
provider, relation, relation_back)
elif provider == 'file':
# todo - add files
new_path = os.path.join(parent_id, request.POST.get('name'))
if os.path.isdir(parent_id) and os.path.isdir(new_path):
os.mkdir(new_path)
make_relation(request.user, parent_id, parent_provider, new_path,
provider, relation, relation_back)
return HttpResponseRedirect(request.META['HTTP_REFERER'])
def detail_url(short_url):
url = Url.query.filter_by(short_url=short_url).first()
form = EditURLForm(obj=url)
if form.validate_on_submit():
new_short_url = form.short_url.data
if Url.short_url_exists(new_short_url) and short_url != new_short_url:
flash("Short url is already in use.")
return redirect(url_for("main.detail_url",
short_url=url.short_url))
url.short_url = new_short_url
url.element_text = form.element_text.data
return redirect(url_for("main.detail_url", short_url=url.short_url))
return render_template("detail_url.html", url=url, form=form)
def create_short_url():
json_data = request.get_json()
if not json_data:
return {"message": "No input data provided"}, 400
schema = UrlSchema()
try:
data = schema.load(json_data)
letters = string.ascii_lowercase
result_str = ''.join(random.choice(letters) for i in range(6))
link = Url(long_url=data['long_url'], short_url=result_str)
db.session.add(link)
db.session.commit()
return jsonify(
short_link="http://localhost:5000/{}".format(result_str)), 201
except ValidationError as err:
return err.messages, 422
def test_set_info(self):
url = 'www.google.com'
request = self.factory.get('/app/')
url_obj = Url()
url_obj.set_info(request, url)
url_obj.save()
self.assertEqual(url_obj.long_url, url)
self.assertEqual(url_obj.ip, '127.0.0.1')
self.assertIsNotNone(url_obj.time)
self.assertIsNotNone(url_obj.shortened_url)
def count_words_at_url(url, to_store_in_db=1):
errors = []
resp = requests.get(url)
count = len(resp.text.split())
try:
result = Url(url=url, count=count)
if (to_store_in_db):
db.session.add(result)
db.session.commit()
print(result.id)
return result.id
return result
except:
errors.append("Unable to add url to database.")
return {"error": errors}
def test_redirecting_speed_without_redis(self):
for i in range(1000):
url_obj = Url()
url_obj.long_url = f'www.google.com/{i}'
url_obj.time = datetime.now(pytz.utc)
url_obj.shortened_url = url_obj.generate_id(6)
url_obj.save()
url_obj_500 = Url.objects.filter(long_url='www.google.com/500').first()
start = process_time()
for i in range(1000):
response = self.client.get(f'/app/detail_without_redis/{url_obj_500.shortened_url}/')
self.assertEqual(response.status_code, 302)
self.assertEqual(response.url, 'https://www.google.com/500')
end = process_time()
print(f'Process time without Redis: {end - start}')
def post(self, name, url):
"""
POST request method for class GetUrlContents
URL: '/url/<name>/<url>'
:param name: Name used for identifying content in the system
:param url: URL which content will be saved in the system
:return: Task ID of the started task
"""
if not url.startswith("http"):
url = "https://" + url
new_url_object = Url(name=name.encode("utf-8"),
url=url.encode("utf-8"))
db.session.add(new_url_object)
commit_to_database()
url_schema = UrlSchema()
result = get_data_from_url.delay(url_schema.dump(new_url_object))
return result.task_id, 200
def index():
form = UrlForm()
link = Url()
if form.validate_on_submit():
if form.submit.data:
tmp = form.url_link.data
if re.match(
r'http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+',
tmp):
generate_short_link()
else:
flash('Invalid url')
return render_template('index.html', form=form, link=link)
elif form.submit2.data:
pass
link = Url.query.filter_by(url_link=form.url_link.data).all()
return render_template(
'index.html',
form=form,
link=link,
)
def index():
urls = Url.query.all()
form = URLForm()
if form.validate_on_submit():
full_url = form.full_url.data
short_url = form.short_url.data
if Url.short_url_exists(short_url):
flash("Short url is already exist. Try another one.")
return redirect(url_for("main.index"))
new_url = Url(full_url=full_url, clicks=0, created=datetime.today())
element_text = new_url.check_url()
if not element_text:
flash("Check url!!! Failed request %s" % new_url.full_url)
return redirect(url_for("main.index"))
new_url.element_text = element_text
db.session.add(new_url)
db.session.commit()
new_url.store_short_url(short_url)
flash("Your url has been created successfully.")
return redirect(url_for("main.index"))
return render_template("index.html", urls=urls, form=form)
def handle(self):
"""
Handle TCP datagrams
"""
data = loads(self.request.recv(1024).strip())
print "Recebido de '%s': %s \n" % (self.client_address[0], data)
try:
obj = Url.objects.filter(title=data['hostname']).update(ip=data['ip'],urls=data['urls'],slug=data['customer'])
if obj:
self.request.sendall(dumps({'return':'Successfully updated'}))
else:
obj = Url(title=data['hostname'],ip=data['ip'],urls=data['urls'],slug=data['customer'])
obj.save()
self.request.sendall(dumps({'return':'Successfully created'}))
except Url.DoesNotExist:
obj = Url(title=data['hostname'],ip=data['ip'],urls=data['urls'],slug=data['customer'])
obj.save()
self.request.sendall(dumps({'return':'Successfully created'}))
def test_get_shorted_url(self):
url_obj = Url()
url_obj.shortened_url = 'AF45Dd'
shortened_url = url_obj.get_shorted_url()
self.assertEqual(shortened_url, f'localhost:8000/app/AF45Dd')
def new_url_deets():
url = url = Url(url="https://www.google.com/", count=313)
return url
parser = argparse.ArgumentParser()
parser.add_argument('-t',
'--generate-test-data',
action='store_true',
help='if present, generate test data for db')
parser.add_argument('-d',
'--drop-tables',
action='store_true',
help='drop all exited tables')
parser.add_argument('-v',
'--verbose',
action='store_true',
help='turn on debug for flask app')
args = parser.parse_args()
if args.drop_tables:
db.drop_all()
db.create_all()
if args.generate_test_data:
habr_urls = [
'https://habrahabr.ru/all/page{}/'.format(i)
for i in range(1, 100)
]
for url in habr_urls:
url = Url(long_url=url)
db.session.add(url)
db.session.flush()
url.short_url = encode_id(url.id)
db.session.add(url)
db.session.commit()
app.run(debug=args.verbose)
def insert(short_id, origin_url, remote_addr):
url = Url(short_link=short_id, long_link=origin_url, ip=remote_addr)
db.session.add(url)
db.session.commit()
def test_generate_id(self):
size = 6
id = Url.generate_id(size)
self.assertEqual(len(id), size)
def write_database(post_id, user_id, text, created_at, created_at_dt,
reply_to_user_id, reply_to_scrname, reply_to_status_id,
retweet_count, favorite_count, is_retweet,
original_tweet_id, original_tweet_retweets, original_text,
original_tweet_created_at, original_tweet_likes,
original_author_id, original_author_scrname, polarity,
polarity_val, tag_list, url_list, user_scrname, user_name,
user_location, user_created, user_followers, user_friends,
user_statuses, query):
#print("starting db entry for postid = {}".format(post_id))
if query[4:7] == 'Sen':
district = 'sen'
district_name = query[2:7]
dist_type = 2
else:
district = query[4:6]
district_name = query[2:6]
dist_type = 1
#POST TABLE:
#If tweet ID not already in database, add to Post table
if db.session.query(Post).filter(Post.post_id == post_id).count() == 0:
#print('adding post')
#USER table
#If User already in User table, update dynamic elements, associate with this post
this_user = db.session.query(User).filter(
User.user_id == user_id).first()
if this_user != None:
this_user.user_location = user_location
this_user.user_followers = user_followers
this_user.user_friends = user_friends
this_user.user_statuses = user_statuses
db.session.add(this_user)
#Otherise, add User to user table, associate with this post
else:
this_user = User(user_id, user_scrname, user_name, user_location,\
user_created, user_followers, user_friends, user_statuses)
db.session.add(this_user)
#POST table
new_post = Post(post_id, user_id, text, created_at, created_at_dt,
reply_to_user_id, reply_to_scrname, reply_to_status_id,
retweet_count, favorite_count, is_retweet,
original_tweet_id, original_tweet_retweets,
original_text, original_tweet_created_at,
original_tweet_likes, original_author_id,
original_author_scrname, polarity, polarity_val)
db.session.add(new_post)
#If original tweet is in database, update its retweeted count. If not, do nothing
if original_tweet_id != None:
orig_tweet = db.session.query(Post).\
filter(Post.post_id == original_tweet_id).first()
if orig_tweet != None:
orig_tweet.retweet_count = original_tweet_retweets
db.session.add(orig_tweet)
#HASHTAG TABLE
# If tweet is being added, iterate through tag/url list, and create a
# Hashtag/Url table row for each tag
for item in tag_list:
#If hashtag is not already in Hashtag table, create new row
hash_search = db.session.query(Hashtag).\
filter(Hashtag.hashtag == item).first()
if hash_search == None:
new_hashtag = Hashtag(item)
db.session.add(new_hashtag)
else:
new_hashtag = hash_search
#Add association to posthash_assoc_table
new_post.hashtags.append(new_hashtag)
#db.session.add(posthash_assoc.hashtag)
#add one row to Post_extended per hashtag
# NOTE: this means number of rows per post_id = cartesian product
# of hashtags times districts (if picked up in search for every dist)
new_row = Post_extended(
post_id, user_id, created_at, created_at_dt, retweet_count,
is_retweet, original_tweet_id, original_text,
original_tweet_created_at, original_author_id,
original_author_scrname, polarity, polarity_val, item,
district_name, dist_type, user_scrname)
db.session.add(new_row)
#print("added newrow for hash {}".format(item))
#DISTRICT TABLE
#capture District_id from 1st query term:
state = query[2:4].lower()
#Handle Senate districts differently than congressional
# if query[4:7] == 'Sen':
# district = 'sen'
# district_name = query[2:7]
# dist_type = 2
# else:
# district = query[4:6]
# district_name = query[2:6]
# dist_type = 1
#Check if district is in DB, add if not
district_search = db.session.query(District).\
filter(District.district_name == district_name).first()
if district_search == None:
new_district = District(state, district, district_name, dist_type)
db.session.add(new_district)
else:
new_district = district_search
#Add association to postdist_assoc_table
new_post.districts.append(new_district)
#URL TABLE
#if URLS exist, add to db
if len(url_list) > 0:
for item in url_list:
url_search = db.session.query(Url).filter(
Url.url == item).first()
if url_search == None:
new_url = Url(item)
db.session.add(new_url)
else:
new_url = url_search
#Add association to postDistAssoc_table
new_post.urls.append(new_url)
#associate user with post
this_user.user_posts.append(new_post)
#If tweet ID in db (from another dist query), add new association to Post table
else:
#print('ID there, trying plan B')
district_check = db.session.query(District.district_name).\
join(Post.districts).\
filter(Post.post_id==post_id).all()
check = 0
for result in district_check: #iterate through associated dists
if result[0] == district_name:
check = 1
#print("already there") #if find match, check = 1, do nothing
if check == 0:
#print("adding newdist") # if no match, add to postdist_assoc
sql_command = '''INSERT INTO postdist_assoc (post_id, district_name)
VALUES (post_id, district_name);'''
conn = db.engine.connect()
conn.execute(sql_command)
conn.close()
