def _save_item(admin_id, file_id=None, last_updated=None, bucketname=None):
table = get_dynamodb_table(table_name='geoadmin-file-storage')
item = None
if last_updated is not None:
timestamp = last_updated.strftime('%Y-%m-%d %X')
else:
timestamp = time.strftime('%Y-%m-%d %X', time.localtime())
if file_id is not None:
try:
table.put_item(
data={
'adminId': admin_id,
'fileId': file_id,
'timestamp': timestamp,
'bucket': bucketname
}
)
except Exception as e:
raise exc.HTTPBadRequest('Error during put item %s' % e)
return True
else:
try:
item = table.get_item(adminId=str(admin_id))
except ItemNotFound:
return False
try:
item['timestamp'] = timestamp
item.save()
except Exception as e:
raise exc.HTTPBadRequest('Timestamp of %s not updated: %s' % (admin_id, e))
def shortener(request):
url = check_url(
request.params.get('url'), request.registry.settings
)
if len(url) >= 2046:
# we only accept URL shorter or equal to 2046 characters
# Index restriction in DynamoDB
url_short = 'toolong'
else:
# DynamoDB v2 high-level abstraction
try:
table = get_dynamodb_table(table_name='shorturl')
except Exception as e:
raise exc.HTTPBadRequest('Error during connection %s' % e)
url_short = _add_item(table, url)
# Use env specific URLs
if request.host not in ('api.geo.admin.ch', 'api3.geo.admin.ch'):
host_url = make_api_url(request) + '/shorten/'
else:
host_url = ''.join((request.scheme, '://s.geo.admin.ch/'))
return {
'shorturl': host_url + url_short
}
def _is_admin_id(admin_id):
table = get_dynamodb_table(table_name='geoadmin-file-storage')
try:
table.get_item(adminId=str(admin_id))
except ItemNotFound:
return False
return True
def _get_file_id_from_admin_id(admin_id):
fileId = None
table = get_dynamodb_table(table_name='geoadmin-file-storage')
try:
item = table.get_item(adminId=str(admin_id))
fileId = item.get('fileId')
except Exception as e:
raise exc.HTTPBadRequest('The id %s doesn\'t exist. Error is: %s' % (admin_id, e))
if fileId is None:
return False
return fileId
def shorten_redirect(request):
url_short = request.matchdict.get('id')
if url_short is None:
raise exc.HTTPBadRequest('Please provide an id')
table = get_dynamodb_table(table_name='shorturl')
if url_short == 'toolong':
raise exc.HTTPFound(location='http://map.geo.admin.ch')
try:
url_short = table.get_item(url_short=url_short)
url = url_short.get('url')
except Exception as e:
raise exc.HTTPBadRequest('This short url doesn\'t exist: s.geo.admin.ch/%s Error is: %s' % (url_short, e))
raise exc.HTTPMovedPermanently(location=url)
def _add_item(id, file_id=False):
table = get_dynamodb_table(table_name='geoadmin-file-storage')
try:
table.put_item(
data={
'adminId': id,
'fileId': file_id,
'timestamp': time.strftime('%Y-%m-%d %X', time.localtime())
}
)
except Exception as e:
raise exc.HTTPBadRequest('Error during put item %s' % e)
return True
def kml_load(api_url='//api3.geo.admin.ch', bucket_name='public.geo.admin.ch'):
now = datetime.datetime.now()
date = now.strftime('%Y-%m-%d')
table = get_dynamodb_table(table_name='geoadmin-file-storage')
fileids = []
results = table.query_2(bucket__eq=bucket_name, timestamp__beginswith=date, index='bucketTimestampIndex', limit=LIMIT * 4, reverse=True)
for f in results:
try:
resp = requests.head("http:" + api_url + "/files/" + f['fileId'])
if int(resp.status_code) == 200:
fileids.append((f['fileId'], f['adminId'], f['timestamp']))
except RequestException:
pass
if len(fileids) >= LIMIT:
return fileids
return fileids
def kml_load(api_url='//api3.geo.admin.ch', bucket_name=None):
now = datetime.datetime.now()
date = now.strftime('%Y-%m-%d')
table = get_dynamodb_table(table_name='geoadmin-file-storage')
fileids = []
results = table.query_2(bucket__eq=bucket_name, timestamp__beginswith=date, index='bucketTimestampIndex', limit=LIMIT * 4, reverse=True)
for f in results:
try:
resp = requests.head("http:" + api_url + "/files/" + f['fileId'], headers={'User-Agent': 'mf-geoadmin/python'})
if int(resp.status_code) == 200:
fileids.append((f['fileId'], f['adminId'], f['timestamp']))
except RequestException:
pass
if len(fileids) >= LIMIT:
return fileids
return fileids
def kml_load(api_url="//api3.geo.admin.ch", bucket_name="public.geo.admin.ch"):
now = datetime.datetime.now()
date = now.strftime("%Y-%m-%d")
table = get_dynamodb_table(table_name="geoadmin-file-storage")
fileids = []
results = table.query_2(
bucket__eq=bucket_name, timestamp__beginswith=date, index="bucketTimestampIndex", limit=LIMIT * 4, reverse=True
)
for f in results:
try:
resp = requests.head("http:" + api_url + "/files/" + f["fileId"])
if int(resp.status_code) == 200:
fileids.append((f["fileId"], f["adminId"], f["timestamp"]))
except RequestException:
pass
if len(fileids) >= LIMIT:
return fileids
return fileids
def shorten_redirect(request):
url_short = request.matchdict.get('id')
if url_short == 'toolong':
raise exc.HTTPFound(location='http://map.geo.admin.ch')
table = get_dynamodb_table(table_name='shorturl')
try:
url_short = table.get_item(url_short=url_short)
url = url_short.get('url')
except boto_exc.ItemNotFound as e:
raise exc.HTTPNotFound('This short url doesn\'t exist: s.geo.admin.ch/%s Error is: %s' % (url_short, e))
except boto_exc.ProvisionedThroughputExceededException as e: # pragma: no cover
raise exc.HTTPInternalServerError('Read units exceeded: %s' % e)
except Exception as e: # pragma: no cover
raise exc.HTTPInternalServerError('Unexpected internal server error: %s' % e)
raise exc.HTTPMovedPermanently(location=url)
def shorten_redirect(request):
url_short = request.matchdict.get('id')
if url_short == 'toolong':
raise exc.HTTPFound(location='http://map.geo.admin.ch')
table = get_dynamodb_table(table_name='shorturl')
try:
url_short = table.get_item(url_short=url_short)
url = url_short.get('url')
except boto_exc.ItemNotFound as e:
raise exc.HTTPNotFound('This short url doesn\'t exist: s.geo.admin.ch/%s Error is: %s' % (url_short, e))
except boto_exc.ProvisionedThroughputExceededException as e: # pragma: no cover
raise exc.HTTPInternalServerError('Read units exceeded: %s' % e)
except Exception as e: # pragma: no cover
raise exc.HTTPInternalServerError('Unexpected internal server error: %s' % e)
raise exc.HTTPMovedPermanently(location=url)
def shortener(request):
url = request.params.get('url')
if len(url) >= 2046:
# we only accept URL shorter or equal to 2046 characters
# Index restriction in DynamoDB
url_short = 'toolong'
else: # pragma: no cover
url_short = check_url(url, request.registry.settings)
# DynamoDB v2 high-level abstraction
try:
table = get_dynamodb_table(table_name='shorturl')
except Exception as e:
raise exc.HTTPInternalServerError('Error during connection %s' % e)
url_short = _add_item(table, url)
# Use env specific URLs
if request.host not in ('api.geo.admin.ch', 'api3.geo.admin.ch'):
host_url = make_api_url(request) + '/shorten/'
else:
host_url = ''.join((request.scheme, '://s.geo.admin.ch/'))
return {'shorturl': host_url + url_short}
def __init__(self, table_name, bucket_name):
# We use instance roles
self.table = get_dynamodb_table(table_name=table_name)
self.bucket_name = bucket_name
def __init__(self, table_name, bucket_name):
# We use instance roles
self.table = get_dynamodb_table(table_name=table_name)
self.bucket_name = bucket_name
def test_get_dynamodb_table(self):
result = get_dynamodb_table('shortenurl')
self.assertNotEqual(result, exc.HTTPBadRequest)
def test_get_dynamodb_table(self):
result = get_dynamodb_table('shortenurl')
self.assertNotEqual(result, exc.HTTPBadRequest)
