def _save_item(admin_id, file_id=None, last_updated=None, bucketname=None): table = get_dynamodb_table(table_name='geoadmin-file-storage') item = None if last_updated is not None: timestamp = last_updated.strftime('%Y-%m-%d %X') else: timestamp = time.strftime('%Y-%m-%d %X', time.localtime()) if file_id is not None: try: table.put_item( data={ 'adminId': admin_id, 'fileId': file_id, 'timestamp': timestamp, 'bucket': bucketname } ) except Exception as e: raise exc.HTTPBadRequest('Error during put item %s' % e) return True else: try: item = table.get_item(adminId=str(admin_id)) except ItemNotFound: return False try: item['timestamp'] = timestamp item.save() except Exception as e: raise exc.HTTPBadRequest('Timestamp of %s not updated: %s' % (admin_id, e))
def shortener(request): url = check_url( request.params.get('url'), request.registry.settings ) if len(url) >= 2046: # we only accept URL shorter or equal to 2046 characters # Index restriction in DynamoDB url_short = 'toolong' else: # DynamoDB v2 high-level abstraction try: table = get_dynamodb_table(table_name='shorturl') except Exception as e: raise exc.HTTPBadRequest('Error during connection %s' % e) url_short = _add_item(table, url) # Use env specific URLs if request.host not in ('api.geo.admin.ch', 'api3.geo.admin.ch'): host_url = make_api_url(request) + '/shorten/' else: host_url = ''.join((request.scheme, '://s.geo.admin.ch/')) return { 'shorturl': host_url + url_short }
def _is_admin_id(admin_id): table = get_dynamodb_table(table_name='geoadmin-file-storage') try: table.get_item(adminId=str(admin_id)) except ItemNotFound: return False return True
def _get_file_id_from_admin_id(admin_id): fileId = None table = get_dynamodb_table(table_name='geoadmin-file-storage') try: item = table.get_item(adminId=str(admin_id)) fileId = item.get('fileId') except Exception as e: raise exc.HTTPBadRequest('The id %s doesn\'t exist. Error is: %s' % (admin_id, e)) if fileId is None: return False return fileId
def shorten_redirect(request): url_short = request.matchdict.get('id') if url_short is None: raise exc.HTTPBadRequest('Please provide an id') table = get_dynamodb_table(table_name='shorturl') if url_short == 'toolong': raise exc.HTTPFound(location='http://map.geo.admin.ch') try: url_short = table.get_item(url_short=url_short) url = url_short.get('url') except Exception as e: raise exc.HTTPBadRequest('This short url doesn\'t exist: s.geo.admin.ch/%s Error is: %s' % (url_short, e)) raise exc.HTTPMovedPermanently(location=url)
def _add_item(id, file_id=False): table = get_dynamodb_table(table_name='geoadmin-file-storage') try: table.put_item( data={ 'adminId': id, 'fileId': file_id, 'timestamp': time.strftime('%Y-%m-%d %X', time.localtime()) } ) except Exception as e: raise exc.HTTPBadRequest('Error during put item %s' % e) return True
def kml_load(api_url='//api3.geo.admin.ch', bucket_name='public.geo.admin.ch'): now = datetime.datetime.now() date = now.strftime('%Y-%m-%d') table = get_dynamodb_table(table_name='geoadmin-file-storage') fileids = [] results = table.query_2(bucket__eq=bucket_name, timestamp__beginswith=date, index='bucketTimestampIndex', limit=LIMIT * 4, reverse=True) for f in results: try: resp = requests.head("http:" + api_url + "/files/" + f['fileId']) if int(resp.status_code) == 200: fileids.append((f['fileId'], f['adminId'], f['timestamp'])) except RequestException: pass if len(fileids) >= LIMIT: return fileids return fileids
def kml_load(api_url='//api3.geo.admin.ch', bucket_name=None): now = datetime.datetime.now() date = now.strftime('%Y-%m-%d') table = get_dynamodb_table(table_name='geoadmin-file-storage') fileids = [] results = table.query_2(bucket__eq=bucket_name, timestamp__beginswith=date, index='bucketTimestampIndex', limit=LIMIT * 4, reverse=True) for f in results: try: resp = requests.head("http:" + api_url + "/files/" + f['fileId'], headers={'User-Agent': 'mf-geoadmin/python'}) if int(resp.status_code) == 200: fileids.append((f['fileId'], f['adminId'], f['timestamp'])) except RequestException: pass if len(fileids) >= LIMIT: return fileids return fileids
def kml_load(api_url="//api3.geo.admin.ch", bucket_name="public.geo.admin.ch"): now = datetime.datetime.now() date = now.strftime("%Y-%m-%d") table = get_dynamodb_table(table_name="geoadmin-file-storage") fileids = [] results = table.query_2( bucket__eq=bucket_name, timestamp__beginswith=date, index="bucketTimestampIndex", limit=LIMIT * 4, reverse=True ) for f in results: try: resp = requests.head("http:" + api_url + "/files/" + f["fileId"]) if int(resp.status_code) == 200: fileids.append((f["fileId"], f["adminId"], f["timestamp"])) except RequestException: pass if len(fileids) >= LIMIT: return fileids return fileids
def shorten_redirect(request): url_short = request.matchdict.get('id') if url_short == 'toolong': raise exc.HTTPFound(location='http://map.geo.admin.ch') table = get_dynamodb_table(table_name='shorturl') try: url_short = table.get_item(url_short=url_short) url = url_short.get('url') except boto_exc.ItemNotFound as e: raise exc.HTTPNotFound('This short url doesn\'t exist: s.geo.admin.ch/%s Error is: %s' % (url_short, e)) except boto_exc.ProvisionedThroughputExceededException as e: # pragma: no cover raise exc.HTTPInternalServerError('Read units exceeded: %s' % e) except Exception as e: # pragma: no cover raise exc.HTTPInternalServerError('Unexpected internal server error: %s' % e) raise exc.HTTPMovedPermanently(location=url)
def shorten_redirect(request): url_short = request.matchdict.get('id') if url_short == 'toolong': raise exc.HTTPFound(location='http://map.geo.admin.ch') table = get_dynamodb_table(table_name='shorturl') try: url_short = table.get_item(url_short=url_short) url = url_short.get('url') except boto_exc.ItemNotFound as e: raise exc.HTTPNotFound('This short url doesn\'t exist: s.geo.admin.ch/%s Error is: %s' % (url_short, e)) except boto_exc.ProvisionedThroughputExceededException as e: # pragma: no cover raise exc.HTTPInternalServerError('Read units exceeded: %s' % e) except Exception as e: # pragma: no cover raise exc.HTTPInternalServerError('Unexpected internal server error: %s' % e) raise exc.HTTPMovedPermanently(location=url)
def shortener(request): url = request.params.get('url') if len(url) >= 2046: # we only accept URL shorter or equal to 2046 characters # Index restriction in DynamoDB url_short = 'toolong' else: # pragma: no cover url_short = check_url(url, request.registry.settings) # DynamoDB v2 high-level abstraction try: table = get_dynamodb_table(table_name='shorturl') except Exception as e: raise exc.HTTPInternalServerError('Error during connection %s' % e) url_short = _add_item(table, url) # Use env specific URLs if request.host not in ('api.geo.admin.ch', 'api3.geo.admin.ch'): host_url = make_api_url(request) + '/shorten/' else: host_url = ''.join((request.scheme, '://s.geo.admin.ch/')) return {'shorturl': host_url + url_short}
def __init__(self, table_name, bucket_name): # We use instance roles self.table = get_dynamodb_table(table_name=table_name) self.bucket_name = bucket_name
def __init__(self, table_name, bucket_name): # We use instance roles self.table = get_dynamodb_table(table_name=table_name) self.bucket_name = bucket_name
def test_get_dynamodb_table(self): result = get_dynamodb_table('shortenurl') self.assertNotEqual(result, exc.HTTPBadRequest)
def test_get_dynamodb_table(self): result = get_dynamodb_table('shortenurl') self.assertNotEqual(result, exc.HTTPBadRequest)