def make_app(debug): app = tornado.web.Application( api_to_handlers(v0) + api_to_handlers(v1) + [(r'/status', handlers.StatusHandler)], debug=debug, ) app.sentry_client = AioSentryClient(settings.get('SENTRY_DSN', None)) return app
def make_app(debug): app = tornado.web.Application( [ (r'/file', crud.CRUDHandler), (r'/data', metadata.MetadataHandler), (r'/status', status.StatusHandler), (r'/revisions', revisions.RevisionHandler), ], debug=debug, ) app.sentry_client = AioSentryClient(settings.get('SENTRY_DSN', None)) return app
def make_app(debug): app = tornado.web.Application( [ (r'/file', crud.CRUDHandler), (r'/data', metadata.MetadataHandler), (r'/status', status.StatusHandler), (r'/ops/copy', copy.CopyHandler), (r'/ops/move', move.MoveHandler), (r'/revisions', revisions.RevisionHandler), (r'/zip', zip.ZipHandler), ], debug=debug, ) app.sentry_client = AioSentryClient(settings.get('SENTRY_DSN', None)) return app
try: from waterbutler import settings except ImportError: settings = {} # type: ignore config = settings.get('OWNCLOUD_PROVIDER_CONFIG', {}) # type: ignore
import hashlib try: from waterbutler import settings except ImportError: settings = {} config = settings.get('OSFSTORAGE_PROVIDER_CONFIG', {}) FILE_PATH_PENDING = config.get('FILE_PATH_PENDING', '/tmp/pending') FILE_PATH_COMPLETE = config.get('FILE_PATH_COMPLETE', '/tmp/complete') RUN_TASKS = config.get('RUN_TASKS', False) HMAC_ALGORITHM = getattr(hashlib, config.get('HMAC_ALGORITHM', 'sha256')) HMAC_SECRET = config.get('HMAC_SECRET', None) if not settings.DEBUG: assert HMAC_SECRET, 'HMAC_SECRET must be specified when not in debug mode' HMAC_SECRET = (HMAC_SECRET or 'changeme').encode('utf-8') # Retry options UPLOAD_RETRY_ATTEMPTS = config.get('UPLOAD_RETRY_ATTEMPTS', 1) UPLOAD_RETRY_INIT_DELAY = config.get('UPLOAD_RETRY_INIT_DELAY', 30) UPLOAD_RETRY_MAX_DELAY = config.get('UPLOAD_RETRY_MAX_DELAY', 60 * 60) UPLOAD_RETRY_BACKOFF = config.get('UPLOAD_RETRY_BACKOFF', 2) UPLOAD_RETRY_WARN_IDX = config.get('UPLOAD_RETRY_WARN_IDX', 5) HOOK_RETRY_ATTEMPTS = config.get('HOOK_RETRY_ATTEMPTS ', 1) HOOK_RETRY_INIT_DELAY = config.get('HOOK_RETRY_INIT_DELAY', 30)
try: from waterbutler import settings except ImportError: settings = {} config = settings.get('S3_PROVIDER_CONFIG', {}) TEMP_URL_SECS = config.get('TEMP_URL_SECS', 100)
import hashlib try: from waterbutler import settings except ImportError: settings = {} config = settings.get('SERVER_CONFIG', {}) ADDRESS = config.get('ADDRESS', '127.0.0.1') PORT = config.get('PORT', 7777) DEBUG = config.get('DEBUG', True) SSL_CERT_FILE = config.get('SSL_CERT_FILE', None) SSL_KEY_FILE = config.get('SSL_KEY_FILE', None) XHEADERS = config.get('XHEADERS', False) CORS_ALLOW_ORIGIN = config.get('CORS_ALLOW_ORIGIN', '*') CHUNK_SIZE = config.get('CHUNK_SIZE', 65536) # 64KB MAX_BODY_SIZE = config.get('MAX_BODY_SIZE', int(4.9 * (1024 ** 3))) # 4.9 GB AUTH_HANDLERS = config.get('AUTH_HANDLERS', [ 'osf', ]) HMAC_ALGORITHM = getattr(hashlib, config.get('HMAC_ALGORITHM', 'sha256')) HMAC_SECRET = config.get('HMAC_SECRET', 'changeme').encode('utf-8')
try: from waterbutler import settings except ImportError: settings = {} config = settings.get('GOOGLEDRIVE_PROVIDER_CONFIG', {}) BASE_URL = config.get('BASE_URL', 'https://www.googleapis.com/drive/v2') BASE_UPLOAD_URL = config.get('BASE_UPLOAD_URL', 'https://www.googleapis.com/upload/drive/v2') DRIVE_IGNORE_VERSION = config.get('DRIVE_IGNORE_VERSION', '0000000000000000000000000000000000000')
try: from waterbutler import settings except ImportError: settings = {} config = settings.get("OSF_AUTH_CONFIG", {}) JWT_EXPIRATION = config.get("JWT_EXPIRATION", 15) JWT_ALGORITHM = config.get("JWT_ALGORITHM", "HS256") API_URL = config.get("API_URL", "http://localhost:5000/api/v1/files/auth/") JWE_SALT = config.get("JWE_SALT") JWE_SECRET = config.get("JWE_SECRET") JWT_SECRET = config.get("JWT_SECRET") if not settings.DEBUG: assert JWE_SALT, "JWE_SALT must be specified when not in debug mode" assert JWE_SECRET, "JWE_SECRET must be specified when not in debug mode" assert JWT_SECRET, "JWT_SECRET must be specified when not in debug mode" JWE_SALT = JWE_SALT or "yusaltydough" JWE_SECRET = JWE_SECRET or "CirclesAre4Squares" JWT_SECRET = JWT_SECRET or "ILiekTrianglesALot"
import os from pkg_resources import iter_entry_points from kombu import Queue, Exchange try: from waterbutler import settings except ImportError: settings = {} config = settings.get('TASKS_CONFIG', {}) BROKER_URL = config.get( 'BROKER_URL', 'amqp://{}:{}//'.format( os.environ.get('RABBITMQ_PORT_5672_TCP_ADDR', ''), os.environ.get('RABBITMQ_PORT_5672_TCP_PORT', ''), ) ) WAIT_TIMEOUT = config.get('WAIT_TIMEOUT', 15) WAIT_INTERVAL = config.get('WAIT_INTERVAL', 0.5) ADHOC_BACKEND_PATH = config.get('ADHOC_BACKEND_PATH', '/tmp') CELERY_CREATE_MISSING_QUEUES = config.get('CELERY_CREATE_MISSING_QUEUES', False) CELERY_DEFAULT_QUEUE = config.get('CELERY_DEFAULT_QUEUE', 'waterbutler') CELERY_QUEUES = ( Queue('waterbutler', Exchange('waterbutler'), routing_key='waterbutler'), ) # CELERY_ALWAYS_EAGER = config.get('CELERY_ALWAYS_EAGER', True)
try: from waterbutler import settings except ImportError: settings = {} config = settings.get('DATAVERSE_PROVIDER_CONFIG', {}) EDIT_MEDIA_BASE_URL = config.get( 'EDIT_MEDIA_BASE_URL', "/dvn/api/data-deposit/v1.1/swordv2/edit-media/") DOWN_BASE_URL = config.get('DOWN_BASE_URL', "/api/access/datafile/") METADATA_BASE_URL = config.get( 'METADATA_BASE_URL', "/dvn/api/data-deposit/v1.1/swordv2/statement/study/") JSON_BASE_URL = config.get('JSON_BASE_URL', "/api/datasets/{0}/versions/:{1}")
try: from waterbutler import settings except ImportError: settings = {} config = settings.get('DROPBOX_PROVIDER_CONFIG', {}) BASE_URL = config.get('BASE_URL', 'https://api.dropbox.com/1/') BASE_CONTENT_URL = config.get('BASE_CONTENT_URL', 'https://api-content.dropbox.com/1/')
try: from waterbutler import settings except ImportError: settings = {} config = settings.get('FIGSHARE_PROVIDER_CONFIG', {}) BASE_URL = config.get('BASE_URL', 'http://api.figshare.com/v1/my_data') VIEW_URL = config.get('VIEW_URL', 'http://figshare.com/')
try: from waterbutler import settings except ImportError: settings = {} config = settings.get('CLOUDFILES_PROVIDER_CONFIG', {}) TEMP_URL_SECS = config.get('TEMP_URL_SECS', 100) AUTH_URL = config.get('AUTH_URL', 'https://identity.api.rackspacecloud.com/v2.0/tokens')
import os from pkg_resources import iter_entry_points from kombu import Queue, Exchange try: from waterbutler import settings except ImportError: settings = {} config = settings.get('TASKS_CONFIG', {}) BROKER_URL = config.get( 'BROKER_URL', 'amqp://{}:{}//'.format( os.environ.get('RABBITMQ_PORT_5672_TCP_ADDR', ''), os.environ.get('RABBITMQ_PORT_5672_TCP_PORT', ''), )) WAIT_TIMEOUT = config.get('WAIT_TIMEOUT', 15) WAIT_INTERVAL = config.get('WAIT_INTERVAL', 0.5) ADHOC_BACKEND_PATH = config.get('ADHOC_BACKEND_PATH', '/tmp') CELERY_CREATE_MISSING_QUEUES = config.get('CELERY_CREATE_MISSING_QUEUES', False) CELERY_DEFAULT_QUEUE = config.get('CELERY_DEFAULT_QUEUE', 'waterbutler') CELERY_QUEUES = (Queue('waterbutler', Exchange('waterbutler'), routing_key='waterbutler'), ) # CELERY_ALWAYS_EAGER = config.get('CELERY_ALWAYS_EAGER', True) CELERY_ALWAYS_EAGER = config.get('CELERY_ALWAYS_EAGER', False) # CELERY_RESULT_BACKEND = config.get('CELERY_RESULT_BACKEND', 'redis://')
try: from waterbutler import settings except ImportError: settings = {} config = settings.get('FILESYSTEM_PROVIDER_CONFIG', {}) CHUNK_SIZE = config.get('CHUNK_SIZE', 65536) # 64KB
import aiohttp from raven import Client from stevedore import driver from waterbutler import settings from waterbutler.core import exceptions from waterbutler.tasks import settings as task_settings from waterbutler.server import settings as server_settings from waterbutler.core.signing import Signer logger = logging.getLogger(__name__) signer = Signer(server_settings.HMAC_SECRET, server_settings.HMAC_ALGORITHM) sentry_dsn = settings.get('SENTRY_DSN', None) client = Client(sentry_dsn) if sentry_dsn else None def make_provider(name, auth, credentials, settings, **kwargs): """Returns an instance of :class:`waterbutler.core.provider.BaseProvider` :param str name: The name of the provider to instantiate. (s3, box, etc) :param dict auth: :param dict credentials: :param dict settings: :param dict \*\*kwargs: currently there to absorb ``callback_url`` :rtype: :class:`waterbutler.core.provider.BaseProvider` """ try:
import functools # from concurrent.futures import ProcessPoolExecutor TODO Get this working import aiohttp from raven.contrib.tornado import AsyncSentryClient from stevedore import driver from waterbutler import settings from waterbutler.server import settings as server_settings from waterbutler.core import exceptions from waterbutler.core.signing import Signer logger = logging.getLogger(__name__) sentry_dns = settings.get('SENTRY_DSN', None) signer = Signer(server_settings.HMAC_SECRET, server_settings.HMAC_ALGORITHM) class AioSentryClient(AsyncSentryClient): def send_remote(self, url, data, headers=None, callback=None): headers = headers or {} if not self.state.should_try(): message = self._get_log_message(data) self.error_logger.error(message) return future = aiohttp.request('POST', url, data=data, headers=headers) asyncio. async (future)
import hashlib try: from waterbutler import settings except ImportError: settings = {} config = settings.get('SERVER_CONFIG', {}) ADDRESS = config.get('ADDRESS', '127.0.0.1') PORT = config.get('PORT', 7777) DOMAIN = config.get('DOMAIN', "http://127.0.0.1:7777") DEBUG = config.get('DEBUG', True) SSL_CERT_FILE = config.get('SSL_CERT_FILE', None) SSL_KEY_FILE = config.get('SSL_KEY_FILE', None) XHEADERS = config.get('XHEADERS', False) CORS_ALLOW_ORIGIN = config.get('CORS_ALLOW_ORIGIN', '*') CHUNK_SIZE = config.get('CHUNK_SIZE', 65536) # 64KB MAX_BODY_SIZE = config.get('MAX_BODY_SIZE', int(4.9 * (1024 ** 3))) # 4.9 GB AUTH_HANDLERS = config.get('AUTH_HANDLERS', [ 'osf', ]) HMAC_ALGORITHM = getattr(hashlib, config.get('HMAC_ALGORITHM', 'sha256'))
try: from waterbutler import settings except ImportError: settings = {} config = settings.get('GITHUB_PROVIDER_CONFIG', {}) BASE_URL = config.get('BASE_URL', 'https://api.github.com/') VIEW_URL = config.get('VIEW_URL', 'https://github.com/') MOVE_MESSAGE = config.get('MOVE_MESSAGE', 'Moved on behalf of WaterButler') COPY_MESSAGE = config.get('COPY_MESSAGE', 'Copied on behalf of WaterButler') DELETE_FILE_MESSAGE = config.get('DELETE_FILE_MESSAGE', 'File deleted on behalf of WaterButler') UPDATE_FILE_MESSAGE = config.get('UPDATE_FILE_MESSAGE', 'File updated on behalf of WaterButler') UPLOAD_FILE_MESSAGE = config.get('UPLOAD_FILE_MESSAGE', 'File uploaded on behalf of WaterButler') DELETE_FOLDER_MESSAGE = config.get('DELETE_FOLDER_MESSAGE', 'Folder deleted on behalf of WaterButler')
try: from waterbutler import settings except ImportError: settings = {} config = settings.get('OSF_AUTH_CONFIG', {}) API_URL = config.get('API_URL', 'http://127.0.0.1:5000/api/v1/files/auth/')
import hashlib try: from waterbutler import settings except ImportError: settings = {} config = settings.get('OSFSTORAGE_PROVIDER_CONFIG', {}) FILE_PATH_PENDING = config.get('FILE_PATH_PENDING', '/tmp/pending') FILE_PATH_COMPLETE = config.get('FILE_PATH_COMPLETE', '/tmp/complete') RUN_TASKS = config.get('RUN_TASKS', False) HMAC_ALGORITHM = getattr(hashlib, config.get('HMAC_ALGORITHM', 'sha256')) HMAC_SECRET = config.get('HMAC_SECRET', 'changeme').encode('utf-8') # Retry options UPLOAD_RETRY_ATTEMPTS = config.get('UPLOAD_RETRY_ATTEMPTS', 1) UPLOAD_RETRY_INIT_DELAY = config.get('UPLOAD_RETRY_INIT_DELAY', 30) UPLOAD_RETRY_MAX_DELAY = config.get('UPLOAD_RETRY_MAX_DELAY', 60 * 60) UPLOAD_RETRY_BACKOFF = config.get('UPLOAD_RETRY_BACKOFF', 2) UPLOAD_RETRY_WARN_IDX = config.get('UPLOAD_RETRY_WARN_IDX', 5) HOOK_RETRY_ATTEMPTS = config.get('HOOK_RETRY_ATTEMPTS ', 1) HOOK_RETRY_INIT_DELAY = config.get('HOOK_RETRY_INIT_DELAY', 30) HOOK_RETRY_MAX_DELAY = config.get('HOOK_RETRY_MAX_DELAY', 60 * 60) HOOK_RETRY_BACKOFF = config.get('HOOK_RETRY_BACKOFF', 2) HOOK_RETRY_WARN_IDX = config.get('HOOK_RETRY_WARN_IDX', None)
try: from waterbutler import settings except ImportError: settings = {} config = settings.get('BOX_PROVIDER_CONFIG', {}) BASE_URL = config.get('BASE_URL', 'https://api.box.com/2.0') BASE_UPLOAD_URL = config.get('BASE_CONTENT_URL', 'https://upload.box.com/api/2.0')
try: from waterbutler import settings except ImportError: settings = {} config = settings.get('DATAVERSE_PROVIDER_CONFIG', {}) EDIT_MEDIA_BASE_URL = config.get('EDIT_MEDIA_BASE_URL', "/dvn/api/data-deposit/v1.1/swordv2/edit-media/") DOWN_BASE_URL = config.get('DOWN_BASE_URL', "/api/access/datafile/") METADATA_BASE_URL = config.get('METADATA_BASE_URL', "/dvn/api/data-deposit/v1.1/swordv2/statement/study/") JSON_BASE_URL = config.get('JSON_BASE_URL', "/api/datasets/{0}/versions/:{1}")