Ejemplo n.º 1
0
def make_app(debug):
    app = tornado.web.Application(
        api_to_handlers(v0) + api_to_handlers(v1) +
        [(r'/status', handlers.StatusHandler)],
        debug=debug,
    )
    app.sentry_client = AioSentryClient(settings.get('SENTRY_DSN', None))
    return app
Ejemplo n.º 2
0
def make_app(debug):
    app = tornado.web.Application(
        api_to_handlers(v0) +
        api_to_handlers(v1) +
        [(r'/status', handlers.StatusHandler)],
        debug=debug,
    )
    app.sentry_client = AioSentryClient(settings.get('SENTRY_DSN', None))
    return app
Ejemplo n.º 3
0
def make_app(debug):
    app = tornado.web.Application(
        [
            (r'/file', crud.CRUDHandler),
            (r'/data', metadata.MetadataHandler),
            (r'/status', status.StatusHandler),
            (r'/revisions', revisions.RevisionHandler),
        ],
        debug=debug,
    )
    app.sentry_client = AioSentryClient(settings.get('SENTRY_DSN', None))
    return app
Ejemplo n.º 4
0
def make_app(debug):
    app = tornado.web.Application(
        [
            (r'/file', crud.CRUDHandler),
            (r'/data', metadata.MetadataHandler),
            (r'/status', status.StatusHandler),
            (r'/ops/copy', copy.CopyHandler),
            (r'/ops/move', move.MoveHandler),
            (r'/revisions', revisions.RevisionHandler),
            (r'/zip', zip.ZipHandler),
        ],
        debug=debug,
    )
    app.sentry_client = AioSentryClient(settings.get('SENTRY_DSN', None))
    return app
Ejemplo n.º 5
0
try:
    from waterbutler import settings
except ImportError:
    settings = {}  # type: ignore

config = settings.get('OWNCLOUD_PROVIDER_CONFIG', {})  # type: ignore
Ejemplo n.º 6
0
import hashlib

try:
    from waterbutler import settings
except ImportError:
    settings = {}

config = settings.get('OSFSTORAGE_PROVIDER_CONFIG', {})

FILE_PATH_PENDING = config.get('FILE_PATH_PENDING', '/tmp/pending')
FILE_PATH_COMPLETE = config.get('FILE_PATH_COMPLETE', '/tmp/complete')

RUN_TASKS = config.get('RUN_TASKS', False)

HMAC_ALGORITHM = getattr(hashlib, config.get('HMAC_ALGORITHM', 'sha256'))

HMAC_SECRET = config.get('HMAC_SECRET', None)

if not settings.DEBUG:
    assert HMAC_SECRET, 'HMAC_SECRET must be specified when not in debug mode'
HMAC_SECRET = (HMAC_SECRET or 'changeme').encode('utf-8')

# Retry options
UPLOAD_RETRY_ATTEMPTS = config.get('UPLOAD_RETRY_ATTEMPTS', 1)
UPLOAD_RETRY_INIT_DELAY = config.get('UPLOAD_RETRY_INIT_DELAY', 30)
UPLOAD_RETRY_MAX_DELAY = config.get('UPLOAD_RETRY_MAX_DELAY', 60 * 60)
UPLOAD_RETRY_BACKOFF = config.get('UPLOAD_RETRY_BACKOFF', 2)
UPLOAD_RETRY_WARN_IDX = config.get('UPLOAD_RETRY_WARN_IDX', 5)

HOOK_RETRY_ATTEMPTS = config.get('HOOK_RETRY_ATTEMPTS ', 1)
HOOK_RETRY_INIT_DELAY = config.get('HOOK_RETRY_INIT_DELAY', 30)
Ejemplo n.º 7
0
try:
    from waterbutler import settings
except ImportError:
    settings = {}

config = settings.get('S3_PROVIDER_CONFIG', {})

TEMP_URL_SECS = config.get('TEMP_URL_SECS', 100)
Ejemplo n.º 8
0
import hashlib

try:
    from waterbutler import settings
except ImportError:
    settings = {}

config = settings.get('SERVER_CONFIG', {})


ADDRESS = config.get('ADDRESS', '127.0.0.1')
PORT = config.get('PORT', 7777)

DEBUG = config.get('DEBUG', True)

SSL_CERT_FILE = config.get('SSL_CERT_FILE', None)
SSL_KEY_FILE = config.get('SSL_KEY_FILE', None)

XHEADERS = config.get('XHEADERS', False)
CORS_ALLOW_ORIGIN = config.get('CORS_ALLOW_ORIGIN', '*')

CHUNK_SIZE = config.get('CHUNK_SIZE', 65536)  # 64KB
MAX_BODY_SIZE = config.get('MAX_BODY_SIZE', int(4.9 * (1024 ** 3)))  # 4.9 GB

AUTH_HANDLERS = config.get('AUTH_HANDLERS', [
    'osf',
])

HMAC_ALGORITHM = getattr(hashlib, config.get('HMAC_ALGORITHM', 'sha256'))
HMAC_SECRET = config.get('HMAC_SECRET', 'changeme').encode('utf-8')
Ejemplo n.º 9
0
try:
    from waterbutler import settings
except ImportError:
    settings = {}

config = settings.get('GOOGLEDRIVE_PROVIDER_CONFIG', {})


BASE_URL = config.get('BASE_URL', 'https://www.googleapis.com/drive/v2')
BASE_UPLOAD_URL = config.get('BASE_UPLOAD_URL', 'https://www.googleapis.com/upload/drive/v2')
DRIVE_IGNORE_VERSION = config.get('DRIVE_IGNORE_VERSION', '0000000000000000000000000000000000000')
Ejemplo n.º 10
0
try:
    from waterbutler import settings
except ImportError:
    settings = {}

config = settings.get("OSF_AUTH_CONFIG", {})

JWT_EXPIRATION = config.get("JWT_EXPIRATION", 15)
JWT_ALGORITHM = config.get("JWT_ALGORITHM", "HS256")
API_URL = config.get("API_URL", "http://localhost:5000/api/v1/files/auth/")

JWE_SALT = config.get("JWE_SALT")
JWE_SECRET = config.get("JWE_SECRET")
JWT_SECRET = config.get("JWT_SECRET")

if not settings.DEBUG:
    assert JWE_SALT, "JWE_SALT must be specified when not in debug mode"
    assert JWE_SECRET, "JWE_SECRET must be specified when not in debug mode"
    assert JWT_SECRET, "JWT_SECRET must be specified when not in debug mode"

JWE_SALT = JWE_SALT or "yusaltydough"
JWE_SECRET = JWE_SECRET or "CirclesAre4Squares"
JWT_SECRET = JWT_SECRET or "ILiekTrianglesALot"
Ejemplo n.º 11
0
import os

from pkg_resources import iter_entry_points
from kombu import Queue, Exchange

try:
    from waterbutler import settings
except ImportError:
    settings = {}

config = settings.get('TASKS_CONFIG', {})


BROKER_URL = config.get(
    'BROKER_URL',
    'amqp://{}:{}//'.format(
        os.environ.get('RABBITMQ_PORT_5672_TCP_ADDR', ''),
        os.environ.get('RABBITMQ_PORT_5672_TCP_PORT', ''),
    )
)

WAIT_TIMEOUT = config.get('WAIT_TIMEOUT', 15)
WAIT_INTERVAL = config.get('WAIT_INTERVAL', 0.5)
ADHOC_BACKEND_PATH = config.get('ADHOC_BACKEND_PATH', '/tmp')

CELERY_CREATE_MISSING_QUEUES = config.get('CELERY_CREATE_MISSING_QUEUES', False)
CELERY_DEFAULT_QUEUE = config.get('CELERY_DEFAULT_QUEUE', 'waterbutler')
CELERY_QUEUES = (
    Queue('waterbutler', Exchange('waterbutler'), routing_key='waterbutler'),
)
# CELERY_ALWAYS_EAGER = config.get('CELERY_ALWAYS_EAGER', True)
Ejemplo n.º 12
0
try:
    from waterbutler import settings
except ImportError:
    settings = {}

config = settings.get('DATAVERSE_PROVIDER_CONFIG', {})

EDIT_MEDIA_BASE_URL = config.get(
    'EDIT_MEDIA_BASE_URL', "/dvn/api/data-deposit/v1.1/swordv2/edit-media/")
DOWN_BASE_URL = config.get('DOWN_BASE_URL', "/api/access/datafile/")
METADATA_BASE_URL = config.get(
    'METADATA_BASE_URL', "/dvn/api/data-deposit/v1.1/swordv2/statement/study/")
JSON_BASE_URL = config.get('JSON_BASE_URL', "/api/datasets/{0}/versions/:{1}")
Ejemplo n.º 13
0
try:
    from waterbutler import settings
except ImportError:
    settings = {}

config = settings.get('DROPBOX_PROVIDER_CONFIG', {})


BASE_URL = config.get('BASE_URL', 'https://api.dropbox.com/1/')
BASE_CONTENT_URL = config.get('BASE_CONTENT_URL', 'https://api-content.dropbox.com/1/')
Ejemplo n.º 14
0
try:
    from waterbutler import settings
except ImportError:
    settings = {}

config = settings.get('FIGSHARE_PROVIDER_CONFIG', {})

BASE_URL = config.get('BASE_URL', 'http://api.figshare.com/v1/my_data')
VIEW_URL = config.get('VIEW_URL', 'http://figshare.com/')
Ejemplo n.º 15
0
try:
    from waterbutler import settings
except ImportError:
    settings = {}

config = settings.get('CLOUDFILES_PROVIDER_CONFIG', {})


TEMP_URL_SECS = config.get('TEMP_URL_SECS', 100)
AUTH_URL = config.get('AUTH_URL', 'https://identity.api.rackspacecloud.com/v2.0/tokens')
Ejemplo n.º 16
0
import os

from pkg_resources import iter_entry_points
from kombu import Queue, Exchange

try:
    from waterbutler import settings
except ImportError:
    settings = {}

config = settings.get('TASKS_CONFIG', {})

BROKER_URL = config.get(
    'BROKER_URL', 'amqp://{}:{}//'.format(
        os.environ.get('RABBITMQ_PORT_5672_TCP_ADDR', ''),
        os.environ.get('RABBITMQ_PORT_5672_TCP_PORT', ''),
    ))

WAIT_TIMEOUT = config.get('WAIT_TIMEOUT', 15)
WAIT_INTERVAL = config.get('WAIT_INTERVAL', 0.5)
ADHOC_BACKEND_PATH = config.get('ADHOC_BACKEND_PATH', '/tmp')

CELERY_CREATE_MISSING_QUEUES = config.get('CELERY_CREATE_MISSING_QUEUES',
                                          False)
CELERY_DEFAULT_QUEUE = config.get('CELERY_DEFAULT_QUEUE', 'waterbutler')
CELERY_QUEUES = (Queue('waterbutler',
                       Exchange('waterbutler'),
                       routing_key='waterbutler'), )
# CELERY_ALWAYS_EAGER = config.get('CELERY_ALWAYS_EAGER', True)
CELERY_ALWAYS_EAGER = config.get('CELERY_ALWAYS_EAGER', False)
# CELERY_RESULT_BACKEND = config.get('CELERY_RESULT_BACKEND', 'redis://')
Ejemplo n.º 17
0
try:
    from waterbutler import settings
except ImportError:
    settings = {}

config = settings.get('FILESYSTEM_PROVIDER_CONFIG', {})


CHUNK_SIZE = config.get('CHUNK_SIZE', 65536)  # 64KB
Ejemplo n.º 18
0
import aiohttp
from raven import Client
from stevedore import driver

from waterbutler import settings
from waterbutler.core import exceptions
from waterbutler.tasks import settings as task_settings
from waterbutler.server import settings as server_settings
from waterbutler.core.signing import Signer


logger = logging.getLogger(__name__)

signer = Signer(server_settings.HMAC_SECRET, server_settings.HMAC_ALGORITHM)

sentry_dsn = settings.get('SENTRY_DSN', None)
client = Client(sentry_dsn) if sentry_dsn else None


def make_provider(name, auth, credentials, settings, **kwargs):
    """Returns an instance of :class:`waterbutler.core.provider.BaseProvider`

    :param str name: The name of the provider to instantiate. (s3, box, etc)
    :param dict auth:
    :param dict credentials:
    :param dict settings:
    :param dict \*\*kwargs: currently there to absorb ``callback_url``

    :rtype: :class:`waterbutler.core.provider.BaseProvider`
    """
    try:
Ejemplo n.º 19
0
import functools
# from concurrent.futures import ProcessPoolExecutor  TODO Get this working

import aiohttp

from raven.contrib.tornado import AsyncSentryClient
from stevedore import driver

from waterbutler import settings
from waterbutler.server import settings as server_settings
from waterbutler.core import exceptions
from waterbutler.core.signing import Signer

logger = logging.getLogger(__name__)

sentry_dns = settings.get('SENTRY_DSN', None)
signer = Signer(server_settings.HMAC_SECRET, server_settings.HMAC_ALGORITHM)


class AioSentryClient(AsyncSentryClient):
    def send_remote(self, url, data, headers=None, callback=None):
        headers = headers or {}
        if not self.state.should_try():
            message = self._get_log_message(data)
            self.error_logger.error(message)
            return

        future = aiohttp.request('POST', url, data=data, headers=headers)
        asyncio. async (future)

Ejemplo n.º 20
0
import hashlib

try:
    from waterbutler import settings
except ImportError:
    settings = {}

config = settings.get('SERVER_CONFIG', {})


ADDRESS = config.get('ADDRESS', '127.0.0.1')
PORT = config.get('PORT', 7777)
DOMAIN = config.get('DOMAIN', "http://127.0.0.1:7777")

DEBUG = config.get('DEBUG', True)

SSL_CERT_FILE = config.get('SSL_CERT_FILE', None)
SSL_KEY_FILE = config.get('SSL_KEY_FILE', None)

XHEADERS = config.get('XHEADERS', False)
CORS_ALLOW_ORIGIN = config.get('CORS_ALLOW_ORIGIN', '*')

CHUNK_SIZE = config.get('CHUNK_SIZE', 65536)  # 64KB
MAX_BODY_SIZE = config.get('MAX_BODY_SIZE', int(4.9 * (1024 ** 3)))  # 4.9 GB

AUTH_HANDLERS = config.get('AUTH_HANDLERS', [
    'osf',
])

HMAC_ALGORITHM = getattr(hashlib, config.get('HMAC_ALGORITHM', 'sha256'))
Ejemplo n.º 21
0
try:
    from waterbutler import settings
except ImportError:
    settings = {}

config = settings.get('GITHUB_PROVIDER_CONFIG', {})

BASE_URL = config.get('BASE_URL', 'https://api.github.com/')
VIEW_URL = config.get('VIEW_URL', 'https://github.com/')

MOVE_MESSAGE = config.get('MOVE_MESSAGE', 'Moved on behalf of WaterButler')
COPY_MESSAGE = config.get('COPY_MESSAGE', 'Copied on behalf of WaterButler')
DELETE_FILE_MESSAGE = config.get('DELETE_FILE_MESSAGE',
                                 'File deleted on behalf of WaterButler')
UPDATE_FILE_MESSAGE = config.get('UPDATE_FILE_MESSAGE',
                                 'File updated on behalf of WaterButler')
UPLOAD_FILE_MESSAGE = config.get('UPLOAD_FILE_MESSAGE',
                                 'File uploaded on behalf of WaterButler')
DELETE_FOLDER_MESSAGE = config.get('DELETE_FOLDER_MESSAGE',
                                   'Folder deleted on behalf of WaterButler')
Ejemplo n.º 22
0
try:
    from waterbutler import settings
except ImportError:
    settings = {}

config = settings.get('OSF_AUTH_CONFIG', {})

API_URL = config.get('API_URL', 'http://127.0.0.1:5000/api/v1/files/auth/')
Ejemplo n.º 23
0
try:
    from waterbutler import settings
except ImportError:
    settings = {}

config = settings.get('GITHUB_PROVIDER_CONFIG', {})


BASE_URL = config.get('BASE_URL', 'https://api.github.com/')
VIEW_URL = config.get('VIEW_URL', 'https://github.com/')

MOVE_MESSAGE = config.get('MOVE_MESSAGE', 'Moved on behalf of WaterButler')
COPY_MESSAGE = config.get('COPY_MESSAGE', 'Copied on behalf of WaterButler')
DELETE_FILE_MESSAGE = config.get('DELETE_FILE_MESSAGE', 'File deleted on behalf of WaterButler')
UPDATE_FILE_MESSAGE = config.get('UPDATE_FILE_MESSAGE', 'File updated on behalf of WaterButler')
UPLOAD_FILE_MESSAGE = config.get('UPLOAD_FILE_MESSAGE', 'File uploaded on behalf of WaterButler')
DELETE_FOLDER_MESSAGE = config.get('DELETE_FOLDER_MESSAGE', 'Folder deleted on behalf of WaterButler')
Ejemplo n.º 24
0
try:
    from waterbutler import settings
except ImportError:
    settings = {}

config = settings.get('GOOGLEDRIVE_PROVIDER_CONFIG', {})

BASE_URL = config.get('BASE_URL', 'https://www.googleapis.com/drive/v2')
BASE_UPLOAD_URL = config.get('BASE_UPLOAD_URL',
                             'https://www.googleapis.com/upload/drive/v2')
DRIVE_IGNORE_VERSION = config.get('DRIVE_IGNORE_VERSION',
                                  '0000000000000000000000000000000000000')
Ejemplo n.º 25
0
import hashlib

try:
    from waterbutler import settings
except ImportError:
    settings = {}

config = settings.get('OSFSTORAGE_PROVIDER_CONFIG', {})


FILE_PATH_PENDING = config.get('FILE_PATH_PENDING', '/tmp/pending')
FILE_PATH_COMPLETE = config.get('FILE_PATH_COMPLETE', '/tmp/complete')

RUN_TASKS = config.get('RUN_TASKS', False)

HMAC_ALGORITHM = getattr(hashlib, config.get('HMAC_ALGORITHM', 'sha256'))
HMAC_SECRET = config.get('HMAC_SECRET', 'changeme').encode('utf-8')

# Retry options
UPLOAD_RETRY_ATTEMPTS = config.get('UPLOAD_RETRY_ATTEMPTS', 1)
UPLOAD_RETRY_INIT_DELAY = config.get('UPLOAD_RETRY_INIT_DELAY', 30)
UPLOAD_RETRY_MAX_DELAY = config.get('UPLOAD_RETRY_MAX_DELAY', 60 * 60)
UPLOAD_RETRY_BACKOFF = config.get('UPLOAD_RETRY_BACKOFF', 2)
UPLOAD_RETRY_WARN_IDX = config.get('UPLOAD_RETRY_WARN_IDX', 5)

HOOK_RETRY_ATTEMPTS = config.get('HOOK_RETRY_ATTEMPTS ', 1)
HOOK_RETRY_INIT_DELAY = config.get('HOOK_RETRY_INIT_DELAY', 30)
HOOK_RETRY_MAX_DELAY = config.get('HOOK_RETRY_MAX_DELAY', 60 * 60)
HOOK_RETRY_BACKOFF = config.get('HOOK_RETRY_BACKOFF', 2)
HOOK_RETRY_WARN_IDX = config.get('HOOK_RETRY_WARN_IDX', None)
Ejemplo n.º 26
0
try:
    from waterbutler import settings
except ImportError:
    settings = {}

config = settings.get('BOX_PROVIDER_CONFIG', {})


BASE_URL = config.get('BASE_URL', 'https://api.box.com/2.0')
BASE_UPLOAD_URL = config.get('BASE_CONTENT_URL', 'https://upload.box.com/api/2.0')
Ejemplo n.º 27
0
try:
    from waterbutler import settings
except ImportError:
    settings = {}

config = settings.get('DATAVERSE_PROVIDER_CONFIG', {})

EDIT_MEDIA_BASE_URL = config.get('EDIT_MEDIA_BASE_URL', "/dvn/api/data-deposit/v1.1/swordv2/edit-media/")
DOWN_BASE_URL = config.get('DOWN_BASE_URL', "/api/access/datafile/")
METADATA_BASE_URL = config.get('METADATA_BASE_URL', "/dvn/api/data-deposit/v1.1/swordv2/statement/study/")
JSON_BASE_URL = config.get('JSON_BASE_URL', "/api/datasets/{0}/versions/:{1}")