from waterbutler import settings
config = settings.child('GOOGLEDRIVE_PROVIDER_CONFIG')
BASE_URL = config.get('BASE_URL', 'https://www.googleapis.com/drive/v2')
BASE_UPLOAD_URL = config.get('BASE_UPLOAD_URL',
'https://www.googleapis.com/upload/drive/v2')
DRIVE_IGNORE_VERSION = config.get('DRIVE_IGNORE_VERSION',
'0000000000000000000000000000000000000')
from waterbutler import settings
config = settings.child('OSF_AUTH_CONFIG')
JWT_EXPIRATION = int(config.get('JWT_EXPIRATION', 15))
JWT_ALGORITHM = config.get('JWT_ALGORITHM', 'HS256')
API_URL = config.get('API_URL', 'http://localhost:5000/api/v1/files/auth/')
JWE_SALT = config.get('JWE_SALT')
JWE_SECRET = config.get('JWE_SECRET')
JWT_SECRET = config.get('JWT_SECRET')
if not settings.DEBUG:
assert JWE_SALT, 'JWE_SALT must be specified when not in debug mode'
assert JWE_SECRET, 'JWE_SECRET must be specified when not in debug mode'
assert JWT_SECRET, 'JWT_SECRET must be specified when not in debug mode'
JWE_SALT = (JWE_SALT or 'yusaltydough')
JWE_SECRET = (JWE_SECRET or 'CirclesAre4Squares')
JWT_SECRET = (JWT_SECRET or 'ILiekTrianglesALot')
from waterbutler import settings
config = settings.child('DATAVERSE_PROVIDER_CONFIG')
EDIT_MEDIA_BASE_URL = config.get('EDIT_MEDIA_BASE_URL', "/dvn/api/data-deposit/v1.1/swordv2/edit-media/")
DOWN_BASE_URL = config.get('DOWN_BASE_URL', "/api/access/datafile/")
METADATA_BASE_URL = config.get('METADATA_BASE_URL', "/dvn/api/data-deposit/v1.1/swordv2/statement/study/")
JSON_BASE_URL = config.get('JSON_BASE_URL', "/api/v1/datasets/{0}/versions/:{1}")
import hashlib
from waterbutler import settings
config = settings.child('SERVER_CONFIG')
ADDRESS = config.get('ADDRESS', 'localhost')
PORT = config.get('PORT', 7777)
DOMAIN = config.get('DOMAIN', "http://localhost:7777")
DEBUG = config.get_bool('DEBUG', True)
SSL_CERT_FILE = config.get_nullable('SSL_CERT_FILE', None)
SSL_KEY_FILE = config.get_nullable('SSL_KEY_FILE', None)
XHEADERS = config.get_bool('XHEADERS', False)
CORS_ALLOW_ORIGIN = config.get('CORS_ALLOW_ORIGIN', '*')
CHUNK_SIZE = int(config.get('CHUNK_SIZE', 65536)) # 64KB
MAX_BODY_SIZE = int(config.get('MAX_BODY_SIZE',
int(4.9 * (1024**3)))) # 4.9 GB
AUTH_HANDLERS = config.get('AUTH_HANDLERS', [
'osf',
])
HMAC_ALGORITHM = getattr(hashlib, config.get('HMAC_ALGORITHM', 'sha256'))
HMAC_SECRET = config.get('HMAC_SECRET')
if not settings.DEBUG:
assert HMAC_SECRET, 'HMAC_SECRET must be specified when not in debug mode'
from waterbutler import settings
config = settings.child('S3_PROVIDER_CONFIG')
TEMP_URL_SECS = int(config.get('TEMP_URL_SECS', 100))
CONTIGUOUS_UPLOAD_SIZE_LIMIT = int(config.get('CONTIGUOUS_UPLOAD_SIZE_LIMIT', 128000000)) # 128 MB
CHUNK_SIZE = int(config.get('CHUNK_SIZE', 64000000)) # 64 MB
CHUNKED_UPLOAD_MAX_ABORT_RETRIES = int(config.get('CHUNKED_UPLOAD_MAX_ABORT_RETRIES', 2))
import os
from pkg_resources import iter_entry_points
from kombu import Queue, Exchange
from waterbutler import settings
config = settings.child('TASKS_CONFIG')
BROKER_URL = config.get(
'BROKER_URL',
'amqp://{}:{}//'.format(
os.environ.get('RABBITMQ_PORT_5672_TCP_ADDR', ''),
os.environ.get('RABBITMQ_PORT_5672_TCP_PORT', ''),
)
)
WAIT_TIMEOUT = int(config.get('WAIT_TIMEOUT', 15))
WAIT_INTERVAL = float(config.get('WAIT_INTERVAL', 0.5))
ADHOC_BACKEND_PATH = config.get('ADHOC_BACKEND_PATH', '/tmp')
CELERY_CREATE_MISSING_QUEUES = config.get_bool('CELERY_CREATE_MISSING_QUEUES', False)
CELERY_DEFAULT_QUEUE = config.get('CELERY_DEFAULT_QUEUE', 'waterbutler')
CELERY_QUEUES = (
Queue('waterbutler', Exchange('waterbutler'), routing_key='waterbutler'),
)
# CELERY_ALWAYS_EAGER = config.get('CELERY_ALWAYS_EAGER', True)
CELERY_ALWAYS_EAGER = config.get_bool('CELERY_ALWAYS_EAGER', False)
# CELERY_RESULT_BACKEND = config.get('CELERY_RESULT_BACKEND', 'redis://')
CELERY_RESULT_BACKEND = config.get_nullable('CELERY_RESULT_BACKEND', None)
import zlib
from waterbutler import settings
config = settings.child('STREAMS_CONFIG')
ZIP_EXTENSIONS = config.get(
'ZIP_EXTENSIONS', '.zip .gz .bzip .bzip2 .rar .xz .bz2 .7z').split(' ')
# Compression level to apply to zipped files. Value must be an integer from 0 to 9, where
# lower values represent less compression. -1 is also allowed, meaning the default level
# (approximately equivalent to a 6). See the zlib docs for more:
# https://docs.python.org/3/library/zlib.html#zlib.compressobj
ZIP_COMPRESSION_LEVEL = int(
config.get('ZIP_COMPRESSION_LEVEL', zlib.Z_DEFAULT_COMPRESSION))
from waterbutler import settings
config = settings.child('ONEDRIVE_PROVIDER_CONFIG')
BASE_URL = config.get('BASE_URL', 'https://api.onedrive.com/v1.0')
BASE_DRIVE_URL = config.get('BASE_DRIVE_URL', 'https://api.onedrive.com/v1.0/drive')
ONEDRIVE_COPY_ITERATION_COUNT = int(config.get('ONEDRIVE_COPY_ITERATION_COUNT', 30))
ONEDRIVE_COPY_SLEEP_INTERVAL = int(config.get('ONEDRIVE_COPY_SLEEP_INTERVAL', 3))
from waterbutler import settings
config = settings.child('CLOUDFILES_PROVIDER_CONFIG')
TEMP_URL_SECS = int(config.get('TEMP_URL_SECS', 100))
AUTH_URL = config.get('AUTH_URL', 'https://identity.api.rackspacecloud.com/v2.0/tokens')
from waterbutler import settings
config = settings.child('DATAVERSE_PROVIDER_CONFIG')
EDIT_MEDIA_BASE_URL = config.get(
'EDIT_MEDIA_BASE_URL', "/dvn/api/data-deposit/v1.1/swordv2/edit-media/")
DOWN_BASE_URL = config.get('DOWN_BASE_URL', "/api/access/datafile/")
# TODO: double check and remove this unused API URL / endpoint
METADATA_BASE_URL = config.get(
'METADATA_BASE_URL', "/dvn/api/data-deposit/v1.1/swordv2/statement/study/")
JSON_BASE_URL = config.get('JSON_BASE_URL',
"/api/v1/datasets/{0}/versions/:{1}")
import os
from pkg_resources import iter_entry_points
from kombu import Queue, Exchange
from waterbutler import settings
config = settings.child('TASKS_CONFIG')
BROKER_URL = config.get(
'BROKER_URL', 'amqp://{}:{}//'.format(
os.environ.get('RABBITMQ_PORT_5672_TCP_ADDR', ''),
os.environ.get('RABBITMQ_PORT_5672_TCP_PORT', ''),
))
WAIT_TIMEOUT = int(config.get('WAIT_TIMEOUT', 20))
WAIT_INTERVAL = float(config.get('WAIT_INTERVAL', 0.5))
ADHOC_BACKEND_PATH = config.get('ADHOC_BACKEND_PATH', '/tmp')
CELERY_CREATE_MISSING_QUEUES = config.get_bool('CELERY_CREATE_MISSING_QUEUES',
False)
CELERY_DEFAULT_QUEUE = config.get('CELERY_DEFAULT_QUEUE', 'waterbutler')
CELERY_QUEUES = (Queue('waterbutler',
Exchange('waterbutler'),
routing_key='waterbutler'), )
CELERY_ALWAYS_EAGER = config.get_bool('CELERY_ALWAYS_EAGER', False)
CELERY_RESULT_BACKEND = config.get_nullable('CELERY_RESULT_BACKEND', 'amqp')
CELERY_RESULT_PERSISTENT = config.get_bool('CELERY_RESULT_PERSISTENT', True)
CELERY_DISABLE_RATE_LIMITS = config.get_bool('CELERY_DISABLE_RATE_LIMITS',
True)
CELERY_TASK_RESULT_EXPIRES = int(config.get('CELERY_TASK_RESULT_EXPIRES', 60))
import hashlib
from waterbutler import settings
config = settings.child('OSFSTORAGE_PROVIDER_CONFIG')
FILE_PATH_PENDING = config.get('FILE_PATH_PENDING', '/tmp/pending')
FILE_PATH_COMPLETE = config.get('FILE_PATH_COMPLETE', '/tmp/complete')
RUN_TASKS = config.get_bool('RUN_TASKS', False)
HMAC_ALGORITHM = getattr(hashlib, config.get('HMAC_ALGORITHM', 'sha256'))
HMAC_SECRET = config.get_nullable('HMAC_SECRET', None)
if not settings.DEBUG:
assert HMAC_SECRET, 'HMAC_SECRET must be specified when not in debug mode'
HMAC_SECRET = (HMAC_SECRET or 'changeme').encode('utf-8')
# Retry options
UPLOAD_RETRY_ATTEMPTS = int(config.get('UPLOAD_RETRY_ATTEMPTS', 1))
UPLOAD_RETRY_INIT_DELAY = int(config.get('UPLOAD_RETRY_INIT_DELAY', 30))
UPLOAD_RETRY_MAX_DELAY = int(config.get('UPLOAD_RETRY_MAX_DELAY', 60 * 60))
UPLOAD_RETRY_BACKOFF = int(config.get('UPLOAD_RETRY_BACKOFF', 2))
UPLOAD_RETRY_WARN_IDX = int(config.get('UPLOAD_RETRY_WARN_IDX', 5))
HOOK_RETRY_ATTEMPTS = int(config.get('HOOK_RETRY_ATTEMPTS ', 1))
HOOK_RETRY_INIT_DELAY = int(config.get('HOOK_RETRY_INIT_DELAY', 30))
HOOK_RETRY_MAX_DELAY = int(config.get('HOOK_RETRY_MAX_DELAY', 60 * 60))
HOOK_RETRY_BACKOFF = int(config.get('HOOK_RETRY_BACKOFF', 2))
import hashlib
from waterbutler import settings
config = settings.child('SERVER_CONFIG')
ADDRESS = config.get('ADDRESS', 'localhost')
PORT = config.get('PORT', 7777)
DOMAIN = config.get('DOMAIN', "http://localhost:7777")
DEBUG = config.get_bool('DEBUG', True)
SSL_CERT_FILE = config.get_nullable('SSL_CERT_FILE', None)
SSL_KEY_FILE = config.get_nullable('SSL_KEY_FILE', None)
XHEADERS = config.get_bool('XHEADERS', False)
CORS_ALLOW_ORIGIN = config.get('CORS_ALLOW_ORIGIN', '*')
CHUNK_SIZE = int(config.get('CHUNK_SIZE', 65536)) # 64KB
MAX_BODY_SIZE = int(config.get('MAX_BODY_SIZE', int(4.9 * (1024 ** 3)))) # 4.9 GB
AUTH_HANDLERS = config.get('AUTH_HANDLERS', [
'osf',
])
HMAC_ALGORITHM = getattr(hashlib, config.get('HMAC_ALGORITHM', 'sha256'))
HMAC_SECRET = config.get('HMAC_SECRET')
if not settings.DEBUG:
assert HMAC_SECRET, 'HMAC_SECRET must be specified when not in debug mode'
from waterbutler import settings
config = settings.child('GOOGLECLOUD_PROVIDER_SETTINGS')
# BASE URL
BASE_URL = config.get('BASE_URL', 'https://storage.googleapis.com')
# The expiration time (in seconds) for a signed request
SIGNATURE_EXPIRATION = int(config.get('SIGNATURE_EXPIRATION', 60))
from waterbutler import settings
config = settings.child('FILESYSTEM_PROVIDER_CONFIG')
CHUNK_SIZE = int(config.get('CHUNK_SIZE', 65536)) # 64KB
import hashlib
from waterbutler import settings
config = settings.child('OSFSTORAGE_PROVIDER_CONFIG')
FILE_PATH_PENDING = config.get('FILE_PATH_PENDING', '/tmp/pending')
FILE_PATH_COMPLETE = config.get('FILE_PATH_COMPLETE', '/tmp/complete')
RUN_TASKS = config.get_bool('RUN_TASKS', False)
HMAC_ALGORITHM = getattr(hashlib, config.get('HMAC_ALGORITHM', 'sha256'))
HMAC_SECRET = config.get_nullable('HMAC_SECRET', None)
if not settings.DEBUG:
assert HMAC_SECRET, 'HMAC_SECRET must be specified when not in debug mode'
HMAC_SECRET = (HMAC_SECRET or 'changeme').encode('utf-8')
# Retry options
UPLOAD_RETRY_ATTEMPTS = int(config.get('UPLOAD_RETRY_ATTEMPTS', 1))
UPLOAD_RETRY_INIT_DELAY = int(config.get('UPLOAD_RETRY_INIT_DELAY', 30))
UPLOAD_RETRY_MAX_DELAY = int(config.get('UPLOAD_RETRY_MAX_DELAY', 60 * 60))
UPLOAD_RETRY_BACKOFF = int(config.get('UPLOAD_RETRY_BACKOFF', 2))
UPLOAD_RETRY_WARN_IDX = int(config.get('UPLOAD_RETRY_WARN_IDX', 5))
HOOK_RETRY_ATTEMPTS = int(config.get('HOOK_RETRY_ATTEMPTS ', 1))
HOOK_RETRY_INIT_DELAY = int(config.get('HOOK_RETRY_INIT_DELAY', 30))
HOOK_RETRY_MAX_DELAY = int(config.get('HOOK_RETRY_MAX_DELAY', 60 * 60))
HOOK_RETRY_BACKOFF = int(config.get('HOOK_RETRY_BACKOFF', 2))
HOOK_RETRY_WARN_IDX = config.get_nullable('HOOK_RETRY_WARN_IDX', None)
from waterbutler import settings
config = settings.child('FIGSHARE_PROVIDER_CONFIG')
BASE_URL = config.get('BASE_URL', 'https://api.figshare.com/v2')
VIEW_URL = config.get('VIEW_URL', 'https://figshare.com/')
DOWNLOAD_URL = config.get('VIEW_URL', 'https://ndownloader.figshare.com/')
VALID_CONTAINER_TYPES = ['project', 'collection', 'article', 'fileset']
FOLDER_TYPES = [4] # Figshare ID for filesets
PRIVATE_IDENTIFIER = 'https://api.figshare.com/v2/account/'
ARTICLE_TYPE_IDENTIFIER = 'https://api.figshare.com/v2/account/articles/'
# During initial testing this was set to 2 because file was not instantly ready after receiving HTTP 201
FILE_CREATE_WAIT = 0.1 # seconds passed to time.sleep
# project/collection article listings are paginated. Specify max number of results returned per page.
MAX_PAGE_SIZE = int(config.get('MAX_PAGE_SIZE', 100))
from waterbutler import settings
config = settings.child('CLOUDFILES_PROVIDER_CONFIG')
TEMP_URL_SECS = int(config.get('TEMP_URL_SECS', 100))
AUTH_URL = config.get('AUTH_URL',
'https://identity.api.rackspacecloud.com/v2.0/tokens')
from waterbutler import settings
config = settings.child("S3_PROVIDER_CONFIG")
TEMP_URL_SECS = int(config.get("TEMP_URL_SECS", 100))
from waterbutler import settings
config = settings.child('BITBUCKET_PROVIDER_CONFIG')
BASE_URL = config.get('BASE_URL', 'https://api.bitbucket.org/')
VIEW_URL = config.get('VIEW_URL', 'https://bitbucket.org/')
MOVE_MESSAGE = config.get('MOVE_MESSAGE', 'Moved on behalf of WaterButler')
COPY_MESSAGE = config.get('COPY_MESSAGE', 'Copied on behalf of WaterButler')
DELETE_FILE_MESSAGE = config.get('DELETE_FILE_MESSAGE', 'File deleted on behalf of WaterButler')
UPDATE_FILE_MESSAGE = config.get('UPDATE_FILE_MESSAGE', 'File updated on behalf of WaterButler')
UPLOAD_FILE_MESSAGE = config.get('UPLOAD_FILE_MESSAGE', 'File uploaded on behalf of WaterButler')
DELETE_FOLDER_MESSAGE = config.get('DELETE_FOLDER_MESSAGE', 'Folder deleted on behalf of WaterButler')
from waterbutler import settings
config = settings.child('FIGSHARE_PROVIDER_CONFIG')
BASE_URL = config.get('BASE_URL', 'https://api.figshare.com/v2')
VIEW_URL = config.get('VIEW_URL', 'https://figshare.com/')
DOWNLOAD_URL = config.get('VIEW_URL', 'https://ndownloader.figshare.com/')
VALID_CONTAINER_TYPES = ['project', 'collection', 'article', 'fileset']
FOLDER_TYPES = [4] # Figshare ID for filesets
PRIVATE_IDENTIFIER = 'https://api.figshare.com/v2/account/'
ARTICLE_TYPE_IDENTIFIER = 'https://api.figshare.com/v2/account/articles/'
# During initial testing this was set to 2 because file was not instantly ready after receiving HTTP 201
FILE_CREATE_WAIT = 0.1 # seconds passed to time.sleep
# project/collection article listings are paginated. Specify max number of results returned per page.
MAX_PAGE_SIZE = int(config.get('MAX_PAGE_SIZE', 100))
from waterbutler import settings
config = settings.child('ONEDRIVE_PROVIDER_CONFIG')
BASE_URL = config.get('BASE_URL', 'https://api.onedrive.com/v1.0')
BASE_DRIVE_URL = config.get('BASE_DRIVE_URL',
'https://api.onedrive.com/v1.0/drive')
ONEDRIVE_COPY_ITERATION_COUNT = int(
config.get('ONEDRIVE_COPY_ITERATION_COUNT', 30))
ONEDRIVE_COPY_SLEEP_INTERVAL = int(
config.get('ONEDRIVE_COPY_SLEEP_INTERVAL', 3))
from waterbutler import settings
config = settings.child('DROPBOX_PROVIDER_CONFIG')
BASE_URL = config.get('BASE_URL', 'https://api.dropboxapi.com/2')
BASE_CONTENT_URL = config.get('BASE_CONTENT_URL',
'https://content.dropboxapi.com/2/')
from waterbutler import settings
config = settings.child('GOOGLEDRIVE_PROVIDER_CONFIG')
BASE_URL = config.get('BASE_URL', 'https://www.googleapis.com/drive/v2')
BASE_UPLOAD_URL = config.get('BASE_UPLOAD_URL', 'https://www.googleapis.com/upload/drive/v2')
DRIVE_IGNORE_VERSION = config.get('DRIVE_IGNORE_VERSION', '0000000000000000000000000000000000000')
from waterbutler import settings
config = settings.child('DROPBOX_PROVIDER_CONFIG')
BASE_URL = config.get('BASE_URL', 'https://api.dropboxapi.com/2')
BASE_CONTENT_URL = config.get('BASE_CONTENT_URL', 'https://content.dropboxapi.com/2/')
CONTIGUOUS_UPLOAD_SIZE_LIMIT = int(config.get('CONTIGUOUS_UPLOAD_SIZE_LIMIT', 150000000)) # 150 MB
CHUNK_SIZE = int(config.get('CHUNK_SIZE', 4000000)) # 4 MB
import zlib
from waterbutler import settings
config = settings.child('STREAMS_CONFIG')
ZIP_EXTENSIONS = config.get('ZIP_EXTENSIONS', '.zip .gz .bzip .bzip2 .rar .xz .bz2 .7z').split(' ')
# Compression level to apply to zipped files. Value must be an integer from 0 to 9, where
# lower values represent less compression. -1 is also allowed, meaning the default level
# (approximately equivalent to a 6). See the zlib docs for more:
# https://docs.python.org/3/library/zlib.html#zlib.compressobj
ZIP_COMPRESSION_LEVEL = int(config.get('ZIP_COMPRESSION_LEVEL', zlib.Z_DEFAULT_COMPRESSION))
from waterbutler import settings
config = settings.child('GITLAB_PROVIDER_CONFIG')
from waterbutler import settings
config = settings.child('S3_PROVIDER_CONFIG')
TEMP_URL_SECS = int(config.get('TEMP_URL_SECS', 100))