from waterbutler import settings config = settings.child('GOOGLEDRIVE_PROVIDER_CONFIG') BASE_URL = config.get('BASE_URL', 'https://www.googleapis.com/drive/v2') BASE_UPLOAD_URL = config.get('BASE_UPLOAD_URL', 'https://www.googleapis.com/upload/drive/v2') DRIVE_IGNORE_VERSION = config.get('DRIVE_IGNORE_VERSION', '0000000000000000000000000000000000000')
from waterbutler import settings config = settings.child('OSF_AUTH_CONFIG') JWT_EXPIRATION = int(config.get('JWT_EXPIRATION', 15)) JWT_ALGORITHM = config.get('JWT_ALGORITHM', 'HS256') API_URL = config.get('API_URL', 'http://localhost:5000/api/v1/files/auth/') JWE_SALT = config.get('JWE_SALT') JWE_SECRET = config.get('JWE_SECRET') JWT_SECRET = config.get('JWT_SECRET') if not settings.DEBUG: assert JWE_SALT, 'JWE_SALT must be specified when not in debug mode' assert JWE_SECRET, 'JWE_SECRET must be specified when not in debug mode' assert JWT_SECRET, 'JWT_SECRET must be specified when not in debug mode' JWE_SALT = (JWE_SALT or 'yusaltydough') JWE_SECRET = (JWE_SECRET or 'CirclesAre4Squares') JWT_SECRET = (JWT_SECRET or 'ILiekTrianglesALot')
from waterbutler import settings config = settings.child('DATAVERSE_PROVIDER_CONFIG') EDIT_MEDIA_BASE_URL = config.get('EDIT_MEDIA_BASE_URL', "/dvn/api/data-deposit/v1.1/swordv2/edit-media/") DOWN_BASE_URL = config.get('DOWN_BASE_URL', "/api/access/datafile/") METADATA_BASE_URL = config.get('METADATA_BASE_URL', "/dvn/api/data-deposit/v1.1/swordv2/statement/study/") JSON_BASE_URL = config.get('JSON_BASE_URL', "/api/v1/datasets/{0}/versions/:{1}")
import hashlib from waterbutler import settings config = settings.child('SERVER_CONFIG') ADDRESS = config.get('ADDRESS', 'localhost') PORT = config.get('PORT', 7777) DOMAIN = config.get('DOMAIN', "http://localhost:7777") DEBUG = config.get_bool('DEBUG', True) SSL_CERT_FILE = config.get_nullable('SSL_CERT_FILE', None) SSL_KEY_FILE = config.get_nullable('SSL_KEY_FILE', None) XHEADERS = config.get_bool('XHEADERS', False) CORS_ALLOW_ORIGIN = config.get('CORS_ALLOW_ORIGIN', '*') CHUNK_SIZE = int(config.get('CHUNK_SIZE', 65536)) # 64KB MAX_BODY_SIZE = int(config.get('MAX_BODY_SIZE', int(4.9 * (1024**3)))) # 4.9 GB AUTH_HANDLERS = config.get('AUTH_HANDLERS', [ 'osf', ]) HMAC_ALGORITHM = getattr(hashlib, config.get('HMAC_ALGORITHM', 'sha256')) HMAC_SECRET = config.get('HMAC_SECRET') if not settings.DEBUG: assert HMAC_SECRET, 'HMAC_SECRET must be specified when not in debug mode'
from waterbutler import settings config = settings.child('S3_PROVIDER_CONFIG') TEMP_URL_SECS = int(config.get('TEMP_URL_SECS', 100)) CONTIGUOUS_UPLOAD_SIZE_LIMIT = int(config.get('CONTIGUOUS_UPLOAD_SIZE_LIMIT', 128000000)) # 128 MB CHUNK_SIZE = int(config.get('CHUNK_SIZE', 64000000)) # 64 MB CHUNKED_UPLOAD_MAX_ABORT_RETRIES = int(config.get('CHUNKED_UPLOAD_MAX_ABORT_RETRIES', 2))
import os from pkg_resources import iter_entry_points from kombu import Queue, Exchange from waterbutler import settings config = settings.child('TASKS_CONFIG') BROKER_URL = config.get( 'BROKER_URL', 'amqp://{}:{}//'.format( os.environ.get('RABBITMQ_PORT_5672_TCP_ADDR', ''), os.environ.get('RABBITMQ_PORT_5672_TCP_PORT', ''), ) ) WAIT_TIMEOUT = int(config.get('WAIT_TIMEOUT', 15)) WAIT_INTERVAL = float(config.get('WAIT_INTERVAL', 0.5)) ADHOC_BACKEND_PATH = config.get('ADHOC_BACKEND_PATH', '/tmp') CELERY_CREATE_MISSING_QUEUES = config.get_bool('CELERY_CREATE_MISSING_QUEUES', False) CELERY_DEFAULT_QUEUE = config.get('CELERY_DEFAULT_QUEUE', 'waterbutler') CELERY_QUEUES = ( Queue('waterbutler', Exchange('waterbutler'), routing_key='waterbutler'), ) # CELERY_ALWAYS_EAGER = config.get('CELERY_ALWAYS_EAGER', True) CELERY_ALWAYS_EAGER = config.get_bool('CELERY_ALWAYS_EAGER', False) # CELERY_RESULT_BACKEND = config.get('CELERY_RESULT_BACKEND', 'redis://') CELERY_RESULT_BACKEND = config.get_nullable('CELERY_RESULT_BACKEND', None)
import zlib from waterbutler import settings config = settings.child('STREAMS_CONFIG') ZIP_EXTENSIONS = config.get( 'ZIP_EXTENSIONS', '.zip .gz .bzip .bzip2 .rar .xz .bz2 .7z').split(' ') # Compression level to apply to zipped files. Value must be an integer from 0 to 9, where # lower values represent less compression. -1 is also allowed, meaning the default level # (approximately equivalent to a 6). See the zlib docs for more: # https://docs.python.org/3/library/zlib.html#zlib.compressobj ZIP_COMPRESSION_LEVEL = int( config.get('ZIP_COMPRESSION_LEVEL', zlib.Z_DEFAULT_COMPRESSION))
from waterbutler import settings config = settings.child('ONEDRIVE_PROVIDER_CONFIG') BASE_URL = config.get('BASE_URL', 'https://api.onedrive.com/v1.0') BASE_DRIVE_URL = config.get('BASE_DRIVE_URL', 'https://api.onedrive.com/v1.0/drive') ONEDRIVE_COPY_ITERATION_COUNT = int(config.get('ONEDRIVE_COPY_ITERATION_COUNT', 30)) ONEDRIVE_COPY_SLEEP_INTERVAL = int(config.get('ONEDRIVE_COPY_SLEEP_INTERVAL', 3))
from waterbutler import settings config = settings.child('CLOUDFILES_PROVIDER_CONFIG') TEMP_URL_SECS = int(config.get('TEMP_URL_SECS', 100)) AUTH_URL = config.get('AUTH_URL', 'https://identity.api.rackspacecloud.com/v2.0/tokens')
from waterbutler import settings config = settings.child('DATAVERSE_PROVIDER_CONFIG') EDIT_MEDIA_BASE_URL = config.get( 'EDIT_MEDIA_BASE_URL', "/dvn/api/data-deposit/v1.1/swordv2/edit-media/") DOWN_BASE_URL = config.get('DOWN_BASE_URL', "/api/access/datafile/") # TODO: double check and remove this unused API URL / endpoint METADATA_BASE_URL = config.get( 'METADATA_BASE_URL', "/dvn/api/data-deposit/v1.1/swordv2/statement/study/") JSON_BASE_URL = config.get('JSON_BASE_URL', "/api/v1/datasets/{0}/versions/:{1}")
import os from pkg_resources import iter_entry_points from kombu import Queue, Exchange from waterbutler import settings config = settings.child('TASKS_CONFIG') BROKER_URL = config.get( 'BROKER_URL', 'amqp://{}:{}//'.format( os.environ.get('RABBITMQ_PORT_5672_TCP_ADDR', ''), os.environ.get('RABBITMQ_PORT_5672_TCP_PORT', ''), )) WAIT_TIMEOUT = int(config.get('WAIT_TIMEOUT', 20)) WAIT_INTERVAL = float(config.get('WAIT_INTERVAL', 0.5)) ADHOC_BACKEND_PATH = config.get('ADHOC_BACKEND_PATH', '/tmp') CELERY_CREATE_MISSING_QUEUES = config.get_bool('CELERY_CREATE_MISSING_QUEUES', False) CELERY_DEFAULT_QUEUE = config.get('CELERY_DEFAULT_QUEUE', 'waterbutler') CELERY_QUEUES = (Queue('waterbutler', Exchange('waterbutler'), routing_key='waterbutler'), ) CELERY_ALWAYS_EAGER = config.get_bool('CELERY_ALWAYS_EAGER', False) CELERY_RESULT_BACKEND = config.get_nullable('CELERY_RESULT_BACKEND', 'amqp') CELERY_RESULT_PERSISTENT = config.get_bool('CELERY_RESULT_PERSISTENT', True) CELERY_DISABLE_RATE_LIMITS = config.get_bool('CELERY_DISABLE_RATE_LIMITS', True) CELERY_TASK_RESULT_EXPIRES = int(config.get('CELERY_TASK_RESULT_EXPIRES', 60))
import hashlib from waterbutler import settings config = settings.child('OSFSTORAGE_PROVIDER_CONFIG') FILE_PATH_PENDING = config.get('FILE_PATH_PENDING', '/tmp/pending') FILE_PATH_COMPLETE = config.get('FILE_PATH_COMPLETE', '/tmp/complete') RUN_TASKS = config.get_bool('RUN_TASKS', False) HMAC_ALGORITHM = getattr(hashlib, config.get('HMAC_ALGORITHM', 'sha256')) HMAC_SECRET = config.get_nullable('HMAC_SECRET', None) if not settings.DEBUG: assert HMAC_SECRET, 'HMAC_SECRET must be specified when not in debug mode' HMAC_SECRET = (HMAC_SECRET or 'changeme').encode('utf-8') # Retry options UPLOAD_RETRY_ATTEMPTS = int(config.get('UPLOAD_RETRY_ATTEMPTS', 1)) UPLOAD_RETRY_INIT_DELAY = int(config.get('UPLOAD_RETRY_INIT_DELAY', 30)) UPLOAD_RETRY_MAX_DELAY = int(config.get('UPLOAD_RETRY_MAX_DELAY', 60 * 60)) UPLOAD_RETRY_BACKOFF = int(config.get('UPLOAD_RETRY_BACKOFF', 2)) UPLOAD_RETRY_WARN_IDX = int(config.get('UPLOAD_RETRY_WARN_IDX', 5)) HOOK_RETRY_ATTEMPTS = int(config.get('HOOK_RETRY_ATTEMPTS ', 1)) HOOK_RETRY_INIT_DELAY = int(config.get('HOOK_RETRY_INIT_DELAY', 30)) HOOK_RETRY_MAX_DELAY = int(config.get('HOOK_RETRY_MAX_DELAY', 60 * 60)) HOOK_RETRY_BACKOFF = int(config.get('HOOK_RETRY_BACKOFF', 2))
import hashlib from waterbutler import settings config = settings.child('SERVER_CONFIG') ADDRESS = config.get('ADDRESS', 'localhost') PORT = config.get('PORT', 7777) DOMAIN = config.get('DOMAIN', "http://localhost:7777") DEBUG = config.get_bool('DEBUG', True) SSL_CERT_FILE = config.get_nullable('SSL_CERT_FILE', None) SSL_KEY_FILE = config.get_nullable('SSL_KEY_FILE', None) XHEADERS = config.get_bool('XHEADERS', False) CORS_ALLOW_ORIGIN = config.get('CORS_ALLOW_ORIGIN', '*') CHUNK_SIZE = int(config.get('CHUNK_SIZE', 65536)) # 64KB MAX_BODY_SIZE = int(config.get('MAX_BODY_SIZE', int(4.9 * (1024 ** 3)))) # 4.9 GB AUTH_HANDLERS = config.get('AUTH_HANDLERS', [ 'osf', ]) HMAC_ALGORITHM = getattr(hashlib, config.get('HMAC_ALGORITHM', 'sha256')) HMAC_SECRET = config.get('HMAC_SECRET') if not settings.DEBUG: assert HMAC_SECRET, 'HMAC_SECRET must be specified when not in debug mode'
from waterbutler import settings config = settings.child('GOOGLECLOUD_PROVIDER_SETTINGS') # BASE URL BASE_URL = config.get('BASE_URL', 'https://storage.googleapis.com') # The expiration time (in seconds) for a signed request SIGNATURE_EXPIRATION = int(config.get('SIGNATURE_EXPIRATION', 60))
from waterbutler import settings config = settings.child('FILESYSTEM_PROVIDER_CONFIG') CHUNK_SIZE = int(config.get('CHUNK_SIZE', 65536)) # 64KB
import hashlib from waterbutler import settings config = settings.child('OSFSTORAGE_PROVIDER_CONFIG') FILE_PATH_PENDING = config.get('FILE_PATH_PENDING', '/tmp/pending') FILE_PATH_COMPLETE = config.get('FILE_PATH_COMPLETE', '/tmp/complete') RUN_TASKS = config.get_bool('RUN_TASKS', False) HMAC_ALGORITHM = getattr(hashlib, config.get('HMAC_ALGORITHM', 'sha256')) HMAC_SECRET = config.get_nullable('HMAC_SECRET', None) if not settings.DEBUG: assert HMAC_SECRET, 'HMAC_SECRET must be specified when not in debug mode' HMAC_SECRET = (HMAC_SECRET or 'changeme').encode('utf-8') # Retry options UPLOAD_RETRY_ATTEMPTS = int(config.get('UPLOAD_RETRY_ATTEMPTS', 1)) UPLOAD_RETRY_INIT_DELAY = int(config.get('UPLOAD_RETRY_INIT_DELAY', 30)) UPLOAD_RETRY_MAX_DELAY = int(config.get('UPLOAD_RETRY_MAX_DELAY', 60 * 60)) UPLOAD_RETRY_BACKOFF = int(config.get('UPLOAD_RETRY_BACKOFF', 2)) UPLOAD_RETRY_WARN_IDX = int(config.get('UPLOAD_RETRY_WARN_IDX', 5)) HOOK_RETRY_ATTEMPTS = int(config.get('HOOK_RETRY_ATTEMPTS ', 1)) HOOK_RETRY_INIT_DELAY = int(config.get('HOOK_RETRY_INIT_DELAY', 30)) HOOK_RETRY_MAX_DELAY = int(config.get('HOOK_RETRY_MAX_DELAY', 60 * 60)) HOOK_RETRY_BACKOFF = int(config.get('HOOK_RETRY_BACKOFF', 2)) HOOK_RETRY_WARN_IDX = config.get_nullable('HOOK_RETRY_WARN_IDX', None)
from waterbutler import settings config = settings.child('FIGSHARE_PROVIDER_CONFIG') BASE_URL = config.get('BASE_URL', 'https://api.figshare.com/v2') VIEW_URL = config.get('VIEW_URL', 'https://figshare.com/') DOWNLOAD_URL = config.get('VIEW_URL', 'https://ndownloader.figshare.com/') VALID_CONTAINER_TYPES = ['project', 'collection', 'article', 'fileset'] FOLDER_TYPES = [4] # Figshare ID for filesets PRIVATE_IDENTIFIER = 'https://api.figshare.com/v2/account/' ARTICLE_TYPE_IDENTIFIER = 'https://api.figshare.com/v2/account/articles/' # During initial testing this was set to 2 because file was not instantly ready after receiving HTTP 201 FILE_CREATE_WAIT = 0.1 # seconds passed to time.sleep # project/collection article listings are paginated. Specify max number of results returned per page. MAX_PAGE_SIZE = int(config.get('MAX_PAGE_SIZE', 100))
from waterbutler import settings config = settings.child("S3_PROVIDER_CONFIG") TEMP_URL_SECS = int(config.get("TEMP_URL_SECS", 100))
from waterbutler import settings config = settings.child('BITBUCKET_PROVIDER_CONFIG') BASE_URL = config.get('BASE_URL', 'https://api.bitbucket.org/') VIEW_URL = config.get('VIEW_URL', 'https://bitbucket.org/') MOVE_MESSAGE = config.get('MOVE_MESSAGE', 'Moved on behalf of WaterButler') COPY_MESSAGE = config.get('COPY_MESSAGE', 'Copied on behalf of WaterButler') DELETE_FILE_MESSAGE = config.get('DELETE_FILE_MESSAGE', 'File deleted on behalf of WaterButler') UPDATE_FILE_MESSAGE = config.get('UPDATE_FILE_MESSAGE', 'File updated on behalf of WaterButler') UPLOAD_FILE_MESSAGE = config.get('UPLOAD_FILE_MESSAGE', 'File uploaded on behalf of WaterButler') DELETE_FOLDER_MESSAGE = config.get('DELETE_FOLDER_MESSAGE', 'Folder deleted on behalf of WaterButler')
from waterbutler import settings config = settings.child('ONEDRIVE_PROVIDER_CONFIG') BASE_URL = config.get('BASE_URL', 'https://api.onedrive.com/v1.0') BASE_DRIVE_URL = config.get('BASE_DRIVE_URL', 'https://api.onedrive.com/v1.0/drive') ONEDRIVE_COPY_ITERATION_COUNT = int( config.get('ONEDRIVE_COPY_ITERATION_COUNT', 30)) ONEDRIVE_COPY_SLEEP_INTERVAL = int( config.get('ONEDRIVE_COPY_SLEEP_INTERVAL', 3))
from waterbutler import settings config = settings.child('DROPBOX_PROVIDER_CONFIG') BASE_URL = config.get('BASE_URL', 'https://api.dropboxapi.com/2') BASE_CONTENT_URL = config.get('BASE_CONTENT_URL', 'https://content.dropboxapi.com/2/')
from waterbutler import settings config = settings.child('DROPBOX_PROVIDER_CONFIG') BASE_URL = config.get('BASE_URL', 'https://api.dropboxapi.com/2') BASE_CONTENT_URL = config.get('BASE_CONTENT_URL', 'https://content.dropboxapi.com/2/') CONTIGUOUS_UPLOAD_SIZE_LIMIT = int(config.get('CONTIGUOUS_UPLOAD_SIZE_LIMIT', 150000000)) # 150 MB CHUNK_SIZE = int(config.get('CHUNK_SIZE', 4000000)) # 4 MB
import zlib from waterbutler import settings config = settings.child('STREAMS_CONFIG') ZIP_EXTENSIONS = config.get('ZIP_EXTENSIONS', '.zip .gz .bzip .bzip2 .rar .xz .bz2 .7z').split(' ') # Compression level to apply to zipped files. Value must be an integer from 0 to 9, where # lower values represent less compression. -1 is also allowed, meaning the default level # (approximately equivalent to a 6). See the zlib docs for more: # https://docs.python.org/3/library/zlib.html#zlib.compressobj ZIP_COMPRESSION_LEVEL = int(config.get('ZIP_COMPRESSION_LEVEL', zlib.Z_DEFAULT_COMPRESSION))
from waterbutler import settings config = settings.child('GITLAB_PROVIDER_CONFIG')
from waterbutler import settings config = settings.child('S3_PROVIDER_CONFIG') TEMP_URL_SECS = int(config.get('TEMP_URL_SECS', 100))