def upload_datafile():
# setup tus client
tus_client = client.TusClient(
upload_endpoint, headers={'Authorization': 'Token token=' + token})
# for multipart uploads, s3 requires that all chunks except last be at least 5 MB
five_mb = 5 * 1024 * 1024 # 5MB
# set up tus client uploader
uploader = tus_client.uploader(filepath, chunk_size=five_mb)
# upload the entire file, chunk by chunk
uploader.upload()
# get the tus_url from the tus client uploader
tus_url = uploader.url
create_response = requests.post(
create_endpoint,
headers={'Authorization': 'Token token=' + token},
data={
'filename': filename,
'tus_url': tus_url,
'size': size,
'dataset_key': dataset_key
},
verify=True)
print(create_response.text)
def upload_file(self, source_filename, destination_path,
destination_filename):
"""
Upload a file to the server.
Parameters
----------
source_filename: string
Local path to the file to upload
destination_path: string
path (without filename) on the server
destination_filename: string
filename to use on the server
"""
uploadToken = self.request_upload(destination_path,
destination_filename, None)
uploadClient = client.TusClient(
self.end_point.replace('/Api/', '/files/'))
uploader = uploadClient.uploader(source_filename,
chunk_size=10 * 1000 * 1000,
metadata={
'uploadtoken': uploadToken,
'apitoken': self.api_token
})
# Uploads the entire file.
# This uploads chunk by chunk.
uploader.upload()
self.finish_upload(uploadToken, uploader.url)
def setUp(self):
self.client = client.TusClient('http://tusd.tusdemo.net/files/')
self.url = 'http://tusd.tusdemo.net/files/15acd89eabdf5738ffc'
responses.add(responses.HEAD,
self.url,
adding_headers={"upload-offset": "0"})
self.uploader = self.client.uploader('./LICENSE', url=self.url)
def upload_video(request):
# if this is a POST request we need to process the form data
if request.method == 'POST':
# create a form instance and populate it with data from the request:
form = VideoUploadForm(request.POST, request.FILES)
video_file_url = request.FILES['video_file_url']
# check whether form it's valid:
if form.is_valid():
form.save()
TUS_ENDPOINT = "https://api.cloudflare.com/client/v4/zones/{0}/media".format(
CLOUDFARE_ZONE_ID)
HEADERS = {
'X-Auth-Key': CLOUDFARE_API,
'X-Auth-Email': CLOUDFARE_USER
}
CHUNK_SIZE = 5242880
my_client = client.TusClient(TUS_ENDPOINT, headers=HEADERS)
uploader = my_client.uploader(file_stream=video_file_url,
chunk_size=CHUNK_SIZE)
uploader.upload()
# redirect to a new URL:
return HttpResponseRedirect('/lessons/')
# if a GET (or any other method) we'll create a blank form
else:
form = VideoUploadForm()
return render(request, 'upload_video_form.html', {'form': form})
def __perform_tus_upload(self, filename, attempt):
"""Take an upload attempt and perform the actual upload via tus.
https://tus.io/
Args:
attempt (:obj): requests object
path (string): path on disk to file
filename (string): name of the video file on vimeo.com
Returns:
string: The Vimeo Video URI of your uploaded video.
Raises:
VideoUploadFailure: If unknown errors occured when uploading your
video.
"""
upload_link = attempt.get('upload').get('upload_link')
try:
with io.open(filename, 'rb') as fs:
tus_client = client.TusClient('https://files.tus.vimeo.com')
uploader = tus_client.uploader(file_stream=fs, url=upload_link)
uploader.upload()
except Exception as e:
raise exceptions.VideoUploadFailure(
e,
'Unexpected error when uploading through tus.'
)
return attempt.get('uri')
def __perform_tus_upload(self,
filename,
attempt,
chunk_size=DEFAULT_CHUNK_SIZE):
"""Take an upload attempt and perform the actual upload via tus.
https://tus.io/
Args:
filename (string): name of the video file on vimeo.com
attempt (:obj): requests object
chunk_size (int): size of each chunk. defaults to DEFAULT_CHUNK_SIZE
Returns:
string: The Vimeo Video URI of your uploaded video.
Raises:
VideoUploadFailure: If unknown errors occured when uploading your
video.
"""
upload_link = attempt.get('upload').get('upload_link')
try:
with io.open(filename, 'rb') as fs:
tus_client = client.TusClient('https://files.tus.vimeo.com')
uploader = tus_client.uploader(chunk_size=chunk_size,
file_stream=fs,
retries=3,
url=upload_link)
uploader.upload()
except Exception as e:
raise exceptions.VideoUploadFailure(
e, 'Unexpected error when uploading through tus.')
return attempt.get('uri')
def _do_tus_upload(self, assembly_url, tus_url, retries):
tus_client = tus.TusClient(tus_url)
metadata = {'assembly_url': assembly_url}
for key in self.files:
metadata['fieldname'] = key
metadata['filename'] = os.path.basename(self.files[key].name)
tus_client.uploader(file_stream=self.files[key],
chunk_size=5 * 1024 * 1024,
metadata=metadata,
retries=retries).upload()
def setUp(self):
self.client = client.TusClient('http://master.tus.io/files/')
self.url = 'http://master.tus.io/files/15acd89eabdf5738ffc'
responses.add(responses.HEAD,
self.url,
adding_headers={"upload-offset": "0"})
self.loop = asyncio.new_event_loop()
self.async_uploader = self.client.async_uploader('./LICENSE',
url=self.url,
io_loop=self.loop)
def upload(self, connection: Connection, data: FileIO, file_name: Optional[str] = None) -> None:
url = connection.url("modules/%s/upload" % self.id)
metadata = {"filename" : file_name}
# Initialize the client for the TUS upload protocol. Apply the authentication header.
client = tus_client.TusClient(url)
connection.auth(client)
uploader = client.uploader(file_stream=data, chunk_size=200)
uploader.upload()
def upload_file(url, path, api_key, history_id):
filename = os.path.basename(path)
metadata = {
'filename': filename,
'history_id': history_id,
}
my_client = client.TusClient(url, headers={'x-api-key': api_key})
# Upload a file to a tus server.
uploader = my_client.uploader(path, metadata=metadata)
uploader.upload()
return uploader.url.rsplit('/', 1)[1]
def upload(file_path=None, url=None, chunk_size=None, log_func=None):
try:
my_client = client.TusClient(url=url)
uploader = my_client.uploader(file_path=file_path,
chunk_size=chunk_size,
log_func=log_func)
uploader.upload()
url_storage = uploader.url
tus_url = get_tus_from_url(url_storage)
return UploadStatus(url=tus_url, status='Ok')
except:
print('Error uploading file to ' + url)
raise
def pushFileToDocumentManager(cls, file_content, filename, mine, document_category,
authorization_header):
folder, pretty_folder = cls._parse_upload_folders(mine, document_category)
data = {
'folder': folder,
'pretty_folder': pretty_folder,
'filename': filename,
'authorization': authorization_header
}
my_client = client.TusClient(cls.document_manager_url, headers=data)
uploader = my_client.uploader(file_stream=io.BytesIO(file_content), chunk_size=2048)
uploader.upload()
document_manager_guid = uploader.url.rsplit('/', 1)[-1]
return document_manager_guid
def upload_video():
FILE_PATH = "/Users/jonathan/Desktop/docker_apps/aprende24/content/"
SINGLE_FILE_PATH = "/Users/jonathan/Desktop/docker_apps/aprende24/content/proposal.mp4"
TUS_ENDPOINT = "https://api.cloudflare.com/client/v4/zones/{0}/media".format(
CLOUDFARE_ZONE_ID)
HEADERS = {'X-Auth-Key': CLOUDFARE_API, 'X-Auth-Email': CLOUDFARE_USER}
CHUNK_SIZE = 5242880
my_client = client.TusClient(TUS_ENDPOINT, headers=HEADERS)
uploader = my_client.uploader(SINGLE_FILE_PATH, chunk_size=CHUNK_SIZE)
uploader.upload()
print(uploader)
def __init__(self, configs):
self.sets = configs
self.name_uuid_len = self.sets.get('name_uuid_len', 5)
self.my_client = client.TusClient(url=self.sets['url'],
headers=self.sets.get('headers', {}))
self.uploader = None
self.rename_tries = self.sets.get('rename_tries', 5)
self.storing_file = self.sets.get('storing_file', 'tus_url_storage')
self.opt_conf = dict(chunk_size=self.sets.get('chunk_size', 500),
retries=self.sets.get('retries', 5),
retry_delay=self.sets.get('retry_delay', 10),
upload_checksum=self.sets.get(
'upload_checksum', True))
def form_valid(self, form):
# This method is called when valid form data has been POSTed.
# It should return an HttpResponse.
# create a form instance and populate it with data from the request:
video_file_url = form.cleaned_data.get('video_file_path')
TUS_ENDPOINT = "https://api.cloudflare.com/client/v4/zones/{0}/media".format(
CLOUDFARE_ZONE_ID)
HEADERS = {'X-Auth-Key': CLOUDFARE_API, 'X-Auth-Email': CLOUDFARE_USER}
CHUNK_SIZE = 5242880
my_client = client.TusClient(TUS_ENDPOINT, headers=HEADERS)
uploader = my_client.uploader(file_stream=video_file_url,
chunk_size=CHUNK_SIZE)
uploader.upload()
# redirect to a new URL:
return super().form_valid(form)
def test_upload_as_secretariat(self):
submission, token = self.create_submission(self.secretariat_user)
tus = client.TusClient(self.tus_host)
stream = io.BytesIO(FILE_CONTENT)
uploader = tus.uploader(
file_stream=stream, chunk_size=200,
metadata={
"filename": "text.txt",
"description": "description",
"token": token.token,
}
)
uploader.upload()
self.assertTrue(uploader.verify_upload())
# XXX Wait to (hopefully) ensure that tusd has finished
time.sleep(2)
self.assertEqual(submission.files.first().name, "text.txt")
self.assertEqual(submission.files.first().description, "description")
self.assertEqual(submission.files.first().file.read(), FILE_CONTENT)
def upload(file_path=None, url=None, chunk_size=None, log_func=None):
"""
Returns upload status and url using tus protocol
:fileUrl:
Url address where to upload the file
:Args:
see tusclient.uploader.Uploader for required and optional arguments.
"""
try:
my_client = client.TusClient(url=url)
uploader = my_client.uploader(file_path=file_path, chunk_size=chunk_size, log_func=log_func)
uploader.upload()
url_storage = uploader.url
tus_url = get_tus_from_url(url_storage)
return UploadStatus(url=tus_url, status='Ok', error='')
except:
print('Error uploading file to ' + url)
raise
def upload_file(url, path, api_key, history_id, file_type='auto', dbkey='?', filename=None, storage=None):
headers = {'x-api-key': api_key}
my_client = client.TusClient(f"{url}{UPLOAD_ENDPOINT}", headers=headers)
filename = filename or os.path.basename(path)
metadata = {
'filename': filename,
'history_id': history_id,
'file_type': file_type,
'dbkey': dbkey,
}
# Upload a file to a tus server.
if storage:
storage = filestorage.FileStorage(storage)
uploader = my_client.uploader(path, metadata=metadata, url_storage=storage)
uploader.chunk_size = CHUNK_SIZE
uploader.upload()
# Extract session from created upload URL
session_id = uploader.url.rsplit('/', 1)[1]
payload = {
'history_id': history_id,
'targets': json.dumps([
{
"destination": {"type": "hdas"},
"elements": [
{
"src": "files",
"ext": file_type,
"dbkey": dbkey,
"name": filename
}
]
}
]),
}
response = requests.post(f"{url}{SUBMISSION_ENDPOINT}", data=payload, files={'files_0|file_data': json.dumps({"session_id": session_id})}, headers=headers)
response.raise_for_status()
def setUp(self):
self.client = client.TusClient('http://master.tus.io/files/',
headers={'foo': 'bar'})
# TODO - one could guess mime type here
'filetype': ''
}
filename=os.path.basename(file_to_upload)
metadata = {
'filename': filename
}
if upload_type == 'attachments':
url = f'{https}{FAIR_API_ENDPOINT}files/datasets/{dataset_code}/attachments'
elif upload_type == 'data':
url = f'{https}{FAIR_API_ENDPOINT}files/datasets/{dataset_code}/data'
print(f'Dataset code: {dataset_code}')
print(f'File: {file_to_upload}')
print(f'Upload type: {upload_type}')
print(f'Destination: {url}')
# Documentation on tusclient: https://tus-py-client.readthedocs.io/en/latest/
try:
print(f'Uploading file ... {filename} ...')
client = client.TusClient(url, headers=headers)
client.set_headers(headers)
uploader = client.uploader(file_to_upload, chunk_size=5242880, metadata=metadata)
uploader.upload()
print(f'Uploading file ... {filename} ... Done')
except Exception as e:
print(f'Problem uploading: {e} ({type(e)})')
#!/usr/bin/env python
import os
from tusclient import client
import utils
utils.enable_logging_with_headers()
dir = os.path.dirname(os.path.realpath(__file__))
fname = os.path.realpath(os.path.join(dir, '../schema.txt'))
my_client = client.TusClient('http://localhost:4200/api/files')
my_client.set_headers(
{'Upload-Metadata': 'filename {0}'.format(os.path.basename(fname))})
uploader = my_client.uploader(fname, chunk_size=1024)
uploader.upload()
def _setup_client(self):
self.logger.info(f"Creating TusClient with token={self.token}")
self.tus_client = client.TusClient(
self.upload_endpoint,
headers={"Authorization": f"Token token={self.token}"})
from tusclient import client
# Establecer encabezados de autorización si es necesario
# by the tus server.
my_client = client.TusClient('http://master.tus.io/files/',
headers={'Authorization': 'Basic xxyyZZAAbbCC='})
# set more headers
my_client.set_headers({'HEADER_NAME': 'HEADER_VALUE'})
uploader = my_client.uploader('path/to/file.ext', chunk_size=200)
# También se puede pasar una secuencia de archivo en lugar de una ruta de archivo.
fs = open('path/to/file.ext')
uploader = my_client.uploader(file_stream=fs, chunk_size=200)
# upload a chunk i.e 200 bytes
uploader.upload_chunk()
# uploads the entire file.
# This uploads chunk by chunk.
uploader.upload()
# you could increase the chunk size to reduce the
# number of upload_chunk cycles.
uploader.chunk_size = 800
uploader.upload()
# Continue uploading chunks till total chunks uploaded reaches 1000 bytes.
uploader.upload(stop_at=1000)
S3_ROOT_PATH = os.getenv('S3_ROOT_PATH')
S3_PUBLIC_PATH = os.getenv('S3_PUBLIC_PATH')
CLOUDFLARE_EMAIL = os.getenv('CLOUDFLARE_EMAIL')
CLOUDFLARE_API_KEY = os.getenv('CLOUDFLARE_API_KEY')
CLOUDFLARE_ACCOUNT_ID = os.getenv('CLOUDFLARE_ACCOUNT_ID')
SECRET_KEY = os.getenv('FLASK_SECRET_KEY')
ALLOWED_EXTENSIONS = set(['jpg', 'jpeg', 'mp4', 'mov', 'png', 'heic'])
s3_session = boto3.session.Session()
s3_client = s3_session.client('s3',
region_name=S3_REGION,
endpoint_url=S3_ENDPOINT,
aws_access_key_id=S3_KEY,
aws_secret_access_key=S3_SECRET)
cf_client = client.TusClient(
'https://api.cloudflare.com/client/v4/accounts/{account_id}/media'.format(
account_id=CLOUDFLARE_ACCOUNT_ID),
headers={
'X-Auth-Email': CLOUDFLARE_EMAIL,
'X-Auth-Key': CLOUDFLARE_API_KEY
})
db = SqliteDatabase(DB_LOCATION)
if __name__ == '__main__':
from uploader.models import create_tables
create_tables()
import sys
from tusclient import client
username = sys.argv[1]
jwt = sys.argv[2]
upload_path = sys.argv[3]
upload_path_local = sys.argv[4]
my_client = client.TusClient('http://localhost:1080/files/')
upload_metadata = {
'Authorization': jwt,
'username': username,
'desiredPath': upload_path
}
uploader = my_client.uploader(upload_path_local,
metadata=upload_metadata,
chunk_size=1000)
uploader.upload()
def test_1():
from tusclient import client
my_client = client.TusClient('http://localhost:8080/upload')
uploader = my_client.uploader('testfile', chunk_size=200000)
uploader.upload()
def setUp(self):
self.client = client.TusClient('http://tusd.tusdemo.net/files/',
headers={'foo': 'bar'})
