def checksums_download(request, dataset_id, **kwargs):
dataset = Dataset.objects.get(id=dataset_id)
if not dataset:
return return_response_not_found(request)
type = request.GET.get('type', 'md5')
format = request.GET.get('format', 'text')
checksums = _get_dataset_checksums(dataset, type)
if format == 'text':
checksum_doc = ''.join(["%s %s\n" % c for c in checksums])
checksum_doc += '\n'
response = HttpResponse(checksum_doc, content_type='text/plain')
response['Content-Disposition'] = \
'%s; filename="%s-manifest-md5.txt"' % (
'attachment',
get_filesystem_safe_dataset_name(dataset))
return response
elif format == 'json':
jdict = {'checksums': []}
for c in checksums:
jdict['checksums'].append({
'checksum': c[0],
'file': c[1],
'type': type
})
return JsonResponse(jdict)
else:
raise ValueError("Invalid format. Valid formats are 'text' or 'json'")
def checksums_download(request, dataset_id, **kwargs):
dataset = Dataset.objects.get(id=dataset_id)
if not dataset:
return return_response_not_found(request)
type = request.GET.get('type', 'md5')
format = request.GET.get('format', 'text')
checksums = _get_dataset_checksums(dataset, type)
if format == 'text':
checksum_doc = ''.join(["%s %s\n" % c for c in checksums])
checksum_doc += '\n'
response = HttpResponse(checksum_doc, content_type='text/plain')
response['Content-Disposition'] = \
'%s; filename="%s-manifest-md5.txt"' % (
'attachment',
get_filesystem_safe_dataset_name(dataset))
return response
elif format == 'json':
jdict = {'checksums': []}
for c in checksums:
jdict['checksums'].append({'checksum': c[0], 'file': c[1], 'type': type})
return JsonResponse(jdict)
else:
raise ValueError("Invalid format. Valid formats are 'text' or 'json'")
def streaming_download_dataset(request, dataset_id, comptype='tgz',
organization=DEFAULT_ORGANIZATION):
dataset = Dataset.objects.get(id=dataset_id)
rootdir = get_filesystem_safe_dataset_name(dataset)
filename = '%s-complete.tar' % rootdir
datafiles = DataFile.objects.filter(dataset=dataset)
return _streaming_downloader(request, datafiles, rootdir, filename,
comptype, organization)
def streaming_download_dataset(request,
dataset_id,
comptype='tgz',
organization=DEFAULT_ORGANIZATION):
dataset = Dataset.objects.get(id=dataset_id)
rootdir = get_filesystem_safe_dataset_name(dataset)
filename = '%s-complete.tar' % rootdir
datafiles = DataFile.objects.filter(dataset=dataset)
return _streaming_downloader(request, datafiles, rootdir, filename,
comptype, organization)
def push_dataset_to_host(user_id, credential_id, remote_host_id, dataset_id,
base_dir=None):
try:
files_to_copy = []
datasets = Dataset.objects.filter(pk=dataset_id)
for ds in datasets:
datafiles = DataFile.objects.filter(dataset=ds)
dataset_description = get_filesystem_safe_dataset_name(ds)
path = [dataset_description]
for df in datafiles:
files_to_copy.append((path, df))
do_file_copy(credential_id, remote_host_id, files_to_copy, base_dir)
notify_user(user_id, remote_host_id, success=True)
except:
notify_user(user_id, remote_host_id, success=False)
raise
def push_dataset_to_host(user_id,
credential_id,
remote_host_id,
dataset_id,
base_dir=None):
try:
files_to_copy = []
datasets = Dataset.objects.filter(pk=dataset_id)
for ds in datasets:
datafiles = DataFile.objects.filter(dataset=ds)
dataset_description = get_filesystem_safe_dataset_name(ds)
path = [dataset_description]
for df in datafiles:
files_to_copy.append((path, df))
do_file_copy(credential_id, remote_host_id, files_to_copy, base_dir)
notify_user(user_id, remote_host_id, success=True)
except:
notify_user(user_id, remote_host_id, success=False)
raise
def process_request(request_id, idle=0):
req = Request.objects.get(pk=request_id)
files = Progress.objects.filter(request=req, status=0, retry__lt=10)
no_errors = True
try:
ssh = req.credential.get_client_for_host(req.host)
# https://github.com/paramiko/paramiko/issues/175#issuecomment-24125451
transport = ssh.get_transport()
transport.default_window_size = 2147483647
transport.packetizer.REKEY_BYTES = pow(2, 40)
transport.packetizer.REKEY_PACKETS = pow(2, 40)
sftp = ssh.open_sftp()
except Exception as err:
# Authentication failed (expired?)
req.message = "Can't connect: %s" % str(err)
req.save()
return
remote_base_dir = []
if req.base_dir is not None:
remote_base_dir.append(req.base_dir)
remote_base_dir.append("mytardis-{}".format(req.id))
if req.object_type == "experiment":
experiment = Experiment.objects.get(pk=req.object_id)
remote_base_dir.append(get_filesystem_safe_experiment_name(experiment))
make_dirs(sftp, remote_base_dir)
for file in files:
file.timestamp = timezone.now()
src_file = file.datafile.get_absolute_filepath()
if src_file is not None and os.path.exists(src_file):
try:
path = [
get_filesystem_safe_dataset_name(file.datafile.dataset)
]
if file.datafile.directory is not None:
path += file.datafile.directory.split('/')
path = remote_base_dir + path
make_dirs(sftp, path)
path_str = "/".join(path + [file.datafile.filename])
sftp.putfo(file.datafile.get_file(), path_str,
file.datafile.size)
file.status = 1
except Exception as e:
no_errors = False
file.retry += 1
file.message = str(e)
else:
no_errors = False
file.retry += 1
file.message = "Can't find source file."
file.save()
if not no_errors and file.message is not None and (
"Socket is closed" in file.message
or "Server connection dropped" in file.message):
break
sftp.close()
ssh.close()
if no_errors:
complete_request(req.id)
else:
process_request.apply_async(args=[req.id, idle + 1],
countdown=(idle + 1) * 60)