def directory_children_proxy_to_storage_server(request, location_uuid, basePath=False):
path = ""
if basePath:
path = b64decode_string(basePath)
path = path + b64decode_string(request.GET.get("base_path", ""))
path = path + b64decode_string(request.GET.get("path", ""))
response = storage_service.browse_location(location_uuid, path)
response = _prepare_browse_response(response)
return helpers.json_response(response)
def _assert_file_node_properties_match_record(file_node, record):
assert file_node["type"] == "file"
assert file_node["id"] == record["fileuuid"]
# the relative path of the node is encoded
assert b64decode_string(
file_node["relative_path"]) == record["relative_path"]
# the node title is the encoded file name
assert b64decode_string(file_node["title"]) == os.path.basename(
record["relative_path"])
assert file_node["size"] == record["size"]
assert file_node["tags"] == record["tags"]
# files are draggable into arrangement by default
assert not file_node["not_draggable"]
def copy_metadata_files(request):
"""
Copy files from list `source_paths` to sip_uuid's metadata folder.
sip_uuid: UUID of the SIP to put files in
paths: List of files to be copied, base64 encoded, in the format
'source_location_uuid:full_path'
"""
sip_uuid = request.POST.get("sip_uuid")
paths = request.POST.getlist("source_paths[]")
if not sip_uuid or not paths:
response = {
"error": True,
"message": "sip_uuid and source_paths[] both required.",
}
return helpers.json_response(response, status_code=400)
paths = [b64decode_string(p) for p in paths]
sip = models.SIP.objects.get(uuid=sip_uuid)
relative_path = sip.currentpath.replace("%sharedPath%", "", 1)
relative_path = os.path.join(relative_path, "metadata")
error, message = _copy_from_transfer_sources(paths, relative_path)
if not error:
message = _("Metadata files added successfully.")
status_code = 201
else:
status_code = 500
response = {"error": error, "message": message}
return helpers.json_response(response, status_code=status_code)
def download_fs(request):
shared_dir = os.path.realpath(django_settings.SHARED_DIRECTORY)
filepath = b64decode_string(request.GET.get("filepath", ""))
requested_filepath = os.path.realpath("/" + filepath)
# respond with 404 if a non-Archivematica file is requested
try:
if requested_filepath.index(shared_dir) == 0:
return helpers.send_file(request, requested_filepath)
else:
raise django.http.Http404
except ValueError:
raise django.http.Http404
def delete_arrange(request, filepath=None):
if filepath is None:
try:
filepath = b64decode_string(request.POST["filepath"])
except KeyError:
response = {
"success": False,
"message": _("No filepath to delete was provided!"),
}
return helpers.json_response(response, status_code=400)
# Delete access mapping if found
models.SIPArrangeAccessMapping.objects.filter(arrange_path=filepath).delete()
models.SIPArrange.objects.filter(arrange_path__startswith=filepath).delete()
return helpers.json_response({"message": _("Delete successful.")})
def browse_location(uuid, path):
"""
Browse files in a location. Encodes path in base64 for transimission, returns decoded entries.
"""
path = b64encode_string(path)
url = _storage_service_url() + "location/" + uuid + "/browse/"
params = {"path": path}
with ss_api_timer(function="browse_location"):
response = _storage_api_session().get(url, params=params)
browse = response.json()
browse["entries"] = list(map(b64decode_string, browse["entries"]))
browse["directories"] = list(map(b64decode_string, browse["directories"]))
browse["properties"] = {
b64decode_string(k): v
for k, v in browse.get("properties", {}).items()
}
return browse
def download_ss(request):
filepath = b64decode_string(request.GET.get("filepath", "")).lstrip("/")
logger.info("download filepath: %s", filepath)
if not filepath.startswith(DEFAULT_BACKLOG_PATH):
return django.http.HttpResponseBadRequest()
filepath = filepath.replace(DEFAULT_BACKLOG_PATH, "", 1)
# Get UUID
uuid_regex = r"[\w]{8}(-[\w]{4}){3}-[\w]{12}"
transfer_uuid = re.search(uuid_regex, filepath).group()
# Get relative path
# Find first /, should be at the end of the transfer name/uuid, rest is relative ptah
relative_path = filepath[filepath.find("/") + 1:]
redirect_url = storage_service.extract_file_url(transfer_uuid,
relative_path)
return helpers.stream_file_from_storage_service(
redirect_url, "Storage service returned {}; check logs?")
def copy_from_arrange_to_completed(request,
filepath=None,
sip_uuid=None,
sip_name=None):
"""Create a SIP from the information stored in the SIPArrange table.
Get all the files in the new SIP, and all their associated metadata, and
move to the processing space. Create needed database entries for the SIP
and start the microservice chain.
"""
if filepath is None:
filepath = b64decode_string(request.POST.get("filepath", ""))
logger.info("copy_from_arrange_to_completed: filepath: %s", filepath)
# can optionally pass in the UUID to an unstarted SIP entity
if sip_uuid is None:
sip_uuid = request.POST.get("uuid")
status_code, response = copy_from_arrange_to_completed_common(
filepath, sip_uuid, sip_name)
return helpers.json_response(response, status_code=status_code)
def copy_to_arrange(request,
sources=None,
destinations=None,
fetch_children=False):
"""
Add files to in-progress SIPs being arranged.
Files being copied can be located in either the backlog or in another SIP being arranged.
If sources or destinations are strs not a list, they will be converted into a list and fetch_children will be set to True.
:param list sources: List of paths relative to this pipelines backlog. If None, will look for filepath[] or filepath
:param list destinations: List of paths within arrange folder. All paths should start with DEFAULT_ARRANGE_PATH
:param bool fetch_children: If True, will fetch all children of the provided path(s) to copy to the destination.
"""
if isinstance(sources, six.string_types) or isinstance(
destinations, six.string_types):
fetch_children = True
sources = [sources]
destinations = [destinations]
if sources is None or destinations is None:
# List of sources & destinations
if "filepath[]" in request.POST or "destination[]" in request.POST:
sources = list(
map(b64decode_string, request.POST.getlist("filepath[]", [])))
destinations = list(
map(b64decode_string,
request.POST.getlist("destination[]", [])))
# Single path representing tree
else:
fetch_children = True
sources = [b64decode_string(request.POST.get("filepath", ""))]
destinations = [
b64decode_string(request.POST.get("destination", ""))
]
logger.info("sources: %s", sources)
logger.info("destinations: %s", destinations)
# The DEFAULT_BACKLOG_PATH constant is missing a leading slash for
# historical reasons; TODO change this at some point.
# External paths passed into these views are in the format
# /originals/, whereas copy_from_arrange_to_completed constructs
# paths without a leading slash as an implementation detail
# (to communicate with the Storage Service).
# Possibly the constant used to refer to externally-constructed
# paths and the one used solely internally should be two different
# constants.
if sources[0].startswith("/" + DEFAULT_BACKLOG_PATH):
action = "copy"
backlog_uuid = storage_service.get_location(purpose="BL")[0]["uuid"]
elif sources[0].startswith(DEFAULT_ARRANGE_PATH):
action = "move"
else:
logger.error(
"Filepath %s is not in base backlog path nor arrange path",
sources[0])
return helpers.json_response({
"error": True,
"message":
_("%(path)s is not in base backlog path nor arrange path") % {
"path": sources[0]
},
})
entries_to_copy = []
try:
for source, dest in zip(sources, destinations):
if action == "copy":
entries = _copy_files_to_arrange(
source,
dest,
fetch_children=fetch_children,
backlog_uuid=backlog_uuid,
)
for entry in entries:
entries_to_copy.append(
models.SIPArrange(
original_path=entry["original_path"],
arrange_path=entry["arrange_path"],
file_uuid=entry["file_uuid"],
transfer_uuid=entry["transfer_uuid"],
))
elif action == "move":
_move_files_within_arrange(source, dest)
response = {"message": _("SIP files successfully moved.")}
status_code = 200
if entries_to_copy:
models.SIPArrange.create_many(entries_to_copy)
except ValueError as e:
logger.exception("Failed copying %s to %s", source, dest)
response = {"message": str(e), "error": True}
status_code = 400
else:
response = {"message": _("Files added to the SIP.")}
status_code = 201
return helpers.json_response(response, status_code=status_code)
def arrange_contents(request, path=None):
if path is None:
path = request.GET.get("path", "")
try:
base_path = b64decode_string(path)
except TypeError:
response = {
"success": False,
"message":
_("Could not base64-decode provided path: %(path)s") % {
"path": path
},
}
return helpers.json_response(response, status_code=400)
else:
base_path = path
# Must indicate that base_path is a folder by ending with /
if not base_path.endswith("/"):
base_path += "/"
if not base_path.startswith(DEFAULT_ARRANGE_PATH):
base_path = DEFAULT_ARRANGE_PATH
# Query SIP Arrangement for results
# Get all the paths that are not in SIPs and start with base_path. We don't
# need the objects, just the arrange_path
paths = (models.SIPArrange.objects.filter(sip_created=False).filter(
aip_created=False).filter(
arrange_path__startswith=base_path).order_by("arrange_path"))
if len(paths) == 0 and base_path != DEFAULT_ARRANGE_PATH:
response = {
"success": False,
"message":
_("No files or directories found under path: %(path)s") % {
"path": base_path
},
}
return helpers.json_response(response, status_code=404)
# Convert the response into an entries [] and directories []
# 'entries' contains everything (files and directories)
entries = set()
directories = set()
properties = {}
for item in paths:
# Strip common prefix
path_parts = item.arrange_path.replace(base_path, "", 1).split("/")
entry = path_parts[0]
if not entry:
continue
entries.add(entry)
# Specify level of description if set
if item.level_of_description:
properties[entry] = properties.get(entry, {}) # Default empty dict
# Don't overwrite if already exists
properties[entry]["levelOfDescription"] = (
properties[entry].get("levelOfDescription")
or item.level_of_description)
if item.file_uuid:
properties[entry] = properties.get(entry, {}) # Default empty dict
properties[entry]["file_uuid"] = item.file_uuid
if len(path_parts) > 1: # Path is a directory
directories.add(entry)
# Don't add directories to the object count
if path_parts[-1]:
properties[entry] = properties.get(entry,
{}) # Default empty dict
properties[entry]["object count"] = (
properties[entry].get("object count", 0) + 1
) # Increment object count
response = {
"entries": list(entries),
"directories": list(directories),
"properties": properties,
}
response = _prepare_browse_response(response)
return helpers.json_response(response)
