def directory_to_dict(path, directory={}, entry=False):
# if starting traversal, set entry to directory root
if entry is False:
entry = directory
# remove leading slash
entry["parent"] = b64encode_string(os.path.dirname(path)[1:])
# set standard entry properties
entry["name"] = b64encode_string(os.path.basename(path))
entry["children"] = []
# define entries
entries = sorted_directory_list(path)
for file in entries:
new_entry = None
if file[0] != ".":
new_entry = {}
new_entry["name"] = b64encode_string(file)
entry["children"].append(new_entry)
# if entry is a directory, recurse
child_path = os.path.join(path, file)
if (new_entry is not None and os.path.isdir(child_path)
and os.access(child_path, os.R_OK)):
directory_to_dict(child_path, directory, new_entry)
# return fully traversed data
return directory
def _es_results_to_directory_tree(path, return_list, not_draggable=False):
# Helper function for transfer_backlog
# Paths MUST be input in sorted order
# Otherwise the same directory might end up with multiple entries
parts = path.split("/", 1)
if _is_hidden(parts[0]):
not_draggable = True
if len(parts) == 1: # path is a file
return_list.append({
"name": b64encode_string(parts[0]),
"properties": {
"not_draggable": not_draggable
},
})
else:
node, others = parts
node = b64encode_string(node)
if not return_list or return_list[-1]["name"] != node:
return_list.append({
"name": node,
"properties": {
"not_draggable": not_draggable,
"object count": 0
},
"children": [],
})
this_node = return_list[-1]
# Populate children list
_es_results_to_directory_tree(others,
this_node["children"],
not_draggable=not_draggable)
# Generate count of all non-directory objects in this tree
object_count = sum(e["properties"].get("object count", 0)
for e in this_node["children"])
object_count += len(
[e for e in this_node["children"] if not e.get("children")])
this_node["properties"]["object count"] = object_count
this_node["properties"]["display_string"] = "{} objects".format(
object_count)
# If any children of a dir are draggable, the whole dir should be
# Otherwise, directories have the draggability of their first child
this_node["properties"]["not_draggable"] = (
this_node["properties"]["not_draggable"] and not_draggable)
def _prepare_browse_response(response):
"""
Additional common processing before passing a browse response back to JS.
Input should be a dictionary with keys 'entries', 'directories' and 'properties'.
'entries' is a list of strings, one for each entry in that directory, both file-like and folder-like.
'directories' is a list of strings for each folder-like entry. Each entry should also be listed in 'entries'.
'properties' is an optional dictionary that may contain additional information for the entries. Keys are the entry name found in 'entries', values are a dictionary containing extra information. 'properties' may not contain all values from 'entries'.
Output will be the input dictionary with the following transforms applied:
* All filenames will be base64 encoded
* 'properties' dicts may have a new entry of 'display_string' with relevant information to display to the user.
:param dict response: Dict response from a browse call. See above.
:return: Dict response ready to be returned to file-browser JS.
"""
# Generate display string based on properties
for entry, prop in response.get("properties", {}).items():
logger.debug("Properties for %s: %s", entry, prop)
if "levelOfDescription" in prop:
prop["display_string"] = prop["levelOfDescription"]
elif "verbose name" in prop:
prop["display_string"] = prop["verbose name"].strip()
elif "object count" in prop:
try:
prop["display_string"] = ungettext(
"%(count)d object", "%(count)d objects",
prop["object count"]) % {
"count": prop["object count"]
}
except TypeError: # 'object_count' val can be a string, see SS:space.py
prop["display_string"] = _("%(count)s objects") % {
"count": prop["object count"]
}
elif "size" in prop:
prop[
"display_string"] = django.template.defaultfilters.filesizeformat(
prop["size"])
response["entries"] = list(map(b64encode_string, response["entries"]))
response["directories"] = list(
map(b64encode_string, response["directories"]))
response["properties"] = {
b64encode_string(k): v
for k, v in response.get("properties", {}).items()
}
return response
def browse_location(uuid, path):
"""
Browse files in a location. Encodes path in base64 for transimission, returns decoded entries.
"""
path = b64encode_string(path)
url = _storage_service_url() + "location/" + uuid + "/browse/"
params = {"path": path}
with ss_api_timer(function="browse_location"):
response = _storage_api_session().get(url, params=params)
browse = response.json()
browse["entries"] = list(map(b64decode_string, browse["entries"]))
browse["directories"] = list(map(b64decode_string, browse["directories"]))
browse["properties"] = {
b64decode_string(k): v
for k, v in browse.get("properties", {}).items()
}
return browse
def test_copy_metadata_files(mocker):
# Mock helper that actually copies files from the transfer source locations
_copy_from_transfer_sources_mock = mocker.patch(
"components.filesystem_ajax.views._copy_from_transfer_sources",
return_value=(None, ""),
)
# Create a SIP
sip_uuid = str(uuid.uuid4())
models.SIP.objects.create(
uuid=sip_uuid,
currentpath="%sharedPath%more/path/metadataReminder/mysip-{}/".format(
sip_uuid),
)
# Call the view with a mocked request
request = mocker.Mock(
**{
"POST.get.return_value":
sip_uuid,
"POST.getlist.return_value":
[b64encode_string("locationuuid:/some/path")],
})
result = views.copy_metadata_files(request)
# Verify the contents of the response
assert result.status_code == 201
assert result["Content-Type"] == "application/json"
assert json.loads(result.content) == {
"message": "Metadata files added successfully.",
"error": None,
}
# Verify the copier helper was called with the right parameters
_copy_from_transfer_sources_mock.assert_called_once_with(
["locationuuid:/some/path"],
"more/path/metadataReminder/mysip-{}/metadata".format(sip_uuid),
)
class TestAPIv2(TestCase):
fixtures = ["test_user"]
# This is valid path value that we're going to pass to the API server.
path = b64encode_string("{location_uuid}:{relative_path}".format(
**{
"location_uuid": "671643e1-5bec-4a5f-b244-abb76fedb0c4",
"relative_path": "foo/bar.jpg",
}))
def setUp(self):
self.client = Client()
self.client.login(username="******", password="******")
helpers.set_setting("dashboard_uuid", "test-uuid")
def test_headers(self):
resp = self.client.get("/api/v2beta/package/")
assert resp.get("X-Archivematica-Version") == get_full_version()
assert resp.get("X-Archivematica-ID") == "test-uuid"
def test_package_list(self):
resp = self.client.get("/api/v2beta/package/")
assert resp.status_code == 501 # Not implemented yet.
def test_package_create_with_errors(self):
# Missing payload
resp = self.client.post("/api/v2beta/package/",
content_type="application/json")
assert resp.status_code == 400
# Invalid document
resp = self.client.post("/api/v2beta/package/",
"INVALID-JSON",
content_type="application/json")
assert resp.status_code == 400
# Invalid path
resp = self.client.post(
"/api/v2beta/package/",
json.dumps({"path": "invalid"}),
content_type="application/json",
)
assert resp.status_code == 400
@mock.patch("components.api.views.MCPClient", return_value=MCPClientMock())
def test_package_create_mcpclient_ok(self, patcher):
resp = self.client.post(
"/api/v2beta/package/",
json.dumps({"path": self.path}),
content_type="application/json",
)
assert resp.status_code == 202
assert resp.content.decode() == json.dumps(
{"id": "59402c61-3aba-4af7-966a-996073c0601d"})
@mock.patch("components.api.views.MCPClient",
return_value=MCPClientMock(fails=True))
def test_package_create_mcpclient_fails(self, patcher):
resp = self.client.post(
"/api/v2beta/package/",
json.dumps({"path": self.path}),
content_type="application/json",
)
assert resp.status_code == 500
payload = json.loads(resp.content.decode())
assert payload["error"] is True
assert payload["message"] == "Package cannot be created"
def _es_results_to_appraisal_tab_format(record,
record_map,
directory_list,
not_draggable=False):
"""
Given a set of records from Elasticsearch, produces a list of records suitable for use with the appraisal tab.
This function mutates a provided `directory_list`; it does not return a value.
Elasticsearch results index only files; directories are inferred by splitting the filename and generating directory entries for each presumed directory.
:param dict record: A record from Elasticsearch.
This must be in the new format defined for Archivematica 1.6.
:param dict record_map: A dictionary to be used to track created directory objects.
This ensures that duplicate directories aren't created when processing multiple files.
:param dict directory_list: A list of top-level directories to return in the results.
This only contains directories which are not themselves contained within any other directories, e.g. transfers.
This will be appended to by this function in lieu of returning a value.
:param bool not_draggable: This property determines whether or not a given file should be able to be dragged in the user interface; passing this will override the default logic for determining if a file is draggable.
"""
dir, fn = record["relative_path"].rsplit("/", 1)
# Recursively create elements for this item's parent directory,
# if not already present in the record map
components = dir.split("/")
directories = []
while len(components) > 0:
directories.insert(0, "/".join(components))
components.pop(-1)
parent = None
for node in directories:
node_parts = node.split("/")
is_transfer = len(node_parts) == 1
draggable = not not_draggable and (_is_draggable(
node_parts) if not is_transfer else not not_draggable)
if node not in record_map:
dir_record = {
"type": "transfer" if is_transfer else "directory",
# have to artificially create directory IDs, since we don't assign those
"id": str(uuid.uuid4()),
"title": b64encode_string(os.path.basename(node)),
"relative_path": b64encode_string(node),
"not_draggable": not draggable,
"object_count": 0,
"children": [],
}
record_map[node] = dir_record
# directory_list should consist only of top-level records
if is_transfer:
directory_list.append(dir_record)
else:
dir_record = record_map[node]
if parent is not None and dir_record not in parent["children"]:
parent["children"].append(dir_record)
parent["object_count"] += 1
parent = dir_record
dir_parts = dir.split("/")
draggable = not not_draggable and (_is_draggable(dir_parts) if
(len(dir_parts) > 1) else
not not_draggable)
child = {
"type": "file",
"id": record["fileuuid"],
"title": b64encode_string(fn),
"relative_path": b64encode_string(record["relative_path"]),
"size": record["size"],
"tags": record["tags"],
"bulk_extractor_reports": record["bulk_extractor_reports"],
"not_draggable": _is_hidden(fn) or not draggable,
}
if record["modification_date"]:
child["last_modified"] = record["modification_date"]
if record["format"]:
format = record["format"][
0] # TODO handle multiple format identifications
child["format"] = format["format"]
child["group"] = format["group"]
child["puid"] = format["puid"]
record_map[dir]["children"].append(child)
record_map[dir]["object_count"] += 1
