def directory_to_dict(path, directory={}, entry=False): # if starting traversal, set entry to directory root if entry is False: entry = directory # remove leading slash entry["parent"] = b64encode_string(os.path.dirname(path)[1:]) # set standard entry properties entry["name"] = b64encode_string(os.path.basename(path)) entry["children"] = [] # define entries entries = sorted_directory_list(path) for file in entries: new_entry = None if file[0] != ".": new_entry = {} new_entry["name"] = b64encode_string(file) entry["children"].append(new_entry) # if entry is a directory, recurse child_path = os.path.join(path, file) if (new_entry is not None and os.path.isdir(child_path) and os.access(child_path, os.R_OK)): directory_to_dict(child_path, directory, new_entry) # return fully traversed data return directory
def _es_results_to_directory_tree(path, return_list, not_draggable=False): # Helper function for transfer_backlog # Paths MUST be input in sorted order # Otherwise the same directory might end up with multiple entries parts = path.split("/", 1) if _is_hidden(parts[0]): not_draggable = True if len(parts) == 1: # path is a file return_list.append({ "name": b64encode_string(parts[0]), "properties": { "not_draggable": not_draggable }, }) else: node, others = parts node = b64encode_string(node) if not return_list or return_list[-1]["name"] != node: return_list.append({ "name": node, "properties": { "not_draggable": not_draggable, "object count": 0 }, "children": [], }) this_node = return_list[-1] # Populate children list _es_results_to_directory_tree(others, this_node["children"], not_draggable=not_draggable) # Generate count of all non-directory objects in this tree object_count = sum(e["properties"].get("object count", 0) for e in this_node["children"]) object_count += len( [e for e in this_node["children"] if not e.get("children")]) this_node["properties"]["object count"] = object_count this_node["properties"]["display_string"] = "{} objects".format( object_count) # If any children of a dir are draggable, the whole dir should be # Otherwise, directories have the draggability of their first child this_node["properties"]["not_draggable"] = ( this_node["properties"]["not_draggable"] and not_draggable)
def _prepare_browse_response(response): """ Additional common processing before passing a browse response back to JS. Input should be a dictionary with keys 'entries', 'directories' and 'properties'. 'entries' is a list of strings, one for each entry in that directory, both file-like and folder-like. 'directories' is a list of strings for each folder-like entry. Each entry should also be listed in 'entries'. 'properties' is an optional dictionary that may contain additional information for the entries. Keys are the entry name found in 'entries', values are a dictionary containing extra information. 'properties' may not contain all values from 'entries'. Output will be the input dictionary with the following transforms applied: * All filenames will be base64 encoded * 'properties' dicts may have a new entry of 'display_string' with relevant information to display to the user. :param dict response: Dict response from a browse call. See above. :return: Dict response ready to be returned to file-browser JS. """ # Generate display string based on properties for entry, prop in response.get("properties", {}).items(): logger.debug("Properties for %s: %s", entry, prop) if "levelOfDescription" in prop: prop["display_string"] = prop["levelOfDescription"] elif "verbose name" in prop: prop["display_string"] = prop["verbose name"].strip() elif "object count" in prop: try: prop["display_string"] = ungettext( "%(count)d object", "%(count)d objects", prop["object count"]) % { "count": prop["object count"] } except TypeError: # 'object_count' val can be a string, see SS:space.py prop["display_string"] = _("%(count)s objects") % { "count": prop["object count"] } elif "size" in prop: prop[ "display_string"] = django.template.defaultfilters.filesizeformat( prop["size"]) response["entries"] = list(map(b64encode_string, response["entries"])) response["directories"] = list( map(b64encode_string, response["directories"])) response["properties"] = { b64encode_string(k): v for k, v in response.get("properties", {}).items() } return response
def browse_location(uuid, path): """ Browse files in a location. Encodes path in base64 for transimission, returns decoded entries. """ path = b64encode_string(path) url = _storage_service_url() + "location/" + uuid + "/browse/" params = {"path": path} with ss_api_timer(function="browse_location"): response = _storage_api_session().get(url, params=params) browse = response.json() browse["entries"] = list(map(b64decode_string, browse["entries"])) browse["directories"] = list(map(b64decode_string, browse["directories"])) browse["properties"] = { b64decode_string(k): v for k, v in browse.get("properties", {}).items() } return browse
def test_copy_metadata_files(mocker): # Mock helper that actually copies files from the transfer source locations _copy_from_transfer_sources_mock = mocker.patch( "components.filesystem_ajax.views._copy_from_transfer_sources", return_value=(None, ""), ) # Create a SIP sip_uuid = str(uuid.uuid4()) models.SIP.objects.create( uuid=sip_uuid, currentpath="%sharedPath%more/path/metadataReminder/mysip-{}/".format( sip_uuid), ) # Call the view with a mocked request request = mocker.Mock( **{ "POST.get.return_value": sip_uuid, "POST.getlist.return_value": [b64encode_string("locationuuid:/some/path")], }) result = views.copy_metadata_files(request) # Verify the contents of the response assert result.status_code == 201 assert result["Content-Type"] == "application/json" assert json.loads(result.content) == { "message": "Metadata files added successfully.", "error": None, } # Verify the copier helper was called with the right parameters _copy_from_transfer_sources_mock.assert_called_once_with( ["locationuuid:/some/path"], "more/path/metadataReminder/mysip-{}/metadata".format(sip_uuid), )
class TestAPIv2(TestCase): fixtures = ["test_user"] # This is valid path value that we're going to pass to the API server. path = b64encode_string("{location_uuid}:{relative_path}".format( **{ "location_uuid": "671643e1-5bec-4a5f-b244-abb76fedb0c4", "relative_path": "foo/bar.jpg", })) def setUp(self): self.client = Client() self.client.login(username="******", password="******") helpers.set_setting("dashboard_uuid", "test-uuid") def test_headers(self): resp = self.client.get("/api/v2beta/package/") assert resp.get("X-Archivematica-Version") == get_full_version() assert resp.get("X-Archivematica-ID") == "test-uuid" def test_package_list(self): resp = self.client.get("/api/v2beta/package/") assert resp.status_code == 501 # Not implemented yet. def test_package_create_with_errors(self): # Missing payload resp = self.client.post("/api/v2beta/package/", content_type="application/json") assert resp.status_code == 400 # Invalid document resp = self.client.post("/api/v2beta/package/", "INVALID-JSON", content_type="application/json") assert resp.status_code == 400 # Invalid path resp = self.client.post( "/api/v2beta/package/", json.dumps({"path": "invalid"}), content_type="application/json", ) assert resp.status_code == 400 @mock.patch("components.api.views.MCPClient", return_value=MCPClientMock()) def test_package_create_mcpclient_ok(self, patcher): resp = self.client.post( "/api/v2beta/package/", json.dumps({"path": self.path}), content_type="application/json", ) assert resp.status_code == 202 assert resp.content.decode() == json.dumps( {"id": "59402c61-3aba-4af7-966a-996073c0601d"}) @mock.patch("components.api.views.MCPClient", return_value=MCPClientMock(fails=True)) def test_package_create_mcpclient_fails(self, patcher): resp = self.client.post( "/api/v2beta/package/", json.dumps({"path": self.path}), content_type="application/json", ) assert resp.status_code == 500 payload = json.loads(resp.content.decode()) assert payload["error"] is True assert payload["message"] == "Package cannot be created"
def _es_results_to_appraisal_tab_format(record, record_map, directory_list, not_draggable=False): """ Given a set of records from Elasticsearch, produces a list of records suitable for use with the appraisal tab. This function mutates a provided `directory_list`; it does not return a value. Elasticsearch results index only files; directories are inferred by splitting the filename and generating directory entries for each presumed directory. :param dict record: A record from Elasticsearch. This must be in the new format defined for Archivematica 1.6. :param dict record_map: A dictionary to be used to track created directory objects. This ensures that duplicate directories aren't created when processing multiple files. :param dict directory_list: A list of top-level directories to return in the results. This only contains directories which are not themselves contained within any other directories, e.g. transfers. This will be appended to by this function in lieu of returning a value. :param bool not_draggable: This property determines whether or not a given file should be able to be dragged in the user interface; passing this will override the default logic for determining if a file is draggable. """ dir, fn = record["relative_path"].rsplit("/", 1) # Recursively create elements for this item's parent directory, # if not already present in the record map components = dir.split("/") directories = [] while len(components) > 0: directories.insert(0, "/".join(components)) components.pop(-1) parent = None for node in directories: node_parts = node.split("/") is_transfer = len(node_parts) == 1 draggable = not not_draggable and (_is_draggable( node_parts) if not is_transfer else not not_draggable) if node not in record_map: dir_record = { "type": "transfer" if is_transfer else "directory", # have to artificially create directory IDs, since we don't assign those "id": str(uuid.uuid4()), "title": b64encode_string(os.path.basename(node)), "relative_path": b64encode_string(node), "not_draggable": not draggable, "object_count": 0, "children": [], } record_map[node] = dir_record # directory_list should consist only of top-level records if is_transfer: directory_list.append(dir_record) else: dir_record = record_map[node] if parent is not None and dir_record not in parent["children"]: parent["children"].append(dir_record) parent["object_count"] += 1 parent = dir_record dir_parts = dir.split("/") draggable = not not_draggable and (_is_draggable(dir_parts) if (len(dir_parts) > 1) else not not_draggable) child = { "type": "file", "id": record["fileuuid"], "title": b64encode_string(fn), "relative_path": b64encode_string(record["relative_path"]), "size": record["size"], "tags": record["tags"], "bulk_extractor_reports": record["bulk_extractor_reports"], "not_draggable": _is_hidden(fn) or not draggable, } if record["modification_date"]: child["last_modified"] = record["modification_date"] if record["format"]: format = record["format"][ 0] # TODO handle multiple format identifications child["format"] = format["format"] child["group"] = format["group"] child["puid"] = format["puid"] record_map[dir]["children"].append(child) record_map[dir]["object_count"] += 1