def _infoFromMetadata(self, metadata): # pylint: disable=no-self-use
isFolder = (metadata["mimeType"] == _folderMimeType)
rfc3339 = "%Y-%m-%dT%H:%M:%S.%fZ"
rawInfo = {
"basic": {
"name": metadata["name"],
"is_dir": isFolder,
},
"details": {
"accessed":
None, # not supported by Google Drive API
"created":
datetime_to_epoch(
datetime.strptime(metadata["createdTime"], rfc3339)),
"metadata_changed":
None, # not supported by Google Drive API
"modified":
datetime_to_epoch(
datetime.strptime(metadata["modifiedTime"], rfc3339)),
"size":
metadata["size"]
if isFolder is False else None, # folders have no size
"type":
1 if isFolder else 0
}
}
# there is also file-type-specific metadata like imageMediaMetadata
return Info(rawInfo)
def _info_from_object(self, obj, namespaces):
"""Make an info dict from an s3 Object."""
key = obj.key
path = self._key_to_path(key)
name = basename(path.rstrip('/'))
is_dir = key.endswith(self.delimiter)
info = {"basic": {"name": name, "is_dir": is_dir}}
if 'details' in namespaces:
_type = int(
ResourceType.directory if is_dir else ResourceType.file)
info['details'] = {
'accessed': None,
'modified': datetime_to_epoch(obj.last_modified),
'size': obj.content_length,
'type': _type
}
if 's3' in namespaces:
s3info = info['s3'] = {}
for name in self._object_attributes:
value = getattr(obj, name, None)
if isinstance(value, datetime):
value = datetime_to_epoch(value)
s3info[name] = value
if 'urls' in namespaces:
url = self.client.generate_presigned_url(ClientMethod='get_object',
Params={
'Bucket':
self._bucket_name,
'Key': key
})
info['urls'] = {'download': url}
return info
def _infoFromMetadata(self, metadata): # pylint: disable=no-self-use
isRoot = isinstance(metadata, list) or metadata == _rootMetadata
isFolder = isRoot or (metadata["mimeType"] == _folderMimeType)
rfc3339 = "%Y-%m-%dT%H:%M:%S.%fZ"
permissions = metadata.get("permissions", None)
rawInfo = {
"basic": {
"name": "" if isRoot else metadata["name"],
"is_dir": isFolder
},
"details": {
"accessed": None, # not supported by Google Drive API
"created": None if isRoot else datetime_to_epoch(datetime.strptime(metadata["createdTime"], rfc3339)),
"metadata_changed": None, # not supported by Google Drive API
"modified": None if isRoot else datetime_to_epoch(datetime.strptime(metadata["modifiedTime"], rfc3339)),
"size": int(metadata["size"]) if "size" in metadata else None, # folders, native google documents etc have no size
"type": ResourceType.directory if isFolder else ResourceType.file
},
"sharing": {
"id": None if isRoot else metadata["id"],
"permissions": permissions,
"is_shared": len(permissions) > 1 if permissions is not None else None
}
}
if "contentHints" in metadata and "indexableText" in metadata["contentHints"]:
rawInfo.update({"google": {"indexableText": metadata["contentHints"]["indexableText"]}})
if "appProperties" in metadata:
rawInfo.update({"google": {"appProperties": metadata["appProperties"]}})
# there is also file-type-specific metadata like imageMediaMetadata
return Info(rawInfo)
def _info_from_object(self, obj, namespaces):
"""Make an info dict from an s3 Object."""
key = obj.key
path = self._key_to_path(key)
name = basename(path.rstrip("/"))
is_dir = key.endswith(self.delimiter)
info = {"basic": {"name": name, "is_dir": is_dir}}
if "details" in namespaces:
_type = int(ResourceType.directory if is_dir else ResourceType.file)
info["details"] = {
"accessed": None,
"modified": datetime_to_epoch(obj.last_modified),
"size": obj.content_length,
"type": _type,
}
if "s3" in namespaces:
s3info = info["s3"] = {}
for name in self._object_attributes:
value = getattr(obj, name, None)
if isinstance(value, datetime):
value = datetime_to_epoch(value)
s3info[name] = value
if "urls" in namespaces:
url = self.client.generate_presigned_url(
ClientMethod="get_object",
Params={"Bucket": self._bucket_name, "Key": key},
)
info["urls"] = {"download": url}
return info
def _info_from_object(self, obj, namespaces):
"""Make an info dict from an s3 Object."""
key = obj.key
path = self._key_to_path(key)
name = basename(path.rstrip('/'))
is_dir = key.endswith(self.delimiter)
info = {"basic": {"name": name, "is_dir": is_dir}}
if 'details' in namespaces:
_type = int(
ResourceType.directory if is_dir else ResourceType.file)
info['details'] = {
'accessed': None,
'modified': datetime_to_epoch(obj.last_modified),
'size': obj.content_length,
'type': _type
}
if 's3' in namespaces:
s3info = info['s3'] = {}
for name in self._object_attributes:
value = getattr(obj, name, None)
if isinstance(value, datetime):
value = datetime_to_epoch(value)
s3info[name] = value
return info
def _infoFromMetadata(self, metadata): # pylint: disable=no-self-use
isFolder = (metadata["mimeType"] == _folderMimeType)
rfc3339 = "%Y-%m-%dT%H:%M:%S.%fZ"
rawInfo = {
"basic": {
"name": metadata["name"],
"is_dir": isFolder
},
"details": {
"accessed":
None, # not supported by Google Drive API
"created":
datetime_to_epoch(
datetime.strptime(metadata["createdTime"], rfc3339)),
"metadata_changed":
None, # not supported by Google Drive API
"modified":
datetime_to_epoch(
datetime.strptime(metadata["modifiedTime"], rfc3339)),
"size":
int(metadata["size"])
if isFolder is False else None, # folders have no size
"type":
ResourceType.directory if isFolder else ResourceType.file
},
"sharing": {
"id": metadata["id"],
"permissions": metadata["permissions"],
"is_shared":
True if len(metadata["permissions"]) > 1 else False
}
}
# there is also file-type-specific metadata like imageMediaMetadata
return Info(rawInfo)
def getattr(self, path, fh=None):
try:
info = self.fs.getinfo(path, namespaces=['access', 'details'])
st = {
'st_atime':
datetime_to_epoch(info.accessed) if info.accessed else 0,
'st_ctime':
datetime_to_epoch(info.created) if info.created else 0,
'st_mtime':
datetime_to_epoch(info.modified) if info.modified else 0,
'st_size': info.size or 0,
'st_uid': self.uid,
'st_gid': self.gid
}
if info.type == ResourceType.directory:
st['st_nlink'] = 2
st['st_mode'] = (stat.S_IFDIR | 0o755)
elif info.type == ResourceType.file:
st['st_nlink'] = 1
st['st_mode'] = (stat.S_IFREG | 0o644)
return (st)
except fs.errors.ResourceNotFound:
raise FuseOSError(errno.ENOENT)
def _infoFromMetadata(self, metadata): # pylint: disable=no-self-use
rawInfo = {
"basic": {
"name": metadata.name,
"is_dir": isinstance(metadata, FolderMetadata),
}
}
if isinstance(metadata, FileMetadata):
rawInfo.update({
"details": {
"accessed": None, # not supported by Dropbox API
"created": None, # not supported by Dropbox API?,
"metadata_changed": None, # not supported by Dropbox
"modified": datetime_to_epoch(metadata.server_modified), # API documentation says that this is reliable
"size": metadata.size,
"type": 0
},
"dropbox": {
"content_hash": metadata.content_hash, # see https://www.dropbox.com/developers/reference/content-hash
"rev": metadata.rev,
"client_modified": metadata.client_modified # unverified value coming from dropbox clients
}
})
if metadata.media_info is not None and metadata.media_info.is_metadata() is True:
media_info_metadata = metadata.media_info.get_metadata()
if media_info_metadata.time_taken is not None:
rawInfo.update({
"media_info": {
"taken_date_time": datetime_to_epoch(media_info_metadata.time_taken)
}
})
if media_info_metadata.location is not None:
rawInfo.update({
"media_info": {
"location_latitude": media_info_metadata.location.latitude,
"location_longitude": media_info_metadata.location.longitude
}
})
# Dropbox doesn't parse some jpgs properly
if media_info_metadata.dimensions is not None:
rawInfo.update({
"media_info": {
"dimensions_height": media_info_metadata.dimensions.height,
"dimensions_width": media_info_metadata.dimensions.width
}
})
elif isinstance(metadata, FolderMetadata):
rawInfo.update({
"details": {
"accessed": None, # not supported by Dropbox API
"created": None, # not supported by Dropbox API,
"metadata_changed": None, # not supported by Dropbox
"modified": None, # not supported for folders
"size": None, # not supported for folders
"type": 1
}})
else:
assert False, f"{metadata.name}, {metadata}, {type(metadata)}"
return Info(rawInfo)
def test_setinfo2(self):
self.fs.touch("file")
modifiedTime = datetime(2000, 1, 1, 14, 42, 42)
self.fs.setinfo("file", {"details": {"modified": datetime_to_epoch(modifiedTime)}})
info_ = self.fs.getinfo("file")
assert datetime_to_epoch(info_.modified) == datetime_to_epoch(modifiedTime), f"{info_.modified}"
createdTime = datetime(1999, 1, 1, 14, 42, 42)
with self.fs.openbin("file2", "wb", createdDateTime=createdTime) as f:
f.write(b"file2")
info_ = self.fs.getinfo("file2")
assert datetime_to_epoch(info_.created) == datetime_to_epoch(createdTime), f"{info_.created}"
def _InfoFromMetadata(metadata):
isRoot = (metadata == _rootMetadata)
isFolder = (metadata['mimeType'] == _folderMimeType)
rfc3339 = '%Y-%m-%dT%H:%M:%S.%fZ'
permissions = metadata.get('permissions', None)
rawInfo = {
'basic': {
'name': '' if isRoot else metadata['name'],
'is_dir': isFolder
},
'details': {
'accessed':
None, # not supported by Google Drive API
'created':
None if isRoot else datetime_to_epoch(
datetime.strptime(metadata['createdTime'], rfc3339)),
'metadata_changed':
None, # not supported by Google Drive API
'modified':
None if isRoot else datetime_to_epoch(
datetime.strptime(metadata['modifiedTime'], rfc3339)),
'size':
int(metadata['size']) if 'size' in metadata else
None, # folders, native google documents etc have no size
'type':
ResourceType.directory if isFolder else ResourceType.file
},
'sharing': {
'id': metadata['id'],
'permissions': permissions,
'is_shared':
len(permissions) > 1 if permissions is not None else False
}
}
googleMetadata = {}
if 'contentHints' in metadata and 'indexableText' in metadata[
'contentHints']:
googleMetadata.update(
{'indexableText': metadata['contentHints']['indexableText']})
if 'appProperties' in metadata:
googleMetadata.update({'appProperties': metadata['appProperties']})
if 'md5Checksum' in metadata:
rawInfo.update({'hashes': {'MD5': metadata['md5Checksum']}})
if 'mimeType' in metadata:
googleMetadata.update(
{'isShortcut': metadata['mimeType'] == _shortcutMimeType})
rawInfo.update({'google': googleMetadata})
# there is also file-type-specific metadata like imageMediaMetadata
return Info(rawInfo)
def write(self, file_source):
"""Given a DownloadFileSource, write each entry to the tarfile.
Args:
file_source (DownloadFileSource): The file source
"""
for target in file_source:
try:
fileobj = file_source.open(target)
try:
tarinfo = tarfile.TarInfo()
tarinfo.name = target.dst_path.lstrip('/')
tarinfo.size = target.size
tarinfo.mtime = datetime_to_epoch(target.modified)
self._addfile(tarinfo, fileobj)
finally:
fileobj.close()
except OSError:
msg = ("Error happened during sending file content in archive stream, file path: %s, "
"container: %s/%s, archive path: %s" % (target.src_path, target.container_type,
target.container_id, target.dst_path))
self.log.critical(msg)
self.log.exception("Error opening file for streaming")
# Write a placeholder instead
tarinfo = tarfile.TarInfo()
tarinfo.name = target.dst_path + '.MISSING'
self._tarfile.addfile(tarinfo)
def test_details(self):
dates = [
datetime.datetime(2016, 7, 5, tzinfo=pytz.UTC),
datetime.datetime(2016, 7, 6, tzinfo=pytz.UTC),
datetime.datetime(2016, 7, 7, tzinfo=pytz.UTC),
datetime.datetime(2016, 7, 8, tzinfo=pytz.UTC)
]
epochs = [datetime_to_epoch(d) for d in dates]
info = Info({
"details": {
"accessed": epochs[0],
"modified": epochs[1],
"created": epochs[2],
"metadata_changed": epochs[3],
"type": int(ResourceType.file)
}
})
self.assertEqual(info.accessed, dates[0])
self.assertEqual(info.modified, dates[1])
self.assertEqual(info.created, dates[2])
self.assertEqual(info.metadata_changed, dates[3])
self.assertIsInstance(info.type, ResourceType)
self.assertEqual(info.type, ResourceType.file)
self.assertEqual(info.type, 2)
def getinfo(self, path, namespaces=None):
if path == "/":
return Info({
"basic": {
"name": "",
"is_dir": True
},
"details": {
"type": int(ResourceType.directory)
}
})
f = self._c.get_file_by_path(self._project_id, path)
is_dir = f.mime_type == "directory"
return Info({
"basic": {
"name": f.name,
"is_dir": is_dir
},
"modified":
datetime_to_epoch(f.mtime),
"size":
f.size,
"type":
int(ResourceType.directory if is_dir else ResourceType.file)
})
def _itemInfo(self, item): # pylint: disable=no-self-use
# Looks like the dates returned directly in item.file_system_info (i.e. not file_system_info) are UTC naive-datetimes
# We're supposed to return timestamps, which the framework can convert to UTC aware-datetimes
rawInfo = {
"basic": {
"name": item.name,
"is_dir": item.folder is not None,
},
"details": {
"accessed": None, # not supported by OneDrive
"created": datetime_to_epoch(item.created_date_time),
"metadata_changed": None, # not supported by OneDrive
"modified": datetime_to_epoch(item.last_modified_date_time),
"size": item.size,
"type": 1 if item.folder is not None else 0,
},
"file_system_info": {
"client_created": datetime_to_epoch(item.file_system_info.created_date_time),
"client_modified": datetime_to_epoch(item.file_system_info.last_modified_date_time)
}
}
if item.photo is not None:
rawInfo.update({"photo":
{
"camera_make": item.photo.camera_make,
"camera_model": item.photo.camera_model,
"exposure_denominator": item.photo.exposure_denominator,
"exposure_numerator": item.photo.exposure_numerator,
"focal_length": item.photo.focal_length,
"f_number": item.photo.f_number,
"taken_date_time": item.photo.taken_date_time,
"iso": item.photo.iso
}})
if item.location is not None:
rawInfo.update({"location":
{
"altitude": item.location.altitude,
"latitude": item.location.latitude,
"longitude": item.location.longitude
}})
if item.tags is not None:
# doesn't work
rawInfo.update({"tags":
{
"tags": list(item.tags.tags)
}})
return Info(rawInfo)
def test_setinfo2(self):
self.fs.touch('file')
modifiedTime = datetime(2000, 1, 1, 14, 42, 42)
self.fs.setinfo(
'file', {'details': {
'modified': datetime_to_epoch(modifiedTime)
}})
info_ = self.fs.getinfo('file')
assert datetime_to_epoch(info_.modified) == datetime_to_epoch(
modifiedTime), f'{info_.modified}'
createdTime = datetime(1999, 1, 1, 14, 42, 42)
with self.fs.openbin('file2', 'wb', createdDateTime=createdTime) as f:
f.write(b'file2')
info_ = self.fs.getinfo('file2')
assert datetime_to_epoch(info_.created) == datetime_to_epoch(
createdTime), f'{info_.created}'
def _info_from_blob(blob: Blob,
namespaces: Optional[List[str]] = None) -> Info:
"""Make an info dict from a GCS object."""
path = blob.name
name = basename(path.rstrip("/"))
info = {"basic": {"name": name, "is_dir": False}}
if "details" in namespaces:
info["details"] = {
"accessed": None,
"modified": datetime_to_epoch(blob.updated),
"size": blob.size,
"type": int(ResourceType.file)
}
# TODO more namespaces: basic, urls, gcs, ...
return Info(info)
def archivestream(self, ticket):
stream = cStringIO.StringIO()
with tarfile.open(mode='w|', fileobj=stream):
for filepath, arcpath, cont_name, cont_id, f_size, f_modified in ticket[
'target']:
tarinfo = TarInfo()
tarinfo.name = arcpath.lstrip('/')
tarinfo.size = f_size
tarinfo.mtime = datetime_to_epoch(f_modified)
tarinfo_buf = tarinfo.tobuf()
signed_url = None
try:
signed_url = files.get_signed_url(filepath, config.fs)
except fs.errors.ResourceNotFound:
pass
if signed_url:
content_generator = self.stream_file_signed_url(
signed_url, tarinfo_buf,
(filepath, cont_name, cont_id, arcpath))
else:
content_generator = self.stream_regular_file(
filepath, tarinfo_buf,
(filepath, cont_name, cont_id, arcpath))
for chunk in content_generator:
yield chunk
self.log_user_access(
AccessType.download_file,
cont_name=cont_name,
cont_id=cont_id,
filename=os.path.basename(arcpath),
origin_override=ticket['origin'],
download_ticket=ticket['_id']) # log download
yield stream.getvalue() # get tar stream trailer
stream.close()
def _itemInfo(self, item): # pylint: disable=no-self-use
# Looks like the dates returned directly in item.file_system_info (i.e. not file_system_info) are UTC naive-datetimes
# We're supposed to return timestamps, which the framework can convert to UTC aware-datetimes
rawInfo = {
"basic": {
"name": item["name"],
"is_dir": "folder" in item,
},
"details": {
"accessed":
None, # not supported by OneDrive
"created":
datetime_to_epoch(_ParseDateTime(item["createdDateTime"])),
"metadata_changed":
None, # not supported by OneDrive
"modified":
datetime_to_epoch(_ParseDateTime(
item["lastModifiedDateTime"])),
"size":
item["size"],
"type":
ResourceType.directory
if "folder" in item else ResourceType.file,
},
"file_system_info": {
"client_created":
datetime_to_epoch(
_ParseDateTime(item["fileSystemInfo"]["createdDateTime"])),
"client_modified":
datetime_to_epoch(
_ParseDateTime(
item["fileSystemInfo"]["lastModifiedDateTime"]))
}
}
if "photo" in item:
rawInfo["photo"] = {}
rawInfo["photo"].update(
_UpdateDict(item["photo"], "cameraMake", "camera_make"))
rawInfo["photo"].update(
_UpdateDict(item["photo"], "cameraModel", "camera_model"))
rawInfo["photo"].update(
_UpdateDict(item["photo"], "exposureDenominator",
"exposure_denominator"))
rawInfo["photo"].update(
_UpdateDict(item["photo"], "exposureNumerator",
"exposure_numerator"))
rawInfo["photo"].update(
_UpdateDict(item["photo"], "focalLength", "focal_length"))
rawInfo["photo"].update(
_UpdateDict(item["photo"], "fNumber", "f_number"))
rawInfo["photo"].update(
_UpdateDict(item["photo"], "takenDateTime", "taken_date_time",
_ParseDateTime))
rawInfo["photo"].update(_UpdateDict(item["photo"], "iso", "iso"))
if "image" in item:
rawInfo["image"] = {}
rawInfo["image"].update(
_UpdateDict(item["image"], "width", "width"))
rawInfo["image"].update(
_UpdateDict(item["image"], "height", "height"))
if "location" in item:
rawInfo["location"] = {}
rawInfo["location"].update(
_UpdateDict(item["location"], "altitude", "altitude"))
rawInfo["location"].update(
_UpdateDict(item["location"], "latitude", "latitude"))
rawInfo["location"].update(
_UpdateDict(item["location"], "longitude", "longitude"))
if "file" in item:
if "hashes" in item["file"]:
rawInfo["hashes"] = {}
# The spec is at https://docs.microsoft.com/en-us/onedrive/developer/rest-api/resources/hashes?view=odsp-graph-online
# CRC32 appears in the spec but not in the implementation
rawInfo["hashes"].update(
_UpdateDict(item["file"]["hashes"], "crc32Hash", "CRC32"))
# Standard SHA1
rawInfo["hashes"].update(
_UpdateDict(item["file"]["hashes"], "sha1Hash", "SHA1"))
# proprietary hash for change detection
rawInfo["hashes"].update(
_UpdateDict(item["file"]["hashes"], "quickXorHash",
"quickXorHash"))
if "tags" in item:
# doesn't work
rawInfo.update({"tags": {"tags": item["tags"]["tags"]}})
return Info(rawInfo)
def test_datetime_to_epoch(self):
self.assertEqual(
datetime_to_epoch(datetime(1974, 7, 5, tzinfo=pytz.UTC)),
142214400)
def test_datetime_to_epoch(self):
self.assertEqual(
datetime_to_epoch(datetime(1974, 7, 5, tzinfo=timezone.utc)),
142214400)
