def stat(self, remote_path):
obj = self._bucket.Object(remote_path)
try:
return {
'size': obj.content_length,
'last_modified': datetime_to_timestamp(obj.last_modified)
}
except botocore.exceptions.ClientError:
return False
def listdir(self, remote_path, recursive=False, is_file=False):
if not is_file and not remote_path.endswith('/'):
remote_path += '/'
listdir = {}
data = {
'directory': self._create_path_from_root(remote_path),
'accountId': self.account_id
}
if recursive:
data = {
'prefix': self._create_path_from_root(remote_path),
'accountId': self.account_id
}
response = requests.get(self.host_url + '/corpus/list', params=data)
list_objects = response.json()
if 'directories' in list_objects:
for key in list_objects['directories']:
new_dir = os.path.join(remote_path, key) + '/'
if new_dir.startswith('/'):
new_dir = new_dir[1:]
if new_dir != '':
listdir[new_dir] = {'is_dir': True, 'type': self.resource_type}
if 'files' in list_objects:
for key in list_objects['files']:
if remote_path in key['filename']:
date_time = datetime.strptime(key["createdAt"].strip(), "%a %b %d %H:%M:%S %Y")
filename = key["filename"][len(self.root_folder) + 1:]
listdir[filename] = {'entries': int(key.get('nbSegments')) if key.get('nbSegments') else None,
'format': key.get('format'),
'id': key.get('id'),
'type': self.resource_type,
'status': key.get('status'),
'sourceLanguage': key.get('sourceLanguage'),
'targetLanguages': key.get('targetLanguages'),
'last_modified': datetime_to_timestamp(
date_time),
'alias_names': [filename + "." + key.get('sourceLanguageCode', ''),
filename + "." + (key.get('targetLanguageCodes')[0]
if key.get('targetLanguageCodes') else '')]}
if recursive:
folder = os.path.dirname(key['filename'][len(self.root_folder) + 1:])
all_dirs = folder.split("/")
for folder_index, folder in enumerate(all_dirs):
new_dir = "/".join(all_dirs[:folder_index + 1]) + "/"
if new_dir != '' and remote_path in new_dir:
listdir[new_dir] = {'is_dir': True, 'type': self.resource_type}
return listdir
def listdir(self, remote_path, recursive=False):
listdir = {}
delimiter = '/'
if recursive:
delimiter = ''
list_objects = self._s3.meta.client.list_objects_v2(
Bucket=self._bucket_name, Delimiter=delimiter, Prefix=remote_path)
if 'CommonPrefixes' in list_objects:
for key in list_objects['CommonPrefixes']:
listdir[key['Prefix']] = {'is_dir': True}
if 'Contents' in list_objects:
for key in list_objects['Contents']:
listdir[key['Key']] = {
'size': key['Size'],
'last_modified': datetime_to_timestamp(key['LastModified'])
}
if key['Key'].endswith('/'):
listdir[key['Key']]['is_dir'] = True
return listdir
def listdir(self, remote_path, recursive=False):
options = {"prefix": remote_path}
if not recursive:
options["delimiter"] = "/"
list_parts_gen = self._client.list(container=self._container,
options=options)
lsdir = {}
for page in list_parts_gen:
if page["success"]:
for item in page["listing"]:
if "subdir" in item:
lsdir[item["subdir"]] = {'is_dir': True}
else:
path = item["name"]
last_modified = datetime.strptime(
item["last_modified"], '%Y-%m-%dT%H:%M:%S.%f')
lsdir[path] = {
'size': item["bytes"],
'last_modified':
datetime_to_timestamp(last_modified)
}
return lsdir
def listdir(self, remote_path, recursive=False, is_file=False):
if not is_file and remote_path != '' and not remote_path.endswith('/'):
remote_path += '/'
listdir = {}
delimiter = '/'
if recursive:
delimiter = ''
list_objects_args = {
'Bucket': self._bucket_name,
'Delimiter': delimiter,
'Prefix': remote_path
}
while True:
list_objects = self._s3.meta.client.list_objects_v2(
**list_objects_args)
if 'CommonPrefixes' in list_objects:
for key in list_objects['CommonPrefixes']:
listdir[key['Prefix']] = {'is_dir': True}
if 'Contents' in list_objects:
for key in list_objects['Contents']:
listdir[key['Key']] = {
'size': key['Size'],
'last_modified':
datetime_to_timestamp(key['LastModified']),
'etag': key['ETag']
}
if key['Key'].endswith('/'):
listdir[key['Key']]['is_dir'] = True
if 'NextContinuationToken' not in list_objects:
break
list_objects_args['ContinuationToken'] = list_objects[
'NextContinuationToken']
return listdir
