def separate_uploads(uploads, prefix, delimiter): """ separate_uploads will separate uploads into non_delimited_uploads (a subset of uploads) and common_prefixes according to the specified delimiter. non_delimited_uploads is a list of uploads which exclude the delimiter. common_prefixes is a set of prefixes prior to the specified delimiter. Note that the prefix in the common_prefixes includes the delimiter itself. i.e. if '/' delimiter specified and then the uploads is consists of ['foo', 'foo/bar'], this function will return (['foo'], ['foo/']). :param uploads: A list of uploads dictionary :param prefix: A string of prefix reserved on the upload path. (i.e. the delimiter must be searched behind the prefix) :param delimiter: A string of delimiter to split the path in each upload :return (non_delimited_uploads, common_prefixes) """ (prefix, delimiter) = \ utf8encode(prefix, delimiter) non_delimited_uploads = [] common_prefixes = set() for upload in uploads: key = upload['key'] end = key.find(delimiter, len(prefix)) if end >= 0: common_prefix = key[:end + len(delimiter)] common_prefixes.add(common_prefix) else: non_delimited_uploads.append(upload) return non_delimited_uploads, sorted(common_prefixes)
def separate_uploads(uploads, prefix, delimiter): """ separate_uploads will separate uploads into non_delimited_uploads (a subset of uploads) and common_prefixes according to the specified delimiter. non_delimited_uploads is a list of uploads which exclude the delimiter. common_prefixes is a set of prefixes prior to the specified delimiter. Note that the prefix in the common_prefixes includes the delimiter itself. i.e. if '/' delimiter specified and then the uploads is consists of ['foo', 'foo/bar'], this function will return (['foo'], ['foo/']). :param uploads: A list of uploads dictionary :param prefix: A string of prefix reserved on the upload path. (i.e. the delimiter must be searched behind the prefix) :param delimiter: A string of delimiter to split the path in each upload :return (non_delimited_uploads, common_prefixes) """ if six.PY2: (prefix, delimiter) = utf8encode(prefix, delimiter) non_delimited_uploads = [] common_prefixes = set() for upload in uploads: key = upload['key'] end = key.find(delimiter, len(prefix)) if end >= 0: common_prefix = key[:end + len(delimiter)] common_prefixes.add(common_prefix) else: non_delimited_uploads.append(upload) return non_delimited_uploads, sorted(common_prefixes)
def list_containers_iter(self, limit, marker, end_marker, prefix, delimiter, reverse=False): """ Get a list of containers sorted by name starting at marker onward, up to limit entries. Entries will begin with the prefix and will not have the delimiter after the prefix. :param limit: maximum number of entries to get :param marker: marker query :param end_marker: end marker query :param prefix: prefix query :param delimiter: delimiter for query :param reverse: reverse the result order. :returns: list of tuples of (name, object_count, bytes_used, 0) """ delim_force_gte = False (marker, end_marker, prefix, delimiter) = utf8encode(marker, end_marker, prefix, delimiter) if reverse: # Reverse the markers if we are reversing the listing. marker, end_marker = end_marker, marker self._commit_puts_stale_ok() if delimiter and not prefix: prefix = "" if prefix: end_prefix = prefix[:-1] + chr(ord(prefix[-1]) + 1) orig_marker = marker with self.get() as conn: results = [] while len(results) < limit: query = """ SELECT name, object_count, bytes_used, 0 FROM container WHERE """ query_args = [] if end_marker and (not prefix or end_marker < end_prefix): query += " name < ? AND" query_args.append(end_marker) elif prefix: query += " name < ? AND" query_args.append(end_prefix) if delim_force_gte: query += " name >= ? AND" query_args.append(marker) # Always set back to False delim_force_gte = False elif marker and marker >= prefix: query += " name > ? AND" query_args.append(marker) elif prefix: query += " name >= ? AND" query_args.append(prefix) if self.get_db_version(conn) < 1: query += " +deleted = 0" else: query += " deleted = 0" query += " ORDER BY name %s LIMIT ?" % ("DESC" if reverse else "") query_args.append(limit - len(results)) curs = conn.execute(query, query_args) curs.row_factory = None # Delimiters without a prefix is ignored, further if there # is no delimiter then we can simply return the result as # prefixes are now handled in the SQL statement. if prefix is None or not delimiter: return [r for r in curs] # We have a delimiter and a prefix (possibly empty string) to # handle rowcount = 0 for row in curs: rowcount += 1 name = row[0] if reverse: end_marker = name else: marker = name if len(results) >= limit: curs.close() return results end = name.find(delimiter, len(prefix)) if end > 0: if reverse: end_marker = name[: end + 1] else: marker = name[:end] + chr(ord(delimiter) + 1) # we want result to be inclusive of delim+1 delim_force_gte = True dir_name = name[: end + 1] if dir_name != orig_marker: results.append([dir_name, 0, 0, 1]) curs.close() break results.append(row) if not rowcount: break return results
def list_objects_iter(self, limit, marker, end_marker, prefix, delimiter, path=None): """ Get a list of objects sorted by name starting at marker onward, up to limit entries. Entries will begin with the prefix and will not have the delimiter after the prefix. :param limit: maximum number of entries to get :param marker: marker query :param end_marker: end marker query :param prefix: prefix query :param delimiter: delimiter for query :param path: if defined, will set the prefix and delimter based on the path :returns: list of tuples of (name, created_at, size, content_type, etag) """ delim_force_gte = False (marker, end_marker, prefix, delimiter, path) = utf8encode(marker, end_marker, prefix, delimiter, path) self._commit_puts_stale_ok() if path is not None: prefix = path if path: prefix = path = path.rstrip('/') + '/' delimiter = '/' elif delimiter and not prefix: prefix = '' orig_marker = marker with self.get() as conn: results = [] while len(results) < limit: query = '''SELECT name, created_at, size, content_type, etag FROM object WHERE''' query_args = [] if end_marker: query += ' name < ? AND' query_args.append(end_marker) if delim_force_gte: query += ' name >= ? AND' query_args.append(marker) # Always set back to False delim_force_gte = False elif marker and marker >= prefix: query += ' name > ? AND' query_args.append(marker) elif prefix: query += ' name >= ? AND' query_args.append(prefix) if self.get_db_version(conn) < 1: query += ' +deleted = 0' else: query += ' deleted = 0' query += ' ORDER BY name LIMIT ?' query_args.append(limit - len(results)) curs = conn.execute(query, query_args) curs.row_factory = None if prefix is None: # A delimiter without a specified prefix is ignored return [r for r in curs] if not delimiter: if not prefix: # It is possible to have a delimiter but no prefix # specified. As above, the prefix will be set to the # empty string, so avoid performing the extra work to # check against an empty prefix. return [r for r in curs] else: return [r for r in curs if r[0].startswith(prefix)] # We have a delimiter and a prefix (possibly empty string) to # handle rowcount = 0 for row in curs: rowcount += 1 marker = name = row[0] if len(results) >= limit or not name.startswith(prefix): curs.close() return results end = name.find(delimiter, len(prefix)) if path is not None: if name == path: continue if end >= 0 and len(name) > end + len(delimiter): marker = name[:end] + chr(ord(delimiter) + 1) curs.close() break elif end > 0: marker = name[:end] + chr(ord(delimiter) + 1) # we want result to be inclusinve of delim+1 delim_force_gte = True dir_name = name[:end + 1] if dir_name != orig_marker: results.append([dir_name, '0', 0, None, '']) curs.close() break results.append(row) if not rowcount: break return results
def list_containers_iter(self, limit, marker, end_marker, prefix, delimiter): """ Get a list of containers sorted by name starting at marker onward, up to limit entries. Entries will begin with the prefix and will not have the delimiter after the prefix. :param limit: maximum number of entries to get :param marker: marker query :param end_marker: end marker query :param prefix: prefix query :param delimiter: delimiter for query :returns: list of tuples of (name, object_count, bytes_used, 0) """ (marker, end_marker, prefix, delimiter) = utf8encode(marker, end_marker, prefix, delimiter) self._commit_puts_stale_ok() if delimiter and not prefix: prefix = '' orig_marker = marker with self.get() as conn: results = [] while len(results) < limit: query = """ SELECT name, object_count, bytes_used, 0 FROM container WHERE deleted = 0 AND """ query_args = [] if end_marker: query += ' name < ? AND' query_args.append(end_marker) if marker and marker >= prefix: query += ' name > ? AND' query_args.append(marker) elif prefix: query += ' name >= ? AND' query_args.append(prefix) if self.get_db_version(conn) < 1: query += ' +deleted = 0' else: query += ' deleted = 0' query += ' ORDER BY name LIMIT ?' query_args.append(limit - len(results)) curs = conn.execute(query, query_args) curs.row_factory = None if prefix is None: # A delimiter without a specified prefix is ignored return [r for r in curs] if not delimiter: if not prefix: # It is possible to have a delimiter but no prefix # specified. As above, the prefix will be set to the # empty string, so avoid performing the extra work to # check against an empty prefix. return [r for r in curs] else: return [r for r in curs if r[0].startswith(prefix)] # We have a delimiter and a prefix (possibly empty string) to # handle rowcount = 0 for row in curs: rowcount += 1 marker = name = row[0] if len(results) >= limit or not name.startswith(prefix): curs.close() return results end = name.find(delimiter, len(prefix)) if end > 0: marker = name[:end] + chr(ord(delimiter) + 1) dir_name = name[:end + 1] if dir_name != orig_marker: results.append([dir_name, 0, 0, 1]) curs.close() break results.append(row) if not rowcount: break return results
def list_objects_iter(self, limit, marker, end_marker, prefix, delimiter, path=None): """ Get a list of objects sorted by name starting at marker onward, up to limit entries. Entries will begin with the prefix and will not have the delimiter after the prefix. :param limit: maximum number of entries to get :param marker: marker query :param end_marker: end marker query :param prefix: prefix query :param delimiter: delimiter for query :param path: if defined, will set the prefix and delimter based on the path :returns: list of tuples of (name, created_at, size, content_type, etag) """ delim_force_gte = False (marker, end_marker, prefix, delimiter, path) = utf8encode( marker, end_marker, prefix, delimiter, path) self._commit_puts_stale_ok() if path is not None: prefix = path if path: prefix = path = path.rstrip('/') + '/' delimiter = '/' elif delimiter and not prefix: prefix = '' orig_marker = marker with self.get() as conn: results = [] while len(results) < limit: query_meta = '''SELECT name, created_at, size, content_type, etag, metadata FROM object WHERE''' query_old = '''SELECT name, created_at, size, content_type, etag, '{}' FROM object WHERE''' query = '' query_args = [] if end_marker: query += ' name < ? AND' query_args.append(end_marker) if delim_force_gte: query += ' name >= ? AND' query_args.append(marker) # Always set back to False delim_force_gte = False elif marker and marker >= prefix: query += ' name > ? AND' query_args.append(marker) elif prefix: query += ' name >= ? AND' query_args.append(prefix) if self.get_db_version(conn) < 1: query += ' +deleted = 0' else: query += ' deleted = 0' query += ' ORDER BY name LIMIT ?' query_args.append(limit - len(results)) try: curs = conn.execute(query_meta + query, query_args) except sqlite3.OperationalError as err: if 'no such column: metadata' \ not in str(err): raise curs = conn.execute(query_old + query, query_args) curs.row_factory = None if prefix is None: # A delimiter without a specified prefix is ignored return [r for r in curs] if not delimiter: if not prefix: # It is possible to have a delimiter but no prefix # specified. As above, the prefix will be set to the # empty string, so avoid performing the extra work to # check against an empty prefix. return [r for r in curs] else: return [r for r in curs if r[0].startswith(prefix)] # We have a delimiter and a prefix (possibly empty string) to # handle rowcount = 0 for row in curs: rowcount += 1 marker = name = row[0] if len(results) >= limit or not name.startswith(prefix): curs.close() return results end = name.find(delimiter, len(prefix)) if path is not None: if name == path: continue if end >= 0 and len(name) > end + len(delimiter): marker = name[:end] + chr(ord(delimiter) + 1) curs.close() break elif end > 0: marker = name[:end] + chr(ord(delimiter) + 1) # we want result to be inclusinve of delim+1 delim_force_gte = True dir_name = name[:end + 1] if dir_name != orig_marker: results.append([dir_name, '0', 0, None, '', '{}']) curs.close() break results.append(row) if not rowcount: break return results
def list_objects_iter(self, limit, marker, end_marker, prefix, delimiter, path=None, storage_policy_index=0, reverse=False): """ Get a list of objects sorted by name starting at marker onward, up to limit entries. Entries will begin with the prefix and will not have the delimiter after the prefix. :param limit: maximum number of entries to get :param marker: marker query :param end_marker: end marker query :param prefix: prefix query :param delimiter: delimiter for query :param path: if defined, will set the prefix and delimiter based on the path :param storage_policy_index: storage policy index for query :param reverse: reverse the result order. :returns: list of tuples of (name, created_at, size, content_type, etag) """ delim_force_gte = False (marker, end_marker, prefix, delimiter, path) = utf8encode(marker, end_marker, prefix, delimiter, path) self._commit_puts_stale_ok() if reverse: # Reverse the markers if we are reversing the listing. marker, end_marker = end_marker, marker if path is not None: prefix = path if path: prefix = path = path.rstrip('/') + '/' delimiter = '/' elif delimiter and not prefix: prefix = '' if prefix: end_prefix = prefix[:-1] + chr(ord(prefix[-1]) + 1) orig_marker = marker with self.get() as conn: results = [] while len(results) < limit: query = '''SELECT name, created_at, size, content_type, etag FROM object WHERE''' query_args = [] if end_marker and (not prefix or end_marker < end_prefix): query += ' name < ? AND' query_args.append(end_marker) elif prefix: query += ' name < ? AND' query_args.append(end_prefix) if delim_force_gte: query += ' name >= ? AND' query_args.append(marker) # Always set back to False delim_force_gte = False elif marker and marker >= prefix: query += ' name > ? AND' query_args.append(marker) elif prefix: query += ' name >= ? AND' query_args.append(prefix) if self.get_db_version(conn) < 1: query += ' +deleted = 0' else: query += ' deleted = 0' orig_tail_query = ''' ORDER BY name %s LIMIT ? ''' % ('DESC' if reverse else '') orig_tail_args = [limit - len(results)] # storage policy filter policy_tail_query = ''' AND storage_policy_index = ? ''' + orig_tail_query policy_tail_args = [storage_policy_index] + orig_tail_args tail_query, tail_args = \ policy_tail_query, policy_tail_args try: curs = conn.execute(query + tail_query, tuple(query_args + tail_args)) except sqlite3.OperationalError as err: if 'no such column: storage_policy_index' not in str(err): raise tail_query, tail_args = \ orig_tail_query, orig_tail_args curs = conn.execute(query + tail_query, tuple(query_args + tail_args)) curs.row_factory = None # Delimiters without a prefix is ignored, further if there # is no delimiter then we can simply return the result as # prefixes are now handled in the SQL statement. if prefix is None or not delimiter: return [self._transform_record(r) for r in curs] # We have a delimiter and a prefix (possibly empty string) to # handle rowcount = 0 for row in curs: rowcount += 1 name = row[0] if reverse: end_marker = name else: marker = name if len(results) >= limit: curs.close() return results end = name.find(delimiter, len(prefix)) if path is not None: if name == path: continue if end >= 0 and len(name) > end + len(delimiter): if reverse: end_marker = name[:end + 1] else: marker = name[:end] + chr(ord(delimiter) + 1) curs.close() break elif end >= 0: if reverse: end_marker = name[:end + 1] else: marker = name[:end] + chr(ord(delimiter) + 1) # we want result to be inclusive of delim+1 delim_force_gte = True dir_name = name[:end + 1] if dir_name != orig_marker: results.append([dir_name, '0', 0, None, '']) curs.close() break results.append(self._transform_record(row)) if not rowcount: break return results
def list_containers_iter(self, limit, marker, end_marker, prefix, delimiter): """ Get a list of containers sorted by name starting at marker onward, up to limit entries. Entries will begin with the prefix and will not have the delimiter after the prefix. :param limit: maximum number of entries to get :param marker: marker query :param end_marker: end marker query :param prefix: prefix query :param delimiter: delimiter for query :returns: list of tuples of (name, object_count, bytes_used, 0) """ (marker, end_marker, prefix, delimiter) = utf8encode( marker, end_marker, prefix, delimiter) self._commit_puts_stale_ok() if delimiter and not prefix: prefix = '' orig_marker = marker with self.get() as conn: results = [] while len(results) < limit: query = """ SELECT name, object_count, bytes_used, 0 FROM container WHERE deleted = 0 AND """ query_args = [] if end_marker: query += ' name < ? AND' query_args.append(end_marker) if marker and marker >= prefix: query += ' name > ? AND' query_args.append(marker) elif prefix: query += ' name >= ? AND' query_args.append(prefix) if self.get_db_version(conn) < 1: query += ' +deleted = 0' else: query += ' deleted = 0' query += ' ORDER BY name LIMIT ?' query_args.append(limit - len(results)) curs = conn.execute(query, query_args) curs.row_factory = None if prefix is None: # A delimiter without a specified prefix is ignored return [r for r in curs] if not delimiter: if not prefix: # It is possible to have a delimiter but no prefix # specified. As above, the prefix will be set to the # empty string, so avoid performing the extra work to # check against an empty prefix. return [r for r in curs] else: return [r for r in curs if r[0].startswith(prefix)] # We have a delimiter and a prefix (possibly empty string) to # handle rowcount = 0 for row in curs: rowcount += 1 marker = name = row[0] if len(results) >= limit or not name.startswith(prefix): curs.close() return results end = name.find(delimiter, len(prefix)) if end > 0: marker = name[:end] + chr(ord(delimiter) + 1) dir_name = name[:end + 1] if dir_name != orig_marker: results.append([dir_name, 0, 0, 1]) curs.close() break results.append(row) if not rowcount: break return results
def list_containers_iter(self, limit, marker, end_marker, prefix, delimiter, reverse=False): """ Get a list of containers sorted by name starting at marker onward, up to limit entries. Entries will begin with the prefix and will not have the delimiter after the prefix. :param limit: maximum number of entries to get :param marker: marker query :param end_marker: end marker query :param prefix: prefix query :param delimiter: delimiter for query :param reverse: reverse the result order. :returns: list of tuples of (name, object_count, bytes_used, put_timestamp, 0) """ delim_force_gte = False (marker, end_marker, prefix, delimiter) = utf8encode(marker, end_marker, prefix, delimiter) if reverse: # Reverse the markers if we are reversing the listing. marker, end_marker = end_marker, marker self._commit_puts_stale_ok() if delimiter and not prefix: prefix = '' if prefix: end_prefix = prefix[:-1] + chr(ord(prefix[-1]) + 1) orig_marker = marker with self.get() as conn: results = [] while len(results) < limit: query = """ SELECT name, object_count, bytes_used, put_timestamp, 0 FROM container WHERE """ query_args = [] if end_marker and (not prefix or end_marker < end_prefix): query += ' name < ? AND' query_args.append(end_marker) elif prefix: query += ' name < ? AND' query_args.append(end_prefix) if delim_force_gte: query += ' name >= ? AND' query_args.append(marker) # Always set back to False delim_force_gte = False elif marker and marker >= prefix: query += ' name > ? AND' query_args.append(marker) elif prefix: query += ' name >= ? AND' query_args.append(prefix) if self.get_db_version(conn) < 1: query += ' +deleted = 0' else: query += ' deleted = 0' query += ' ORDER BY name %s LIMIT ?' % \ ('DESC' if reverse else '') query_args.append(limit - len(results)) curs = conn.execute(query, query_args) curs.row_factory = None # Delimiters without a prefix is ignored, further if there # is no delimiter then we can simply return the result as # prefixes are now handled in the SQL statement. if prefix is None or not delimiter: return [r for r in curs] # We have a delimiter and a prefix (possibly empty string) to # handle rowcount = 0 for row in curs: rowcount += 1 name = row[0] if reverse: end_marker = name else: marker = name if len(results) >= limit: curs.close() return results end = name.find(delimiter, len(prefix)) if end > 0: if reverse: end_marker = name[:end + 1] else: marker = name[:end] + chr(ord(delimiter) + 1) # we want result to be inclusive of delim+1 delim_force_gte = True dir_name = name[:end + 1] if dir_name != orig_marker: results.append([dir_name, 0, 0, '0', 1]) curs.close() break results.append(row) if not rowcount: break return results
def list_objects_iter(self, limit, marker, end_marker, prefix, delimiter, path=None, storage_policy_index=0, reverse=False): """ Get a list of objects sorted by name starting at marker onward, up to limit entries. Entries will begin with the prefix and will not have the delimiter after the prefix. :param limit: maximum number of entries to get :param marker: marker query :param end_marker: end marker query :param prefix: prefix query :param delimiter: delimiter for query :param path: if defined, will set the prefix and delimiter based on the path :param storage_policy_index: storage policy index for query :param reverse: reverse the result order. :returns: list of tuples of (name, created_at, size, content_type, etag) """ delim_force_gte = False (marker, end_marker, prefix, delimiter, path) = utf8encode( marker, end_marker, prefix, delimiter, path) self._commit_puts_stale_ok() if reverse: # Reverse the markers if we are reversing the listing. marker, end_marker = end_marker, marker if path is not None: prefix = path if path: prefix = path = path.rstrip('/') + '/' delimiter = '/' elif delimiter and not prefix: prefix = '' if prefix: end_prefix = prefix[:-1] + chr(ord(prefix[-1]) + 1) orig_marker = marker with self.get() as conn: results = [] while len(results) < limit: query = '''SELECT name, created_at, size, content_type, etag FROM object WHERE''' query_args = [] if end_marker and (not prefix or end_marker < end_prefix): query += ' name < ? AND' query_args.append(end_marker) elif prefix: query += ' name < ? AND' query_args.append(end_prefix) if delim_force_gte: query += ' name >= ? AND' query_args.append(marker) # Always set back to False delim_force_gte = False elif marker and marker >= prefix: query += ' name > ? AND' query_args.append(marker) elif prefix: query += ' name >= ? AND' query_args.append(prefix) if self.get_db_version(conn) < 1: query += ' +deleted = 0' else: query += ' deleted = 0' orig_tail_query = ''' ORDER BY name %s LIMIT ? ''' % ('DESC' if reverse else '') orig_tail_args = [limit - len(results)] # storage policy filter policy_tail_query = ''' AND storage_policy_index = ? ''' + orig_tail_query policy_tail_args = [storage_policy_index] + orig_tail_args tail_query, tail_args = \ policy_tail_query, policy_tail_args try: curs = conn.execute(query + tail_query, tuple(query_args + tail_args)) except sqlite3.OperationalError as err: if 'no such column: storage_policy_index' not in str(err): raise tail_query, tail_args = \ orig_tail_query, orig_tail_args curs = conn.execute(query + tail_query, tuple(query_args + tail_args)) curs.row_factory = None # Delimiters without a prefix is ignored, further if there # is no delimiter then we can simply return the result as # prefixes are now handled in the SQL statement. if prefix is None or not delimiter: return [self._transform_record(r) for r in curs] # We have a delimiter and a prefix (possibly empty string) to # handle rowcount = 0 for row in curs: rowcount += 1 name = row[0] if reverse: end_marker = name else: marker = name if len(results) >= limit: curs.close() return results end = name.find(delimiter, len(prefix)) if path is not None: if name == path: continue if end >= 0 and len(name) > end + len(delimiter): if reverse: end_marker = name[:end + 1] else: marker = name[:end] + chr(ord(delimiter) + 1) curs.close() break elif end > 0: if reverse: end_marker = name[:end + 1] else: marker = name[:end] + chr(ord(delimiter) + 1) # we want result to be inclusive of delim+1 delim_force_gte = True dir_name = name[:end + 1] if dir_name != orig_marker: results.append([dir_name, '0', 0, None, '']) curs.close() break results.append(self._transform_record(row)) if not rowcount: break return results