def calculate_bounders(num_items, num_groups=None, max_size=None):
"""
Calculate bounders to split a list.
Use num_groups or max_size.
:param num_items: Total number os items to be splitted
:param num_groups: Number of chunks
:param max_size: Max size per chunk
:return: List of tuples with the indexes to split
"""
if num_groups or max_size:
if max_size:
num_groups = int(ceil(float(num_items) / float(max_size)))
else:
num_groups = num_items if num_items < num_groups else num_groups
size = int(num_items / num_groups)
rest = num_items % num_groups
bounders = []
end = 0
for _ in range(num_groups):
start = end
end += size
if rest:
end += 1
rest -= 1
bounders.append((start, end))
return bounders
else:
raise InvalidArguments("You must give num_groups or max_size!")
def delete_table_if_exists(self,
table: str = None,
database: Optional[str] = None):
if database is None and self._session.athena_database is not None:
database = self._session.athena_database
if database is None:
raise InvalidArguments(
"You must pass a valid database or have one defined in your Session!"
)
try:
self._client_glue.delete_table(DatabaseName=database, Name=table)
except self._client_glue.exceptions.EntityNotFoundException:
pass
def calculate_bounders(num_items, num_groups=None, max_size=None):
if num_groups or max_size:
if max_size:
num_groups = int(ceil(float(num_items) / float(max_size)))
else:
num_groups = num_items if num_items < num_groups else num_groups
size = int(num_items / num_groups)
rest = num_items % num_groups
bounders = []
end = 0
for _ in range(num_groups):
start = end
end += size
if rest:
end += 1
rest -= 1
bounders.append((start, end))
return bounders
else:
raise InvalidArguments("You must give num_groups or max_size!")