def parse_part_size( s ):
i = human2bytes( s )
if i < min_part_size:
raise argparse.ArgumentTypeError( "Part size must be at least %i" % min_part_size )
if i > max_part_size:
raise argparse.ArgumentTypeError( "Part size must not exceed %i" % max_part_size )
return i
def parse_upload_part_size( s ):
i = human2bytes( s )
if i < min_part_size:
raise argparse.ArgumentTypeError( "Part size must be at least %i" % min_part_size )
if i > max_part_size:
raise argparse.ArgumentTypeError( "Part size must not exceed %i" % max_part_size )
return i
from urlparse import urlparse from StringIO import StringIO import boto.s3 from boto.s3.connection import S3Connection from boto.s3.multipart import MultiPartUpload, Part from s3am import me, log, UserError, WorkerException from s3am.boto_utils import work_around_dots_in_bucket_names from s3am.humanize import bytes2human, human2bytes max_part_per_page = 1000 # http://docs.aws.amazon.com/AmazonS3/latest/dev/qfacts.html max_uploads_per_page = 1000 min_part_size = human2bytes( "5M" ) max_part_size = human2bytes( "5G" ) max_parts_per_upload = 10000 # The multiprocessing module needs semaphores to be declared at the top level. I'm assuming this # applies to events, too. The semaphore's initial value depends on a command line option so we # need to instantiate it later, in stream(). The second reason for initializing them later is # that we want stream() to be called multiple times per program invocation, e.g. for unit tests. # download_slots_semaphore = None done_event = None error_event = None num_cores = multiprocessing.cpu_count( )
def parse_download_part_size( s ):
i = human2bytes( s )
if i < 1:
raise argparse.ArgumentTypeError( "Part size must be at least 1" )
return i