def upload_helper(self, num_kilobytes): key_name = str(uuid.uuid4()) bucket = self.conn.create_bucket(self.bucket_name) md5_expected = md5_from_file(kb_file_gen(num_kilobytes)) file_obj = kb_file_gen(num_kilobytes) key = Key(bucket, key_name) key.set_contents_from_file(file_obj, md5=key.get_md5_from_hexdigest(md5_expected)) self.assertEqual(md5_expected, remove_double_quotes(key.etag))
def upload_s3(file_path, bucket_name, file_key, force=False, acl='private'): """Upload a local file to S3. """ file_path = path(file_path) bucket = open_s3(bucket_name) if file_path.isdir(): # Upload the contents of the dir path. paths = file_path.listdir() paths_keys = zip(paths, ['%s/%s' % (file_key, p.name) for p in paths]) else: # Upload just the given file path. paths_keys = [(file_path, file_key)] for p, k in paths_keys: headers = {} s3_key = bucket.get_key(k) if not s3_key: from boto.s3.key import Key s3_key = Key(bucket, k) content_type = mimetypes.guess_type(p)[0] if content_type: headers['Content-Type'] = content_type file_size = p.stat().st_size file_data = p.bytes() file_md5, file_md5_64 = s3_key.get_md5_from_hexdigest( hashlib.md5(file_data).hexdigest()) # Check the hash. if s3_key.etag: s3_md5 = s3_key.etag.replace('"', '') if s3_md5 == file_md5: info('Hash is the same. Skipping %s' % file_path) continue elif not force: # Check if file on S3 is older than local file. s3_datetime = datetime.datetime(*time.strptime( s3_key.last_modified, '%a, %d %b %Y %H:%M:%S %Z')[0:6]) local_datetime = datetime.datetime.utcfromtimestamp( p.stat().st_mtime) if local_datetime < s3_datetime: info("File %s hasn't been modified since last " \ "being uploaded" % (file_key)) continue # File is newer, let's process and upload info("Uploading %s..." % (file_key)) try: s3_key.set_contents_from_string(file_data, headers, policy=acl, replace=True, md5=(file_md5, file_md5_64)) except Exception as e: error("Failed: %s" % e) raise
def upload_parts_helper(self, zipped_parts_and_md5s, expected_md5): key_name = str(uuid.uuid4()) bucket = self.conn.create_bucket(self.bucket_name) upload = bucket.initiate_multipart_upload(key_name) key = Key(bucket, key_name) for idx, (part, md5_of_part) in enumerate(zipped_parts_and_md5s): upload.upload_part_from_file(part, idx + 1, md5=key.get_md5_from_hexdigest(md5_of_part)) upload.complete_upload() actual_md5 = md5_from_key(key) self.assertEqual(expected_md5, actual_md5)
def upload_s3(file_path, bucket_name, file_key, force=False, acl='private'): """Upload a local file to S3. """ file_path = path(file_path) bucket = open_s3(bucket_name) if file_path.isdir(): # Upload the contents of the dir path. paths = file_path.listdir() paths_keys = zip(paths, ['%s/%s' % (file_key, p.name) for p in paths]) else: # Upload just the given file path. paths_keys = [(file_path, file_key)] for p, k in paths_keys: headers = {} s3_key = bucket.get_key(k) if not s3_key: from boto.s3.key import Key s3_key = Key(bucket, k) content_type = mimetypes.guess_type(p)[0] if content_type: headers['Content-Type'] = content_type file_size = p.stat().st_size file_data = p.bytes() file_md5, file_md5_64 = s3_key.get_md5_from_hexdigest(hashlib.md5(file_data).hexdigest()) # Check the hash. if s3_key.etag: s3_md5 = s3_key.etag.replace('"', '') if s3_md5 == file_md5: info('Hash is the same. Skipping %s' % file_path) continue elif not force: # Check if file on S3 is older than local file. s3_datetime = datetime.datetime(*time.strptime( s3_key.last_modified, '%a, %d %b %Y %H:%M:%S %Z')[0:6]) local_datetime = datetime.datetime.utcfromtimestamp(p.stat().st_mtime) if local_datetime < s3_datetime: info("File %s hasn't been modified since last " \ "being uploaded" % (file_key)) continue # File is newer, let's process and upload info("Uploading %s..." % (file_key)) try: s3_key.set_contents_from_string(file_data, headers, policy=acl, replace=True, md5=(file_md5, file_md5_64)) except Exception as e: error("Failed: %s" % e) raise