def test_write_s3_check_consistency_wrong_items_count(self): # given items_to_write = self.get_batch() options = self.get_writer_config() options['options']['check_consistency'] = True # when: try: writer = S3Writer(options, ExportMeta(options)) writer.write_batch(items_to_write) writer.flush() finally: writer.close() bucket = self.s3_conn.get_bucket('fake_bucket') key = bucket.get_key('tests/0.jl.gz') content = key.get_contents_as_string() bucket.delete_key('tests/0.jl.gz') new_key = bucket.new_key('tests/0.jl.gz') new_key.update_metadata({'total': 999}) new_key.set_contents_from_string(content) # then: with self.assertRaisesRegexp(InconsistentWriteState, 'Unexpected number of records'): writer.finish_writing()
def __get_file_contents_list_from_bucket(bucket, prefix, bucket_name): json_files_list = [] for key in bucket.list(prefix=prefix): if key.name.endswith('/') or key.name.endswith('-done'): continue try: new_key_name = "{}-done".format(key.name) bucket.copy_key(new_key_name=new_key_name, src_bucket_name=bucket_name, src_key_name=key.name) bucket.delete_key(key.name) new_key = bucket.get_key(new_key_name) new_key.get_contents_to_filename(filename="tmp.json.gz") f = gzip.open('tmp.json.gz', 'rb') json_files_list.append(f.read()) f.close() except Exception as ex: Logger.log("warning", "{} FAILED: {}".format(key.name, ex.message)) return json_files_list
def find_key(self, bucket_names, key_name): """ Takes a list of bucket and searches across all of them for a given key. Returns a list of keys found, as duplicate key names can be used in different buckets. """ keys = [] for bucket_name in bucket_names: try: bucket = self.get_bucket(bucket_name) except boto.exception.S3ResponseError: continue key = bucket.get_key(key_name) if key: keys.append(key) return keys
def test_write_s3_check_consistency_wrong_size(self): # given items_to_write = self.get_batch() options = self.get_writer_config() options['options']['check_consistency'] = True # when: try: writer = S3Writer(options, ExportMeta(options)) writer.write_batch(items_to_write) writer.flush() finally: writer.close() bucket = self.s3_conn.get_bucket('fake_bucket') key = bucket.get_key('tests/0.jl.gz') key.set_contents_from_string('fake contents') # then: with self.assertRaisesRegexp(InconsistentWriteState, 'has unexpected size'): writer.finish_writing()
calling_format = boto.s3.connection.OrdinaryCallingFormat(), ) bucket = conn.get_bucket(s3BktLdrBucketName) # user/group setup for perms subprocess.call(['groupadd', '-g', gid, groupname]) subprocess.call(['useradd', '-M', '-u', uid, '-g', gid, groupname]) # setup dirs subprocess.call(['mkdir', '-p', nfsLocalMountRoot]) subprocess.call(['mkdir', '-p', s3MountRoot]) subprocess.call(['mkdir', '-p', s3BktLdrInstallRoot]) # pull down software key = bucket.get_key(s3BktLdrJar) key.get_contents_to_filename(s3BktLdrInstallRoot+'/'+s3BktLdrJar) key = bucket.get_key(s3BktLdrProps) key.get_contents_to_filename(s3BktLdrInstallRoot+'/'+s3BktLdrProps) key = bucket.get_key(fusepyRPM) key.get_contents_to_filename(s3BktLdrInstallRoot+'/'+fusepyRPM) key = bucket.get_key(yas3fsRPM) key.get_contents_to_filename(s3BktLdrInstallRoot+'/'+yas3fsRPM) # perms subprocess.call(['chown', '-R', ('root:'+groupname), s3BktLdrInstallRoot]) subprocess.call(['chmod', '-R', '770', s3BktLdrInstallRoot])