def test_schema(): fields = [ pa.field('foo', pa.int32()), pa.field('bar', pa.string()), pa.field('baz', pa.list_(pa.int8())) ] sch = pa.schema(fields) assert sch.names == ['foo', 'bar', 'baz'] assert sch.types == [pa.int32(), pa.string(), pa.list_(pa.int8())] assert len(sch) == 3 assert sch[0].name == 'foo' assert sch[0].type == fields[0].type assert sch.field_by_name('foo').name == 'foo' assert sch.field_by_name('foo').type == fields[0].type assert repr(sch) == """\ foo: int32 bar: string baz: list<item: int8> child 0, item: int8""" with pytest.raises(TypeError): pa.schema([None])
def test_table_unsafe_casting(): data = [ pa.array(range(5), type=pa.int64()), pa.array([-10, -5, 0, 5, 10], type=pa.int32()), pa.array([1.1, 2.2, 3.3, 4.4, 5.5], type=pa.float64()), pa.array(['ab', 'bc', 'cd', 'de', 'ef'], type=pa.string()) ] table = pa.Table.from_arrays(data, names=tuple('abcd')) expected_data = [ pa.array(range(5), type=pa.int32()), pa.array([-10, -5, 0, 5, 10], type=pa.int16()), pa.array([1, 2, 3, 4, 5], type=pa.int64()), pa.array(['ab', 'bc', 'cd', 'de', 'ef'], type=pa.string()) ] expected_table = pa.Table.from_arrays(expected_data, names=tuple('abcd')) target_schema = pa.schema([ pa.field('a', pa.int32()), pa.field('b', pa.int16()), pa.field('c', pa.int64()), pa.field('d', pa.string()) ]) with pytest.raises(pa.ArrowInvalid, match='Floating point value truncated'): table.cast(target_schema) casted_table = table.cast(target_schema, safe=False) assert casted_table.equals(expected_table)
def test_table_safe_casting(): data = [ pa.array(range(5), type=pa.int64()), pa.array([-10, -5, 0, 5, 10], type=pa.int32()), pa.array([1.0, 2.0, 3.0, 4.0, 5.0], type=pa.float64()), pa.array(['ab', 'bc', 'cd', 'de', 'ef'], type=pa.string()) ] table = pa.Table.from_arrays(data, names=tuple('abcd')) expected_data = [ pa.array(range(5), type=pa.int32()), pa.array([-10, -5, 0, 5, 10], type=pa.int16()), pa.array([1, 2, 3, 4, 5], type=pa.int64()), pa.array(['ab', 'bc', 'cd', 'de', 'ef'], type=pa.string()) ] expected_table = pa.Table.from_arrays(expected_data, names=tuple('abcd')) target_schema = pa.schema([ pa.field('a', pa.int32()), pa.field('b', pa.int16()), pa.field('c', pa.int64()), pa.field('d', pa.string()) ]) casted_table = table.cast(target_schema) assert casted_table.equals(expected_table)
def test_cast_from_null(): in_data = [None] * 3 in_type = pa.null() out_types = [ pa.null(), pa.uint8(), pa.float16(), pa.utf8(), pa.binary(), pa.binary(10), pa.list_(pa.int16()), pa.decimal128(19, 4), pa.timestamp('us'), pa.timestamp('us', tz='UTC'), pa.timestamp('us', tz='Europe/Paris'), pa.struct([pa.field('a', pa.int32()), pa.field('b', pa.list_(pa.int8())), pa.field('c', pa.string())]), ] for out_type in out_types: _check_cast_case((in_data, in_type, in_data, out_type)) out_types = [ pa.dictionary(pa.int32(), pa.string()), pa.union([pa.field('a', pa.binary(10)), pa.field('b', pa.string())], mode=pa.lib.UnionMode_DENSE), pa.union([pa.field('a', pa.binary(10)), pa.field('b', pa.string())], mode=pa.lib.UnionMode_SPARSE), ] in_arr = pa.array(in_data, type=pa.null()) for out_type in out_types: with pytest.raises(NotImplementedError): in_arr.cast(out_type)
def test_orcfile_empty(): from pyarrow import orc f = orc.ORCFile(path_for_orc_example('TestOrcFile.emptyFile')) table = f.read() assert table.num_rows == 0 schema = table.schema expected_schema = pa.schema([ ('boolean1', pa.bool_()), ('byte1', pa.int8()), ('short1', pa.int16()), ('int1', pa.int32()), ('long1', pa.int64()), ('float1', pa.float32()), ('double1', pa.float64()), ('bytes1', pa.binary()), ('string1', pa.string()), ('middle', pa.struct([ ('list', pa.list_(pa.struct([ ('int1', pa.int32()), ('string1', pa.string()), ]))), ])), ('list', pa.list_(pa.struct([ ('int1', pa.int32()), ('string1', pa.string()), ]))), ('map', pa.list_(pa.struct([ ('key', pa.string()), ('value', pa.struct([ ('int1', pa.int32()), ('string1', pa.string()), ])), ]))), ]) assert schema == expected_schema
def test_type_schema_pickling(): cases = [ pa.int8(), pa.string(), pa.binary(), pa.binary(10), pa.list_(pa.string()), pa.struct([ pa.field('a', 'int8'), pa.field('b', 'string') ]), pa.time32('s'), pa.time64('us'), pa.date32(), pa.date64(), pa.timestamp('ms'), pa.timestamp('ns'), pa.decimal(12, 2), pa.field('a', 'string', metadata={b'foo': b'bar'}) ] for val in cases: roundtripped = pickle.loads(pickle.dumps(val)) assert val == roundtripped fields = [] for i, f in enumerate(cases): if isinstance(f, pa.Field): fields.append(f) else: fields.append(pa.field('_f{}'.format(i), f)) schema = pa.schema(fields, metadata={b'foo': b'bar'}) roundtripped = pickle.loads(pickle.dumps(schema)) assert schema == roundtripped
def test_custom_nulls(self): # Infer nulls with custom values opts = ConvertOptions(null_values=['Xxx', 'Zzz']) rows = b"a,b,c,d\nZzz,Xxx,1,2\nXxx,#N/A,,Zzz\n" table = self.read_bytes(rows, convert_options=opts) schema = pa.schema([('a', pa.null()), ('b', pa.string()), ('c', pa.string()), ('d', pa.int64())]) assert table.schema == schema assert table.to_pydict() == { 'a': [None, None], 'b': [u"Xxx", u"#N/A"], 'c': [u"1", u""], 'd': [2, None], } opts = ConvertOptions(null_values=[]) rows = b"a,b\n#N/A,\n" table = self.read_bytes(rows, convert_options=opts) schema = pa.schema([('a', pa.string()), ('b', pa.string())]) assert table.schema == schema assert table.to_pydict() == { 'a': [u"#N/A"], 'b': [u""], }
def test_sequence_utf8_to_unicode(): # ARROW-1225 data = [b'foo', None, b'bar'] arr = pa.array(data, type=pa.string()) assert arr[0].as_py() == u'foo' # test a non-utf8 unicode string val = (u'maƱana').encode('utf-16-le') with pytest.raises(pa.ArrowInvalid): pa.array([val], type=pa.string())
def test_is_binary_string(): assert types.is_binary(pa.binary()) assert not types.is_binary(pa.string()) assert types.is_string(pa.string()) assert types.is_unicode(pa.string()) assert not types.is_string(pa.binary()) assert types.is_fixed_size_binary(pa.binary(5)) assert not types.is_fixed_size_binary(pa.binary())
def test_convert_options(): cls = ConvertOptions opts = cls() assert opts.check_utf8 is True opts.check_utf8 = False assert opts.check_utf8 is False assert opts.strings_can_be_null is False opts.strings_can_be_null = True assert opts.strings_can_be_null is True assert opts.column_types == {} # Pass column_types as mapping opts.column_types = {'b': pa.int16(), 'c': pa.float32()} assert opts.column_types == {'b': pa.int16(), 'c': pa.float32()} opts.column_types = {'v': 'int16', 'w': 'null'} assert opts.column_types == {'v': pa.int16(), 'w': pa.null()} # Pass column_types as schema schema = pa.schema([('a', pa.int32()), ('b', pa.string())]) opts.column_types = schema assert opts.column_types == {'a': pa.int32(), 'b': pa.string()} # Pass column_types as sequence opts.column_types = [('x', pa.binary())] assert opts.column_types == {'x': pa.binary()} with pytest.raises(TypeError, match='DataType expected'): opts.column_types = {'a': None} with pytest.raises(TypeError): opts.column_types = 0 assert isinstance(opts.null_values, list) assert '' in opts.null_values assert 'N/A' in opts.null_values opts.null_values = ['xxx', 'yyy'] assert opts.null_values == ['xxx', 'yyy'] assert isinstance(opts.true_values, list) opts.true_values = ['xxx', 'yyy'] assert opts.true_values == ['xxx', 'yyy'] assert isinstance(opts.false_values, list) opts.false_values = ['xxx', 'yyy'] assert opts.false_values == ['xxx', 'yyy'] opts = cls(check_utf8=False, column_types={'a': pa.null()}, null_values=['N', 'nn'], true_values=['T', 'tt'], false_values=['F', 'ff'], strings_can_be_null=True) assert opts.check_utf8 is False assert opts.column_types == {'a': pa.null()} assert opts.null_values == ['N', 'nn'] assert opts.false_values == ['F', 'ff'] assert opts.true_values == ['T', 'tt'] assert opts.strings_can_be_null is True
def test_schema_equals_propagates_check_metadata(): # ARROW-4088 schema1 = pa.schema([ pa.field('foo', pa.int32()), pa.field('bar', pa.string()) ]) schema2 = pa.schema([ pa.field('foo', pa.int32()), pa.field('bar', pa.string(), metadata={'a': 'alpha'}), ]) assert not schema1.equals(schema2) assert schema1.equals(schema2, check_metadata=False)
def test_table_pickle(): data = [ pa.chunked_array([[1, 2], [3, 4]], type=pa.uint32()), pa.chunked_array([["some", "strings", None, ""]], type=pa.string()), ] schema = pa.schema([pa.field('ints', pa.uint32()), pa.field('strs', pa.string())], metadata={b'foo': b'bar'}) table = pa.Table.from_arrays(data, schema=schema) result = pickle.loads(pickle.dumps(table)) result._validate() assert result.equals(table)
def test_array_mixed_unicode_bytes(): values = [u'qux', b'foo', bytearray(b'barz')] b_values = [b'qux', b'foo', b'barz'] u_values = [u'qux', u'foo', u'barz'] arr = pa.array(values) expected = pa.array(b_values, type=pa.binary()) assert arr.type == pa.binary() assert arr.equals(expected) arr = pa.array(values, type=pa.string()) expected = pa.array(u_values, type=pa.string()) assert arr.type == pa.string() assert arr.equals(expected)
def test_type_to_pandas_dtype(): M8_ns = np.dtype('datetime64[ns]') cases = [ (pa.null(), np.float64), (pa.bool_(), np.bool_), (pa.int8(), np.int8), (pa.int16(), np.int16), (pa.int32(), np.int32), (pa.int64(), np.int64), (pa.uint8(), np.uint8), (pa.uint16(), np.uint16), (pa.uint32(), np.uint32), (pa.uint64(), np.uint64), (pa.float16(), np.float16), (pa.float32(), np.float32), (pa.float64(), np.float64), (pa.date32(), M8_ns), (pa.date64(), M8_ns), (pa.timestamp('ms'), M8_ns), (pa.binary(), np.object_), (pa.binary(12), np.object_), (pa.string(), np.object_), (pa.list_(pa.int8()), np.object_), ] for arrow_type, numpy_type in cases: assert arrow_type.to_pandas_dtype() == numpy_type
def test_unicode(self): data = [u("foo"), u("bar"), None, u("arrow")] arr = pyarrow.from_pylist(data) assert len(arr) == 4 assert arr.null_count == 1 assert arr.type == pyarrow.string() assert arr.to_pylist() == [u("foo"), u("bar"), None, u("arrow")]
def test_field(self): t = arrow.string() f = arrow.field('foo', t) assert f.name == 'foo' assert f.type is t assert repr(f) == "Field('foo', type=string)"
def test_dictionary_type(): ty0 = pa.dictionary(pa.int32(), pa.string()) assert ty0.index_type == pa.int32() assert ty0.value_type == pa.string() assert ty0.ordered is False ty1 = pa.dictionary(pa.int8(), pa.float64(), ordered=True) assert ty1.index_type == pa.int8() assert ty1.value_type == pa.float64() assert ty1.ordered is True # construct from non-arrow objects ty2 = pa.dictionary('int8', 'string') assert ty2.index_type == pa.int8() assert ty2.value_type == pa.string() assert ty2.ordered is False
def test_struct_from_tuples(): ty = pa.struct([pa.field('a', pa.int32()), pa.field('b', pa.string()), pa.field('c', pa.bool_())]) data = [(5, 'foo', True), (6, 'bar', False)] expected = [{'a': 5, 'b': 'foo', 'c': True}, {'a': 6, 'b': 'bar', 'c': False}] arr = pa.array(data, type=ty) data_as_ndarray = np.empty(len(data), dtype=object) data_as_ndarray[:] = data arr2 = pa.array(data_as_ndarray, type=ty) assert arr.to_pylist() == expected assert arr.equals(arr2) # With omitted values data = [(5, 'foo', None), None, (6, None, False)] expected = [{'a': 5, 'b': 'foo', 'c': None}, None, {'a': 6, 'b': None, 'c': False}] arr = pa.array(data, type=ty) assert arr.to_pylist() == expected # Invalid tuple size for tup in [(5, 'foo'), (), ('5', 'foo', True, None)]: with pytest.raises(ValueError, match="(?i)tuple size"): pa.array([tup], type=ty)
def test_is_union(): for mode in [pa.lib.UnionMode_SPARSE, pa.lib.UnionMode_DENSE]: assert types.is_union(pa.union([pa.field('a', pa.int32()), pa.field('b', pa.int8()), pa.field('c', pa.string())], mode=mode)) assert not types.is_union(pa.list_(pa.int32()))
def test_empty_cast(): types = [ pa.null(), pa.bool_(), pa.int8(), pa.int16(), pa.int32(), pa.int64(), pa.uint8(), pa.uint16(), pa.uint32(), pa.uint64(), pa.float16(), pa.float32(), pa.float64(), pa.date32(), pa.date64(), pa.binary(), pa.binary(length=4), pa.string(), ] for (t1, t2) in itertools.product(types, types): try: # ARROW-4766: Ensure that supported types conversion don't segfault # on empty arrays of common types pa.array([], type=t1).cast(t2) except pa.lib.ArrowNotImplementedError: continue
def test_sequence_unicode(): data = [u'foo', u'bar', None, u'maƱana'] arr = pa.array(data) assert len(arr) == 4 assert arr.null_count == 1 assert arr.type == pa.string() assert arr.to_pylist() == data
def test_unicode(self): data = [u'foo', u'bar', None, u'maƱana'] arr = pyarrow.from_pylist(data) assert len(arr) == 4 assert arr.null_count == 1 assert arr.type == pyarrow.string() assert arr.to_pylist() == data
def test_struct_from_dicts_inference(): expected_type = pa.struct([pa.field('a', pa.int64()), pa.field('b', pa.string()), pa.field('c', pa.bool_())]) data = [{'a': 5, 'b': u'foo', 'c': True}, {'a': 6, 'b': u'bar', 'c': False}] arr = pa.array(data) check_struct_type(arr.type, expected_type) assert arr.to_pylist() == data # With omitted values data = [{'a': 5, 'c': True}, None, {}, {'a': None, 'b': u'bar'}] expected = [{'a': 5, 'b': None, 'c': True}, None, {'a': None, 'b': None, 'c': None}, {'a': None, 'b': u'bar', 'c': None}] arr = pa.array(data) data_as_ndarray = np.empty(len(data), dtype=object) data_as_ndarray[:] = data arr2 = pa.array(data) check_struct_type(arr.type, expected_type) assert arr.to_pylist() == expected assert arr.equals(arr2) # Nested expected_type = pa.struct([ pa.field('a', pa.struct([pa.field('aa', pa.list_(pa.int64())), pa.field('ab', pa.bool_())])), pa.field('b', pa.string())]) data = [{'a': {'aa': [5, 6], 'ab': True}, 'b': 'foo'}, {'a': {'aa': None, 'ab': False}, 'b': None}, {'a': None, 'b': 'bar'}] arr = pa.array(data) assert arr.to_pylist() == data # Edge cases arr = pa.array([{}]) assert arr.type == pa.struct([]) assert arr.to_pylist() == [{}] # Mixing structs and scalars is rejected with pytest.raises((pa.ArrowInvalid, pa.ArrowTypeError)): pa.array([1, {'a': 2}])
def test_type_list(): value_type = pa.int32() list_type = pa.list_(value_type) assert str(list_type) == 'list<item: int32>' field = pa.field('my_item', pa.string()) l2 = pa.list_(field) assert str(l2) == 'list<my_item: string>'
def test_infer_lists(self): data = OrderedDict([ ('nan_ints', [[None, 1], [2, 3]]), ('ints', [[0, 1], [2, 3]]), ('strs', [[None, u'b'], [u'c', u'd']]), ('nested_strs', [[[None, u'b'], [u'c', u'd']], None]) ]) df = pd.DataFrame(data) expected_schema = pa.schema([ pa.field('nan_ints', pa.list_(pa.int64())), pa.field('ints', pa.list_(pa.int64())), pa.field('strs', pa.list_(pa.string())), pa.field('nested_strs', pa.list_(pa.list_(pa.string()))) ]) self._check_pandas_roundtrip(df, expected_schema=expected_schema)
def test_unicode(self): repeats = 1000 values = [u'foo', None, u'bar', u'maƱana', np.nan] df = pd.DataFrame({'strings': values * repeats}) field = pa.field('strings', pa.string()) schema = pa.schema([field]) self._check_pandas_roundtrip(df, expected_schema=schema)
def test_chunked_array_basics(): data = pa.chunked_array([], type=pa.string()) assert data.type == pa.string() assert data.to_pylist() == [] with pytest.raises(ValueError): pa.chunked_array([]) data = pa.chunked_array([ [1, 2, 3], [4, 5, 6], [7, 8, 9] ]) assert isinstance(data.chunks, list) assert all(isinstance(c, pa.lib.Int64Array) for c in data.chunks) assert all(isinstance(c, pa.lib.Int64Array) for c in data.iterchunks()) assert len(data.chunks) == 3
def test_struct_from_mixed_sequence(): # It is forbidden to mix dicts and tuples when initializing a struct array ty = pa.struct([pa.field('a', pa.int32()), pa.field('b', pa.string()), pa.field('c', pa.bool_())]) data = [(5, 'foo', True), {'a': 6, 'b': 'bar', 'c': False}] with pytest.raises(TypeError): pa.array(data, type=ty)
def test_schema_from_tuples(): fields = [ ('foo', pa.int32()), ('bar', pa.string()), ('baz', pa.list_(pa.int8())), ] sch = pa.schema(fields) assert sch.names == ['foo', 'bar', 'baz'] assert sch.types == [pa.int32(), pa.string(), pa.list_(pa.int8())] assert len(sch) == 3 assert repr(sch) == """\ foo: int32 bar: string baz: list<item: int8> child 0, item: int8""" with pytest.raises(TypeError): pa.schema([('foo', None)])
def test_cast_binary_to_utf8(): binary_arr = pa.array([b'foo', b'bar', b'baz'], type=pa.binary()) utf8_arr = binary_arr.cast(pa.utf8()) expected = pa.array(['foo', 'bar', 'baz'], type=pa.utf8()) assert utf8_arr.equals(expected) non_utf8_values = [(u'maƱana').encode('utf-16-le')] non_utf8_binary = pa.array(non_utf8_values) assert non_utf8_binary.type == pa.binary() with pytest.raises(ValueError): non_utf8_binary.cast(pa.string()) non_utf8_all_null = pa.array(non_utf8_values, mask=np.array([True]), type=pa.binary()) # No error casted = non_utf8_all_null.cast(pa.string()) assert casted.null_count == 1
"""Conversion between different types of arrays""" import numpy as np import pyarrow as pa import vaex.utils supported_arrow_array_types = (pa.Array, pa.ChunkedArray) supported_array_types = (np.ndarray, ) + supported_arrow_array_types string_types = [pa.string(), pa.large_string()] def full(n, value, dtype): from .datatype import DataType dtype = DataType(dtype) values = np.full(n, value, dtype=dtype.numpy) if dtype.is_arrow: return pa.array(values) else: return values def is_arrow_array(ar): return isinstance(ar, supported_arrow_array_types) def is_numpy_array(ar): return isinstance(ar, np.ndarray) def filter(ar, boolean_mask): if isinstance(ar, supported_arrow_array_types):
tf.io.FixedLenFeature((1, ), tf.int64, default_value=[0]), 'image/class/text': tf.io.FixedLenFeature((1, ), tf.string, default_value=[b'']), 'image/format': tf.io.FixedLenFeature((1, ), tf.string, default_value=[b'']), 'image/filename': tf.io.FixedLenFeature((1, ), tf.string, default_value=[b'']), 'image/encoded': tf.io.FixedLenFeature((1, ), tf.string, default_value=[b'']) } parquet_schema = { 'image/height': pa.int64(), 'image/width': pa.int64(), 'image/channels': pa.int64(), 'image/colorspace': pa.string(), 'image/class/label': pa.int64(), 'image/class/text': pa.string(), 'image/format': pa.string(), 'image/filename': pa.string(), 'image/encoded': pa.binary() } def reformat_row(row): import pyarrow as pa out_row = {} for key, val in row.items(): out_type = parquet_schema[key] np_val = val.numpy()
def test_try_incompatible_extension_type(self): arr = pa.array(TypedSequence(["foo", "bar"], try_type=Array2DExtensionType((1, 3), "int64"))) self.assertEqual(arr.type, pa.string())
def lambda_handler(event, _): """Lambda entry point""" source_path = 's3://' + event['bucket'] + '/' + event['source_key'] target_path = 's3://' + event['bucket'] + '/' + event['target_key'] print('Source: ' + source_path) print('Target: ' + target_path) s3_client = boto3.client('s3') found = False # Ensure file exists before we actually run the conversion while not found: try: s3_client.head_object(Bucket=event['bucket'], Key=event['source_key']) except botocore.exceptions.ClientError: print('Waiting for: "' + source_path + '" to exist') time.sleep(10) else: print('Found: "' + source_path + '"') found = True s3fs_source = s3fs.S3FileSystem() s3fs_target = s3fs.S3FileSystem() with s3fs_source.open(source_path, 'rb') as source_file, \ s3fs_target.open(target_path, 'wb') as target_file: # Open a stream reader for the csv file csv_stream = pd.read_csv(source_file, skiprows=0, compression='gzip', dtype=object, iterator=True, chunksize=100000) parquet_writer = None for i, chunk in enumerate(csv_stream): print('Reading chunk: ' + str(i)) # First chunck, get schema and setup writer if not parquet_writer: # Fetch columns from header, hardcodes type to string columns = [ pa.field(column, pa.string()) for column in chunk.columns ] # Generate schema from columns parquet_schema = pa.schema(columns) # Open a writer to S3 parquet_writer = pq.ParquetWriter(target_file, parquet_schema, compression='snappy') # Read the first chunk table = pa.Table.from_pandas(chunk, preserve_index=False) print('Writing chunk: ' + str(i)) parquet_writer.write_table(table) parquet_writer.close() print('Done processing "' + source_path + '"') return event['target_key']
def test_string(self): data = ['foo', b'bar', None, 'arrow'] arr = pyarrow.from_pylist(data) assert len(arr) == 4 assert arr.null_count == 1 assert arr.type == pyarrow.string()
def test_str_length(array, expected, offset): array = pa.array(array, pa.string())[offset:] np.testing.assert_array_equal( str_length(NumbaStringArray.make(array)), np.asarray(expected[offset:], dtype=np.int32), )
def test_is_datetime(): assert is_datetime(pyarrow.timestamp("us", tz=None)) assert not is_datetime(pyarrow.timestamp("ms", tz=None)) assert not is_datetime(pyarrow.timestamp("us", tz="UTC")) assert not is_datetime(pyarrow.timestamp("ns", tz="UTC")) assert not is_datetime(pyarrow.string())
import pyarrow as pa PQ_SCHEMAS = dict() # site_visits fields = [ pa.field('visit_id', pa.int64(), nullable=False), pa.field('crawl_id', pa.uint32(), nullable=False), pa.field('instance_id', pa.uint32(), nullable=False), pa.field('site_url', pa.string(), nullable=False), pa.field('site_rank', pa.uint32()) ] PQ_SCHEMAS['site_visits'] = pa.schema(fields) # flash_cookies fields = [ pa.field('crawl_id', pa.uint32(), nullable=False), pa.field('visit_id', pa.int64(), nullable=False), pa.field('instance_id', pa.uint32(), nullable=False), pa.field('domain', pa.string()), pa.field('filename', pa.string()), pa.field('local_path', pa.string()), pa.field('key', pa.string()), pa.field('content', pa.string()) ] PQ_SCHEMAS['flash_cookies'] = pa.schema(fields) # crawl_history fields = [ pa.field('crawl_id', pa.uint32(), nullable=False), pa.field('visit_id', pa.int64(), nullable=False),
def test_sql(redshift_table, postgresql_table, mysql_table, databases_parameters, db_type): if db_type == "postgresql": table = postgresql_table elif db_type == "mysql": table = mysql_table else: table = redshift_table df = get_df() if db_type == "redshift": df.drop(["binary"], axis=1, inplace=True) engine = wr.catalog.get_engine(connection=f"aws-data-wrangler-{db_type}", echo=False) index = True if engine.name == "redshift" else False wr.db.to_sql( df=df, con=engine, name=table, schema=databases_parameters[db_type]["schema"], if_exists="replace", index=index, index_label=None, chunksize=None, method=None, dtype={"iint32": sqlalchemy.types.Integer}, ) df = wr.db.read_sql_query( sql=f"SELECT * FROM {databases_parameters[db_type]['schema']}.{table}", con=engine) ensure_data_types(df, has_list=False) engine = wr.db.get_engine( db_type=db_type, host=databases_parameters[db_type]["host"], port=databases_parameters[db_type]["port"], database=databases_parameters[db_type]["database"], user=databases_parameters["user"], password=databases_parameters["password"], echo=False, ) dfs = wr.db.read_sql_query( sql=f"SELECT * FROM {databases_parameters[db_type]['schema']}.{table}", con=engine, chunksize=1, dtype={ "iint8": pa.int8(), "iint16": pa.int16(), "iint32": pa.int32(), "iint64": pa.int64(), "float": pa.float32(), "double": pa.float64(), "decimal": pa.decimal128(3, 2), "string_object": pa.string(), "string": pa.string(), "date": pa.date32(), "timestamp": pa.timestamp(unit="ns"), "binary": pa.binary(), "category": pa.float64(), }, ) for df in dfs: ensure_data_types(df, has_list=False) if db_type != "redshift": account_id = boto3.client("sts").get_caller_identity().get("Account") engine = wr.catalog.get_engine( connection=f"aws-data-wrangler-{db_type}", catalog_id=account_id) wr.db.to_sql( df=pd.DataFrame({"col0": [1, 2, 3]}, dtype="Int32"), con=engine, name=table, schema=databases_parameters[db_type]["schema"], if_exists="replace", index=True, index_label="index", ) schema = None if db_type == "postgresql": schema = databases_parameters[db_type]["schema"] df = wr.db.read_sql_table(con=engine, table=table, schema=schema, index_col="index") assert df.shape == (3, 1)
def _text_replace_case_sensitive(data: pa.Array, pat: str, repl: str, max_repl: int) -> pa.Array: """ Replace occurrences of ``pat`` with ``repl`` in the Series/Index with some other string. For every row, only the first ``max_repl`` replacements will be performed. If ``max_repl = -1`` we consider that we have no limit for the number of replacements. This implementation does basic byte-by-byte comparison and is independent of any locales or encodings. """ # Convert to UTF-8 bytes pat_bytes: bytes = pat.encode() repl_bytes: bytes = repl.encode() offsets_buffer, data_buffer = _extract_string_buffers(data) if data.null_count == 0: valid_buffer = np.empty(0, dtype=np.uint8) else: valid_buffer = _buffer_to_view(data.buffers()[0]) if len(pat) > 0: output_t = _text_replace_case_sensitive_numba( len(data), valid_buffer, data.offset, offsets_buffer, data_buffer, pat_bytes, repl_bytes, max_repl, ) else: output_t = _text_replace_case_sensitive_empty_pattern( len(data), valid_buffer, data.offset, offsets_buffer, data_buffer, repl_bytes, max_repl, ) output_offsets, output_buffer = output_t if data.null_count == 0: output_valid = None else: output_valid = data.buffers()[0].slice(data.offset // 8) if data.offset % 8 != 0: output_valid = shift_unaligned_bitmap(output_valid, data.offset % 8, len(data)) buffers = [ output_valid, pa.py_buffer(output_offsets), pa.py_buffer(output_buffer) ] return pa.Array.from_buffers(pa.string(), len(data), buffers, data.null_count)
def test_type_string(): t = pa.string() assert str(t) == 'string'
def __setitem__(self, key: int | slice | np.ndarray, value: Any) -> None: """Set one or more values inplace. Parameters ---------- key : int, ndarray, or slice When called from, e.g. ``Series.__setitem__``, ``key`` will be one of * scalar int * ndarray of integers. * boolean ndarray * slice object value : ExtensionDtype.type, Sequence[ExtensionDtype.type], or object value or values to be set of ``key``. Returns ------- None """ key = check_array_indexer(self, key) if is_integer(key): key = cast(int, key) if not is_scalar(value): raise ValueError("Must pass scalars with scalar indexer") elif isna(value): value = None elif not isinstance(value, str): raise ValueError("Scalar must be NA or str") # Slice data and insert in-between new_data = [ *self._data[0:key].chunks, pa.array([value], type=pa.string()), *self._data[(key + 1):].chunks, ] self._data = pa.chunked_array(new_data) else: # Convert to integer indices and iteratively assign. # TODO: Make a faster variant of this in Arrow upstream. # This is probably extremely slow. # Convert all possible input key types to an array of integers if isinstance(key, slice): key_array = np.array(range(len(self))[key]) elif is_bool_dtype(key): # TODO(ARROW-9430): Directly support setitem(booleans) key_array = np.argwhere(key).flatten() else: # TODO(ARROW-9431): Directly support setitem(integers) key_array = np.asanyarray(key) if is_scalar(value): value = np.broadcast_to(value, len(key_array)) else: value = np.asarray(value) if len(key_array) != len(value): raise ValueError("Length of indexer and values mismatch") for k, v in zip(key_array, value): self[k] = v
def __init__(self, expressions, dtype=None, shape=None, fill_value=None): self.is_masked = any([e.is_masked for e in expressions]) self.fill_value = fill_value if self.is_masked and fill_value is None: for expression in expressions: if expression.is_masked: try: # fast path self.fill_value = expression[0:1].fill_value break except: # noqa # slower path (we have to evaluate everything) self.fill_value = expression.values.fill_value break else: raise ValueError( 'Concatenating expressions with masked values, but no fill value is found' ) if dtype is None: dtypes = [e.dtype for e in expressions] any_strings = any([is_string_type(dtype) for dtype in dtypes]) if any_strings: self.dtype = pa.string( ) # TODO: how do we know it should not be large_string? else: # np.datetime64/timedelta64 and find_common_type don't mix very well if all([dtype == 'datetime64' for dtype in dtypes]): self.dtype = dtypes[0] elif all([dtype == 'timedelta64' for dtype in dtypes]): self.dtype = dtypes[0] else: if all([dtype == dtypes[0] for dtype in dtypes ]): # find common types doesn't always behave well self.dtype = dtypes[0] if any([dtype.kind in 'SU' for dtype in dtypes ]): # strings are also done manually if all([dtype.kind in 'SU' for dtype in dtypes]): index = np.argmax( [dtype.itemsize for dtype in dtypes]) self.dtype = dtypes[index] else: index = np.argmax([ df.columns[self.column_name].astype( 'O').astype('U').dtype.itemsize for df in dfs ]) self.dtype = dfs[index].columns[ self.column_name].astype('O').astype('U').dtype else: self.dtype = np.find_common_type( [k.numpy for k in dtypes], []) logger.debug("common type for %r is %r", dtypes, self.dtype) # make sure all expression are the same type self.expressions = [ e if vaex.array_types.same_type(e.dtype, self.dtype) else e.astype(self.dtype) for e in expressions ] else: # if dtype is given, we assume every expression/column is the same dtype self.dtype = dtype self.expressions = expressions[:] if shape is not None: self.shape = (len(self), ) + shape else: self.shape = (len(self), ) + self.expressions[0].evaluate( 0, 1, array_type='numpy', parallel=False).shape[1:] for i in range(1, len(self.expressions)): expression = self.expressions[i] shape_i = (len(self), ) + expressions[i].evaluate( 0, 1, array_type='numpy', parallel=False).shape[1:] if self.shape != shape_i: raise ValueError( "shape of of expression %s, array index 0, is %r and is incompatible with the shape of the same column of array index %d, %r" % (self.expressions[0], self.shape, i, shape_i))
assert result2.equals(arr) def test_cast_date64_to_int(): arr = pa.array(np.array([0, 1, 2], dtype='int64'), type=pa.date64()) expected = pa.array([0, 1, 2], type='i8') result = arr.cast('i8') assert result.equals(expected) @pytest.mark.parametrize( ('data', 'typ'), [([True, False, True, True], pa.bool_()), ([1, 2, 4, 6], pa.int64()), ([1.0, 2.5, None], pa.float64()), (['a', None, 'b'], pa.string()), ([], None), ([[1, 2], [3]], pa.list_(pa.int64())), ([['a'], None, ['b', 'c']], pa.list_(pa.string())), ([(1, 'a'), (2, 'c'), None ], pa.struct([pa.field('a', pa.int64()), pa.field('b', pa.string())]))]) def test_array_pickle(data, typ): # Allocate here so that we don't have any Arrow data allocated. # This is needed to ensure that allocator tests can be reliable. array = pa.array(data, type=typ) result = pickle.loads(pickle.dumps(array)) assert array.equals(result) @pytest.mark.parametrize('narr', [ np.arange(10, dtype=np.int64),
(pa.int8(), 'int8'), (pa.int16(), 'int16'), (pa.int32(), 'int32'), (pa.int64(), 'int64'), (pa.uint8(), 'uint8'), (pa.uint16(), 'uint16'), (pa.uint32(), 'uint32'), (pa.uint64(), 'uint64'), (pa.float16(), 'float16'), (pa.float32(), 'float32'), (pa.float64(), 'float64'), (pa.date32(), 'date'), (pa.date64(), 'date'), (pa.binary(), 'bytes'), (pa.binary(length=4), 'bytes'), (pa.string(), 'unicode'), (pa.list_(pa.list_(pa.int16())), 'list[list[int16]]'), (pa.decimal128(18, 3), 'decimal'), (pa.timestamp('ms'), 'datetime'), (pa.timestamp('us', 'UTC'), 'datetimetz'), (pa.time32('s'), 'time'), (pa.time64('us'), 'time') ] ) def test_logical_type(type, expected): assert get_logical_type(type) == expected def test_array_uint64_from_py_over_range(): arr = pa.array([2 ** 63], type=pa.uint64()) expected = pa.array(np.array([2 ** 63], dtype='u8'))
def setUp(self): # Reducing the size of thread pools. Without this test execution may fail in # environments with limited amount of resources. filebasedsource.MAX_NUM_THREADS_FOR_SIZE_ESTIMATION = 2 self.temp_dir = tempfile.mkdtemp() self.RECORDS = [{ 'name': 'Thomas', 'favorite_number': 1, 'favorite_color': 'blue' }, { 'name': 'Henry', 'favorite_number': 3, 'favorite_color': 'green' }, { 'name': 'Toby', 'favorite_number': 7, 'favorite_color': 'brown' }, { 'name': 'Gordon', 'favorite_number': 4, 'favorite_color': 'blue' }, { 'name': 'Emily', 'favorite_number': -1, 'favorite_color': 'Red' }, { 'name': 'Percy', 'favorite_number': 6, 'favorite_color': 'Green' }, { 'name': 'Peter', 'favorite_number': 3, 'favorite_color': None }] self.SCHEMA = pa.schema([('name', pa.string(), False), ('favorite_number', pa.int64(), False), ('favorite_color', pa.string())]) self.SCHEMA96 = pa.schema([('name', pa.string(), False), ('favorite_number', pa.timestamp('ns'), False), ('favorite_color', pa.string())]) self.RECORDS_NESTED = [{ 'items': [ { 'name': 'Thomas', 'favorite_number': 1, 'favorite_color': 'blue' }, { 'name': 'Henry', 'favorite_number': 3, 'favorite_color': 'green' }, ] }, { 'items': [ { 'name': 'Toby', 'favorite_number': 7, 'favorite_color': 'brown' }, ] }] self.SCHEMA_NESTED = pa.schema([ ('items', pa.list_( pa.struct([('name', pa.string(), False), ('favorite_number', pa.int64(), False), ('favorite_color', pa.string())]))) ])
"FLOAT64", pyarrow.float64().id: "FLOAT64", pyarrow.time32("ms").id: "TIME", pyarrow.time64("ns").id: "TIME", pyarrow.timestamp("ns").id: "TIMESTAMP", pyarrow.date32().id: "DATE", pyarrow.date64().id: "DATETIME", # because millisecond resolution pyarrow.binary().id: "BYTES", pyarrow.string().id: "STRING", # also alias for pyarrow.utf8() pyarrow.decimal128(38, scale=9).id: "NUMERIC", # The exact decimal's scale and precision are not important, as only # the type ID matters, and it's the same for all decimal128 instances. } else: # pragma: NO COVER BQ_TO_ARROW_SCALARS = {} # pragma: NO COVER ARROW_SCALAR_IDS_TO_BQ = {} # pragma: NO_COVER def bq_to_arrow_struct_data_type(field): arrow_fields = [] for subfield in field.fields:
def infer_schema(self, data): """ Infer a schema for a given data input. The schema can be used to test with schema validator. This function currently supports DataFrame, Numpy, Dictionary, List and basic python types.:: data = pandas.DataFrame(...) schema = infer_schema(data) This function returns None if it can not infer the schema. """ schema = None if data is None: schema = pa.null() elif isinstance(data, dict): schema = {'type': dict, 'fields': {}} for key, value in data.items(): schema['fields'][key] = self.infer_schema(value) elif isinstance(data, pd.DataFrame): schema = {'type': pd.DataFrame, 'fields': {}} # sample the table to get the schema pa_schema = pa.Table.from_pandas(data[:_SAMPLE_SIZE], preserve_index=False).schema for i, name in enumerate(pa_schema.names): schema['fields'][name] = pa_schema.types[i] elif isinstance(data, pd.Series): schema = { 'type': pd.Series, 'item': pa.Array.from_pandas(data).type, } elif isinstance(data, np.ndarray): pa_type = pa.from_numpy_dtype( data.dtype) if data.dtype.num != 17 else pa.string() if len(data.shape) == 1: # 1d array schema = { 'type': np.ndarray, 'item': pa_type, } else: shape = [ v if i != 0 else None for i, v in enumerate(data.shape) ] schema = { 'type': np.ndarray, 'item': pa_type, 'shape': tuple(shape), } elif isinstance(data, pa.Table): schema = data.schema elif isinstance(data, (list, tuple)) and len(data) > 0: # try to infer type of the data current_type = self.infer_schema(data[0]) for i in range(1, min(len(data), _SAMPLE_SIZE)): new_type = self.infer_schema(data[i]) if new_type != current_type: current_type = None break # does not support multiple type yet if current_type: if isinstance(current_type, pa.DataType): schema = pa.list_(current_type) else: schema = {'type': list, 'item': current_type} elif type(data) in _python_mapping: schema = _python_mapping[type(data)]() else: return {'type': type(data)} return schema
MANY_TYPES = [ pa.null(), pa.bool_(), pa.int32(), pa.time32('s'), pa.time64('us'), pa.date32(), pa.timestamp('us'), pa.timestamp('us', tz='UTC'), pa.timestamp('us', tz='Europe/Paris'), pa.float16(), pa.float32(), pa.float64(), pa.decimal128(19, 4), pa.string(), pa.binary(), pa.binary(10), pa.list_(pa.int32()), pa.struct([ pa.field('a', pa.int32()), pa.field('b', pa.int8()), pa.field('c', pa.string()) ]), pa.union([pa.field('a', pa.binary(10)), pa.field('b', pa.string())], mode=pa.lib.UnionMode_DENSE), pa.union([pa.field('a', pa.binary(10)), pa.field('b', pa.string())], mode=pa.lib.UnionMode_SPARSE), # XXX Needs array pickling
def test_try_incompatible_type(self): arr = pa.array(TypedSequence(["foo", "bar"], try_type=pa.int64())) self.assertEqual(arr.type, pa.string())
import pyarrow as pa PQ_SCHEMAS = dict() # site_visits fields = [ pa.field('visit_id', pa.int64(), nullable=False), pa.field('crawl_id', pa.int32(), nullable=False), pa.field('instance_id', pa.int32(), nullable=False), pa.field('site_url', pa.string(), nullable=False) ] PQ_SCHEMAS['site_visits'] = pa.schema(fields) # flash_cookies fields = [ pa.field('crawl_id', pa.int32(), nullable=False), pa.field('visit_id', pa.int64(), nullable=False), pa.field('instance_id', pa.int32(), nullable=False), pa.field('domain', pa.string()), pa.field('filename', pa.string()), pa.field('local_path', pa.string()), pa.field('key', pa.string()), pa.field('content', pa.string()) ] PQ_SCHEMAS['flash_cookies'] = pa.schema(fields) # profile_cookies fields = [ pa.field('crawl_id', pa.int32(), nullable=False), pa.field('visit_id', pa.int64(), nullable=False), pa.field('instance_id', pa.int32(), nullable=False),
def test_is_dictionary(): assert types.is_dictionary(pa.dictionary(pa.int32(), pa.string())) assert not types.is_dictionary(pa.int32())
MANY_TYPES = [ pa.null(), pa.bool_(), pa.int32(), pa.time32('s'), pa.time64('us'), pa.date32(), pa.timestamp('us'), pa.timestamp('us', tz='UTC'), pa.timestamp('us', tz='Europe/Paris'), pa.float16(), pa.float32(), pa.float64(), pa.decimal128(19, 4), pa.string(), pa.binary(), pa.binary(10), pa.list_(pa.int32()), pa.struct([pa.field('a', pa.int32()), pa.field('b', pa.int8()), pa.field('c', pa.string())]), pa.union([pa.field('a', pa.binary(10)), pa.field('b', pa.string())], mode=pa.lib.UnionMode_DENSE), pa.union([pa.field('a', pa.binary(10)), pa.field('b', pa.string())], mode=pa.lib.UnionMode_SPARSE), # XXX Needs array pickling # pa.dictionary(pa.int32(), pa.array(['a', 'b', 'c'])), ]
def test_isnull(array, expected, offset): array = pa.array(array, pa.string())[offset:] np.testing.assert_array_equal( isnull(NumbaStringArray.make(array)), np.asarray(expected[offset:], dtype=np.bool), )
('uint32', range(0, 10)), ('int32', range(0, 10)), ('uint64', range(0, 10)), ('int64', range(0, 10)), ('float', [0.0, 0.1, 0.2]), ('double', [0.0, 0.1, 0.2]), ('string', ['a', 'b', 'c']), ('binary', [b'a', b'b', b'c']), (pa.binary(3), [b'abc', b'bcd', b'cde'])]) def test_cast_identities(ty, values): arr = pa.array(values, type=ty) assert arr.cast(ty).equals(arr) pickle_test_parametrize = pytest.mark.parametrize( ('data', 'typ'), [([True, False, True, True], pa.bool_()), ([1, 2, 4, 6], pa.int64()), ([1.0, 2.5, None], pa.float64()), (['a', None, 'b'], pa.string()), ([], None), ([[1, 2], [3]], pa.list_(pa.int64())), ([[4, 5], [6]], pa.large_list(pa.int16())), ([['a'], None, ['b', 'c']], pa.list_(pa.string())), ([(1, 'a'), (2, 'c'), None ], pa.struct([pa.field('a', pa.int64()), pa.field('b', pa.string())]))]) @pickle_test_parametrize def test_array_pickle(data, typ): # Allocate here so that we don't have any Arrow data allocated. # This is needed to ensure that allocator tests can be reliable. array = pa.array(data, type=typ) for proto in range(0, pickle.HIGHEST_PROTOCOL + 1): result = pickle.loads(pickle.dumps(array, proto))
def array_inhom_chunks(): chunk1 = pa.array(list("abc"), pa.string()) chunk2 = pa.array(list("12345"), pa.string()) chunk3 = pa.array(list("Z"), pa.string()) chunked_array = pa.chunked_array([chunk1, chunk2, chunk3]) return fr.FletcherChunkedArray(chunked_array)
with self.assertRaises(OverflowError): _ = pa.array(TypedSequence([["x" * 1024]] * ((2 << 20) + 1))) # ListArray with a bit more than 2GB def _check_output(output, expected_num_chunks: int): stream = pa.BufferReader(output) if isinstance(output, pa.Buffer) else pa.memory_map(output) f = pa.ipc.open_stream(stream) pa_table: pa.Table = f.read_all() assert len(pa_table.to_batches()) == expected_num_chunks assert pa_table.to_pydict() == {"col_1": ["foo", "bar"], "col_2": [1, 2]} del pa_table @pytest.mark.parametrize("writer_batch_size", [None, 1, 10]) @pytest.mark.parametrize( "fields", [None, {"col_1": pa.string(), "col_2": pa.int64()}, {"col_1": pa.string(), "col_2": pa.int32()}] ) def test_write(fields, writer_batch_size): output = pa.BufferOutputStream() schema = pa.schema(fields) if fields else None with ArrowWriter(stream=output, schema=schema, writer_batch_size=writer_batch_size) as writer: writer.write({"col_1": "foo", "col_2": 1}) writer.write({"col_1": "bar", "col_2": 2}) num_examples, num_bytes = writer.finalize() assert num_examples == 2 assert num_bytes > 0 if not fields: fields = {"col_1": pa.string(), "col_2": pa.int64()} assert writer._schema == pa.schema(fields, metadata=writer._schema.metadata) _check_output(output.getvalue(), expected_num_chunks=num_examples if writer_batch_size == 1 else 1)
def __call__(self): return pa.string()
('string', ['a', 'b', 'c']), ('binary', [b'a', b'b', b'c']), (pa.binary(3), [b'abc', b'bcd', b'cde']) ]) def test_cast_identities(ty, values): arr = pa.array(values, type=ty) assert arr.cast(ty).equals(arr) pickle_test_parametrize = pytest.mark.parametrize( ('data', 'typ'), [ ([True, False, True, True], pa.bool_()), ([1, 2, 4, 6], pa.int64()), ([1.0, 2.5, None], pa.float64()), (['a', None, 'b'], pa.string()), ([], None), ([[1, 2], [3]], pa.list_(pa.int64())), ([['a'], None, ['b', 'c']], pa.list_(pa.string())), ([(1, 'a'), (2, 'c'), None], pa.struct([pa.field('a', pa.int64()), pa.field('b', pa.string())])) ] ) @pickle_test_parametrize def test_array_pickle(data, typ): # Allocate here so that we don't have any Arrow data allocated. # This is needed to ensure that allocator tests can be reliable. array = pa.array(data, type=typ) for proto in range(0, pickle.HIGHEST_PROTOCOL + 1):
def test_chunked_array_basics(): data = pa.chunked_array([], type=pa.string()) assert data.to_pylist() == [] with pytest.raises(ValueError): pa.chunked_array([])