def test_int_dub(): assert ib.dub([1]) == '1' assert ib.dub([20433930, 20433930]) == '20433930' assert ib.dub([1, 2]) == '1-2' assert ib.dub([1, 3, 2]) == '1-3' assert ib.dub([0, 4, 2]) == '0-4%2' assert ib.dub([0, 4, 2, 8]) == '0-4%2_8' assert ib.dub([0, 4, 2, 8, 10]) == '0-4%2_8-10%2'
def test_str_dub(): assert ib.dub(['foo']) == 'foo' assert ib.dub(['foo~']) == 'foo~' assert ib.dub(['foo', 'bar', 'bar']) == 'bar~foo' assert ib.dub(['foo~', 'bar', 'bar']) == '...%2' assert ib.dub([ 'baaaaaaaaaaaaaaaaaarrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrr', ]) == 'baaaaaaaaaaaaaaaaaarrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrr' assert ib.dub([ 'foo', 'baaaaaaaaaaaaaaaaaarrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrr', 'foo', ]) == '...%2' assert ib.dub(['foo', 'bar', 'bizz', 'foo']) == 'bar~...%2' assert ib.dub(['foo', 'bar', 'bizz', 'foz']) == 'bar~...%3' assert ib.dub([ 'fooooooooooooooooooooooooooooooooooooooooo', 'bar', 'bizz', 'bar', ]) == '...%3' assert ib.dub(['foo~', 'bar', 'bizz', 'bar']) == '...%3'
def make_output_filename(): df = pd.DataFrame.from_records( [kn.unpack(source_filename) for source_filename in sys.argv[1:]], ) out_attrs = { column: ib.dub(df[column]) for column in df.columns if column not in { '_', 'ext', } } out_filename = kn.pack({ **out_attrs, **{ 'concat': str(len(sys.argv) - 1), 'ext': '.csv', }, }) return out_filename
def test_float_dub(): assert ib.dub([0.25]) == '0.25' assert ib.dub([0.1, 0.2, 0.2, 0.3]) == '0.1-0.3%0.1'
def test_empty_dub(): assert ib.dub([]) == 'none'
}, }) if 'stage' in kn.unpack(segment) and 'what' in kn.unpack(segment) else segment for segment in prefix.split('/') ) common_keys = set.intersection(*[ set( kn.unpack(match).keys() ) for match in matches ]) out_filename = kn.pack({ **{ key : ib.dub( kn.unpack(match)[key] for match in matches ) for key in common_keys }, **{ 'ext' : '.csv.xz' }, }) out_path = (out_prefix + '/' + out_filename).replace('//', '/') print(f'upload path will be s3://{bucket}/{out_path}') ################################################################################ print( ) print( 'downloading and concatenating' ) print( '-----------------------------' )