def test_int_dub():
assert ib.dub([1]) == '1'
assert ib.dub([20433930, 20433930]) == '20433930'
assert ib.dub([1, 2]) == '1-2'
assert ib.dub([1, 3, 2]) == '1-3'
assert ib.dub([0, 4, 2]) == '0-4%2'
assert ib.dub([0, 4, 2, 8]) == '0-4%2_8'
assert ib.dub([0, 4, 2, 8, 10]) == '0-4%2_8-10%2'
def test_str_dub():
assert ib.dub(['foo']) == 'foo'
assert ib.dub(['foo~']) == 'foo~'
assert ib.dub(['foo', 'bar', 'bar']) == 'bar~foo'
assert ib.dub(['foo~', 'bar', 'bar']) == '...%2'
assert ib.dub([
'baaaaaaaaaaaaaaaaaarrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrr',
]) == 'baaaaaaaaaaaaaaaaaarrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrr'
assert ib.dub([
'foo',
'baaaaaaaaaaaaaaaaaarrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrr',
'foo',
]) == '...%2'
assert ib.dub(['foo', 'bar', 'bizz', 'foo']) == 'bar~...%2'
assert ib.dub(['foo', 'bar', 'bizz', 'foz']) == 'bar~...%3'
assert ib.dub([
'fooooooooooooooooooooooooooooooooooooooooo',
'bar',
'bizz',
'bar',
]) == '...%3'
assert ib.dub(['foo~', 'bar', 'bizz', 'bar']) == '...%3'
def make_output_filename():
df = pd.DataFrame.from_records(
[kn.unpack(source_filename) for source_filename in sys.argv[1:]], )
out_attrs = {
column: ib.dub(df[column])
for column in df.columns if column not in {
'_',
'ext',
}
}
out_filename = kn.pack({
**out_attrs,
**{
'concat': str(len(sys.argv) - 1),
'ext': '.csv',
},
})
return out_filename
def test_float_dub():
assert ib.dub([0.25]) == '0.25'
assert ib.dub([0.1, 0.2, 0.2, 0.3]) == '0.1-0.3%0.1'
def test_empty_dub():
assert ib.dub([]) == 'none'
},
})
if 'stage' in kn.unpack(segment) and 'what' in kn.unpack(segment)
else segment
for segment in prefix.split('/')
)
common_keys = set.intersection(*[
set( kn.unpack(match).keys() )
for match in matches
])
out_filename = kn.pack({
**{
key : ib.dub(
kn.unpack(match)[key]
for match in matches
)
for key in common_keys
},
**{ 'ext' : '.csv.xz' },
})
out_path = (out_prefix + '/' + out_filename).replace('//', '/')
print(f'upload path will be s3://{bucket}/{out_path}')
################################################################################
print( )
print( 'downloading and concatenating' )
print( '-----------------------------' )