def test_esk602(self):
"""Test Esk-602: Read CSV files into a Spark data frame."""
# check if running in local mode
sc = process_manager.service(SparkManager).get_session().sparkContext
self.assertRegex(
sc.getConf().get('spark.master', ''), 'local\[[.*]\]',
'Spark not running in local mode, required for testing with local files'
)
# run Eskapade
self.eskapade_run(resources.tutorial('esk602_read_csv_to_spark_df.py'))
ds = process_manager.service(DataStore)
# check data frame
self.assertIn('spark_df', ds,
'no object with key "spark_df" in data store')
self.assertIsInstance(ds['spark_df'], pyspark.sql.DataFrame,
'"spark_df" is not a Spark data frame')
self.assertEqual(ds['spark_df'].rdd.getNumPartitions(), 5,
'unexpected number of partitions in data frame')
self.assertEqual(ds['spark_df'].count(), 12,
'unexpected number of rows in data frame')
self.assertListEqual(ds['spark_df'].columns, ['date', 'loc', 'x', 'y'],
'unexpected columns in data frame')
self.assertSetEqual(
set((r['date'], r['loc']) for r in ds['spark_df'].collect()),
set([(20090101, 'a'), (20090102, 'b'), (20090103, 'c'),
(20090104, 'd'), (20090104, 'e'), (20090106, 'a'),
(20090107, 'b'), (20090107, 'c'), (20090107, 'd'),
(20090108, 'e'), (20090109, 'e'), (20090109, 'f')]),
'unexpected values in date/loc columns')
def test_esk609(self):
"""Test Esk-609: Map data-frame groups."""
# run Eskapade
self.eskapade_run(resources.tutorial('esk609_map_df_groups.py'))
ds = process_manager.service(DataStore)
# check input data
for key in ('map_rdd', 'flat_map_rdd'):
self.assertIn(key, ds, 'no data found with key "{}"'.format(key))
self.assertIsInstance(
ds[key], pyspark.RDD,
'object "{0:s}" is not an RDD (type "{1!s}")'.format(
key, type(ds[key])))
# sums of "bar" variable
bar_sums = [(0, 27.5), (1, 77.5), (2, 127.5), (3, 177.5), (4, 227.5),
(5, 277.5), (6, 327.5), (7, 377.5), (8, 427.5), (9, 477.5)]
flmap_rows = [(it, 'foo{:d}'.format(it), (it + 1) / 2.,
bar_sums[it // 10][1]) for it in range(100)]
# check mapped data frames
self.assertListEqual(sorted(ds['map_rdd'].collect()), bar_sums,
'unexpected values in "map_rdd"')
self.assertListEqual(sorted(ds['flat_map_rdd'].collect()), flmap_rows,
'unexpected values in "flat_map_rdd"')
def test_esk611(self):
self.eskapade_run(resources.tutorial('esk611_flatten_time_series.py'))
output_dir = os.path.join(persistence.io_dir('results_data'),
'dummy_time_series_flat.csv')
self.assertTrue(os.path.exists(output_dir))
if os.path.exists(output_dir):
shutil.rmtree(output_dir)
def test_esk607(self):
"""Test Esk-607: Add column to Spark dataframe."""
# check if running in local mode
sc = process_manager.service(SparkManager).get_session().sparkContext
self.assertRegex(
sc.getConf().get('spark.master', ''), 'local\[[.*]\]',
'Spark not running in local mode, required for testing with local files'
)
# run Eskapade
self.eskapade_run(resources.tutorial('esk607_spark_with_column.py'))
ds = process_manager.service(DataStore)
# check data frame
self.assertIn('new_spark_df', ds,
'no object with key "new_spark_df" in data store')
self.assertIsInstance(ds['new_spark_df'], pyspark.sql.DataFrame,
'"new_spark_df" is not a Spark data frame')
self.assertEqual(ds['new_spark_df'].count(), 5,
'unexpected number of rows in filtered data frame')
self.assertListEqual(ds['new_spark_df'].columns,
['dummy', 'date', 'loc', 'x', 'y', 'pow_xy1'],
'unexpected columns in data frame')
self.assertSetEqual(
set(tuple(r) for r in ds['new_spark_df'].collect()),
set([('bla', 20090103, 'c', 5, 7, 78125.0),
('bal', 20090102, 'b', 3, 8, 6561.0),
('flo', 20090104, 'e', 3, 5, 243.0),
('bar', 20090101, 'a', 1, 9, 1.0),
('foo', 20090104, 'd', 1, 6, 1.0)]),
'unexpected values in columns')
def test_esk605(self):
"""Test Esk-605: Create Spark data frame."""
# run Eskapade
self.eskapade_run(resources.tutorial('esk605_create_spark_df.py'))
ds = process_manager.service(DataStore)
# check created data frames
cols = (StructField('index',
LongType()), StructField('foo', StringType()),
StructField('bar', DoubleType()))
rows = [(it, 'foo{:d}'.format(it), (it + 1) / 2.)
for it in range(20, 100)]
for key in ('rows_df', 'rdd_df', 'df_df', 'pd_df'):
self.assertIn(key, ds,
'no object with key {} in data store'.format(key))
df = ds[key]
self.assertIsInstance(
df, pyspark.sql.DataFrame,
'object with key {0:s} is not a data frame (type {1!s})'.
format(key, type(df)))
self.assertTupleEqual(
tuple(df.schema), cols,
'unexpected data-frame schema for {}'.format(key))
self.assertListEqual(
sorted(tuple(r) for r in df.collect()), rows,
'unexpected data-frame content for {}'.format(key))
self.assertTrue(df.is_cached,
'data frame {} is not cached'.format(key))
self.assertLessEqual(
df.rdd.getNumPartitions(), 2,
'unexpected number of data-frame partitions for {}'.format(
key))
def test_esk604(self):
"""Test Esk-604: Execute Spark-SQL query."""
# check if running in local mode
sc = process_manager.service(SparkManager).get_session().sparkContext
self.assertRegex(
sc.getConf().get('spark.master', ''), 'local\[[.*]\]',
'Spark not running in local mode, required for testing with local files'
)
# run Eskapade
self.eskapade_run(resources.tutorial('esk604_spark_execute_query.py'))
ds = process_manager.service(DataStore)
# check data frame
self.assertIn('spark_df_sql', ds,
'no object with key "spark_df_sql" in data store')
self.assertIsInstance(ds['spark_df_sql'], pyspark.sql.DataFrame,
'"spark_df_sql" is not a Spark data frame')
self.assertEqual(ds['spark_df_sql'].count(), 4,
'unexpected number of rows in filtered data frame')
self.assertListEqual(ds['spark_df_sql'].columns,
['loc', 'sumx', 'sumy'],
'unexpected columns in data frame')
self.assertEqual(
ds['spark_df_sql'].schema,
process_manager.get('ApplySQL').get('SparkSQL').schema,
'schema of data frame does not correspond to schema stored in link'
)
self.assertSetEqual(
set(tuple(r) for r in ds['spark_df_sql'].collect()),
set([('e', 10, 15), ('d', 2, 11), ('b', 6, 16), ('a', 2, 18)]),
'unexpected values in loc/sumx/sumy columns')
def test_esk603(self):
"""Test Esk-603: Write Spark data to CSV."""
# check if running in local mode
sc = process_manager.service(SparkManager).get_session().sparkContext
self.assertRegex(
sc.getConf().get('spark.master', ''), 'local\[[.*]\]',
'Spark not running in local mode, required for testing with local files'
)
# run Eskapade
self.eskapade_run(
resources.tutorial('esk603_write_spark_data_to_csv.py'))
# read output data
results_data_path = persistence.io_dir('results_data')
names = []
headers = []
contents = []
csv_dirs = glob.glob('{}/*'.format(results_data_path))
self.assertEqual(len(csv_dirs), 3,
'expected to find three CSV output directories')
for csv_dir in csv_dirs:
names.append(os.path.basename(csv_dir))
csv_files = glob.glob('{}/part*'.format(csv_dir))
self.assertEqual(
len(csv_files), 1,
'expected to find only one CSV file in "{}"'.format(names[-1]))
with open(csv_files[0]) as csv:
contents.append([l.strip().split(',') for l in csv])
self.assertEqual(
len(contents[-1]), 101,
'unexpected number of lines in "{}" CSV'.format(names[-1]))
headers.append(contents[-1][0])
contents[-1] = sorted(contents[-1][1:])
# check output data
self.assertListEqual(headers[0], ['index', 'foo', 'bar'],
'unexpected CSV header for "{}"'.format(names[0]))
self.assertListEqual(
contents[0],
sorted([str(it), 'foo{:d}'.format(it),
str((it + 1) / 2.)] for it in range(100)),
'unexpected CSV content for "{}"'.format(names[0]))
for name, head, cont in zip(names[1:], headers[1:], contents[1:]):
self.assertListEqual(
head, headers[0],
'CSV header of "{0:s}" differs from header of "{1:s}"'.format(
name, names[0]))
self.assertListEqual(
cont, contents[0],
'CSV content of "{0:s}" differs from content of "{1:s}"'.
format(name, names[0]))
def test_esk608(self):
"""Test Esk-608: Execute Spark histogram filling macro."""
# check if required Python and Java libraries are made available to worker nodes
sc = process_manager.service(SparkManager).get_session().sparkContext
self.assertRegex(
sc.getConf().get('spark.master', ''), 'local\[[.*]\]',
'Spark not running in local mode, required for testing with local files'
)
self.assertRegex(
sc.getConf().get('spark.jars.packages', ''),
'org.diana-hep:histogrammar-sparksql_2.11:1.0.4',
'org.diana-hep:histogrammar-sparksql_2.11:1.0.4 missing from spark.jars.packages, test_esk608 will fail'
)
# run Eskapade
self.eskapade_run(resources.tutorial('esk608_spark_histogrammar.py'))
ds = process_manager.service(DataStore)
# check data frame
self.assertIn('spark_df', ds,
'no object with key "spark_df" in data store')
self.assertIsInstance(ds['spark_df'], pyspark.sql.DataFrame,
'"spark_df" is not a Spark data frame')
self.assertEqual(ds['spark_df'].count(), 12,
'unexpected number of rows in data frame')
self.assertListEqual(sorted(ds['spark_df'].columns),
sorted(['date', 'loc', 'x', 'y']),
'unexpected columns in data frame')
# data-generation checks
self.assertIn('hist', ds)
self.assertIsInstance(ds['hist'], dict)
col_names = ['date', 'x', 'y', 'loc', 'x:y']
self.assertListEqual(sorted(ds['hist'].keys()), sorted(col_names))
# data-summary checks
f_bases = ['date', 'x', 'y', 'loc', 'x_vs_y']
file_names = ['report.tex'
] + ['hist_{}.pdf'.format(col) for col in f_bases]
for fname in file_names:
path = persistence.io_path('results_data',
'report/{}'.format(fname))
self.assertTrue(os.path.exists(path))
statinfo = os.stat(path)
self.assertTrue(statinfo.st_size > 0)
def test_esk606(self):
"""Test Esk-606: Convert Spark data frame."""
# run Eskapade
self.eskapade_run(resources.tutorial('esk606_convert_spark_df.py'))
ds = process_manager.service(DataStore)
# define types of stored data sets
data_types = {
'df': pyspark.sql.DataFrame,
'rdd': pyspark.RDD,
'list': list,
'pd': pd.DataFrame
}
# define functions to obtain data-frame content
content_funcs = {
'df': lambda d: sorted(d.rdd.map(tuple).collect()),
'rdd': lambda d: sorted(d.collect()),
'list': lambda d: sorted(d),
'pd': lambda d: sorted(map(tuple, d.values))
}
# check input data
self.assertIn('df', ds, 'no data found with key "df"')
self.assertIsInstance(ds['df'], pyspark.sql.DataFrame,
'unexpected type for input data frame')
# check created data sets
rows = [(it, 'foo{:d}'.format(it), (it + 1) / 2.)
for it in range(20, 100)]
for key, dtype in data_types.items():
# check content
dkey = '{}_output'.format(key)
self.assertIn(dkey, ds, 'no data found with key "{}"'.format(dkey))
self.assertIsInstance(ds[dkey], dtype,
'unexpected type for "{}" data'.format(key))
self.assertListEqual(
content_funcs[key](ds[dkey]), rows,
'unexpected content for "{}" data'.format(key))
# check schema
skey = '{}_schema'.format(key)
self.assertIn(skey, ds, 'no schema found with key {}'.format(skey))
self.assertListEqual(list(ds[skey]), list(ds['df'].schema),
'unexpected schema for "{}" data'.format(key))
def test_esk601(self):
"""Test Esk-601: Configure Spark."""
# ensure no running Spark instance
process_manager.service(SparkManager).finish()
# run Eskapade
self.eskapade_run(resources.tutorial('esk601_spark_configuration.py'))
sc = process_manager.service(SparkManager).get_session().sparkContext
# check configuration
self.assertEqual(sc.getConf().get('spark.app.name'),
'esk601_spark_configuration_link',
'SparkConf.setAll() not picked up correctly')
self.assertEqual(sc.getConf().get('spark.master'), 'local[42]',
'SparkConf.setAll() not picked up correctly')
self.assertEqual(sc.getConf().get('spark.driver.host'), '127.0.0.1',
'SparkConf.setAll() not picked up correctly')
# stop spark manager
process_manager.service(SparkManager).finish()
def test_esk610(self):
"""Test Esk-610: Spark Streaming word count."""
# this test relies on linux shell scripts to create file stream
if (sys.platform != 'linux') and (sys.platform != 'darwin'):
self.logger.debug(
'skipping test_esk610 for non-unix {} platform'.format(
sys.platform))
return
# check if running in local mode
sc = process_manager.service(SparkManager).get_session().sparkContext
self.assertRegex(
sc.getConf().get('spark.master', ''), 'local\[[.*]\]',
'Spark not running in local mode, required for testing with local files'
)
# create test dir
tmpdir = '/tmp/eskapade_stream_test'
os.mkdir(tmpdir)
def remove_tmp():
# clean up files
shutil.rmtree(tmpdir)
self.addCleanup(remove_tmp)
# create a file stream
tmpfile = ''.join(
random.choice(string.ascii_lowercase) for _ in range(8))
cmd = 'for i in $(seq -f \"%05g\" 0 1000); \
do echo \'Hello world\' > "{}"/"{}"_$i.dummy; \
sleep 1; done'.format(tmpdir, tmpfile)
p = subprocess.Popen(cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
# run eskapade
self.eskapade_run(
resources.tutorial('esk610_spark_streaming_wordcount.py'))
ds = process_manager.service(DataStore)
# end file stream
p.kill()
# check if file stream was properly executed
stdout, stderr = p.communicate()
self.assertEqual(stdout, b'',
'unexpected stdout output {}'.format(stdout))
self.assertEqual(stderr, b'',
'unexpected stderr output {}'.format(stderr))
# check if stream was setup correctly (that's all we can do - the data itself is gone)
self.assertIsInstance(ds['dstream'], pyspark.streaming.DStream)
# read and check output data
results_data_path = persistence.io_dir('results_data')
names = []
contents = []
csv_dirs = glob.glob(
'{}/dstream/wordcount-*.txt'.format(results_data_path))
self.assertGreater(len(csv_dirs), 0,
'expected to find CSV output directories')
for csv_dir in csv_dirs:
names.append(os.path.basename(csv_dir))
csv_files = glob.glob('{}/part*'.format(csv_dir))
# self.assertEqual(len(csv_files), 1, 'expected to find exactly one CSV file in "{}"'.format(names[-1]))
if csv_files:
with open(csv_files[0]) as csv:
record = [l for l in csv]
if record: # empty records are allowed (because of timing differences)
self.assertRegex(
record[0], 'Hello',
'Expected \'Hello\' as in \'Hello world\'')
self.assertRegex(
record[1], 'world',
'Expected \'world\' as in \'Hello world\'')
contents.append(record[:])
self.assertGreater(
len(contents), 0,
'expected ~ten items (each second a streaming RDD) - depending on timing'
)