def test_simple_parse(self, yaml_full_path, full_root_path):
output_file_path = os.path.join(self.current_tmp, 'datafile.root')
hepdata_converter.convert(yaml_full_path, output_file_path,
options={'output_format': 'root'})
self.assertNotEqual(os.stat(output_file_path).st_size, 0, 'output root file is empty')
f = ROOT.TFile.Open(output_file_path)
f_orig = ROOT.TFile.Open(full_root_path)
self.assertEqual(list(walk(f)), list(walk(f_orig)))
f.Close()
f_orig.Close()
with open(output_file_path, 'w') as output:
hepdata_converter.convert(yaml_full_path, output,
options={'output_format': 'root'})
self.assertNotEqual(os.stat(output_file_path).st_size, 0, 'output root file is empty')
f = ROOT.TFile.Open(output_file_path)
f_orig = ROOT.TFile.Open(full_root_path)
self.assertEqual(list(walk(f)), list(walk(f_orig)))
for path, dirs, objects in list(walk(f))[1:]:
for obj in objects:
o = f.Get('%s/%s' % (path, obj))
o_orig = f_orig.Get('%s/%s' % (path, obj))
self.assertEqual(o.__class__, o_orig.__class__)
if o.__class__.__name__.startswith('TGraph'):
self.assertEqual(o.GetN(), o_orig.GetN())
for i in xrange(o.GetN()):
self.assertEqual(o.GetX()[i],o_orig.GetX()[i])
self.assertEqual(o.GetY()[i],o_orig.GetY()[i])
f.Close()
f_orig.Close()
def test_select_table(self, submission_filepath):
csv_content = convert(submission_filepath,
options={
'input_format':
'yaml',
'output_format':
'csv',
'table':
os.path.join(submission_filepath,
'data1.yaml')
})
csv_content = convert(submission_filepath,
options={
'input_format': 'yaml',
'output_format': 'csv',
'table': 'Table 1'
})
csv_content = convert(submission_filepath,
options={
'input_format': 'yaml',
'output_format': 'csv',
'table': 0
})
def test_simple_parse(self, yaml_simple_path, yoda_template):
output_file_path = os.path.join(self.current_tmp, 'datafile.yoda')
hepdata_converter.convert(yaml_simple_path, output_file_path,
options={'output_format': 'yoda'})
self.assertNotEqual(os.stat(output_file_path).st_size, 0, 'output yoda file is empty')
with open(output_file_path, 'r') as f:
self.assertMultiLineAlmostEqual(f, yoda_template)
def test_parse_all(self, test_submissions):
for idx, test_submission in enumerate(test_submissions):
output_file_path = os.path.join(self.current_tmp, 'data-{}.root'.format(idx))
hepdata_converter.convert(test_submission, output_file_path,
options={'output_format': 'root'})
self.assertNotEqual(os.stat(output_file_path).st_size, 0, 'output root file is empty')
def test_create_newdir(self, oldhepdata_path, yaml_path):
new_dir = os.path.join(self.current_tmp, 'test', 'dir')
hepdata_converter.convert(oldhepdata_path, new_dir,
options={'input_format': 'oldhepdata',
'output_format': 'yaml'})
self.assertTrue(os.path.exists(new_dir))
self.assertDirsEqual(new_dir, yaml_path)
def test_submission(self):
hepdata_converter.convert(StringIO.StringIO(self.simple_submission),
self.current_tmp,
options={'input_format': 'oldhepdata'})
with open(os.path.join(self.current_tmp,
'submission.yaml')) as submission_file:
self.assertEqual(list(yaml.load_all(submission_file)),
list(yaml.load_all(self.correct_submit_output)))
def test_th1_parse(self, yaml_full_path, full_root_path):
output_file_path = os.path.join(self.current_tmp, 'datafile.root')
hepdata_converter.convert(yaml_full_path,
output_file_path,
options={
'output_format': 'root',
'table': 'data2.yaml'
})
pass
def test_simple_parse(self, yaml_simple_path, yoda_template):
output_file_path = os.path.join(self.current_tmp, 'datafile.yoda')
hepdata_converter.convert(yaml_simple_path,
output_file_path,
options={'output_format': 'yoda'})
self.assertNotEqual(
os.stat(output_file_path).st_size, 0, 'output yoda file is empty')
with open(output_file_path, 'r') as f:
self.assertMultiLineAlmostEqual(f, yoda_template)
def test_multiple_tables_pack(self, table_1_content, table_9_content):
convert(self.submission_filepath, self.current_tmp, options={'input_format': 'yaml',
'output_format': 'csv',
'pack': True})
with open(os.path.join(self.current_tmp, 'Table1.csv'), 'r') as f:
self.assertMultiLineAlmostEqual(table_1_content, f.read())
with open(os.path.join(self.current_tmp, 'Table9.csv'), 'r') as f:
self.assertMultiLineAlmostEqual(table_9_content, f.read())
def test_same_type_conversion(self, yaml_path):
hepdata_converter.convert(yaml_path,
self.current_tmp,
options={
'input_format': 'yaml',
'output_format': 'yaml'
})
# exclude data6.yaml and data7.yaml because they are not listed in submission.yaml
self.assertDirsEqual(yaml_path,
self.current_tmp,
exclude=['data6.yaml', 'data7.yaml'])
def test_parse_all(self, test_submissions):
for idx, test_submission in enumerate(test_submissions):
output_file_path = os.path.join(self.current_tmp,
'data-{}.root'.format(idx))
hepdata_converter.convert(test_submission,
output_file_path,
options={'output_format': 'root'})
self.assertNotEqual(
os.stat(output_file_path).st_size, 0,
'output root file is empty')
def test_multiple_tables_pack(self, table_1_content, table_9_content):
convert(self.submission_filepath,
self.current_tmp,
options={
'input_format': 'yaml',
'output_format': 'csv',
'pack': True
})
with open(os.path.join(self.current_tmp, 'Table1.csv'), 'r') as f:
self.assertMultiLineAlmostEqual(table_1_content, f.read())
with open(os.path.join(self.current_tmp, 'Table9.csv'), 'r') as f:
self.assertMultiLineAlmostEqual(table_9_content, f.read())
def convert():
kwargs = request.get_json(force=True)
input_tar = kwargs['input']
archive_name = kwargs['options'].get('filename',
'hepdata-converter-ws-data')
output_format = kwargs['options'].get('output_format', '')
output, os_handle = BytesIO(), None
if output_format.lower(
) in SINGLEFILE_FORMATS or 'table' in kwargs['options']:
os_handle, tmp_output = tempfile.mkstemp()
else:
tmp_output = tempfile.mkdtemp()
tmp_dir = tempfile.mkdtemp()
try:
conversion_input = os.path.abspath(tmp_dir)
conversion_output = os.path.abspath(tmp_output)
with tarfile.open(mode="r:gz",
fileobj=BytesIO(base64.b64decode(input_tar))) as tar:
tar.extractall(path=conversion_input)
# one file - treat it as one file input format
walked = list(os.walk(tmp_dir))
if len(walked) == 1 and len(walked[0][2]) == 1:
path, dirs, files = walked[0]
conversion_input = os.path.join(path, files[0])
else:
conversion_input = conversion_input + '/' + archive_name + '/'
hepdata_converter.convert(conversion_input, conversion_output,
kwargs.get('options', {}))
if not os.path.isdir(conversion_output):
archive_name = archive_name + '.' + output_format
with tarfile.open(mode='w:gz', fileobj=output) as tar:
tar.add(conversion_output, arcname=archive_name)
finally:
if os_handle:
os.fdopen(os_handle).close()
if os.path.exists(tmp_output):
os.remove(tmp_output)
shutil.rmtree(tmp_dir, ignore_errors=True)
shutil.rmtree(tmp_output, ignore_errors=True)
return Response(output.getvalue(), mimetype='application/x-gzip')
def test_2_qualifiers_2_iv_unpack(self, table_9_content):
csv_content = convert(self.submission_filepath, options={'input_format': 'yaml',
'output_format': 'csv',
'table': 'Table 9',
'separator': ';',
'pack': False})
self.assertMultiLineAlmostEqual(table_9_content, csv_content)
def test_no_qal_parse(self, yaml_path, table_5_noqual):
data = hepdata_converter.convert(yaml_path, options={'input_format': 'yaml',
'output_format': 'csv',
'single_file': True,
'validator_schema_version': '0.1.0'})
self.assertEqual(data, table_5_noqual)
def test_single_file_output(self, oldhepdata_path, oldhepdata_file,
oldhepdata_yaml_file):
data = hepdata_converter.convert(oldhepdata_file,
options={
'input_format':
'oldhepdata',
'output_format':
'yaml',
'single_file':
True,
'hepdata_doi':
'10.17182/hepdata.62535.v1'
})
yaml_single_output_path = os.path.join(self.current_tmp, 'output.yaml')
hepdata_converter._main([
'--input-format', 'oldhepdata', '--output-format', 'yaml',
'--single-file', oldhepdata_path, yaml_single_output_path
])
print('OUTPUT PATH IS {}'.format(yaml_single_output_path))
self.assertTrue(os.path.exists(yaml_single_output_path))
self.assertEqual(oldhepdata_yaml_file, data)
def test_csvwriter_options(self, table_1_content):
csv_content = convert(self.submission_filepath, options={'input_format': 'yaml',
'output_format': 'csv',
'table': 'Table 1',
'pack': True})
self.assertMultiLineAlmostEqual(table_1_content, csv_content)
def test_simple_parse(self, yaml_simple_path, yoda_template):
output_file_path = os.path.join(self.current_tmp, 'datafile.yoda')
hepdata_converter.convert(yaml_simple_path,
output_file_path,
options={
'output_format': 'yoda',
'validator_schema_version': '0.1.0',
'hepdata_doi':
'10.17182/hepdata.62535.v1',
'rivet_analysis_name':
'ATLAS_2012_I1203852'
})
self.assertNotEqual(
os.stat(output_file_path).st_size, 0, 'output yoda file is empty')
with open(output_file_path, 'r') as f:
self.assertMultiLineAlmostEqual(f, yoda_template)
def test_simple_parse(self, yaml_full_path, full_root_path):
output_file_path = os.path.join(self.current_tmp, 'datafile.root')
hepdata_converter.convert(yaml_full_path,
output_file_path,
options={
'output_format': 'root',
'validator_schema_version': '0.1.0'
})
self.assertNotEqual(
os.stat(output_file_path).st_size, 0, 'output root file is empty')
f = ROOT.TFile.Open(output_file_path)
f_orig = ROOT.TFile.Open(full_root_path)
self.assertEqual(list(walk(f)), list(walk(f_orig)))
f.Close()
f_orig.Close()
with open(output_file_path, 'wb') as output:
hepdata_converter.convert(yaml_full_path,
output,
options={
'output_format': 'root',
'validator_schema_version': '0.1.0'
})
self.assertNotEqual(
os.stat(output_file_path).st_size, 0, 'output root file is empty')
f = ROOT.TFile.Open(output_file_path)
f_orig = ROOT.TFile.Open(full_root_path)
self.assertEqual(list(walk(f)), list(walk(f_orig)))
for path, dirs, objects in list(walk(f))[1:]:
for obj in objects:
o = f.Get('%s/%s' % (path, obj))
o_orig = f_orig.Get('%s/%s' % (path, obj))
self.assertEqual(o.__class__, o_orig.__class__)
if o.__class__.__name__.startswith('TGraph'):
self.assertEqual(o.GetN(), o_orig.GetN())
for i in range(o.GetN()):
self.assertEqual(o.GetX()[i], o_orig.GetX()[i])
self.assertEqual(o.GetY()[i], o_orig.GetY()[i])
f.Close()
f_orig.Close()
def test_csvwriter_options(self, table_1_content):
csv_content = convert(self.submission_filepath,
options={
'input_format': 'yaml',
'output_format': 'csv',
'table': 'Table 1',
'pack': True
})
self.assertMultiLineAlmostEqual(table_1_content, csv_content)
def test_2_qualifiers_2_iv_unpack(self, table_9_content):
csv_content = convert(self.submission_filepath,
options={
'input_format': 'yaml',
'output_format': 'csv',
'table': 'Table 9',
'separator': ';',
'pack': False
})
self.assertMultiLineAlmostEqual(table_9_content, csv_content)
def convert():
kwargs = request.get_json(force=True)
input_tar = kwargs["input"]
archive_name = kwargs["options"].get("filename", "hepdata-converter-ws-data")
output_format = kwargs["options"].get("output_format", "")
output, os_handle = StringIO.StringIO(), None
if output_format.lower() in SINGLEFILE_FORMATS or "table" in kwargs["options"]:
os_handle, tmp_output = tempfile.mkstemp()
else:
tmp_output = tempfile.mkdtemp()
tmp_dir = tempfile.mkdtemp()
try:
conversion_input = os.path.abspath(tmp_dir)
conversion_output = os.path.abspath(tmp_output)
with tarfile.open(mode="r:gz", fileobj=StringIO.StringIO(base64.decodestring(input_tar))) as tar:
tar.extractall(path=conversion_input)
# one file - treat it as one file input format
walked = list(os.walk(tmp_dir))
if len(walked) == 1 and len(walked[0][2]) == 1:
path, dirs, files = walked[0]
conversion_input = os.path.join(path, files[0])
else:
conversion_input = conversion_input + "/" + archive_name + "/"
hepdata_converter.convert(conversion_input, conversion_output, kwargs.get("options", {}))
if not os.path.isdir(conversion_output):
archive_name = archive_name + "." + output_format
with tarfile.open(mode="w:gz", fileobj=output) as tar:
tar.add(conversion_output, arcname=archive_name)
finally:
if os_handle:
os.fdopen(os_handle).close()
shutil.rmtree(tmp_dir, ignore_errors=True)
shutil.rmtree(tmp_output, ignore_errors=True)
return Response(output.getvalue(), mimetype="application/x-gzip")
def test_2_qualifiers_2_iv_pack(self, table_9_content):
csv_content = convert(self.submission_filepath,
options={
'input_format': 'yaml',
'output_format': 'csv',
'table': 'Table 9',
'pack': True,
'validator_schema_version': '0.1.0'
})
self.assertMultiLineAlmostEqual(table_9_content, csv_content)
def test_single_file_output(self, oldhepdata_path, oldhepdata_file, oldhepdata_yaml_file):
data = hepdata_converter.convert(oldhepdata_file,
options={
'input_format': 'oldhepdata',
'output_format': 'yaml',
'single_file': True})
yaml_single_output_path = os.path.join(self.current_tmp, 'output.yaml')
hepdata_converter._main(['--input-format', 'oldhepdata', '--output-format', 'yaml', '--single-file',
oldhepdata_path, yaml_single_output_path])
print('OUTPUT PATH IS {}'.format(yaml_single_output_path))
self.assertTrue(os.path.exists(yaml_single_output_path))
self.assertEqual(oldhepdata_yaml_file, data)
def test_th1_parse(self, yaml_full_path, full_root_path):
output_file_path = os.path.join(self.current_tmp, 'datafile.root')
hepdata_converter.convert(yaml_full_path, output_file_path,
options={'output_format': 'root', 'table': 'data2.yaml'})
pass
def test_same_type_conversion(self, yaml_path):
hepdata_converter.convert(yaml_path, self.current_tmp, options={'input_format': 'yaml', 'output_format': 'yaml'})
# exclude data6.yaml and data7.yaml because they are not listed in submission.yaml
self.assertDirsEqual(yaml_path, self.current_tmp, exclude=['data6.yaml', 'data7.yaml'])
def test_no_qal_parse(self, yaml_path, table_5_noqual):
data = hepdata_converter.convert(yaml_path, options={'input_format': 'yaml',
'output_format': 'csv',
'single_file': True})
self.assertEqual(data, table_5_noqual)
def test_submission(self):
hepdata_converter.convert(StringIO.StringIO(self.simple_submission), self.current_tmp, options={'input_format': 'oldhepdata'})
with open(os.path.join(self.current_tmp, 'submission.yaml')) as submission_file:
self.assertEqual(list(yaml.load_all(submission_file)), list(yaml.load_all(self.correct_submit_output)))
