def convert_files(files_to_convert, source_folder, marc_folder,
from_date, to_date):
"""Converts the xml source files to marc xml files"""
converted_files = []
journal_mappings = get_kbs()['journals'][1]
edp = EDPSciencesPackage(journal_mappings)
counter = 1
for filename in files_to_convert:
task_update_progress('Converting files 3/3 \t%s of %s' %
(counter, len(files_to_convert)))
target_file = filename.split('/')[-1]
target_file = join(marc_folder, target_file)
target_folder = dirname(target_file)
if not exists(target_folder):
makedirs(target_folder)
record = ""
datestamp = edp.get_date(filename)
if exists(target_file):
if from_date and to_date:
if datestamp >= from_date and\
datestamp <= to_date:
converted_files.append(target_file)
elif from_date:
if datestamp >= from_date:
converted_files.append(target_file)
elif to_date:
if datestamp <= to_date:
converted_files.append(target_file)
else:
converted_files.append(target_file)
else:
if 'xml_rich' in filename:
record = edp.get_record_rich(filename, refextract)
else:
if from_date and to_date:
if datestamp >= from_date and\
datestamp <= to_date:
record = edp.get_record(filename, refextract)
elif from_date:
if datestamp >= from_date:
record = edp.get_record(filename, refextract)
elif to_date:
if datestamp <= to_date:
record = edp.get_record(filename, refextract)
else:
record = edp.get_record(filename, refextract)
if record:
write_message("Converted file: %s" % (filename,))
with open(target_file, 'w') as out:
out.write(record)
converted_files.append(target_file)
counter += 1
return converted_files
def setUp(self):
self.edp = EDPSciencesPackage(journal_mappings)
self.edp.document = parse(join(dirname(folder), edp_test_record))
def setUp(self):
self.edp = EDPSciencesPackage(journal_mappings)
self.edp.document = parse(join(dirname(folder), edp_test_record))
class EDPSciencesPackageTests(unittest.TestCase):
def setUp(self):
self.edp = EDPSciencesPackage(journal_mappings)
self.edp.document = parse(join(dirname(folder), edp_test_record))
def test_abstract(self):
abstract = (
u'<p><italic>Context. </italic>The depletion of iron and sulphur into dust in the interstellar'
u' medium and the exact nature of interstellar amorphous silicate grains is still an open question.'
u'</p> <p><italic>Aims. </italic>We study the incorporation of iron and sulphur into amorphous '
u'silicates of olivine- and pyroxene-types and their effects on the dust spectroscopy and thermal'
u' emission.</p> <p><italic>Methods. </italic>We used the Maxwell-Garnett effective-medium theory to'
u' construct the optical constants for a mixture of silicates, metallic iron, and iron sulphide. '
u'We also studied the effects of iron and iron sulphide in aggregate grains.</p> <p><italic>Results.'
u' </italic>Iron sulphide inclusions within amorphous silicates that contain iron metal inclusions'
u' show no strong differences in the optical properties of the grains. A mix of amorphous olivine-'
u' and pyroxene-type silicate broadens the silicate features. An amorphous carbon mantle with a'
u' thickness of 10 nm on the silicate grains leads to an increase in absorption on the short-wavelength '
u'side of the 10 <inline-formula specific-use="simple-math"><italic>μ</italic></inline-formula>'
u'm silicate band.</p> <p><italic>Conclusions. </italic>The assumption of amorphous olivine-type and'
u' pyroxene-type silicates and a 10 nm thick amorphous carbon mantle better matches the interstellar'
u' silicate band profiles. Including iron nano-particles leads to an increase in the mid-IR extinction,'
u' while up to 5 ppm of sulphur can be incorporated as Fe/FeS nano inclusions into silicate grains '
u'without leaving a significant trace of its presence.</p>')
self.assertEqual(self.edp._get_abstract(), abstract)
def test_journal(self):
self.assertEqual(self.edp._get_journal(), 'A&A')
def test_publisher(self):
self.assertEqual(self.edp._get_publisher(), 'EDP Sciences')
def test_date(self):
self.assertEqual(self.edp._get_date(), '2014-05-23')
def test_title(self):
title = u'A dearth of small particles in debris disks', 'An energy-constrained smallest fragment size', [u'FN1']
self.assertEqual(self.edp._get_title(), title)
def test_doi(self):
self.assertEqual(self.edp._get_doi(), u'10.1051/0004-6361/201423985')
def test_page_count(self):
self.assertEqual(self.edp._get_page_count(), u'4')
def test_authors(self):
authors = [('Krause, Martin', [u'AFF1', u'AFF2'], []),
('K\xc3\xb6hler, M.', [], []),
('Jones, A.', [], []),
('Ysard, N.', [], [])]
self.assertEqual(self.edp._get_authors(), authors)
def test_pacscodes(self):
self.assertEqual(self.edp._get_pacscodes(), [])
def test_copyright(self):
self.assertEqual(self.edp._get_copyright(), ('ESO', '2014', '\xc2\xa9 ESO, 2014'))
def test_publication_information(self):
publication_information = ('Astron.Astrophys.',
'565',
'',
u'2014',
'2014-05-23',
u'10.1051/0004-6361/201423985',
'L9',
'925',
'')
self.assertEqual(self.edp._get_publication_information(), publication_information)
def test_affiliations(self):
affiliations = {u'AFF1': 'Institut d\xe2\x80\x99Astrophysique Spatiale (IAS), Universit\xc3\xa9 Paris-Sud & CNRS , B\xc3\xa2t. 121 , 91405 Orsay , France',
u'AFF2': 'Excellence Cluster Universe, Technische Universit\xc3\xa4t M\xc3\xbcnchen , Boltzmannstrasse 2 , 85748 Garching , Germany'}
self.assertEqual(self.edp._get_affiliations(), affiliations)
def test_author_emails(self):
author_emails = {u'FN1': ['*****@*****.**']}
self.assertEqual(self.edp._get_author_emails(), author_emails)
def test_subject(self):
self.assertEqual(self.edp._get_subject(), 'Letters')
def test_license(self):
self.assertEqual(self.edp._get_license(), ('Creative Commons Attribution License 2.0',
u'open-access-test',
u'http://creativecommons.org/licenses/by/2.0/'))
def test_keywords(self):
self.assertEqual(self.edp._get_keywords(), [u'dust, extinction', u'ISM: abundances'])
def test_references(self):
references = [(u'2', u'journal', '', '', ['Bradley, J.P.'], '1994', 'Science', '265', '925'),
(u'3', u'journal', '', '', ['Caselli, P.', 'Hasegawa, T.I.', 'Herbst, E.'], '1994', 'Astrophys.J.', '421', '206'),
(u'4', u'journal', '', '', ['Chiar, J.E.', 'Tielens, A.G.G.M.'], '2006', 'Astrophys.J.', '637', '774'),
(u'5', u'journal', '', '', ['Compi\xc3\xa8gne, M.', 'Verstraete, L.', 'Jones, A.'], '2011', 'Astron.Astrophys.', '525', 'A103'),
(u'6', u'journal', '', '', ['Costantini, E.', 'Freyberg, M.J.', 'Predehl, P.'], '2005', 'Astron.Astrophys.', '444', '187'),
(u'7', u'journal', '', '', ['Davoisne, C.', 'Djouadi, Z.', 'Leroux, H.'], '2006', 'Astron.Astrophys.', '448', 'L1'),
(u'8', u'journal', '', '', ['Demyk, K.', 'Carrez, P.', 'Leroux, H.'], '2001', 'Astron.Astrophys.', '368', 'L38'),
(u'9', u'journal', '', '', ['Djouadi, Z.', 'Gattacceca, J.', 'D\xe2\x80\x99Hendecourt, L.'], '2007', 'Astron.Astrophys.', '468', 'L9'),
(u'10', u'journal', '', '', ['Draine, B.'], '1988', 'Astrophys.J.', '333', '848'),
(u'12', u'journal', '', '', ['Jenkins, E.B.'], '2009', 'Astrophys.J.', '700', '1299'),
(u'13', u'journal', '', '', ['Jones, A.P.'], '2000', 'J.Geophys.Res.', '105', '10257'),
(u'14', u'journal', '', '', ['Jones, A.P.'], '2011', 'Astron.Astrophys.', '528', 'A98'),
(u'15', u'journal', '', '', ['Jones, R.V.', 'Spitzer, L.'], '1967', 'Astrophys.J.', '147', '943'),
(u'16', u'journal', '', '', ['Jones, A.P.', 'Fanciullo, L.', 'K\xc3\xb6hler, M.'], '2013', 'Astron.Astrophys.', '558', 'A62'),
(u'17', u'journal', '', '', ['Joseph, C.L.', 'Snow, T.P.', 'Seab, C.G.', 'Crutcher, R.M.'], '1986', 'Astrophys.J.', '309', '771'),
(u'18', u'journal', '', '', ['Keller, L.P.', 'Hony, S.', 'Bradley, J.P.'], '2002', 'Nature', '417', '148'),
(u'19', u'journal', '', '', ['Leroux, H.', 'Roskosz, M.', 'Jacob, D.'], '2009', 'Geochim.Cosmochim.Acta', '73', '767'),
(u'20', u'journal', '', '', ['Mathis, J.S.'], '1990', 'Ann.Rev.Astron.Astrophys.', '28', '37'),
(u'21', u'journal', '', '', ['McClure, M.'], '2009', 'Astrophys.J.', '693', 'L81'),
(u'22', u'journal', '', '', ['Millar, T.J.', 'Herbst, E.'], '1990', 'Astron.Astrophys.', '231', '466'),
(u'23', u'journal', '', '', ['Min, M.', 'Waters, L.B.F.M.', 'de Koter, A.'], '2007', 'Astron.Astrophys.', '462', '667'),
(u'24', u'journal', '', '', ['Nguyen, A.N.', 'Stadermann, F.J.', 'Zinner, E.'], '2007', 'Astrophys.J.', '656', '1223'),
(u'25', u'journal', '', '', ['Ordal, M.A.', 'Bell, R.J.', 'Alexander, R.W.', 'Long, L.L.', 'Querry, M.R.'], '1985', 'Appl.Opt.', '24', '4493'),
(u'26', u'journal', '', '', ['Ordal, M.A.', 'Bell, R.J.', 'Alexander, R.W.', 'Newquist, L.A.', 'Querry, M.R.'], '1988', 'Appl.Opt.', '27', '1203'),
(u'28', u'journal', '', '', ['Pollack, J.B.', 'Hollenbach, D.', 'Beckwith, S.'], '1994', 'Astrophys.J.', '421', '615'),
(u'29', u'journal', '', '', ['Purcell, E.M.', 'Pennypacker, C.R.'], '1973', 'Astrophys.J.', '186', '705'),
(u'30', u'journal', '', '', ['Rieke, G.H.', 'Lebofsky, M.J.'], '1985', 'Astrophys.J.', '288', '618'),
(u'31', u'journal', '', '', ['Roskosz, M.', 'Gillot, J.', 'Capet, F.', 'Roussel, P.', 'Leroux, H.'], '2011', 'Astron.Astrophys.', '529', 'A111'),
(u'32', u'journal', '', '', ['Savage, B.D.', 'Bohlin, R.C.'], '1979', 'Astrophys.J.', '229', '136'),
(u'33', u'journal', '', '', ['Savage, B.D.', 'Mathis, J.S.'], '1979', 'Ann.Rev.Astron.Astrophys.', '17', '73'),
(u'34', u'journal', '', '', ['Scott, A.', 'Duley, W.W.'], '1996', 'Astrophys.J.Suppl.', '105', '401'),
(u'35', u'journal', '', '', ['Sofia, U.J.', 'Joseph, C.L.'], '1995', 'Bull.Am.Astron.Soc.', '27', '860'),
(u'36', u'journal', '', '', ['Ueda, Y.', 'Mitsuda, K.', 'Murakami, H.', 'Matsushita, K.'], '2005', 'Astrophys.J.', '620', '274'),
(u'37', u'journal', '', '', ['Xiang, J.', 'Lee, J.C.', 'Nowak, M.A.', 'Wilms, J.'], '2011', 'Astrophys.J.', '738', '78')]
i = 0
for ref in self.edp._get_references():
self.assertEqual(ref, references[i])
i += 1
def test_article_type(self):
self.assertEqual(self.edp._get_article_type(), 'research-article')
def test_get_record(self):
source_file = join(dirname(folder), edp_test_record)
marc_file = join(dirname(folder), edp_output)
with open(marc_file) as marc:
result = marc.read()
self.assertEqual(self.edp.get_record(source_file), result)
class EDPSciencesPackageTests(unittest.TestCase):
def setUp(self):
self.edp = EDPSciencesPackage(journal_mappings)
self.edp.document = parse(join(dirname(folder), edp_test_record))
def test_abstract(self):
abstract = (
u'<p><italic>Context. </italic>The depletion of iron and sulphur into dust in the interstellar'
u' medium and the exact nature of interstellar amorphous silicate grains is still an open question.'
u'</p> <p><italic>Aims. </italic>We study the incorporation of iron and sulphur into amorphous '
u'silicates of olivine- and pyroxene-types and their effects on the dust spectroscopy and thermal'
u' emission.</p> <p><italic>Methods. </italic>We used the Maxwell-Garnett effective-medium theory to'
u' construct the optical constants for a mixture of silicates, metallic iron, and iron sulphide. '
u'We also studied the effects of iron and iron sulphide in aggregate grains.</p> <p><italic>Results.'
u' </italic>Iron sulphide inclusions within amorphous silicates that contain iron metal inclusions'
u' show no strong differences in the optical properties of the grains. A mix of amorphous olivine-'
u' and pyroxene-type silicate broadens the silicate features. An amorphous carbon mantle with a'
u' thickness of 10 nm on the silicate grains leads to an increase in absorption on the short-wavelength '
u'side of the 10 <inline-formula specific-use="simple-math"><italic>μ</italic></inline-formula>'
u'm silicate band.</p> <p><italic>Conclusions. </italic>The assumption of amorphous olivine-type and'
u' pyroxene-type silicates and a 10 nm thick amorphous carbon mantle better matches the interstellar'
u' silicate band profiles. Including iron nano-particles leads to an increase in the mid-IR extinction,'
u' while up to 5 ppm of sulphur can be incorporated as Fe/FeS nano inclusions into silicate grains '
u'without leaving a significant trace of its presence.</p>')
self.assertEqual(self.edp._get_abstract(), abstract)
def test_journal(self):
self.assertEqual(self.edp._get_journal(), 'A&A')
def test_publisher(self):
self.assertEqual(self.edp._get_publisher(), 'EDP Sciences')
def test_date(self):
self.assertEqual(self.edp._get_date(), '2014-05-23')
def test_title(self):
title = u'A dearth of small particles in debris disks', 'An energy-constrained smallest fragment size', [
u'FN1'
]
self.assertEqual(self.edp._get_title(), title)
def test_doi(self):
self.assertEqual(self.edp._get_doi(), u'10.1051/0004-6361/201423985')
def test_page_count(self):
self.assertEqual(self.edp._get_page_count(), u'4')
def test_authors(self):
authors = [('Krause, Martin', [u'AFF1', u'AFF2'], []),
('K\xc3\xb6hler, M.', [], []), ('Jones, A.', [], []),
('Ysard, N.', [], [])]
self.assertEqual(self.edp._get_authors(), authors)
def test_pacscodes(self):
self.assertEqual(self.edp._get_pacscodes(), [])
def test_copyright(self):
self.assertEqual(self.edp._get_copyright(),
('ESO', '2014', '\xc2\xa9 ESO, 2014'))
def test_publication_information(self):
publication_information = ('Astron.Astrophys.', '565', '', u'2014',
'2014-05-23',
u'10.1051/0004-6361/201423985', 'L9', '925',
'')
self.assertEqual(self.edp._get_publication_information(),
publication_information)
def test_affiliations(self):
affiliations = {
u'AFF1':
'Institut d\xe2\x80\x99Astrophysique Spatiale (IAS), Universit\xc3\xa9 Paris-Sud & CNRS , B\xc3\xa2t. 121 , 91405 Orsay , France',
u'AFF2':
'Excellence Cluster Universe, Technische Universit\xc3\xa4t M\xc3\xbcnchen , Boltzmannstrasse 2 , 85748 Garching , Germany'
}
self.assertEqual(self.edp._get_affiliations(), affiliations)
def test_author_emails(self):
author_emails = {u'FN1': ['*****@*****.**']}
self.assertEqual(self.edp._get_author_emails(), author_emails)
def test_subject(self):
self.assertEqual(self.edp._get_subject(), 'Letters')
def test_license(self):
self.assertEqual(
self.edp._get_license(),
('Creative Commons Attribution License 2.0', u'open-access-test',
u'http://creativecommons.org/licenses/by/2.0/'))
def test_keywords(self):
self.assertEqual(self.edp._get_keywords(),
[u'dust, extinction', u'ISM: abundances'])
def test_references(self):
references = [
(u'2', u'journal', '', '', ['Bradley, J.P.'], '1994', 'Science',
'265', '925'),
(u'3', u'journal', '', '',
['Caselli, P.', 'Hasegawa, T.I.',
'Herbst, E.'], '1994', 'Astrophys.J.', '421', '206'),
(u'4', u'journal', '', '', ['Chiar, J.E.', 'Tielens, A.G.G.M.'],
'2006', 'Astrophys.J.', '637', '774'),
(u'5', u'journal', '', '',
['Compi\xc3\xa8gne, M.', 'Verstraete, L.',
'Jones, A.'], '2011', 'Astron.Astrophys.', '525', 'A103'),
(u'6', u'journal', '', '',
['Costantini, E.', 'Freyberg, M.J.',
'Predehl, P.'], '2005', 'Astron.Astrophys.', '444', '187'),
(u'7', u'journal', '', '',
['Davoisne, C.', 'Djouadi, Z.',
'Leroux, H.'], '2006', 'Astron.Astrophys.', '448', 'L1'),
(u'8', u'journal', '', '',
['Demyk, K.', 'Carrez, P.',
'Leroux, H.'], '2001', 'Astron.Astrophys.', '368', 'L38'),
(u'9', u'journal', '', '',
['Djouadi, Z.', 'Gattacceca, J.', 'D\xe2\x80\x99Hendecourt, L.'
], '2007', 'Astron.Astrophys.', '468', 'L9'),
(u'10', u'journal', '', '', ['Draine, B.'], '1988', 'Astrophys.J.',
'333', '848'),
(u'12', u'journal', '', '', ['Jenkins, E.B.'], '2009',
'Astrophys.J.', '700', '1299'),
(u'13', u'journal', '', '', ['Jones, A.P.'], '2000',
'J.Geophys.Res.', '105', '10257'),
(u'14', u'journal', '', '', ['Jones, A.P.'], '2011',
'Astron.Astrophys.', '528', 'A98'),
(u'15', u'journal', '', '', ['Jones, R.V.', 'Spitzer, L.'], '1967',
'Astrophys.J.', '147', '943'),
(u'16', u'journal', '', '',
['Jones, A.P.', 'Fanciullo, L.',
'K\xc3\xb6hler, M.'], '2013', 'Astron.Astrophys.', '558', 'A62'),
(u'17', u'journal', '', '',
['Joseph, C.L.', 'Snow, T.P.', 'Seab, C.G.',
'Crutcher, R.M.'], '1986', 'Astrophys.J.', '309', '771'),
(u'18', u'journal', '', '',
['Keller, L.P.', 'Hony, S.',
'Bradley, J.P.'], '2002', 'Nature', '417', '148'),
(u'19', u'journal', '', '',
['Leroux, H.', 'Roskosz, M.',
'Jacob, D.'], '2009', 'Geochim.Cosmochim.Acta', '73', '767'),
(u'20', u'journal', '', '', ['Mathis, J.S.'], '1990',
'Ann.Rev.Astron.Astrophys.', '28', '37'),
(u'21', u'journal', '', '', ['McClure, M.'], '2009',
'Astrophys.J.', '693', 'L81'),
(u'22', u'journal', '', '', ['Millar, T.J.', 'Herbst, E.'], '1990',
'Astron.Astrophys.', '231', '466'),
(u'23', u'journal', '', '',
['Min, M.', 'Waters, L.B.F.M.',
'de Koter, A.'], '2007', 'Astron.Astrophys.', '462', '667'),
(u'24', u'journal', '', '',
['Nguyen, A.N.', 'Stadermann, F.J.',
'Zinner, E.'], '2007', 'Astrophys.J.', '656', '1223'),
(u'25', u'journal', '', '', [
'Ordal, M.A.', 'Bell, R.J.', 'Alexander, R.W.', 'Long, L.L.',
'Querry, M.R.'
], '1985', 'Appl.Opt.', '24', '4493'),
(u'26', u'journal', '', '', [
'Ordal, M.A.', 'Bell, R.J.', 'Alexander, R.W.',
'Newquist, L.A.', 'Querry, M.R.'
], '1988', 'Appl.Opt.', '27', '1203'),
(u'28', u'journal', '', '',
['Pollack, J.B.', 'Hollenbach, D.',
'Beckwith, S.'], '1994', 'Astrophys.J.', '421', '615'),
(u'29', u'journal', '', '', ['Purcell, E.M.', 'Pennypacker, C.R.'],
'1973', 'Astrophys.J.', '186', '705'),
(u'30', u'journal', '', '', ['Rieke, G.H.', 'Lebofsky, M.J.'],
'1985', 'Astrophys.J.', '288', '618'),
(u'31', u'journal', '', '', [
'Roskosz, M.', 'Gillot, J.', 'Capet, F.', 'Roussel, P.',
'Leroux, H.'
], '2011', 'Astron.Astrophys.', '529', 'A111'),
(u'32', u'journal', '', '', ['Savage, B.D.', 'Bohlin, R.C.'],
'1979', 'Astrophys.J.', '229', '136'),
(u'33', u'journal', '', '', ['Savage, B.D.', 'Mathis, J.S.'],
'1979', 'Ann.Rev.Astron.Astrophys.', '17', '73'),
(u'34', u'journal', '', '', ['Scott, A.', 'Duley, W.W.'], '1996',
'Astrophys.J.Suppl.', '105', '401'),
(u'35', u'journal', '', '', ['Sofia, U.J.', 'Joseph, C.L.'],
'1995', 'Bull.Am.Astron.Soc.', '27', '860'),
(u'36', u'journal', '', '',
['Ueda, Y.', 'Mitsuda, K.', 'Murakami, H.',
'Matsushita, K.'], '2005', 'Astrophys.J.', '620', '274'),
(u'37', u'journal', '', '',
['Xiang, J.', 'Lee, J.C.', 'Nowak, M.A.',
'Wilms, J.'], '2011', 'Astrophys.J.', '738', '78')
]
i = 0
for ref in self.edp._get_references():
self.assertEqual(ref, references[i])
i += 1
def test_article_type(self):
self.assertEqual(self.edp._get_article_type(), 'research-article')
def test_get_record(self):
source_file = join(dirname(folder), edp_test_record)
marc_file = join(dirname(folder), edp_output)
xml = self.edp.get_record(source_file)
with open(marc_file) as marc:
result = marc.read()
with open("/tmp/test.xml", "w") as marc:
marc.write(xml)
self.assertEqual(xml.strip(), result.strip())
