def test_from_structures(self):
s1 = Structure([[5, 0, 0], [0, 5, 0], [0, 0, 5]], ["Fe"], [[0, 0, 0]])
s2 = Structure([[5, 0, 0], [0, 5, 0], [0, 0, 5]], ["Mn"], [[0, 0, 0]])
remarks = ["unittest"]
authors="Test User <*****@*****.**>"
snl_list = StructureNL.from_structures([s1, s2], authors, remarks=remarks)
self.assertEqual(len(snl_list), 2)
snl1 = snl_list[0]
snl2 = snl_list[1]
self.assertEqual(snl1.remarks, remarks)
self.assertEqual(snl2.remarks, remarks)
self.assertEqual(snl1.authors, [Author.parse_author(authors)])
self.assertEqual(snl2.authors, [Author.parse_author(authors)])
def test_from_structures(self):
s1 = Structure([[5, 0, 0], [0, 5, 0], [0, 0, 5]], ["Fe"], [[0, 0, 0]])
s2 = Structure([[5, 0, 0], [0, 5, 0], [0, 0, 5]], ["Mn"], [[0, 0, 0]])
remarks = ["unittest"]
authors="Test User <*****@*****.**>"
snl_list = StructureNL.from_structures([s1, s2], authors, remarks=remarks)
self.assertEqual(len(snl_list), 2)
snl1 = snl_list[0]
snl2 = snl_list[1]
self.assertEqual(snl1.remarks, remarks)
self.assertEqual(snl2.remarks, remarks)
self.assertEqual(snl1.authors, [Author.parse_author(authors)])
self.assertEqual(snl2.authors, [Author.parse_author(authors)])
def plot(self, contributor_email, cid):
"""make all plots for contribution_id"""
plot_contrib = self.contrib_coll.find_one(
{'contribution_id': cid}, {
'content.data': 1, 'content.plots': 1,
'_id': 0, 'collaborators': 1, 'project': 1
}
)
if 'data' not in plot_contrib['content']:
return None
author = Author.parse_author(contributor_email)
project = str(author.name).translate(None, '.').replace(' ','_') \
if 'project' not in plot_contrib else plot_contrib['project']
subfld = 'contributed_data.%s.plotly_urls.%d' % (project, cid)
data = plot_contrib['content']['data']
df = pd.DataFrame.from_dict(data)
url_list = list(self.mat_coll.find(
{subfld: {'$exists': True}},
{'_id': 0, subfld: 1}
))
urls = []
if len(url_list) > 0:
urls = url_list[0]['contributed_data'][project]['plotly_urls'][str(cid)]
for nplot,plotopts in enumerate(
plot_contrib['content']['plots'].itervalues()
):
filename = 'test%d_%d' % (cid,nplot)
fig, ax = plt.subplots(1, 1)
df.plot(ax=ax, **plotopts)
if len(urls) == len(plot_contrib['content']['plots']):
pyfig = py.get_figure(urls[nplot])
for ti,line in enumerate(ax.get_lines()):
pyfig['data'][ti]['x'] = list(line.get_xdata())
pyfig['data'][ti]['y'] = list(line.get_ydata())
py.plot(pyfig, filename=filename, auto_open=False)
else:
update = dict(
layout=dict(
annotations=[dict(text=' ')],
showlegend=True,
legend=Legend(x=1.05, y=1)
),
)
urls.append(py.plot_mpl(
fig, filename=filename, auto_open=False,
strip_style=True, update=update, resize=True
))
return None if len(url_list) > 0 else urls
def build(self, contributor_email, cid, api_key=None, endpoint=None):
"""update materials/compositions collections with contributed data"""
cid_short, cid_str = get_short_object_id(cid), str(cid)
contrib = self.find_contribution(cid)
if contributor_email not in contrib['collaborators']: raise ValueError(
"Build stopped: building contribution {} not "
"allowed due to insufficient permissions of {}! Ask "
"someone of {} to make you a collaborator on {}.".format(
cid_short, contributor_email, contrib['collaborators'], cid_short))
mpfile = MPFileCore.from_contribution(contrib)
mp_cat_id = mpfile.ids[0]
is_mp_id = mp_id_pattern.match(mp_cat_id)
self.curr_coll = self.materials if is_mp_id else self.compositions
author = Author.parse_author(contributor_email)
project = str(author.name).translate(None, '.') \
if 'project' not in contrib else contrib['project']
nb = nbf.new_notebook()
if isinstance(self.db, dict):
contrib.pop('_id')
contrib['content'].pop('cid')
nb['cells'].append(nbf.new_code_cell(
"from mpcontribs.io.core.mpfile import MPFileCore\n"
"from mpcontribs.io.core.recdict import RecursiveDict\n"
"mpfile = MPFileCore.from_contribution({})\n"
"mpid = '{}'"
.format(contrib, mp_cat_id)
))
else:
nb['cells'].append(nbf.new_code_cell("print 'hello'"))
nb['cells'].append(nbf.new_code_cell(
"from mpcontribs.rest.rester import MPContribsRester"
))
os.environ['MAPI_KEY'] = api_key
os.environ['MAPI_ENDPOINT'] = endpoint
nb['cells'].append(nbf.new_code_cell(
"with MPContribsRester() as mpr:\n"
" mpfile = mpr.find_contribution('{}')\n"
" mpid = mpfile.ids[0]"
.format(cid)
))
nb['cells'].append(nbf.new_markdown_cell(
"# Contribution #{} for {}".format(cid_short, mp_cat_id)
))
nb['cells'].append(nbf.new_markdown_cell(
"## Hierarchical Data"
))
nb['cells'].append(nbf.new_code_cell("mpfile.hdata[mpid]"))
if mpfile.tdata[mp_cat_id]:
nb['cells'].append(nbf.new_markdown_cell("## Tabular Data"))
for table_name, table in mpfile.tdata[mp_cat_id].iteritems():
nb['cells'].append(nbf.new_markdown_cell(
"### {}".format(table_name)
))
nb['cells'].append(nbf.new_code_cell(
"mpfile.tdata[mpid]['{}']".format(table_name)
))
if mpfile.gdata[mp_cat_id]:
nb['cells'].append(nbf.new_markdown_cell("## Graphical Data"))
for plot_name, plot in mpfile.gdata[mp_cat_id].iteritems():
nb['cells'].append(nbf.new_markdown_cell(
"### {}".format(plot_name)
))
nb['cells'].append(nbf.new_code_cell(
"mpfile.gdata[mpid]['{}']".format(plot_name)
))
nbdir = os.path.dirname(os.path.abspath(__file__))
ep = ExecutePreprocessor(timeout=600, kernel_name='python2',
allow_errors=True)
out = ep.preprocess(nb, {'metadata': {'path': nbdir}})
if isinstance(self.db, dict):
return [mp_cat_id, project, cid_short, export_notebook(nb, cid)]
else:
build_doc = RecursiveDict()
build_doc['mp_cat_id'] = mp_cat_id
build_doc['project'] = project
build_doc['nb'] = nb
self.curr_coll.update({'_id': cid}, {'$set': build_doc}, upsert=True)
return '{}/{}'.format( # return URL for contribution page
('materials' if is_mp_id else 'compositions'), cid_str)
def build(self, contributor_email, cids=None):
"""update materials collection with contributed data"""
# NOTE: this build is only for contributions tagged with mp-id
# TODO: in general, distinguish mp cat's by format of mp_cat_id
# TODO check all DB calls, consolidate in aggregation call?
plot_cids = None
for doc in self.contrib_coll.aggregate(self.pipeline, cursor={}):
#if plot_cids is None and doc['num_contribs'] > 2:
# only make plots for one mp-id due to plotly restrictions
plot_cids = doc['contrib_ids']
for cid in doc['contrib_ids']:
if cids is not None and cid not in cids: continue
contrib = self.contrib_coll.find_one({'contribution_id': cid})
if contributor_email not in contrib['collaborators']:
raise ValueError(
"Build stopped: building contribution {} not"
" allowed due to insufficient permissions of {}!"
" Ask someone of {} to make you a collaborator on"
" contribution {}.".format(
cid, contributor_email, contrib['collaborators'], cid
))
author = Author.parse_author(contributor_email)
project = str(author.name).translate(None, '.').replace(' ','_') \
if 'project' not in contrib else contrib['project']
logging.info(doc['_id'])
all_data = {}
if contrib['content']:
all_data.update({
'contributed_data.%s.tree_data.%d' % (project, cid): contrib['content'],
})
if 'data' in contrib['content']:
table_columns, table_rows = None, None
raw_data = contrib['content']['data']
if isinstance(raw_data, dict):
table_columns = [ { 'title': k } for k in raw_data ]
table_rows = [
[ str(raw_data[d['title']][row_index]) for d in table_columns ]
for row_index in xrange(len(
raw_data[table_columns[0]['title']]
))
]
elif isinstance(raw_data, list):
table_columns = [ { 'title': k } for k in raw_data[0] ]
table_rows = [
[ str(row[d['title']]) for d in table_columns ]
for row in raw_data
]
if table_columns is not None:
all_data.update({
'contributed_data.%s.tables.%d.columns' % (project,cid): table_columns,
'contributed_data.%s.tables.%d.rows' % (project,cid): table_rows,
})
logging.info(self.mat_coll.update(
{'task_id': doc['_id']}, { '$set': all_data }
))
if plot_cids is not None and cid in plot_cids:
plotly_urls = self.plot(contributor_email, cid)
if plotly_urls is not None:
for plotly_url in plotly_urls:
logging.info(self.mat_coll.update(
{'task_id': doc['_id']}, { '$push': {
'contributed_data.%s.plotly_urls.%d' % (project,cid): plotly_url
}}
))