def updateExportStatus(self, result):
""" Update export status of HTML/CSV/TSV
"""
setattr(self, 'exportWorks', True)
try:
setattr(self, 'exportWorks', True)
setattr(self, 'exportStatusMessage', '')
sparql2json(result)
except Exception, err:
logger.exception(err)
setattr(self, 'exportWorks', False)
setattr(self, 'exportStatusMessage', err)
def updateExportStatus(self, result):
""" Update export status of HTML/CSV/TSV
"""
setattr(self, 'exportWorks', True)
try:
setattr(self, 'exportWorks', True)
setattr(self, 'exportStatusMessage', '')
sparql2json(result)
except Exception, err:
logger.exception(err)
setattr(self, 'exportWorks', False)
setattr(self, 'exportStatusMessage', err)
def set_sparql_export_attribute(context):
catalog = getToolByName(context, 'portal_catalog')
brains = catalog.searchResults(portal_type='Sparql',
Language='all',
show_inactive=True)
log_total = len(brains)
log_count = 0
migrated = 0
negative_export = 0
no_result = 0
for brain in brains:
log_count += 1
try:
obj = brain.getObject()
except Exception:
logger.info('FAILED getObject %s:%s: %s', log_count, log_total,
brain.getPath())
continue
try:
cached_result = obj.getSparqlCacheResults()
except:
cached_result = None
logger.info('FAILED getgetSparqlCacheResults %s:%s: %s', log_count,
log_total, brain.getPath())
no_result += 1
continue
if cached_result:
logger.info('PATH %s:%s: %s', log_count, log_total,
brain.getPath())
setattr(obj, 'exportWorks', True)
try:
setattr(obj, 'exportWorks', True)
setattr(obj, 'exportStatusMessage', '')
sparql2json(cached_result)
except Exception, err:
logger.info('Export status failed on: %s', brain.getPath())
setattr(obj, 'exportWorks', False)
setattr(obj, 'exportStatusMessage', err)
negative_export += 1
continue
transaction.commit()
migrated += 1
if log_count % 100 == 0:
logger.info('INFO: Transaction committed to zodb (%s/%s)',
log_count, log_total)
def set_sparql_export_attribute(context):
catalog = getToolByName(context, 'portal_catalog')
brains = catalog.searchResults(portal_type='Sparql', Language='all',
show_inactive=True)
log_total = len(brains)
log_count = 0
migrated = 0
negative_export = 0
no_result = 0
for brain in brains:
log_count += 1
try:
obj = brain.getObject()
except Exception:
logger.info('FAILED getObject %s:%s: %s', log_count, log_total,
brain.getPath())
continue
try:
cached_result = obj.getSparqlCacheResults()
except:
cached_result = None
logger.info('FAILED getgetSparqlCacheResults %s:%s: %s', log_count,
log_total, brain.getPath())
no_result += 1
continue
if cached_result:
logger.info('PATH %s:%s: %s', log_count, log_total, brain.getPath())
setattr(obj, 'exportWorks', True)
try:
setattr(obj, 'exportWorks', True)
setattr(obj, 'exportStatusMessage', '')
sparql2json(cached_result)
except Exception, err:
logger.info('Export status failed on: %s', brain.getPath())
setattr(obj, 'exportWorks', False)
setattr(obj, 'exportStatusMessage', err)
negative_export += 1
continue
transaction.commit()
migrated += 1
if log_count % 100 == 0:
logger.info('INFO: Transaction committed to zodb (%s/%s)',
log_count, log_total)
def sparql2exhibit(self):
""" Download sparql results as Exhibit JSON
"""
try:
data = sparql2json(self.context.execute_query(
self.getArgumentMap()))
except Exception, err:
logger.exception(err)
data = {'properties':{}, 'items':{}}
def sparql2exhibit(self):
""" Download sparql results as Exhibit JSON
"""
try:
data = sparql2json(
self.context.execute_query(self.getArgumentMap()))
except Exception, err:
logger.exception(err)
data = {'properties': {}, 'items': {}}
def json(self, **kwargs):
"""json"""
data = self.context.execute_query(self.getArgumentMap())
column_types = kwargs.get('column_types')
annotations = kwargs.get('annotations')
return sortProperties(json.dumps(
sparql2json(data,
column_types=column_types,
annotations=annotations)
))
def sparql2csv(self, dialect='excel'):
""" Download sparql results as Comma Separated File
"""
if not self.getExportStatus():
self.request.response.setStatus(500)
return 'Error: %s' % self.context.exportStatusMessage
try:
data = sparql2json(self.context.execute_query(
self.getArgumentMap()))
except Exception, err:
data = {'properties':{}, 'items':{}}
logger.exception(err)
def sparql2csv(self, dialect='excel'):
""" Download sparql results as Comma Separated File
"""
if not self.getExportStatus():
self.request.response.setStatus(500)
return 'Error: %s' % self.context.exportStatusMessage
try:
data = sparql2json(
self.context.execute_query(self.getArgumentMap()))
except Exception, err:
data = {'properties': {}, 'items': {}}
logger.exception(err)
def sparql2csv(self, dialect='excel'):
""" Download sparql results as Comma Separated File
"""
try:
data = sparql2json(self.context.execute_query(
self.getArgumentMap()))
except Exception:
data = {'properties':{}, 'items':{}}
if dialect == 'excel':
self.request.response.setHeader(
'Content-Type', 'application/csv')
self.request.response.setHeader(
'Content-Disposition',
'attachment; filename="%s.csv"' % self.context.getId())
else:
self.request.response.setHeader(
'Content-Type', 'application/tsv')
self.request.response.setHeader(
'Content-Disposition',
'attachment; filename="%s.tsv"' % self.context.getId())
writter = csv.writer(self.request.response, dialect=dialect)
row = []
properties = []
def_order = 0
for key, item in data['properties'].items():
prop = []
prop.append(item.get('order', def_order))
prop.append(key)
prop.append(item['valueType'])
properties.append(prop)
def_order += 1
properties.sort()
headers = []
for prop in properties:
headers.append(prop[1])
for col in headers:
header = '%s:%s' % (col, data['properties'][col]['valueType'])
row.append(header)
writter.writerow(row)
for item in data['items']:
row = []
for col in headers:
row.append(unicode(item[col]).encode('utf'))
writter.writerow(row)
return ''
def json(self, **kwargs):
"""json"""
data = self.context.execute_query(self.getArgumentMap())
column_types = kwargs.get('column_types')
annotations = kwargs.get('annotations')
try:
return sortProperties(
json.dumps(
sparql2json(data,
column_types=column_types,
annotations=annotations)))
except KeyError, err:
logger.warn(err)
def sparql2exhibit(self):
""" Download sparql results as Exhibit JSON
"""
try:
data = sparql2json(self.context.execute_query(
self.getArgumentMap()))
except Exception:
data = {'properties':{}, 'items':{}}
self.request.response.setHeader(
'Content-Type', 'application/json')
self.request.response.setHeader(
'Content-Disposition',
'attachment; filename="%s.exhibit.json"' % self.context.getId())
return sortProperties(json.dumps(data))
def __call__(self):
self.results = []
self.can_save = False
is_post = self.request.method == 'POST'
if is_post and ('get_data' in self.request.form):
endpoint_url = self.context.getEndpoint_url()
query = self.request.form.get('query')
if not query:
return self.index()
cooked_query = interpolate_query(query, {})
args = (endpoint_url, cooked_query)
results = run_with_timeout(20, query_and_get_result, *args)
print results
self.results = sparql2json(results)
elif is_post and 'relabel' in self.request.form:
blacklist = ['query', 'relabel', 'save']
remap = [(k, v)
for k, v in self.request.form.items()
if (v and (k not in blacklist))]
query = self.request.form.get('query')
sm = IStatusMessage(self.request)
if not query:
sm.add(u"Need a query.", type='warning')
return self.index()
if len(remap) != len(self.choices):
sm.add(u"You don't have enough mapped columns", type='warning')
return self.index()
else:
for rep, sub in remap:
rx = r'(\?' + rep + ')(?!\w)'
query = re.sub(rx, '?' + sub, query)
self.query = query # override the method
self.can_save = True
elif is_post and 'save' in self.request.form:
self.context.setSparql_query(self.query())
return self.request.response.redirect(self.context.absolute_url())
return self.index()
def sparql2html(self):
""" Download sparql results as HTML
"""
try:
data = sparql2json(self.context.execute_query(
self.getArgumentMap()))
except Exception:
data = {'properties':{}, 'items':{}}
result = []
result.append(u"<style type='text/css'>")
result.append(u"table {border-collapse: collapse }")
result.append(u"th, td {border:1px solid black}")
result.append(u"</style>")
result.append(u"<table>")
result.append(u"\t<tr>")
properties = []
def_order = 0
for key, item in data['properties'].items():
prop = []
prop.append(item.get('order', def_order))
prop.append(key)
prop.append(item['valueType'])
properties.append(prop)
def_order += 1
properties.sort()
for col in properties:
result.append(u"\t\t<th>" + col[1] + u"</th>")
result.append(u"\t</tr>")
for row in data['items']:
result.append(u"\t<tr>")
for col in properties:
result.append(u"\t\t<td>" + unicode(row[col[1]]) + "</td>")
result.append(u"\t</tr>")
result.append(u"</table>")
return '\n'.join(result)
