def import_pubmed(self, node, raw_info):
'''
node is the current folder. we leave the validation
of raw_ids to the importer
How much of this code can we reuse for importing BibTex?
The tail end.
'''
raw_ids = self._file_or_text(raw_info)
self.ref_count_before = self.item_count('refs')
try:
importer = PubmedImporter(raw_ids)
records, failed_pmids = importer()
except PubmedError as e:
hub.show_errors(e.message)
return
if len(failed_pmids):
msg = 'nothing retrieved for the following identifiers: %s' % ', '.join(
failed_pmids)
hub.show_errors(msg)
self._import_records(node, records)
def import_doi(self, node, raw_info):
'''
convert a doi to bibtex first, and then import that. should we allow multiple dois?
why not - let's just split across whitespace.
'''
doi = self._file_or_text(raw_info)
dois = doi.split()
l = len(dois)
progress_bar = hub.progress_bar(
l, title="fetching %s record(s) from doi.org" % l)
progress_bar.show()
records = []
failed_dois = []
for i, doi in enumerate(dois):
progress_bar.update(i + 1)
try:
bibtex = self._fetch_bibtex_for_doi(doi)
records.append(bibtex)
except:
failed_dois.append(doi)
progress_bar.dismiss()
if len(failed_dois):
msg = 'retrieval failed for the following identifiers: %s' % ', '.join(
failed_dois)
hub.show_errors(msg)
if len(records):
bibtex = '\n\n'.join(records)
self.import_bibtex(node, bibtex)
def import_bibtex(self, node, raw_info):
'''
parse inputted bibtex and create references from it.
raw_info could be a file name or bibtex text
'''
raw_bibtex = self._file_or_text(raw_info)
try:
p = BibtexParser(raw_bibtex)
records = p()
except Exception as e:
raise
hub.show_errors(getattr(e, 'message', 'bibtexparser foobared'))
return
if len(records) == 0:
msg = 'No records recognized in input'
hub.show_errors(msg)
return
# OK doke, we have at least one valid record. Now what? Just construct
# a new progress bar and invoke the common backend.
self.ref_count_before = self.item_count('refs')
self._import_records(node, records)
def update_branch(self, branch, old_name, new_name):
'''
change name of a folder
'''
_type, branch_id, parent_id = branch
assert _type == hub.BRANCH
error = None
if parent_id in (1, None):
error = "Folder '%s' is protected and can't be renamed" % old_name
elif new_name == "":
error = "Folder name can't be empty"
if error is not None:
hub.show_errors(error, height=10)
return
stmt = "update branches set name=(?) where branch_id=(?)"
self._db.execute(stmt, [new_name, branch[1]])
self._db.commit()
# also flush out the old stored text
self.refresh_tree_item(branch, False)
hub.tree.add_to_history(branch)
def move_selected(self, node=None):
'''
move selected items into current folder. Reset selected to 0.
'''
if node is None: # invocation from menu will trigger this case
node = hub.tree.focus_element()
assert hub.is_branch(node)
node_id = node[1]
# get hold of all selected folders
stmt = 'update branches set parent_id=(?), selected=0 where selected=1'
c = self._db.execute(stmt, [node_id])
moved_items = c.rowcount
errors = 0
stmt = '''
update reflink set branch_id=(?), selected = 0
where selected=1
and not ref_id in
(select ref_id from reflink where branch_id=(?))
'''
try:
c = self._db.execute(stmt, [node_id, node_id])
moved_items += c.rowcount
except IntegrityError:
hub.show_errors('Oopsie - something bad just happened')
if not moved_items:
hub.show_errors('Nothing was moved')
else:
self._db.commit()
hub.clear_cache()
hub.tree.refresh()
def cite_selected_oo(self):
'''
cite all currently selected references.
'''
selected_refs = hub.get_selected_refs(
) # list of (ref__id, branch_id) tuples
if len(selected_refs) == 0:
hub.show_errors("No references selected")
return
seen = set()
for ref_id, branch_id in selected_refs:
if ref_id in seen:
continue
seen.add(ref_id)
fake_node = (hub.REF, ref_id, branch_id)
try:
self.oo_cite(fake_node, status=False)
except ConnectionError:
hub.show_errors("No active Writer document found")
return
suffix = "" if len(seen) == 1 else "s"
hub.set_status_bar('cited %s reference%s' % (len(seen), suffix))
def cite(self, refstring):
'''
push that string.
'''
try:
self.get_connection()
except PushkeyError as e:
hub.show_errors(str(e))
return
msg = self.cite_template % refstring
sock = socket(AF_UNIX, SOCK_STREAM)
sock.connect(self.fn)
sock.send(msg.encode("utf-8"))
sock.send("\n".encode("utf-8"))
# Block until new message arrives
# msg = sock.recv(10)
# forsooth. All sock.recv does is slow everything down ...
# if we axe it, everything works smoothly. We
# can probably rely on getting some exception if the
# connection fails and need not bother with the reply.
# When the socket is closed cleanly, recv unblocks and returns ""
# if not msg:
# print("It seems the other side has closed its connection")
sock.close()
def oo_cite(self, node=None, status=True):
'''
insert a single reference into openoffice
'''
if node is None:
node = hub.tree.focus_element()
assert hub.is_ref(node)
raw_data = self.get_ref_dict(node)
adapted = {}
junk = "branches abstract selected pmid reftype_id purpose ref_id"
for key, value in raw_data.items():
if key in junk:
continue
new_key = self.key_mapping.get(key, key.title())
adapted[new_key] = str(value)
# pprint.pprint(adapted)
try:
self.insert_ref(adapted)
except ConnectionError:
hub.show_errors("No active Writer document found")
if status:
hub.set_status_bar('cited %s' % adapted['Identifier'])
def _import_records(self, node, records):
'''
shared backend for insertion of references retrieved from
pubmed or parsed from bibtex
'''
errors = []
progress_bar = hub.progress_bar(target=len(records),
title="Adding references to database")
progress_bar.show()
for i, record in enumerate(records):
more_errors = self.add_reference(record, node, single_mode=False)
errors.extend(more_errors)
progress_bar.update(i + 1)
self._db.commit()
# hub.refresh_tree_item(node)
# we need to refresh the tree in order to update the Imported pseudo-folder
hub.clear_cache()
hub.tree.refresh()
if len(errors):
hub.show_errors(errors)
refs_imported = self.item_count('refs') - self.ref_count_before
suffix = '' if refs_imported == 1 else 's'
hub.set_status_bar('Imported %s reference%s' % (refs_imported, suffix))
def erase_node(self, node=None):
'''
completely erase a reference. In this case, we just delete from the refs table
and let it propagate to reflink etc. Yes, it works from the command line.
'''
if node is None: # invocation from menu will trigger this case
node = hub.tree.focus_element()
node_type, node_id, parent_id = node
parent = self.get_parent_node(node)
if node_type == hub.BRANCH:
hub.show_errors('Cannot erase folders')
return
#stmt = "delete from refs where ref_id=(?)"
#Well, why take this aggressive approach? Why not just leave it in the trashcan?
#Let's do that instead.
stmt = "delete from reflink where ref_id=(?)"
self._db.execute(stmt, [node_id])
self._db.commit()
hub.tree.set_focus(parent)
hub.tree.add_to_history(parent) # does this make sense? I suppose so.
hub.coredb.clear_cache()
hub.tree.refresh()
hub.set_status_bar('Moved one reference to trash')
def cite_selected_latex(self):
selected_refs = hub.get_selected_bibtexkeys()
if len(selected_refs) == 0:
hub.show_errors("No references selected")
return
self.cite(','.join(selected_refs))
def copy_selected(self, node=None):
'''
copy selected references and folders into current folder. Should we also
deselect? Because if we don't, it's less work, but might be annoying.
Let's wait and see. Actually, we need a 'deselect all' anyway, so we
might just call that at the end.
'''
if node is None: # invocation from menu will trigger this case
node = hub.tree.focus_element()
assert hub.is_branch(node)
node_id = node[1]
# get hold of all selected folders
stmt = 'select * from branches where selected=1'
branches = self._db.execute(stmt).fetchall()
errors = []
success = 0
for branch in branches:
b = (hub.BRANCH, branch['branch_id'], branch['parent_id'])
try:
new_id = self.clone_branch(b, node_id)
success += 1
except RefdbError as e:
errors.append(e.message)
# now on to the references
stmt = 'select ref_id from reflink where selected=1'
ref_ids = self._db.execute(stmt).fetchvalues()
failed_refs = 0
for ref_id in ref_ids:
try:
self._db.insert('reflink',
dict(ref_id=ref_id, branch_id=node_id))
success += 1
except IntegrityError:
failed_refs += 1
if success:
self._db.commit()
hub.clear_cache()
hub.tree.refresh()
if failed_refs:
suffix = '' if failed_refs == 1 else 's'
errors.append("%s duplicate reference%s not copied" %
(failed_refs, suffix))
if errors:
hub.show_errors(errors)
if success == len(errors) == 0:
hub.show_errors('Nothing was selected')
def delete_node(self, node=None):
'''
delete either a folder or a reference. In the case of folders,
we rely on SQLite cascading to delete nested content.
'''
refs_before = self.item_count('reflink')
folders_before = self.item_count('branches')
if node is None: # invocation from menu will trigger this case
node = hub.tree.focus_element()
parent = self.get_parent_node(node)
node_type, node_id, parent_id = node
if node_type == hub.BRANCH:
if node_id in self.special_branch_ids:
hub.show_errors('This folder is protected')
return
stmt = "delete from branches where branch_id=(?)"
values = [node_id]
else:
stmt = "delete from reflink where ref_id=(?) and branch_id=(?)"
values = [node_id, parent_id]
self._db.execute(stmt, values)
self._db.commit()
hub.tree.set_focus(parent)
hub.tree.add_to_history(parent) # does this make sense? I suppose so.
hub.coredb.clear_cache()
hub.tree.refresh()
rdel = refs_before - self.item_count('reflink')
fdel = folders_before - self.item_count('branches')
joiner = " and " if fdel > 0 and rdel > 0 else ""
if fdel > 0:
if fdel == 1:
fs = "1 folder"
else:
fs = "%s folders" % fdel
else:
fs = ""
if rdel > 0:
if rdel == 1:
rs = "1 reference"
else:
rs = "%s references" % rdel
else:
rs = ""
hub.set_status_bar('Deleted %s%s%s' % (fs, joiner, rs))
def show_pdf_bibtexkey(self, bibtexkey):
'''
find and display a pdf file for a bibtexkey
'''
fake_node = self.node_for_bibtexkey(bibtexkey)
if fake_node is None:
hub.show_errors("reference '%s' not found" % bibtexkey)
return
self.show_pdf(fake_node)
def get_selected_bibtexkeys(self):
'''
needed by latex push operations and xsel
'''
data = self.get_selected_with_keys()
if len(data) == 0:
hub.show_errors('Nothing was selected')
return data
keys = [d['bibtexkey'] for d in data]
return sorted(keys)
def write_export_records(self, output, file_name, batch):
'''
write formatted records to file. backend for both bibtex and html
'''
try:
rv = writefile(file_name, output)
except Exception as error:
if not batch:
hub.show_errors(str(error))
else:
traceback.print_exc()
else:
# hub.show_info("File %s written" % file_name)
if not batch:
hub.set_status_bar("Output sent to %s" % rv)
def show_pdf(self, node=None):
'''
try to show a pdf file. If not there, show error; if multiple files, show first one.
if we get annoyed with this, we can always bolt on a menu later
'''
viewer = config['preferences'].get('pdf_viewer', 'xdg-open')
bibtexkey = self.bibtexkey_for_node(node)
path = self.pdf_filepath(bibtexkey)
if path is None:
hub.show_errors('PDF file not found')
return
# prevent subprocess from messing up the screen ...
hub.set_status_bar('Opening PDF ...')
fnull = open(os.devnull, 'w')
subprocess.Popen([viewer, path], stdout=fnull, stderr=fnull)
def update_reference(self, old_data, new_data, add_reference=False):
'''
invoked by RefEdit dialog, but also indirectly by the add reference
dialog.
If we do have errors, do we commit the non-erroneous changes
anyway? I would say yes.
'''
ref_id = old_data['ref_id']
errors = []
updates = False
for key, value in list(new_data.items()):
new_value = value.strip() or None
old_value = old_data.get(key, None)
if new_value != old_value:
error = self.update_field(ref_id, key, old_value, new_value)
if error is None:
updates = True
else:
errors.append(error)
if updates:
self._db.commit()
# also update store.
# Ha, ha, ha. Turns out that is not so easy -
# references can occur in multiple places, and there
# is no unique parent_id associated with it.
s = self.cache['ref_index'][ref_id]
while s:
key = s.pop()
self.cache['node_text'].pop(key, None)
hub.tree.refresh()
if not add_reference:
if errors:
hub.show_errors(errors) # show errors right here
else: # return any errors.
return errors
def _mail_references(self, ref_ids):
'''
shared back-end for mailing selected or current references
'''
stmt = "select bibtexkey from refs where ref_id in (%s)"
keys = self._db.execute_qmarks(stmt, [list(ref_ids)]).fetchvalues()
# I guess we want to keep track of which PDF files were actually found.
# so, we don't just filter
paths = [(key, self.pdf_filepath(key)) for key in keys]
attach_list = []
missing_list = []
for key, path in paths:
if path is None:
missing_list.append(key)
else:
attach_list.append(path)
if not len(attach_list):
hub.show_errors("No PDF files found for selected reference(s)")
return
hub.set_status_bar('composing email')
cmd = ["xdg-email"]
cmd.append('--subject "%s"' %
config['email'].get('subject', 'PDF files'))
body = config['email'].get('body', 'Please see attached')
if len(missing_list):
body += '. PDFs not found for: %s' % ", ".join(missing_list)
cmd.append('--body "%s"' % body)
for fn in attach_list:
cmd.append('--attach "%s"' % str(fn))
cmd = ' '.join(cmd)
os.system("%s > /dev/null 2> /dev/null" % cmd)
hub.set_status_bar('')
def cite_by_key(self, bibtexkey):
'''
find a reference by key and cite it in OpenOffice.
We select a real branch_id here. We could just fake one to
get past is_ref, but the point of using a real one is that
it excludes references in the trash; this is consistent
with the menu restrictions. I guess this should be documented
somewhere.
'''
fake_node = hub.node_for_bibtexkey(bibtexkey)
if fake_node is None:
hub.show_errors("reference '%s' not found" % bibtexkey)
return
try:
self.oo_cite(fake_node)
except ConnectionError:
hub.show_errors("No active Writer document found")
def add_folder(self, parent, new_name):
'''
not really a clipboard operation itself, but related.
Should we guard against duplicated names in same folder? I guess so.
'''
if not new_name:
hub.show_errors("Folder name cannot be empty")
return
if not hub.is_branch(parent):
hub.show_errors("Cannot create folder inside a reference")
return
parent_id = parent[1]
if self.exists_folder(parent, new_name):
hub.show_errors("Subfolder '%s' already exists - aborting" %
new_name)
return
c = self._db.insert('branches', dict(name=new_name,
parent_id=parent_id))
new_id = c.lastrowid
self._db.commit()
# clearing the cache shouldn't be needed, but it is - deleted folder nodes
# somehow linger, and if an id gets reused, the zombies reappear
hub.clear_cache()
hub.tree.refresh()
# now, we should be able to construct the new node signature without going
# back to the database
new_node = (hub.BRANCH, new_id, parent_id)
hub.tree.set_focus(new_node)
hub.tree.add_to_history(new_node)
def show_invalid(self):
hub.show_errors("'Create reference' is not available here")
def show_invalid(self):
hub.show_errors('This folder is protected and cannot be renamed')
def show_invalid(self):
hub.show_errors("'Create folder' is not available here")
def toggle_select(self, node=None):
'''
toggle selection status of reference or folder.
- first, check if the current node is inside a folder that is currently selected.
if it is, tell them that and exit.
- if no parent is selected, then toggle selection status.
When a folder is selected, we also reset the selections of all items below.
Hm. How do we deal with stuff in the trash? With references, we have no reflink
entries, so we have no place to store the 'selected' attribute.
Darn. Why did I feel the need again to change this around? What exactly was wrong
with having the selected attribute on the reference?
'''
if node is None: # invocation from menu will trigger this case
node = hub.tree.focus_element()
node_type, node_id, parent_id = node
is_branch = node_type == hub.BRANCH
parents = self.get_branches_above(parent_id, include_start_branch=True)
selected = sum([p['selected'] for p in parents])
if selected:
hub.show_errors(
'This item is inside a selected folder and cannot be individually selected or deselected.'
)
return
if is_branch:
if node_id in self.special_branch_ids:
hub.show_errors(
'This folder is protected and cannot be moved or copied')
return
this_branch = self._db.execute(
'select * from branches where branch_id=(?)',
[node_id]).fetchone()
currently_selected = this_branch['selected']
if currently_selected == 0: # going to be 1 - deselect all individually selected
# descendants. Is that sensible? Let's try it out.
# deselect references.
stmt = '''
with recursive branch_set( i )
as ( select branch_id from branches where parent_id = (?)
union select branch_id from branches, branch_set
where branches.parent_id = branch_set.i
)
update reflink
set selected = 0
where selected = 1
and branch_id in branch_set or branch_id = (?);
'''
self._db.execute(stmt, [node_id, node_id])
# deselect branches
stmt = '''
with recursive branch_set( i )
as ( select branch_id from branches where parent_id = (?)
union select branch_id from branches, branch_set
where branches.parent_id = branch_set.i
)
update branches
set selected = 0
where selected = 1
and branch_id in branch_set;
'''
self._db.execute(stmt, [node_id])
# update branch itself
stmt = 'update branches set selected=(?) where branch_id=(?)'
self._db.execute(stmt, [1 - currently_selected, node_id])
else:
stmt = '''
update reflink
set selected = case selected when 1 then 0 else 1 end
where ref_id = (?)
and branch_id = (?)
'''
self._db.execute(stmt, [node_id, parent_id])
self._db.commit()
hub.clear_cache()
hub.tree.refresh()
def add_reference(self, new_data, node=None, single_mode=True):
'''
What do we do here if constraints are violated? I think in the case of
missing or reduplicated bibtex keys we just concatenate a random string
and try again. For title, we can just supply a dummy title.
single_mode: we also use this as a backend for importers, so we
don't commit and display errors right here in that case.
'''
if node is None:
node = hub.tree.focus_element()
assert hub.is_branch(node)
values = [_f for _f in list(new_data.values()) if _f]
if not values:
hub.show_errors("No data provided - aborting.")
return
# I guess from here we will insert something, even if it is crap.
# we just keep track of the errors and show them at the end.
errors = []
orig_bibtexkey = new_data.get('bibtexkey')
new_bibtexkey = new_data['bibtexkey'] = self.make_bibtex_key(new_data)
#if orig_bibtexkey and orig_bibtexkey != new_bibtexkey: don't nag the user
#hub.show_message("bibtexkey '%s' changed to '%s'" % (orig_bibtexkey, new_bibtexkey))
new_data['title'] = new_data['title'].rstrip('.')
if not new_data['title']:
new_data['title'] = "Title can't be empty -- please fix"
errors.append(new_data['title'])
if new_data['reftype'] not in self.ref_types:
errors.append("reftype was empty or faulty - set to 'article'")
new_data['reftype'] = 'article'
# at this point, we should have everything in place. Now, we can still fail to
# insert a reference if we have a duplicate bibtexkey, in which case we need to fix.
# I suppose I will fix it up with appending a short random string.
ref_data = dict(reftype_id=self.ref_types[new_data['reftype']],
title=new_data['title'],
bibtexkey=new_bibtexkey)
c = self._db.insert('refs', ref_data)
ref_id = c.lastrowid
try:
errors += self.update_reference(dict(ref_id=ref_id),
new_data,
add_reference=True)
except IntegrityError:
errors.append("Record '%s...' not imported (likely duplicate)" %
new_data['title'][:50])
# also need to revert partial import
self._db.execute('delete from refs where ref_id=(?)', [ref_id])
return errors
# Associate the new reference with the current branch
branch_id = node[1]
for branch_id in (node[1], self.recently_added_id):
self._db.insert('reflink', dict(ref_id=ref_id,
branch_id=branch_id))
if single_mode:
self._db.commit()
hub.refresh_tree_item(node)
if errors:
hub.show_errors(errors)
else: # collect errors for display later on.
return errors
def show_invalid(self):
hub.show_errors("Nothing has been selected")
