def purge_unused_annotations(remote_instance=None, force=False, verbose=True):
if remote_instance in [None, 'source']:
remote_instance = source_project
if verbose:
print('Purging unused annotations from the source project.')
elif remote_instance == 'target':
remote_instance = target_project
if verbose:
print('Purging unused annotations from the target project.')
pid = remote_instance.project_id
tmp_neuron_skids = {2: 6401, 13: 120221, 38: 352154, 59: 665257}
if pid not in tmp_neuron_skids:
raise ValueError(
f'Need to specify a dummy neuron to use for project id {pid}')
tmp_skid = tmp_neuron_skids[pid]
all_annots = pymaid.get_annotation_list(remote_instance=remote_instance)
def add_escapes(s, chars_to_escape='()[]?+'):
for char in chars_to_escape:
s = s.replace(char, '\\' + char)
return s
def remove_escapes(s):
while '\\' in s:
s = s.replace('\\', '')
return s
for name in tqdm(all_annots.name):
name = add_escapes(name)
try:
count = len(
pymaid.get_annotated(name, remote_instance=remote_instance))
if count > 0:
pass #print('{:04d}'.format(count), name)
else:
print(name)
if not force and input('Purge me? [Y/n] ').lower() != 'y':
continue
pymaid.add_annotations(tmp_skid,
remove_escapes(name),
remote_instance=remote_instance)
pymaid.remove_annotations(tmp_skid,
remove_escapes(name),
remote_instance=remote_instance)
except:
print(name)
raise
def __merge_annotations(n, bn, tag, target_instance):
"""Make sure proper annotations are added."""
to_add = []
# Add "{URL} upload {tag} annotation"
if not isinstance(getattr(n, '_remote_instance', None), type(None)):
u = n._remote_instance.server.split('/')[-1] + ' upload'
if isinstance(tag, str):
u += " {}".format(tag)
to_add.append(u)
# Existing annotation (the individual fragments would not have inherited them)
if n.__dict__.get('annotations', []):
to_add += n.annotations
# If anything to add
if to_add:
_ = pymaid.add_annotations(bn, to_add, remote_instance=target_instance)
def upload_or_update_neurons(neurons,
linking_relation='',
annotate_source_neuron=False,
import_connectors=False,
reuse_existing_connectors=True,
refuse_to_update=True,
verbose=False,
fake=True):
server_responses = []
start_day = time.strftime('%Y-%m-%d')
start_time = time.strftime('%Y-%m-%d %I:%M %p')
if type(neurons) is pymaid.core.CatmaidNeuron:
neurons = pymaid.core.CatmaidNeuronList(neurons)
# There are some pesky corner cases where updates will unintentionally create unlinked
# connectors. When that occurs, the user is warned and asked to investigate manually.
unlinked_connectors_start = find_unlinked_connectors(
remote_instance=target_project)
for source_neuron in neurons:
source_project.clear_cache()
target_project.clear_cache()
# Check if a neuron/skeleton with this neuron's name already exists in the target project
# If so, replace that neuron/skeleton's data with this neuron's data.
skid_to_update = None
nid_to_update = None
force_id = False
if linking_relation is '':
linking_annotation_template = 'LINKED NEURON - skeleton id {skid} in project id {pid} on server {server}'
else:
linking_annotation_template = 'LINKED NEURON - {relation} skeleton id {skid} in project id {pid} on server {server}'
linking_annotation_target = linking_annotation_template.format(
relation=linking_relation,
skid=source_neuron.skeleton_id,
name=source_neuron.neuron_name, #Not used currently
pid=source_project.project_id,
server=source_project.server)
if verbose:
print("Linking annotation is: '{linking_annotation_target}'")
try:
linked_neuron_skid = pymaid.get_skids_by_annotation(
add_escapes(linking_annotation_target),
raise_not_found=False,
remote_instance=target_project)
except Exception as e:
# There appears to be a bug in get_skids_by_annotation where it still
# raises exceptions sometimes even with raise_not_found=False, so
# use this block to continue through any of those cases without raising.
#print(e)
linked_neuron_skid = []
source_neuron.annotations = [
annot for annot in source_neuron.annotations
if 'LINKED NEURON' not in annot
]
if len(linked_neuron_skid) is 0: # Prepare to upload neuron as new
print(f'Uploading "{source_neuron.neuron_name}" to project'
f' {target_project.project_id} as a new skeleton.')
source_neuron.annotations.append(linking_annotation_target)
source_neuron.annotations.append(
f'UPDATED FROM LINKED NEURON - {start_time}')
elif len(linked_neuron_skid) is not 1:
print('Found multiple neurons annotated with'
f' "{linking_annotation_target}" in target project.'
' Go fix that! Skipping upload for this neuron.')
else: # Prepare to update the linked neuron
linked_neuron = pymaid.get_neuron(linked_neuron_skid[0],
remote_instance=target_project)
m = ', connectors,' if import_connectors else ''
print(f'{source_neuron.neuron_name}: Found linked neuron with '
f'skeleton ID {linked_neuron.skeleton_id} in target project.'
f' Updating its treenodes{m} and annotations to match the'
' source neuron.')
# Check whether names match
if not source_neuron.neuron_name == linked_neuron.neuron_name:
user_input = input(
'WARNING: The linked neuron\'s name is'
f' "{linked_neuron.neuron_name}" but was expected to be'
f' "{source_neuron.neuron_name}". Continuing will rename'
' the linked neuron to the expected name. Proceed? [Y/n] ')
if user_input not in ('y', 'Y'):
continue
# TODO
# Check whether there are any nodes or connectors in the source
# neuron with edition dates after the previous upload date. If not,
# skip the upload and tell the user.
# Check whether any edited nodes will be overwritten
linked_node_details = pymaid.get_node_details(
linked_neuron.nodes.node_id.values,
remote_instance=target_project)
is_edited = linked_node_details.edition_time != min(
linked_node_details.edition_time)
if is_edited.any():
edited_nodes = linked_node_details.loc[
is_edited, ['node_id', 'edition_time', 'editor']]
users = pymaid.get_user_list(
remote_instance=target_project).set_index('id')
edited_nodes.loc[:, 'editor'] = [
users.loc[user_id, 'login']
for user_id in edited_nodes.editor
]
print('WARNING: The linked neuron has been manually edited,'
f' with {len(edited_nodes)} nodes modified. Those'
' changes will get thrown away if this update is allowed'
' to continue.')
print(edited_nodes)
user_input = input(
'OK to proceed and throw away the above changes? [Y/n] ')
if user_input not in ('y', 'Y'):
print(f'Skipping update for "{source_neuron.neuron_name}"')
continue
if refuse_to_update:
print('refuse_to_update set to true. Skipping.\n')
continue
# This does NOT annotate the source neuron on the server,
# it only appends to the object in memory
source_neuron.annotations.append(
f'UPDATED FROM LINKED NEURON - {start_time}')
# Make sure to preserve all annotations on the target neuron. This will not
# be necessary once # https://github.com/catmaid/CATMAID/issues/2042 is resolved
source_neuron.annotations.extend([
a for a in linked_neuron.annotations
if a not in source_neuron.annotations
])
skid_to_update = linked_neuron.skeleton_id
nid_to_update = pymaid.get_neuron_id(
linked_neuron.skeleton_id,
remote_instance=target_project)[str(linked_neuron.skeleton_id)]
force_id = True
if not fake:
# Actually do the upload/update:
server_responses.append(
pymaid.upload_neuron(
source_neuron,
skeleton_id=skid_to_update,
neuron_id=nid_to_update,
force_id=force_id,
import_tags=True,
import_annotations=True,
import_connectors=import_connectors,
reuse_existing_connectors=reuse_existing_connectors,
remote_instance=target_project))
if annotate_source_neuron:
try:
upload_skid = server_responses[-1]['skeleton_id']
source_annotation = linking_annotation_template.format(
relation=linking_relation,
skid=server_responses[-1]['skeleton_id'],
name=source_neuron.neuron_name, #Not used currently
pid=target_project.project_id,
server=target_project.server)
try:
server_responses[-1][
'source_annotation'] = pymaid.add_annotations(
source_neuron.skeleton_id,
source_annotation,
remote_instance=source_project)
except:
m = ('WARNING: annotate_source_neuron was requested,'
' but failed. You may not have permissions to'
' annotate the source project through the API')
print(m)
input('(Press enter to acknowledge and continue.)')
server_responses[-1]['source_annotation'] = m
except:
print('WARNING: upload was not successful,'
' so could not annotate source neuron.')
input('(Press enter to acknowledge and continue.)')
print(f'{source_neuron.neuron_name}: Done with upload or update.')
print(' ')
if fake:
print('fake was set to True. Set fake=False to actually run'
' upload_or_update_neurons with settings:\n'
f'annotate_source_neuron={annotate_source_neuron}\n'
f'import_connectors={import_connectors},\n'
f'reuse_existing_connectors={reuse_existing_connectors},\n'
f'refuse_to_update={refuse_to_update}')
else:
# There are some pesky corner cases where updates will unintentionally
# create unlinked connectors. When that occurs, the user is warned and
# asked to investigate manually. Note that if a human tracer is
# annotating in catmaid and happens to make an unlinked connector that
# exists when the following lines are run, this will throw an warning
# despite there being nothing to worry about. Not much I can do there.
target_project.clear_cache()
unlinked_connectors_end = find_unlinked_connectors(
remote_instance=target_project)
newly_unlinked_connectors = set(unlinked_connectors_end).difference(
set(unlinked_connectors_start))
if len(newly_unlinked_connectors) != 0:
print("WARNING: This upload caused some connectors in the "
"target project to become unlinked from any skeleton. "
"(This can harmlessly result from deleting connectors from "
"the source project, or it may indicate a bug in the code.) "
"You may want to go clean up the new unlinked connectors:")
print(newly_unlinked_connectors)
input('(Press enter to acknowledge warning and continue.)')
return server_responses