for comp_txtId in component:
dic_txtId__clusterId[comp_txtId]= components
print('comp_PostId=', comp_txtId)
print("---cluster:postids end-----")
print('---histogram size----')
for size, count in dic_clussize__count.items():
print(size, count)
print('---histogram size end----')
total_nodes=set(total_nodes)
print("--- %s seconds ---" % (time.time() - start_time))
print('components', components, 'max', max(list_compSize), 'min', min(list_compSize), 'avg', mean(list_compSize), 'stdev', stdev(list_compSize), 'total_nodes', len(total_nodes))
undirected.clear()
del undirected
#list_compSize.clear()
del list_compSize
total_nodes.clear()
del total_nodes
print("--- %s seconds ---" % (time.time() - start_time))
total_posts=len(dic_txtId__clusterId)
print('total_posts', total_posts)
posts_found=0
f_java_w = open('stackoverflow_r_true_id_title_tags', 'w')
class rxnGraph(object):
def __init__(self, datatables, proteins=None, ligands=None, interactions=None): #, dockdf=None, seadf=None, rsimpn=None):
# Construct graph
self.graph = DiGraph()
self.data = datatables
self.proteins = proteins # list of nodes
self.ligands = ligands # list of nodes
self.protein_members = [] # subset of possible proteins, members in graph
self.protein_choices = [] # subset of possible proteins, not members in graph
self.ligand_members = [] # subset of possible ligands, members in graph
self.ligand_choices = [] # subset of possible ligands, not members in graph
self.transporter = None
self.protein_nodes = [] # moveable nodes
self.ligand_nodes = []
self.protein_allnodes = []
self.ligand_allnodes = []
self.fixed = []
self.keep = []
if proteins is not None and ligands is not None and interactions is not None:
self.construct_graph(proteins, ligands, interactions)
self.protein_members = proteins[:]
self.ligand_members = ligands[:]
def add_choices(self, protein_choices=[], ligand_choices=[]):
self.ligand_choices = ligand_choices
self.protein_choices = protein_choices
def add_transporter(self, transporter):
self.transporter = transporter
#n = self.get_start()
#self.transporter_substrate = self.graph.node[n]['label']
def copy(self):
newGraph = rxnGraph(self.data, proteins=self.proteins[:], ligands = self.ligands[:])
#dockdf=self.dockdf, seadf=self.seadf, rsimpn=self.rsimpn)
newGraph.graph = self.graph.copy()
#newGraph.constraints = self.constraints
newGraph.ligand_members = self.ligand_members[:]
newGraph.protein_members = self.protein_members[:]
newGraph.ligand_choices = self.ligand_choices[:]
newGraph.protein_choices = self.protein_choices[:]
newGraph.protein_nodes = self.protein_nodes
newGraph.ligand_nodes = self.ligand_nodes
newGraph.protein_allnodes = self.protein_allnodes
newGraph.ligand_allnodes = self.ligand_allnodes
newGraph.fixed = self.fixed
newGraph.keep = self.keep
newGraph.transporter = self.transporter
return newGraph
def construct_graph_from_path(self, proteins, ligands):
#self.graph.add_nodes_from(ligands, bipartite=0)
#self.graph.add_nodes_from(proteins, bipartite=1)
interactions = []
for i in range(len(proteins)):
interactions.append([ligands[i], proteins[i]])
if i > 0:
interactions.append([proteins[i-1], ligands[i]])
if len(ligands) == len(proteins) + 1:
interactions.append([proteins[-1], ligands[-1]])
self.construct_graph(proteins, ligands, interactions)
def construct_graph(self, proteins, ligands, interactions):
# interactions: [[protein, ligand], [protein, ligand]]
self.graph.clear()
labelposdict = {}
for i, p in enumerate(proteins):
self.graph.add_node(i, label=p, bipartite=1)
labelposdict[p] = i
self.protein_allnodes.append(i)
for j, l in enumerate(ligands):
self.graph.add_node(j + len(proteins), label=l, bipartite=0)
labelposdict[l] = j + len(proteins)
self.ligand_allnodes.append(j + len(proteins))
#for i in range(len(interactions)):
# node1 = labelposdict[interactions[i][0]]
# node2 = labelposdict[interactions[i][1]]
# self.graph.add_edge(node1, node2)
[self.graph.add_edge(labelposdict[interactions[i][0]], labelposdict[interactions[i][1]]) for i in range(len(interactions))]
self.proteins = proteins
self.ligands = ligands
self.protein_members = proteins[:]
self.ligand_members = ligands[:]
self.ligand_nodes = self.ligand_allnodes[:]
self.protein_nodes = self.protein_allnodes[:]
for i in self.fixed:
if i in self.protein_nodes:
self.protein_nodes.remove(i)
if i in self.ligand_nodes:
self.ligand_nodes.remove(i)
# updates the proteins and ligands in order
# maybe not necessary
def update_proteins_ligands(self):
n = self.get_start()
all_ligands = self.ligand_members[:]
all_ligands.extend(self.ligand_choices[:])
all_proteins = self.protein_members[:]
all_proteins.extend(self.protein_choices[:])
self.ligands = []
self.proteins = []
if n in all_proteins:
self.proteins.append(n)
elif n in all_ligands:
self.ligands.append(n)
while len(self.graph.successors(n)) > 0:
n = self.graph.successors(n)[0]
if n in all_proteins:
self.proteins.append(n)
elif n in all_ligands:
self.ligands.append(n)
# also, maybe not necessary
def get_start(self):
start = None
for n in self.graph.nodes_iter():
if len(self.graph.predecessors(n)) < 1:
start = n
if start is None:
print self.graph.edges()
return start
def get_single_predecessor_node(self, node):
pred = self.graph.predecessors(node)
if len(pred) > 0:
return pred[0]
else:
return None
def get_single_successor_node(self, node):
succ = self.graph.successors(node)
if len(succ) > 0:
return succ[0]
else:
return None
def move_remove_single(self):
all = self.protein_members[:]
all.extend(self.ligand_members[:])
node = self.choose_by_degree(1, all, 1)
if node is not None:
self.graph.remove_node(node)
if node in self.ligand_members:
self.ligand_members.remove(node)
self.ligand_choices.append(node)
elif node in self.protein_members:
self.protein_members.remove(node)
self.protein_choices.append(node)
# Removes a pair of consecutive nodes, a protein node and a ligand node
# Adds edge between predecessors to the successors of the removed nodes
def move_remove_pair(self, pnode):
lnodes = None
if self.graph.in_edges(pnode) is not None and self.graph.out_edges(pnode) is not None:
ppred = self.graph.predecessors(pnode)
psucc = self.graph.successors(pnode)
node_choices = []
if len(ppred) > 0:
node_choices.append(ppred)
if len(psucc) > 0:
node_choices.append(psucc)
if len(node_choices) > 0:
#lnodes = rand.choice(node_choices)
lnode = rand.choice(node_choices)
if len(ppred) > 0 and len(psucc) > 0:
for pp in ppred:
for ps in ppred:
self.graph.add_edge(pp, ps)
self.graph.remove_node(pnode)
self.protein_members.remove(pnode)
self.protein_choices.append(pnode)
if self.graph.in_edges(lnode) is not None and self.graph.out_edges(lnode) is not None:
lpred = self.graph.predecessors(lnode)
lsucc = self.graph.successors(lnode)
if len(lpred) > 0 and len(lsucc) > 0:
for lp in lpred:
for ls in lsucc:
self.graph.add_edge(lp, ls)
self.graph.remove_node(lnode)
self.ligand_members.remove(lnode)
self.ligand_choices.append(lnode)
# choose k nodes from a nodeset with a degree less than the limit
def choose_by_degree_in(self, upperlim_deg, nodeset, k):
choices = []
for node in nodeset:
degree = self.graph.in_degree(node)
if degree < upperlim_deg:
choices.append(node)
if len(choices) >= k:
return rand.sample(choices, k)
else:
return None
def choose_by_degree_out(self, upperlim_deg, nodeset, k):
choices = []
for node in nodeset:
degree = self.graph.out_degree(node)
if degree < upperlim_deg:
choices.append(node)
if len(choices) >= k:
return rand.sample(choices, k)
else:
return None
def choose_by_degree(self, upperlim_deg, nodeset, k):
choices = []
for node in nodeset:
degree = self.graph.degree(node)
if degree < upperlim_deg:
choices.append(node)
if len(choices) >= k:
return rand.sample(choices, k)
else:
return None
# Adds a pair of consecutive nodes to the end of the linear graph
def move_add_pair(self, pnode, lnode):
last = self.choose_by_degree_in(2, self.ligand_members, 1)[0]
if last is not None:
self.graph.add_node(pnode)
self.graph.add_node(lnode)
self.graph.add_edge(last, pnode)
self.graph.add_edge(pnode, lnode)
# Updates lists to reflect the current state of the full system
self.ligand_members.append(lnode)
self.ligand_choices.remove(lnode)
self.protein_members.append(pnode)
self.protein_choices.remove(pnode)
def move_add_edge(self):
if rand.choice([0, 1]):
pnode = self.choose_by_degree_out(2, self.protein_members, 1)
lnode = self.choose_by_degree_in(2, self.ligand_members, 1)
if pnode is not None and lnode is not None:
self.graph.add_edge(pnode[0], lnode[0])
else:
pnode = self.choose_by_degree_in(2, self.protein_members, 1)
lnode = self.choose_by_degree_out(2, self.ligand_members, 1)
if pnode is not None and lnode is not None:
self.graph.add_edge(lnode[0], pnode[0])
def move_swap_pair(self, swapLigands=True):
if swapLigands:
node1, node2 = rand.sample(self.ligand_nodes, 2)
else:
node1, node2 = rand.sample(self.protein_nodes, 2)
if (node1 not in self.fixed) and (node2 not in self.fixed):
label1 = self.graph.node[node1]['label']
label2 = self.graph.node[node2]['label']
self.graph.node[node1]['label'] = label2
self.graph.node[node2]['label'] = label1
def move_swap_neighboring_pair(self, swapLigands=True):
if swapLigands:
node = rand.choice(self.ligand_nodes)
else:
node = rand.choice(self.protein_nodes)
predprot = self.get_single_predecessor_node(node)
succprot = self.get_single_successor_node(node)
options = []
if predprot is not None:
predpred = self.get_single_predecessor_node(predprot)
if predpred is not None:
options.append(predpred)
if succprot is not None:
succsucc = self.get_single_successor_node(succprot)
if succsucc is not None:
options.append(succsucc)
if self.fixed in options:
options.remove(self.fixed)
if len(options) > 0:
swapnode = rand.choice(options)
label1 = self.graph.node[node]['label']
label2 = self.graph.node[swapnode]['label']
self.graph.node[node]['label'] = label2
self.graph.node[swapnode]['label'] = label1
# assumes linear
def move_lp_swap_pairs(self, pnode1, pnode2):
lpred = self.graph.predecessors(pnode1)
if not len(lpred) > 0:
tempnode = pnode1
pnode1 = pnode2
pnode2 = tempnode
lpred = self.graph.predecessors(pnode1)
ppred = self.graph.predecessors(lpred[0])
lsucc = self.graph.successors(pnode1)
if len(ppred) > 0:
self.graph.remove_edge(ppred[0], lpred[0])
if len(lsucc) > 0:
self.graph.remove_edge(pnode1, lsucc[0])
if len(ppred) > 0 and len(lsucc) > 0:
self.graph.add_edge(ppred[0], lsucc[0])
lsucc2 = self.graph.successors(pnode2)
if len(lsucc2) > 0:
self.graph.remove_edge(pnode2, lsucc2[0])
self.graph.add_edge(pnode1, lsucc2[0])
self.graph.add_edge(pnode2, lpred[0])
def move_swap_in(self, swapLigands=True):
if swapLigands:
select = True
while select or onodelabel in self.keep:
onode = rand.choice(self.ligand_nodes)
onodelabel = self.graph.node[onode]['label']
select = False
newnodelabel = rand.choice(self.ligand_choices)
else:
onode = rand.choice(self.protein_nodes)
onodelabel = self.graph.node[onode]['label']
newnodelabel = rand.choice(self.protein_choices)
self.graph.node[onode]['label'] = newnodelabel
if onode in self.ligand_allnodes:
self.ligand_choices.remove(newnodelabel)
self.ligand_choices.append(onodelabel)
elif onode in self.protein_allnodes:
self.protein_choices.append(onodelabel)
self.protein_choices.remove(newnodelabel)
self.protein_members.append(newnodelabel)
self.protein_members.remove(onodelabel)
# lnode is the new node to swap in
# tested, doesn't help at all
def move_swap_in_similar(self, lnode):
npdt = np.dtype([('ligand', np.str_, 35), ('tc', np.float32)])
tcarray = []
tcsum = 0.0
for lig in self.ligand_members:
tc = self.data.chsimdf.get_value(lig, lnode)
tcarray.append((lig, tc))
nptc = np.array(tcarray, dtype = npdt)
cutoff = rand.uniform(0, 0.5)
bool = nptc['tc'] > cutoff
if bool.any():
options = nptc['ligand'][bool]
else:
options = nptc['ligand']
outnode = rand.choice(options)
pred = self.get_single_predecessor_node(outnode)
succ = self.get_single_successor_node(outnode)
self.graph.add_node(lnode)
if pred is not None:
self.graph.add_edge(pred, lnode)
self.graph.remove_edge(pred, outnode)
if succ is not None:
self.graph.add_edge(lnode, succ)
self.graph.remove_edge(outnode, succ)
self.graph.remove_node(outnode)
self.ligand_members.remove(outnode)
self.ligand_choices.append(outnode)
self.ligand_members.append(lnode)
self.ligand_members.remove(lnode)
def get_subgraph_successors(self, pnode, numnodes):
currnode = pnode
subgraph_nodes = [pnode]
for n in range(numnodes - 1):
if currnode is not None:
nextnode = self.get_single_successor_node(currnode)
if nextnode is not None:
currnode = self.get_single_successor_node(nextnode)
if currnode is self.fixed:
break
if currnode is not None:
subgraph_nodes.append(currnode)
else:
return False
return subgraph_nodes
def get_subgraph_predecessors(self, pnode, numnodes):
currnode = pnode
subgraph_nodes = [pnode]
for n in range(numnodes - 1):
if currnode is not None:
nextnode = self.get_single_predecessor_node(currnode)
if nextnode is not None:
currnode = self.get_single_successor_node(nextnode)
if currnode is self.fixed:
break
if currnode is not None:
subgraph_nodes.append(currnode)
else:
return False
return subgraph_nodes.reverse()
def move_subgraph(self):
pnode = rand.choice(self.protein_nodes)
numnodes = rand.choice(range(2, len(self.protein_nodes)/2 + 1))
subgraph_nodes = self.get_subgraph_successors(pnode, numnodes)
if not subgraph_nodes:
subgraph_nodes = self.get_subgraph_predecessors(pnode, numnodes)
if subgraph_nodes:
pnode = subgraph_nodes[0]
qnode = subgraph_nodes[-1]
lnode = self.get_single_predecessor_node(pnode)
rchoices = self.protein_nodes[:]
for s in subgraph_nodes:
if s in rchoices:
rchoices.remove(s)
rnode = rand.choice(rchoices)
# Remove edges
if lnode is not None:
lpred = self.get_single_predecessor_node(lnode)
qsucc = self.get_single_successor_node(qnode)
rsucc = self.get_single_successor_node(rnode)
if lnode == rsucc:
rchoices.remove(rnode)
rnode = rand.choice(rchoices)
rsucc = self.get_single_successor_node(rnode)
if lpred is not None:
self.graph.remove_edge(lpred, lnode)
if qsucc is not None:
self.graph.add_edge(lpred, qsucc)
if qsucc is not None:
self.graph.remove_edge(qnode, qsucc)
if rsucc is not None:
self.graph.remove_edge(rnode, rsucc)
self.graph.add_edge(qnode, rsucc)
self.graph.add_edge(rnode, lnode)
def string_repr(self):
string = ''
for e in self.graph.edges():
string += "('%s', '%s'), " % (e[0], e[1])
return string.rstrip(', ')
def string_interactions(self):
string = ''
for e in self.graph.edges():
string += "'%s', '%s', " % (e[0], e[1])
return string.rstrip(', ')
def linear_to_string(self):
string = ''
if self.transporter is not None and self.transporter in self.data.dockdf.index:
string += '%s -> ' % self.transporter
n = self.get_start()
string += self.graph.node[n]['label']
while len(self.graph.successors(n)) > 0:
n = self.graph.successors(n)[0]
label = self.graph.node[n]['label']
string += ' -> '
string += label
return string
def add_constraint(self, fun):
self.constraints.append(fun)
def test_constraints(self):
for fun in self.constraints:
if not fun(self):
return False
return True
class Network:
def __init__(self):
"""
Networks are containers for all :class:`objects <Network.Object.Object>` that exist in a neural circuit.
"""
self.graph = DiGraph()
self.objects = []
self.idDict = {} # TODO: weak ref dict?
self.displays = []
self._nextUniqueId = -1
self._savePath = None
self._attributes = []
self._displaysAreSynchronized = True
self._weightingFunction = None
self._bulkLoading = False
self._bulkAddObjects = []
self._modified = False
@classmethod
def _fromXMLElement(cls, xmlElement):
network = cls()
network.setBulkLoading(True)
# Load the classes in such an order that any referenced objects are guaranteed to have already been created.
for moduleName, className in [('region', 'Region'), ('pathway', 'Pathway'), ('neuron', 'Neuron'), ('muscle', 'Muscle'), ('arborization', 'Arborization'), ('innervation', 'Innervation'), ('gap_junction', 'GapJunction'), ('synapse', 'Synapse'), ('stimulus', 'Stimulus')]:
elementModule = getattr(sys.modules['network'], moduleName)
elementClass = getattr(elementModule, className)
for element in xmlElement.findall(className):
networkObject = elementClass._fromXMLElement(network, element)
if networkObject is not None:
network.addObject(networkObject)
weightingFunctionElement = xmlElement.find('WeightingFunction')
if weightingFunctionElement is not None:
funcType = weightingFunctionElement.get('type')
funcName = weightingFunctionElement.get('name')
if funcType == 'source':
exec(weightingFunctionElement.text)
network._weightingFunction = eval(funcName)
elif funcType == 'marshal':
code = marshal.loads(eval(weightingFunctionElement.text))
network._weightingFunction = types.FunctionType(code, globals(), funcName or 'weightingFunction')
else:
raise ValueError, gettext('Unknown weighting function type: %s') % (funcType)
for element in xmlElement.findall('Attribute'):
attribute = Attribute._fromXMLElement(network, element)
if attribute is not None:
network._attributes.append(attribute)
network.setBulkLoading(False)
return network
def _toXMLElement(self, parentElement):
networkElement = ElementTree.SubElement(parentElement, 'Network')
for networkObject in self.objects:
# Nested regions are handled by their parents and neurites are handled by their neurons.
if not (isinstance(networkObject, Region) and networkObject.parentRegion is not None) and not isinstance(networkObject, Neurite):
objectElement = networkObject._toXMLElement(networkElement)
if objectElement is None:
pass # TODO: are there any cases where this is NOT an error?
if self._weightingFunction:
weightingFunctionElement = ElementTree.SubElement(networkElement, 'WeightingFunction')
weightingFunctionElement.set('name', self._weightingFunction.func_name)
# First try to get the function source and if that fails then marshal the byte code.
try:
source = inspect.getsource(self._weightingFunction)
weightingFunctionElement.text = source
weightingFunctionElement.set('type', 'source')
except IOError:
weightingFunctionElement.text = repr(marshal.dumps(self._weightingFunction.func_code))
weightingFunctionElement.set('type', 'marshal')
for attribute in self._attributes:
attribute._toXMLElement(networkElement)
return networkElement
def _toScriptFile(self, scriptFile, scriptRefs):
if len(self._attributes) > 0:
scriptFile.write(gettext('# Create the network') + '\n\n')
for attribute in self._attributes:
attribute._toScriptFile(scriptFile, scriptRefs)
if self._weightingFunction:
# First try to get the function source and if that fails then marshal the byte code.
scriptFile.write(gettext('# Add the weighting function') + '\n\n')
funcName = self._weightingFunction.func_name
try:
source = inspect.getsource(self._weightingFunction)
scriptFile.write(source + '\n\nnetwork.setWeightingFunction(' + funcName + ')\n\n')
except IOError:
scriptFile.write('import marshal, types\n')
scriptFile.write('code = marshal.loads(' + repr(marshal.dumps(self._weightingFunction.func_code)) + ')\n')
scriptFile.write('network.setWeightingFunction(types.FunctionType(code, globals(), \'' + funcName + '\'))\n\n')
# Add each network object to the script in an order that guarantees dependent objects will already have been added.
# Neurites will be added by their neurons, sub-regions by their root region.
for objectClass in (Region, Pathway, Muscle, Neuron, Arborization, GapJunction, Innervation, Synapse, Stimulus):
objects = self.objectsOfClass(objectClass)
if len(objects) > 0:
scriptFile.write('\n# ' + gettext('Create each %s') % (objectClass.displayName().lower()) + '\n\n')
for networkObject in objects:
if networkObject._includeInScript(atTopLevel = True):
networkObject._toScriptFile(scriptFile, scriptRefs)
def setBulkLoading(self, bulkLoading):
"""
Indicate whether or not a large quantity of objects are being added to the network.
>>> network.setBulkLoading(True)
>>> # ... add lots of objects ...
>>> network.setBulkLoading(False)
If bulk loading is enabled then various actions are delayed to make loading faster.
"""
if bulkLoading != self._bulkLoading:
self._bulkLoading = bulkLoading
if self._bulkLoading == False:
self._updateGraph()
if any(self._bulkAddObjects):
dispatcher.send('addition', self, affectedObjects = self._bulkAddObjects)
self._bulkAddObjects = []
dispatcher.send(('set', 'bulkLoading'), self)
def setSavePath(self, path):
if path != self._savePath:
self._savePath = path
dispatcher.send(('set', 'savePath'), self)
def savePath(self):
return self._savePath
def name(self):
if self._savePath is None:
return gettext('Untitled Network')
else:
return os.path.splitext(os.path.basename(self._savePath))[0]
def _generateUniqueId(self):
self._nextUniqueId += 1
return self._nextUniqueId
def findObject(self, objectClass, name = None, default = False):
"""
Return the first object of the given class with the given name.
>>> neuron = network.findObject(Neuron, 'AVAL')
Returns an :class:`Object <Network.Object.Object>` or None if there are no matching objects.
If default is True then each object's :meth:`defaultName() <Network.Object.Object.defaultName>` will be queried instead of its name.
"""
if name is not None:
for networkObject in self.objects:
if isinstance(networkObject, objectClass) and ((not default and networkObject.name == name) or (default and networkObject.defaultName() == name)):
return networkObject
return None
def createRegion(self, addSubTerms = False, *args, **keywordArgs):
"""
Create a new region optionally associated with an ontology term.
>>> region = network.createRegion(name = 'Glomerulus 2')
To associate the region with an ontology term pass in a term from an ontology in the library:
>>> flyBrainOnt = library.ontology('flybrain')
>>> ellipsoidBody = network.createRegion(ontologyTerm = flyBrainOnt.findTerm(name = 'Ellipsoid body'))
If addSubTerms is true then sub-regions will be created for all sub-terms in the ontology.
Returns the :class:`region <Network.Region.Region>` that is created.
"""
region = Region(self, *args, **keywordArgs)
self.addObject(region)
if region.ontologyTerm is not None and addSubTerms:
for term in region.ontologyTerm.parts:
self.createRegion(ontologyTerm = term, parentRegion = region, addSubTerms = True)
return region
def findRegion(self, name = None):
"""
Find the first region with the given name.
>>> region = network.findRegion('Ellipsoid body')
Returns a :class:`region <Network.Region.Region>` or None if there are no regions with the name.
"""
return self.findObject(Region, name)
def regions(self):
"""
Return a list of all :class:`regions <Network.Region.Region>` in the network.
>>> for region in network.regions():
... display.setVisibleColor(region, (1.0, 0.0, 0.0))
An empty list will be returned if there are no regions in the network.
"""
return self.objectsOfClass(Region)
def pathways(self):
"""
Return a list of all :class:`pathways <Network.Pathway.Pathway>` in the network.
>>> for pathway in network.pathways():
... display.setVisibleColor(pathway, (1.0, 0.0, 0.0))
An empty list will be returned if there are no pathways in the network.
"""
return self.objectsOfClass(Pathway)
def createNeuron(self, *args, **keywordArgs):
"""
Create a new neuron.
>>> neuron = network.createNeuron(name = 'AVAL')
Returns the :class:`neuron <Network.Neuron.Neuron>` that is created.
"""
neuron = Neuron(self, *args, **keywordArgs)
self.addObject(neuron)
return neuron
def findNeuron(self, name = None):
"""
Find the first neuron with the given name.
>>> neuron = network.findNeuron('AVAL')
Returns a :class:`neuron <Network.Neuron.Neuron>` or None if there are no neurons with the name.
"""
return self.findObject(Neuron, name)
def neurons(self):
"""
Return a list of all :class:`neurons <Network.Neuron.Neuron>` in the network.
>>> for neuron in network.neurons():
... neuron.setHasFunction(Neuron.Function.SENSORY, False)
An empty list will be returned if there are no neurons in the network.
"""
return self.objectsOfClass(Neuron)
def neurites(self):
"""
Return a list of all :class:`neurites <Network.Neurite.Neurite>` in the network.
>>> for neurite in network.neurites():
... neurite.setPathway(None)
An empty list will be returned if there are no neurites in the network.
"""
return self.objectsOfClass(Neurite)
def arborizations(self):
"""
Return a list of all :class:`arborizations <Network.Arborization.Arborization>` in the network.
>>> for arborization in network.arborizations():
... display.setVisibleShape(arborization, shapes['Cone'])
An empty list will be returned if there are no arborizations in the network.
"""
return self.objectsOfClass(Arborization)
def gapJunctions(self):
"""
Return a list of all :class:`gap junctions <Network.GapJunction.GapJunction>` in the network.
>>> for gapJunction in network.gapJunctions():
... display.setVisibleColor(gapJunction, (0, 0, 0))
An empty list will be returned if there are no gap junctions in the network.
"""
return self.objectsOfClass(GapJunction)
def innervations(self):
"""
Return a list of all :class:`innervations <Network.Innervation.Innervation>` in the network.
>>> for innervation in network.innervations():
... display.setVisibleWeight(innervation, 2.0)
An empty list will be returned if there are no innervations in the network.
"""
return self.objectsOfClass(Innervation)
def synapses(self):
"""
Return a list of all :class:`chemical synapses <Network.Synapse.Synapse>` in the network.
>>> for synapse in network.synapses():
... synapse.activation = None
An empty list will be returned if there are no chemical synapses in the network.
"""
return self.objectsOfClass(Synapse)
def createStimulus(self, *args, **keywordArgs):
"""
Create a new stimulus. DEPRECATED: Call :meth:`stimulate() <Network.Object.Object.stimulate>` on the desired target object instead.
>>> stimulus = network.createStimulus(target = neuron1, modality = library.modality('light'))
Returns the :class:`stimulus <Network.Stimulus.Stimulus>` that is created.
"""
target = keywordArgs['target']
del keywordArgs['target']
return target.stimulate(*args, **keywordArgs)
def findStimulus(self, name = None):
"""
Find the first stimulus with the given name.
>>> stimulus = network.findStimulus('Light')
Returns a :class:`stimulus <Network.Stimulus.Stimulus>` or None if there are no stimuli with the name.
"""
return self.findObject(Stimulus, name)
def stimuli(self):
"""
Return a list of all :class:`stimuli <Network.Stimulus.Stimulus>` in the network.
>>> for stimulus in network.stimuli():
... if stimulus.modality == library.modality('light'):
... display.setVisibleColor(stimulus, (1, 1, 1))
An empty list will be returned if there are no stimuli in the network.
"""
return self.objectsOfClass(Stimulus)
def createMuscle(self, *args, **keywordArgs):
"""
Create a new muscle.
>>> muscle = network.createMuscle(name = 'M1')
Returns the :class:`muscle <Network.Muscle.Muscle>` that is created.
"""
muscle = Muscle(self, *args, **keywordArgs)
self.addObject(muscle)
return muscle
def findMuscle(self, name = None):
"""
Find the first muscle with the given name.
>>> muscle = network.findMuscle('M1')
Returns a :class:`muscle <Network.Muscle.Muscle>` or None if there are no muscles with the name.
"""
return self.findObject(Muscle, name)
def muscles(self):
"""
Return a list of all :class:`muscles <Network.Muscle.Muscle>` in the network.
>>> for muscle in network.muscles():
... display.setVisibleOpacity(muscle, 0.5)
An empty list will be returned if there are no muscles in the network.
"""
return self.objectsOfClass(Muscle)
def _updateGraph(self, objectToUpdate = None):
if objectToUpdate is None:
# Rebuild the entire graph.
objectsToUpdate = self.objects
self.graph.clear()
else:
# Just rebuild the connections to the one object.
objectsToUpdate = [objectToUpdate]
# Remove the object if it was there before. This will also delete any edges from the node.
objectId = objectToUpdate.networkId
if objectId in self.graph:
self.graph.remove_node(objectId)
# Maintain a temporary cache of weights so that rebuilding the whole graph doesn't take so long.
objectWeights = {}
def weightOfObject(weightedObject):
if weightedObject.networkId in objectWeights:
objectWeight = objectWeights[weightedObject.networkId]
else:
objectWeight = self.weightOfObject(weightedObject)
objectWeights[weightedObject.networkId] = objectWeight
return objectWeight
for objectToUpdate in objectsToUpdate:
objectId = objectToUpdate.networkId
# (Re-)Add the object to the graph.
self.graph.add_node(objectId)
# Get the weight of this object.
objectWeight = weightOfObject(objectToUpdate)
# Add the connections to other objects already in the graph.
# (Each connection to an object not in the graph will be added when that object is added.)
# The weight of each edge is the average of the weights of the two objects it connects.
inputIds = set([objectInput.networkId for objectInput in objectToUpdate.inputs(recurse = False)])
outputIds = set([objectOutput.networkId for objectOutput in objectToUpdate.outputs(recurse = False)])
unknownIds = set([objectInput.networkId for objectInput in objectToUpdate.connections(recurse = False)]).difference(inputIds).difference(outputIds)
for inputId in inputIds.union(unknownIds):
if inputId in self.graph:
otherWeight = weightOfObject(objectToUpdate)
self.graph.add_edge(inputId, objectId, weight = (objectWeight + otherWeight) / 2.0)
for outputId in outputIds.union(unknownIds):
if outputId in self.graph:
otherWeight = weightOfObject(objectToUpdate)
self.graph.add_edge(objectId, outputId, weight = (objectWeight + otherWeight) / 2.0)
def _objectChanged(self, sender):
if not self._bulkLoading:
self._updateGraph(sender)
if not self._modified:
self._modified = True
dispatcher.send(('set', 'modified'), self)
def simplifiedGraph(self):
"""
Return a simplified version of the NetworkX representation of the network.
This version of the network will have far fewer nodes but will not accurately model edges with more than two end points (hyperedges). This speeds processing when using NetworkX's algorithms.
"""
def addEdge(graph, object1, object2, weight):
node1 = object1.networkId
node2 = object2.networkId
if node1 in graph and node2 in graph[node1]:
if weight < graph[node1][node2]['weight']:
# Use a smaller weight for an existing edge.
graph[node1][node2]['weight'] = weight
else:
# Create a new edge.
graph.add_edge(node1, node2, weight = weight)
simplifiedGraph = DiGraph()
# In self.graph edges are actually nodes to support hyperedges. Convert these to standard edges in the simplified graph.
# TODO: make this object type independent
for arborization in self.arborizations():
if arborization.sendsOutput:
addEdge(simplifiedGraph, arborization.neurite.neuron(), arborization.region, self.weightOfObject(arborization))
if arborization.receivesInput:
addEdge(simplifiedGraph, arborization.region, arborization.neurite.neuron(), self.weightOfObject(arborization))
for synapse in self.synapses():
for postPartner in synapse.postSynapticPartners:
if isinstance(postPartner, Neurite):
postPartner = postPartner.neuron()
addEdge(simplifiedGraph, synapse.preSynapticNeurite.neuron(), postPartner, self.weightOfObject(synapse))
for gapJunction in self.gapJunctions():
neurites = gapJunction.neurites()
addEdge(simplifiedGraph, neurites[0].neuron(), neurites[1].neuron(), self.weightOfObject(gapJunction))
addEdge(simplifiedGraph, neurites[1].neuron(), neurites[0].neuron(), self.weightOfObject(gapJunction))
for innervation in self.innervations():
addEdge(simplifiedGraph, innervation.neurite.neuron(), innervation.muscle, self.weightOfObject(innervation))
for pathway in self.pathways():
region1, region2 = pathway.regions()
weight = self.weightOfObject(pathway)
if pathway.region1Projects:
addEdge(simplifiedGraph, region1, region2, weight)
if pathway.region2Projects:
addEdge(simplifiedGraph, region2, region1, weight)
for stimulus in self.stimuli():
addEdge(simplifiedGraph, stimulus, stimulus.target.rootObject(), self.weightOfObject(stimulus))
return simplifiedGraph
def setModified(self, modified):
"""
Set whether or not this network is dirty and needs to be saved.
"""
if self._modified != modified:
self._modified = modified
dispatcher.send(('set', 'modified'), self)
def isModified(self):
"""
Return whether the network has been modified and needs to be saved.
"""
return self._modified
def addObject(self, objectToAdd):
if objectToAdd.networkId in self.idDict:
raise ValueError, gettext('All objects in a network must have unique identifiers.')
self.objects.append(objectToAdd)
self.idDict[objectToAdd.networkId] = objectToAdd
if objectToAdd.networkId > self._nextUniqueId:
self._nextUniqueId = objectToAdd.networkId
# Update the NetworkX graph representation of the object and its connections.
if not self._bulkLoading:
self._updateGraph(objectToAdd)
# Watch for any changes to the object so we can update our dirty state and the graph.
dispatcher.connect(self._objectChanged, dispatcher.Any, objectToAdd)
# Let anyone who cares know that the network was changed.
if self._bulkLoading:
self._bulkAddObjects += [objectToAdd]
else:
dispatcher.send('addition', self, affectedObjects = [objectToAdd])
def objectWithId(self, objectId):
if (isinstance(objectId, str) or isinstance(objectId, unicode)) and objectId.isdigit():
objectId = int(objectId)
return self.idDict[objectId] if objectId in self.idDict else None
def objectsOfClass(self, objectClass):
objects = []
for networkObject in self.objects:
if isinstance(networkObject, objectClass):
objects.append(networkObject)
return objects
def setWeightingFunction(self, weightingFunction = None):
"""
Set a function to be used to calculate the weight of objects in the network.
The function should accept a single argument (an :class:`object <network.object.Object>` in the network) and return a floating point value indicating the weight of the object. An object with a higher weight is considered more expensive to traverse.
"""
if weightingFunction is not None and not callable(weightingFunction):
raise ValueError, gettext('The function passed to setWeightingFunction must be callable.')
if weightingFunction != self._weightingFunction:
self._weightingFunction = weightingFunction
self._updateGraph()
dispatcher.send(('set', 'weightingFunction'), self)
def weightingFunction(self):
"""
Return the function being used to calculate the weights of objects in the network.
If no function has been set then None will be returned.
"""
return self._weightingFunction
def weightOfObject(self, weightedObject):
"""
Return the weight of the indicated object or 1.0 if no weighting function has been set.
"""
return 1.0 if not self._weightingFunction else self._weightingFunction(weightedObject)
def shortestPath(self, startObject, endObject):
"""
Return one of the shortest paths through the :class:`network <Network.Network.Network>` from the first object to the second.
Returns a list of objects in the path from the first object to the second. If the second object cannot be reached from the first then an empty list will be returned.
"""
if not isinstance(startObject, Object) or startObject.network != self or not isinstance(endObject, Object) or endObject.network != self:
raise ValueError, 'The objects passed to shortestPath() must be from the same network.'
path = []
try:
nodeList = dijkstra_path(self.graph, startObject.networkId, endObject.networkId)
except:
nodeList = []
for nodeID in nodeList:
pathObject = self.objectWithId(nodeID)
if pathObject is not startObject:
path.append(pathObject)
return path
def removeObject(self, networkObject):
"""
Remove the indicated object and any dependent objects from the network and any displays.
>>> network.removeObject(network.findNeuron('AVAL'))
"""
if networkObject in self.objects:
# Determine all of the objects that will need to be removed
objectsToRemove = set([networkObject])
objectsToInspect = [networkObject]
while any(objectsToInspect):
objectToInspect = objectsToInspect.pop(0)
dependentObjects = set(objectToInspect.dependentObjects())
objectsToInspect += list(dependentObjects.difference(objectsToRemove))
objectsToRemove = objectsToRemove.union(dependentObjects)
# Remove all of the objects.
for objectToRemove in objectsToRemove:
objectToRemove.disconnectFromNetwork()
self.objects.remove(objectToRemove)
del self.idDict[objectToRemove.networkId]
# Keep the NetworkX graph in sync.
if objectToRemove.networkId in self.graph:
self.graph.remove_node(objectToRemove.networkId)
# Let anyone who cares know that the network was changed.
dispatcher.send('deletion', self, affectedObjects = objectsToRemove)
def removeAllObjects(self):
"""
Remove all objects from the network and any displays.
"""
removedObjects = list(self.objects)
for networkObject in self.objects:
networkObject.network = None
self.objects = []
self.idDict = {}
self.graph.clear()
# Let anyone who cares know that the network was changed.
dispatcher.send('deletion', self, affectedObjects = removedObjects)
def addDisplay(self, display):
self.displays.append(display)
dispatcher.connect(self._synchronizeDisplays, ('set', 'selection'), display)
def removeDisplay(self, display):
self.displays.remove(display)
dispatcher.disconnect(self._synchronizeDisplays, ('set', 'selection'), display)
def setSynchronizeDisplays(self, synchronize):
if synchronize != self._displaysAreSynchronized:
self._displaysAreSynchronized = synchronize
if synchronize and any(self.displays):
self._synchronizeDisplays(None, self.displays[0])
def _synchronizeDisplays(self, sender):
if self._displaysAreSynchronized:
selection = sender.selectedObjects()
for display in self.displays:
if display != sender:
display.selectObjects(selection)
def addAttribute(self, name = None, type = None, value = None): # pylint: disable=W0622
"""
Add a user-defined attribute to this network.
>>> network.addAttribute('Preliminary', Attribute.BOOLEAN_TYPE, True)
The type parameter should be one of the :class:`Attribute.*_TYPE <Network.Attribute.Attribute>` values.
Returns the attribute object that is created.
"""
if name is None or type is None or value is None:
raise ValueError, gettext('The name, type and value parameters must be specified when adding an attribute.')
if not isinstance(name, str):
raise TypeError, 'The name parameter passed to addAttribute() must be a string.'
if type not in Attribute.TYPES:
raise TypeError, 'The type parameter passed to addAttribute() must be one of the Attribute.*_TYPE values.'
# TODO: validate value based on the type?
attribute = Attribute(self, name, type, value)
self._attributes.append(attribute)
dispatcher.send(('set', 'attributes'), self)
return attribute
def getAttribute(self, name):
"""
Return the first user-defined :class:`attribute <Network.Attribute.Attribute>` of this network with the given name or None if there is no matching attribute.
>>> creationDate = network.getAttribute('Creation Date').value()
"""
for attribute in self._attributes:
if attribute.name() == name:
return attribute
return None
def getAttributes(self, name = None):
"""
Return a list of all user-defined :class:`attributes <Network.Attribute.Attribute>` of this network or only those with the given name.
>>> reviewers = [reviewer.value() for reviewer in network.getAttributes('Reviewed By')]
If there are no attributes then an empty list will be returned.
"""
attributes = []
for attribute in self._attributes:
if name == None or attribute.name() == name:
attributes += [attribute]
return attributes
def removeAttribute(self, attribute):
"""
Remove the given attribute from the network.
"""
if not isinstance(attribute, Attribute) or not attribute in self._attributes:
raise ValueError, 'The attribute passed to removeAttribute() must be an existing attribute of the network.'
self._attributes.remove(attribute)
dispatcher.send(('set', 'attributes'), self)
class MainWindow(QMainWindow, Ui_MainWindow):
def __init__(self, initial_dir: str = None):
super().__init__()
self._input_folder = None
self._network = DiGraph() # explicit targets dependencies
self._shared_states = edict(hovered=None)
self._enable_cross_selection = False
self._meta = None
self._placeholders = defaultdict(
edict) # save placeholder text for folder-specific fields
self._last_tab_index = None
self._executing = False
self._task = None
self._status_owner = None
self.setupUi(self)
self.retranslateUi(self)
self.setupSignals()
# load resources
self.setWindowIcon(_get_icon())
for format in global_config.organizer.output_format:
self.cbox_output_type.addItem(format)
self.cbox_output_type.setCurrentIndex(0)
if initial_dir:
self.txt_input_path.setText(initial_dir)
if global_config.organizer.default_output_dir:
self.txt_output_path.setText(
global_config.organizer.default_output_dir)
def setupSignals(self):
self.btn_input_browse.clicked.connect(self.browseInput)
self.btn_output_browse.clicked.connect(self.browseOutput)
self.btn_close.clicked.connect(self.safeClose)
self.btn_add_folder.clicked.connect(
lambda: self.addOutputFolder(self.txt_folder_name.currentText()))
self.btn_del_folder.clicked.connect(self.removeOutputFolder)
self.btn_reset.clicked.connect(
lambda: self.txt_input_path.clear() or self.reset())
self.btn_apply.clicked.connect(self.applyRequested)
self.btn_apply.enterEvent = self.applyButtonEnter
self.btn_apply.leaveEvent = self.applyButtonLeave
self.txt_input_path.textChanged.connect(self.inputChanged)
self.list_input_files.itemDoubleClicked.connect(self.previewInput)
self.list_input_files.itemPressed.connect(self.updateSelectedInput)
self.list_input_files.itemEntered.connect(self.updateHighlightInput)
self.list_input_files.enterEvent = self.listInputViewEnter
self.list_input_files.leaveEvent = self.listInputViewLeave
self.list_input_files.customContextMenuRequested.connect(
self.inputContextMenu)
self.panel_folder_meta.setVisible(False)
self.btn_expand_meta.clicked.connect(lambda: (
self.panel_folder_meta.setVisible(not self.panel_folder_meta.
isVisible()),
self.btn_expand_meta.setText("Fold Meta" if self.panel_folder_meta.
isVisible() else "Expand Meta")))
self.tab_folders.currentChanged.connect(self.updateSelectedFolder)
def applyButtonEnter(self, event):
if self._status_owner is None:
self._status_owner = self.btn_apply
if self.formattedOutputName:
outpath = Path(self.txt_output_path.text(),
self.formattedOutputName)
self.statusbar.showMessage("Start conversion to " +
str(outpath))
def applyButtonLeave(self, event):
if self._status_owner == self.btn_apply:
self.statusbar.clearMessage()
self._status_owner = None
@property
def formattedOutputName(self) -> str:
self.flushFolderMeta()
if self.cbox_output_type.currentIndex() < 0:
return ""
if not self._meta:
return ""
datestr = self.txt_date.text() or self.txt_date.placeholderText()
if datestr:
try:
x = int(datestr)
not_complete = True
except:
not_complete = False
if not not_complete:
date = date_parse(datestr)
yymmdd = date.strftime("%y%m%d")
else:
yymmdd = ""
else:
yymmdd = ""
partnumbers = [
f.partnumber for f in self._meta.folders.values() if f.partnumber
]
name_args = dict(
title=self.txt_title.text() or self.txt_title.placeholderText(),
artist=self.txt_artists.text()
or self.txt_artists.placeholderText(),
yymmdd=yymmdd,
partnumber=self.combinePartnumber(partnumbers)
if partnumbers else "",
event=self.txt_event.text(),
collaboration="", # TODO: add collaboration option?
)
fmt: str = global_config.organizer.output_format[
self.cbox_output_type.currentText()]
name = fmt.format(**name_args)
# simplify and escape
name = name.replace("()", "").replace("[]", "")
while "[(" in name:
name = name.replace("[(", "(", 1).replace(")]", ")", 1)
name = name.replace(":", "：").replace("/", "／").replace(
"<", "＜").replace(">", "＞").replace("*", "＊") # escape characters
return name.strip().rstrip(
'.') # directory with trailing dot is not supported by windows
def listInputViewEnter(self, event):
if not self._enable_cross_selection and self.currentOutputListView:
self.currentOutputListView.clearSelection()
def listInputViewLeave(self, event):
self._shared_states.hovered = None
def refreshOutputBgcolor(self):
start = self.currentOutputList.createIndex(0, 0)
end = self.currentOutputList.createIndex(-1, 0)
self.currentOutputListView.dataChanged(start, end, [Qt.BackgroundRole])
def refreshInputBgcolor(self):
for i in range(self.list_input_files.count()):
item = self.list_input_files.item(i)
if self._shared_states.hovered is not None and \
item.text() in self._network.predecessors(self._shared_states.hovered):
item.setBackground(PRED_COLOR)
elif len(list(self._network.successors(item.text()))):
item.setBackground(USED_COLOR)
else:
item.setBackground(QBrush())
def updateHighlightOutput(self, index: QModelIndex):
self._shared_states.hovered = self.currentOutputList[index.row()]
self.refreshInputBgcolor()
self.refreshOutputBgcolor()
def updateSelectedFolder(self, index: int):
if self.tab_folders.count() == 0: # happens when reset
return
if self._last_tab_index is not None:
self.flushFolderMeta(self._last_tab_index)
self._last_tab_index = index
# update fields
current_meta = self._meta.folders[self.tab_folders.tabText(index)]
self.txt_catalog.setText(current_meta.catalog)
self.txt_partnumber.setText(current_meta.partnumber)
self.txt_edition.setText(current_meta.edition)
self.txt_tool.setText(current_meta.tool)
self.txt_source.setText(current_meta.source)
self.txt_ripper.setText(current_meta.ripper)
self.txt_comment.setPlainText(current_meta.comment)
# update placeholder text
self.txt_partnumber.setPlaceholderText(
self._placeholders[self.currentOutputFolder].partnumber or "")
self.txt_tool.setPlaceholderText(
self._placeholders[self.currentOutputFolder].tool or "")
def flushFolderMeta(self, index: int = None):
if index is None:
index = self.tab_folders.currentIndex()
if self._meta:
target_meta = self._meta.folders[self.tab_folders.tabText(index)]
target_meta.catalog = self.txt_catalog.text()
target_meta.partnumber = self.txt_partnumber.text(
) or self.txt_partnumber.placeholderText()
target_meta.edition = self.txt_edition.text()
target_meta.tool = self.txt_tool.text()
target_meta.source = self.txt_source.text()
target_meta.ripper = self.txt_ripper.text()
target_meta.comment = self.txt_comment.toPlainText()
def listOutputViewEnter(self, event):
if not self._enable_cross_selection:
self.list_input_files.clearSelection()
def listOutputViewLeave(self, event):
self._shared_states.hovered = None
self.refreshInputBgcolor()
def addOutputFolder(self, name: str):
listview = QListView(self)
listview.setObjectName("tab_" + name.lower())
listview.setSelectionMode(QListView.ExtendedSelection)
listview.setAcceptDrops(True)
listview.setMouseTracking(True)
listview.setModel(
TargetListModel(listview, self._network, self._shared_states))
listview.pressed.connect(self.updateSelectedOutput)
listview.entered.connect(self.updateHighlightOutput)
listview.enterEvent = self.listOutputViewEnter
listview.leaveEvent = self.listOutputViewLeave
listview.setContextMenuPolicy(Qt.CustomContextMenu)
listview.customContextMenuRequested.connect(
lambda pos: self.outputContextMenu(listview, pos))
self._meta.folders[name] = FolderMeta()
self.tab_folders.addTab(listview, name.upper())
self.updateFolderNames()
def removeOutputFolder(self):
self._placeholders.pop(self.currentOutputFolder)
self._meta.folders.pop(self.currentOutputFolder)
self.tab_folders.removeTab(self.tab_folders.currentIndex())
self.updateFolderNames()
def updateFolderNames(self):
# update valid folder names
valid_folders = [
'CD', 'BK', 'DVD', 'DL', 'OL', 'BD', 'MISC', 'PHOTO', 'LRC'
]
for i in range(self.tab_folders.count()):
valid_folders.remove(self.tab_folders.tabText(i))
self.txt_folder_name.clear()
self.txt_folder_name.addItems(valid_folders)
# update availability of delete button
self.btn_del_folder.setEnabled(self.tab_folders.count() > 0)
def updateSelectedInput(self, item: QListWidgetItem):
if not self._enable_cross_selection:
for i in range(self.tab_folders.count()):
self.tab_folders.widget(i).clearSelection()
def updateHighlightInput(self, item: QListWidgetItem):
self._shared_states.hovered = item.text()
self.refreshOutputBgcolor()
@property
def currentOutputFolder(self) -> str:
return self.tab_folders.tabText(self.tab_folders.currentIndex())
@property
def currentOutputListView(self) -> QListView:
return self.tab_folders.currentWidget()
@property
def currentOutputList(self) -> TargetListModel:
return self.currentOutputListView.model()
def updateSelectedOutput(self, index: QModelIndex):
if not self._enable_cross_selection:
self.list_input_files.clearSelection()
for i in range(self.tab_folders.count()):
if i != self.tab_folders.currentIndex():
self.tab_folders.widget(i).clearSelection()
def browseInput(self):
dlg = QFileDialog()
dlg.setFileMode(QFileDialog.Directory)
if self.txt_input_path and Path(self.txt_input_path.text()).exists():
dlg.setDirectory(self.txt_input_path.text())
if dlg.exec_():
self.txt_input_path.setText(dlg.selectedFiles()[0])
def browseOutput(self):
dlg = QFileDialog()
dlg.setFileMode(QFileDialog.Directory)
if self.txt_output_path and Path(self.txt_output_path.text()).exists():
dlg.setDirectory(self.txt_output_path.text())
if dlg.exec_():
self.txt_output_path.setText(dlg.selectedFiles()[0])
def safeClose(self):
check_folder = False
for i in range(self.tab_folders.count()):
if len(self.tab_folders.widget(i).model()) > 0:
check_folder = True
break
if not (check_folder or self._executing):
self.close()
return
msgbox = QMessageBox(self)
msgbox.setWindowTitle("Close")
msgbox.setIcon(QMessageBox.Warning)
if self._executing:
msgbox.setText(
"There are pending jobs. Do you really want to close?")
else:
msgbox.setText("Do you really want to close?")
msgbox.setStandardButtons(QMessageBox.Yes | QMessageBox.No)
msgbox.setDefaultButton(QMessageBox.No)
if msgbox.exec_() == QMessageBox.Yes:
if self._task is not None:
self._task.cancel()
self.close()
def previewInput(self, item: QListWidgetItem):
QDesktopServices.openUrl(
QUrl.fromLocalFile(str(self._input_folder / item.text())))
def addTargetActions(self, menu: QMenu):
selected_items = [
i.text() for i in self.list_input_files.selectedItems()
]
selected_items += [
self.currentOutputList[i.row()]
for i in self.currentOutputListView.selectedIndexes()
]
menu.addSeparator()
# cannot use default arg to force coping the type here, since the signal also provides other inputs
func_create = lambda _t, items: lambda: self.currentOutputList.appendTarget(
_t(items))
for t in target_types:
if t.validate(selected_items):
create_action = QAction(t.description, menu)
create_action.triggered.connect(func_create(t, selected_items))
menu.addAction(create_action)
menu.addSeparator()
func_create_batch = lambda _t, items: lambda: self.currentOutputList.extendTargets(
[_t(i) for i in items])
for t in target_types:
if all(t.validate([item]) for item in selected_items):
create_action = QAction("Batch " + t.description, menu)
create_action.triggered.connect(
func_create_batch(t, selected_items))
menu.addAction(create_action)
def inputContextMenu(self, pos: QPoint):
menu = QMenu()
preview_action = QAction("Preview", menu)
preview_action.triggered.connect(
lambda: self.previewInput(self.list_input_files.currentItem()))
menu.addAction(preview_action)
self.addTargetActions(menu)
action = menu.exec_(self.list_input_files.mapToGlobal(pos))
def combinePartnumber(self, partnumbers: List[str]):
assert len(partnumbers) > 0
if len(partnumbers) == 1:
return partnumbers[0]
try:
prefix = commonprefix(partnumbers)
remain = [pn[len(prefix):] for pn in partnumbers]
remain_num = [int(i) for i in remain]
rmin, rmax = min(remain_num), max(remain_num)
if rmax - rmin + 1 == len(remain):
rlen = len(remain[0])
return prefix + str(rmin).zfill(rlen) + '~' + str(rmax).zfill(
rlen)
else:
return prefix + '&'.join(remain)
except ValueError: # non trivial part numbers
return ','.join(partnumbers)
def fillMetaFromFolder(self):
partnumbers = []
for target in self.currentOutputList._targets:
if isinstance(target, MergeTracksTarget):
if target._meta.title:
self.txt_title.setPlaceholderText(target._meta.title)
if target._meta.full_artist:
self.txt_artists.setPlaceholderText(
target._meta.full_artist)
if target._meta.cuesheet:
if target._meta.cuesheet.rems.get('COMMENT', ''):
comment = target._meta.cuesheet.rems['COMMENT']
if 'Exact Audio Copy' in comment:
self.txt_tool.setPlaceholderText(comment)
self._placeholders[
self.currentOutputList].tool = comment
else:
self.txt_comment.setPlaceholderText(comment)
if target._meta.cuesheet.rems.get('DATE', ''):
self.txt_date.setPlaceholderText(
target._meta.cuesheet.rems['DATE'])
if target._meta.partnumber:
partnumbers.append(target._meta.partnumber)
# combine part numbers
if len(partnumbers) > 1:
pnstr = self.combinePartnumber(partnumbers)
if ',' not in pnstr:
self.txt_partnumber.setPlaceholderText(pnstr)
self._placeholders[self.currentOutputFolder].partnumber = pnstr
elif len(partnumbers) == 1:
self.txt_partnumber.setPlaceholderText(partnumbers[0])
self._placeholders[
self.currentOutputFolder].partnumber = partnumbers[0]
@asyncSlot()
async def editCurrentTarget(self):
await editTarget(self.currentOutputList[
self.currentOutputListView.currentIndex().row()],
input_root=self._input_folder,
output_root=Path(self.txt_output_path.text(),
self.currentOutputFolder))
self.fillMetaFromFolder()
def outputContextMenu(self, listview: QListView, pos: QPoint):
current_model = listview.model()
menu = QMenu()
edit_action = QAction(
"Edit clicked" if len(listview.selectedIndexes()) > 1 else "Edit",
menu)
edit_action.triggered.connect(self.editCurrentTarget)
menu.addAction(edit_action)
delete_action = QAction("Remove", menu)
delete_action.triggered.connect(lambda: current_model.__delitem__(
[i.row() for i in listview.selectedIndexes()]))
menu.addAction(delete_action)
selected_istemp = current_model[
listview.currentIndex().row()].temporary
mark_temp_action = QAction("Mark temp", menu)
mark_temp_action.setCheckable(True)
mark_temp_action.setChecked(selected_istemp)
mark_temp_action.triggered.connect(lambda: [
current_model[i.row()].switch_temporary(not selected_istemp)
for i in listview.selectedIndexes()
])
menu.addAction(mark_temp_action)
self.addTargetActions(menu)
menu.exec_(listview.mapToGlobal(pos))
def inputChanged(self, content):
path = Path(content)
if not (content and path.exists()):
return
if self._input_folder == path:
return
self.reset()
# read file list
self._input_folder = path
glob = (p for p in path.rglob("*") if p.is_file())
files = [str(p.relative_to(path)) for p in islice(glob, 50)]
self.list_input_files.addItems(files)
self._network.add_nodes_from(files)
# read more files
remains = True
try:
remains = next(glob)
except StopIteration:
remains = False
if remains:
msgbox = QMessageBox(self)
msgbox.setIcon(QMessageBox.Warning)
msgbox.setText("There are too many files in the directory.")
msgbox.setInformativeText("Do you still want to list them?")
msgbox.setStandardButtons(QMessageBox.Yes | QMessageBox.No)
msgbox.setDefaultButton(QMessageBox.No)
if msgbox.exec_() != QMessageBox.Yes:
self.reset()
return
self.list_input_files.addItem(str(remains.relative_to(path)))
self._network.add_node(str(remains.relative_to(path)))
files = [str(p.relative_to(path)) for p in glob]
self.list_input_files.addItems(files)
self._network.add_nodes_from(files)
# generate keywords
keywords = set()
keypattern = re.compile(global_config.organizer.keyword_splitter)
keywords.update(k.strip() for k in keypattern.split(path.name)
if k.strip())
if len(os.listdir(path)) == 0:
subf = next(path.iterdir())
if subf.is_dir():
keywords.update(k.strip() for k in keypattern.split(subf.name)
if k.strip())
# TODO: extract metadata from input audio files
self.widget_keywords.extendKeywords(keywords)
# default output path
if not global_config.organizer.default_output_dir:
self.txt_output_path.setText(str(path.parent / "organized"))
def reset(self):
'''
Reset all information except for input/output path and folder list
'''
# clear text
self.list_input_files.clear()
self.widget_keywords.clear()
self.tab_folders.clear()
self.txt_title.setText(None)
self.txt_artists.setText(None)
self.txt_publisher.setText(None)
self.txt_vendor.setText(None)
self.txt_partnumber.setText(None)
self.txt_event.setText(None)
self.txt_date.setText(None)
self.txt_genre.setText(None)
# clear placeholder text
self.txt_title.setPlaceholderText(None)
self.txt_artists.setPlaceholderText(None)
self.txt_partnumber.setPlaceholderText(None)
self.txt_date.setPlaceholderText(None)
self.txt_comment.setPlaceholderText(None)
self._input_folder = None
self._shared_states.hovered = None
self._network.clear()
self._meta = AlbumMeta()
self._last_tab_index = None
self.addOutputFolder("CD")
def keyPressEvent(self, event: QKeyEvent) -> None:
super().keyPressEvent(event)
if event.key() == Qt.Key_Alt:
self.list_input_files.setDragEnabled(True)
elif event.key() in [Qt.Key_Shift, Qt.Key_Control]:
self._enable_cross_selection = True
def keyReleaseEvent(self, event: QKeyEvent) -> None:
super().keyReleaseEvent(event)
if event.key() == Qt.Key_Alt:
self.list_input_files.setDragEnabled(False)
elif event.key() in [Qt.Key_Shift, Qt.Key_Control]:
self._enable_cross_selection = False
@asyncSlot()
async def applyRequested(self):
# check all targets has been initialized
for target in self._network.nodes:
if isinstance(target, OrganizeTarget) and not target.initialized:
msgbox = QMessageBox(self)
msgbox.setWindowTitle("Close")
msgbox.setIcon(QMessageBox.Critical)
msgbox.setText("There are uninitialized targets!")
msgbox.exec_()
return
# flush folder meta
self.flushFolderMeta()
for folder, fmeta in self._meta.folders.items():
fmeta.tool = fmeta.tool or self._placeholders[folder].tool
fmeta.partnumber = fmeta.partnumber or self._placeholders[
folder].partnumber
# flush
self._meta.title = self.txt_title.text(
) or self.txt_title.placeholderText()
artist_text = self.txt_artists.text(
) or self.txt_artists.placeholderText()
self._meta.artists = re.split(global_config.organizer.artist_splitter,
artist_text)
self._meta.publisher = self.txt_publisher.text(
) or self.txt_publisher.placeholderText()
self._meta.vendor = self.txt_vendor.text(
) or self.txt_vendor.placeholderText()
self._meta.event = self.txt_event.text(
) or self.txt_event.placeholderText()
self._meta.date = self.txt_date.text(
) or self.txt_date.placeholderText()
self._meta.genre = self.txt_genre.text(
) or self.txt_genre.placeholderText()
self.statusbar.showMessage("Starting execution...")
self._task = asyncio.ensure_future(self.executeTargets())
self._status_owner = self._task
await self._task
self._task = None
async def executeTargets(self):
self._executing = True
files_to_remove = []
try:
# sort targets
order = topological_sort(self._network)
order = [t for t in order if isinstance(t, OrganizeTarget)]
# get output folder
folder_map = {}
disc_targets = defaultdict(list)
for i in range(self.tab_folders.count()):
folder = self.tab_folders.tabText(i)
for target in self.tab_folders.widget(i).model():
folder_map[target] = folder
if isinstance(target, MergeTracksTarget):
disc_targets[folder].append(target)
# add disc numbers
for folder, targets in disc_targets.items():
targets.sort(key=lambda t: t._outstem.lower())
for i, t in enumerate(targets):
t._meta.discnumber = i + 1
# execute targets
output_path = Path(self.txt_output_path.text(),
self.formattedOutputName)
output_path.mkdir(exist_ok=True, parents=True)
for i, target in enumerate(order):
self._status_owner = target
self.statusbar.showMessage("(%d/%d) Executing: %s" %
(i, len(order), str(target)))
if isinstance(target, str):
continue
output_folder_root = output_path / folder_map[target]
output_folder_root.mkdir(exist_ok=True)
if isinstance(target, MergeTracksTarget):
await target.apply(
self._input_folder, output_folder_root,
lambda q: self.statusbar.showMessage(
"(%d/%d) Executing: %s (%d%%)" %
(i, len(order), str(target), int(q * 100))))
else:
await target.apply(self._input_folder, output_folder_root)
if target.temporary:
files_to_remove.append(output_folder_root /
target.output_name)
# create meta.yaml
meta_dict = self._meta.to_dict()
with Path(output_path,
"meta.yaml").open("w", encoding="utf-8-sig") as fout:
yaml.dump(meta_dict,
fout,
encoding="utf-8",
allow_unicode=True)
self.statusbar.showMessage("Organizing done successfully!")
except Exception as e:
import traceback
stack = traceback.format_exc()
_logger.error("Pipeline execution failed. Full stack:\n" + stack)
msgbox = QMessageBox(self)
msgbox.setWindowTitle("Execution failed")
msgbox.setIcon(QMessageBox.Critical)
msgbox.setText("Reason: " + str(e))
msgbox.exec_()
self.statusbar.showMessage("Organizing failed!")
finally:
# clean up
for f in files_to_remove:
f.unlink()
self._status_owner = None
self._executing = False
