def load_ontology_to_neo4jdb(db, ontology):
"""
Loads the supplied ontology into a neo4j database. Returns a mapping of
neo4j node ID to ontology ID which will be utilized when creating
relationships in the neo4j database.
"""
nodeMap = {}
for term in ontology:
log.info('Loading term %s...' % term.name)
node = db.node()
if term.obsolete:
log.info(' ** Skipping node %s because it is obsolete **' % term.name)
continue
for (attr, value) in term.__dict__.iteritems():
if value and attr not in ["relationships", "synonyms"]:
node.set(attr, value)
elif value and attr == "synonyms":
# Synonyms need to be converted from a list of tuples to a list
# of strings
synonymStrList = [" ".join(x) for x in value]
node.set(attr, synonymStrList)
nodeMap.setdefault(term.id, {})
nodeMap[term.id]['node_id'] = node.id
nodeMap[term.id]['relationships'] = term.relationships
index_neo4j_node(db, node, term.id)
return nodeMap
def load_ontology_to_neo4jdb(db, ontology):
"""
Loads the supplied ontology into a neo4j database. Returns a mapping of
neo4j node ID to ontology ID which will be utilized when creating
relationships in the neo4j database.
"""
nodeMap = {}
for term in ontology:
log.info('Loading term %s...' % term.name)
node = db.node()
if term.obsolete:
log.info(' ** Skipping node %s because it is obsolete **' %
term.name)
continue
for (attr, value) in term.__dict__.iteritems():
if value and attr not in ["relationships", "synonyms"]:
node.set(attr, value)
elif value and attr == "synonyms":
# Synonyms need to be converted from a list of tuples to a list
# of strings
synonymStrList = [" ".join(x) for x in value]
node.set(attr, synonymStrList)
nodeMap.setdefault(term.id, {})
nodeMap[term.id]['node_id'] = node.id
nodeMap[term.id]['relationships'] = term.relationships
index_neo4j_node(db, node, term.id)
return nodeMap
def synchronize(issues):
tw = TaskWarrior()
# Load info about the task database
tasks = tw.load_tasks()
is_bugwarrior_task = lambda task: task['description'].startswith(MARKUP)
# Prune down to only tasks managed by bugwarrior
for key in tasks.keys():
tasks[key] = filter(is_bugwarrior_task, tasks[key])
# Build a list of only the descriptions of those local bugwarrior tasks
local_descs = [t['description'] for t in sum(tasks.values(), [])]
# Now for the remote data.
# Build a list of only the descriptions of those remote issues
remote_descs = [i['description'] for i in issues]
# Build the list of tasks that need to be added
is_new = lambda issue: issue['description'] not in local_descs
new_issues = filter(is_new, issues)
# Build the list of local tasks that need to be completed
is_done = lambda task: task['description'] not in remote_descs
done_tasks = filter(is_done, tasks['pending'])
log.struct(new=len(new_issues), completed=len(done_tasks))
for issue in new_issues:
log.info("Adding task {0}", issue['description'])
tw.task_add(**issue)
for task in done_tasks:
log.info("Completing task {0}", task['description'])
tw.task_done(id=None, uuid=task['uuid'])
def unpickle_all(filename, log=log):
log = log.fields(filename=filename)
if os.path.isfile(filename):
log.info('unpickling')
with open(filename, 'rb') as f:
globals().update(pickle.load(f))
else:
log.info('notfound')
def synchronize(issues):
tw = TaskWarrior()
# Load info about the task database
tasks = tw.load_tasks()
is_bugwarrior_task = lambda task: task.get('description', '').startswith(MARKUP)
# Prune down to only tasks managed by bugwarrior
for key in tasks.keys():
tasks[key] = filter(is_bugwarrior_task, tasks[key])
# Build a list of only the descriptions of those local bugwarrior tasks
local_descs = [t['description'] for t in sum(tasks.values(), []) \
if t['status'] not in ('deleted')]
# Now for the remote data.
# Build a list of only the descriptions of those remote issues
remote_descs = [i['description'] for i in issues]
# Build the list of tasks that need to be added
is_new = lambda issue: issue['description'] not in local_descs
new_issues = filter(is_new, issues)
old_issues = filter(lambda i: not is_new(i), issues)
# Build the list of local tasks that need to be completed
is_done = lambda task: task['description'] not in remote_descs
done_tasks = filter(is_done, tasks['pending'])
log.struct(new=len(new_issues), completed=len(done_tasks))
# Add new issues
for issue in new_issues:
log.info("Adding task {0}", issue['description'].encode("utf-8"))
tw.task_add(**issue)
# Update any issues that may have had new properties added. These are
# usually annotations that come from comments in the issue thread.
pending_descriptions = [t['description'] for t in tasks['pending']]
for upstream_issue in old_issues:
if upstream_issue['description'] not in pending_descriptions:
continue
id, task = tw.get_task(description=upstream_issue['description'])
for key in upstream_issue:
if key not in task:
log.info("Updating {0} on {1}",
key, upstream_issue['description'].encode("utf-8"))
task[key] = upstream_issue[key]
id, task = tw.task_update(task)
# Delete old issues
for task in done_tasks:
log.info("Completing task {0}", task['description'].encode("utf-8"))
tw.task_done(uuid=task['uuid'])
def __init__(self, config, target, shorten):
self.config = config
self.target = target
self.shorten = shorten
log.info("Working on [{0}]", self.target)
def pull(self):
# get last pull
log.info(' [pull]')
for target in self.targets:
log.info(' [{0}]', target)
service = services.service(self.config, target)
synch_date = self.synch_date(target)
service.set_synch_date( synch_date )
# Pull target tasks
tasks = service.tasks()
task_ids = []
for t in tasks: task_ids.append(service.get_id(t))
# Find exist tasks
self.db.where('target','=',target)
self.db.where_in('id_b', task_ids);
exist = {}
for rel in self.db.get('relations').fetchall():
exist[rel['id_b']] = rel['id_a']
# Fill insert/update lists
insert_tasks = []
update_tasks = []
for t in tasks:
task_key = service.get_id(t)
if task_key in exist:
# print t['name'], service.get_date_modified(t), parser.parse(synch_date)
if service.get_date_modified(t) > synch_date:
t['id'] = exist[task_key]
update_tasks.append(t)
else:
insert_tasks.append(t)
log.info(' INSERT:{0} | UPDATE:{1}', len(insert_tasks), len(update_tasks))
for task in update_tasks:
task_id = task['id']
task = service.task(task)
del task['id']
self.db.where('id','=', task_id).update('tasks', task);
for task in insert_tasks:
task = service.task(task)
id_b = task['id']
del task['id']
task['source_target'] = target
task_id = self.db.insert('tasks', task);
self.db.insert('relations', dict(
target = target,
id_b = id_b,
id_a = str(task_id),
type = 'tasks'
));
self.db.commit();
self.synch_date(target, self.date_now())
del task['id']
task['source_target'] = target
task_id = self.db.insert('tasks', task);
self.db.insert('relations', dict(
target = target,
id_b = id_b,
id_a = str(task_id),
type = 'tasks'
));
self.db.commit();
self.synch_date(target, self.date_now())
############################################################################
############################################################################
############################################################################
if __name__ == '__main__':
ztask = zTask()
ztask.pull()
# ztask.display_tasks()
# ztask.display_projects()
log.info(' [END]')
############################################################################
