def wrapper(self, *args, **kwargs): from neomodel import db as transaction transaction_open = True try: try: transaction.begin() log.verbose("Neomodel transaction BEGIN2") except SystemError: transaction_open = False log.debug("Neomodel transaction is already in progress") out = func(self, *args, **kwargs) if transaction_open: transaction.commit() log.verbose("Neomodel transaction COMMIT2") else: log.debug("Skipping neomodel transaction commit") return out except Exception as e: if not transaction_open: log.debug("Skipping neomodel transaction rollback") else: try: log.verbose("Neomodel transaction ROLLBACK") transaction.rollback() except Exception as sub_ex: log.warning("Exception raised during rollback: {}", sub_ex) raise e
def updated_node(node, data, template): """The function given a template and a node, the node and next nodes information is updated""" db.begin() result = updated_node_aux(node, data, template[list(template.keys())[0]]) if result is None: db.rollback() return None else: db.commit() return True
def dglab_node_update(node, identifiers, titles): db.begin() update_titles_result = update_titles(node, titles) update_identifiers_result = update_identifiers(node, identifiers) if update_titles_result and update_identifiers_result: db.commit() return True else: db.rollback() return None
def delete(self, name): try: building = Building.nodes.get_or_none(name=name) if building is not None: db.begin() db.cypher_query( f"MATCH (n) WHERE n.buildingName = '{name}' DETACH DELETE n" ) building.delete() db.commit() return 'Building successfully deleted', 200 else: return "Not Found", 404 except Exception as e: db.rollback() return str(e), 500
def test_rollback_and_commit_transaction(): for p in Person.nodes: p.delete() Person(name="Roger").save() db.begin() Person(name="Terry S").save() db.rollback() assert len(Person.nodes) == 1 db.begin() Person(name="Terry S").save() db.commit() assert len(Person.nodes) == 2
def test_rollback_and_commit_transaction(): for p in APerson.nodes: p.delete() APerson(name="Roger").save() db.begin() APerson(name="Terry S").save() db.rollback() assert len(APerson.nodes) == 1 db.begin() APerson(name="Terry S").save() db.commit() assert len(APerson.nodes) == 2
def test_rollback_and_commit_transaction(): for p in Person.nodes: p.delete() Person(name='Roger').save() db.begin() Person(name='Terry S').save() db.rollback() assert len(Person.nodes) == 1 db.begin() Person(name='Terry S').save() db.commit() assert len(Person.nodes) == 2
def test_rollback_and_commit_transaction(): for p in APerson.nodes: p.delete() APerson(name='Roger').save() db.begin() APerson(name='Terry S').save() db.rollback() assert len(APerson.nodes) == 1 db.begin() APerson(name='Terry S').save() db.commit() assert len(APerson.nodes) == 2
def wrapper(self: Any, *args: Any, **kwargs: Any) -> Any: neo4j_enabled = Connector.check_availability("neo4j") sqlalchemy_enabled = Connector.check_availability("sqlalchemy") if neo4j_enabled: from neomodel import db as neo4j_db if sqlalchemy_enabled: # thanks to connectors cache this should always match the # same instance that will be used from inside the endpoint from restapi.connectors import sqlalchemy alchemy_db = sqlalchemy.get_instance() try: if neo4j_enabled: neo4j_db.begin() # Transaction is already open... # if sqlalchemy_enabled: # pass out = func(self, *args, **kwargs) if neo4j_enabled: neo4j_db.commit() if sqlalchemy_enabled: alchemy_db.session.commit() return out except Exception as e: log.debug("Rolling backend database transaction") try: if neo4j_enabled: neo4j_db.rollback() if sqlalchemy_enabled: alchemy_db.session.rollback() except Exception as sub_ex: # pragma: no cover log.warning("Exception raised during rollback: {}", sub_ex) raise e
def wrapper(self, *args, **kwargs): try: from neomodel import db as transaction transaction.begin() log.verbose("Neomodel transaction BEGIN") out = func(self, *args, **kwargs) transaction.commit() log.verbose("Neomodel transaction COMMIT") return out except Exception as e: log.verbose("Neomodel transaction ROLLBACK") try: transaction.rollback() except Exception as sub_ex: log.warning("Exception raised during rollback: %s", sub_ex) raise e
def wrapper(self, *args, **kwargs): from neomodel import db try: db.begin() out = func(self, *args, **kwargs) db.commit() return out except Exception as e: log.debug("Neomodel transaction ROLLBACK") try: db.rollback() except Exception as sub_ex: log.warning("Exception raised during rollback: {}", sub_ex) raise e
def post(self): name = request.json.get('name') email = request.json.get('email') or "" education = request.json.get('education') or "" job_title = request.json.get('job_title') or "" disciplines = request.json.get('disciplines') or [] degrees = request.json.get('degrees') or [] interests = request.json.get('interests') or [] publications = request.json.get('publications') or [] department = request.json.get('department') or [] db.begin() try: employee = Employee( name=name, email=email, education=education, job_title=job_title, ) employee.save() department = Department.nodes.get_or_none(name=department) if not department: return {"error": "Department doesn't exist"}, 400 department.employees.connect(employee, {'job_title': job_title}) for d in disciplines: discipline = Discipline.nodes.get_or_none( name=d['discipline']['name']) if not discipline: discipline = Discipline( name=d['discipline']['name']).save() for lesson in d['lessons']: employee.disciplines.connect(discipline, {**lesson}) for d in degrees: degree = Degree.nodes.get_or_none(content=d['content']) if not degree: degree = Degree(content=d['content']).save() employee.degrees.connect(degree) for i in interests: interest = Interest.nodes.get_or_none(content=i['content']) if not interest: interest = Interest(content=i['content']).save() employee.interests.connect(interest) for p in publications: publication = Publication.nodes.get_or_none( content=p['content']) if not publication: publication = Publication(content=p['content']).save() employee.publications.connect(publication) db.commit() except Exception as e: db.rollback() return {"error": str(e)}, 400 response = { **employee.__properties__, 'job_title': department.employees.relationship(employee).job_title, 'disciplines': [{ 'discipline': discipline.__properties__, 'lessons': getLessons(employee.id, discipline.id) } for discipline in getUniqueDisciplines(employee.disciplines.all()) ], 'degrees': [degree.__properties__ for degree in employee.degrees.all()], 'interests': [interest.__properties__ for interest in employee.interests.all()], 'publications': [ publication.__properties__ for publication in employee.publications.all() ], } return response
def extract_br_annotations(self, item, analyze_dir_path): ''' Extract building recognition results given by automatic analysis tool and ingest valueable annotations as 'automatic' TAGs. ''' brf_results_filename = 'brf.xml' brf_results_path = os.path.join(os.path.dirname(analyze_dir_path), brf_results_filename) if not os.path.exists(brf_results_path): raise IOError("Analyze results does not exist in the path %s", brf_results_path) log.info('get automatic building recognition from file [%s]' % brf_results_path) parser = ORF_XMLParser() frames = parser.parse(brf_results_path) # get the shot list of the item shots = item.shots.all() shot_list = {} for s in shots: shot_list[s.shot_num] = set() if item.item_type == 'Image': # consider a image like single frame shot shot_list[0] = set() object_ids = set() concepts = set() report = {} obj_cat_report = {} for timestamp, bf_list in frames.items(): ''' A frame is a <dict> with timestamp<int> as key and a <list> as value. Each element in the list is a <tuple> that contains: (obj_id<str>, concept_label<str>, confidence<float>, region<list>). ''' if bf_list: log.debug("timestamp: {0}, element: {1}".format( timestamp, bf_list)) for recognized_building in bf_list: concepts.add(recognized_building[1]) if recognized_building[0] not in object_ids: report[recognized_building[1]] = report.get( recognized_building[1], 0) + 1 object_ids.add(recognized_building[0]) if item.item_type == 'Video': shot_uuid, shot_num = shot_lookup(self, shots, timestamp) elif item.item_type == 'Image': shot_num = 0 if shot_num is not None: shot_list[shot_num].add(recognized_building[1]) # collect timestamp for keys: # (cat,shot) --> [(id1, t1, confidence1), (id2, t2, confidence2), ...] tmp = obj_cat_report.get((recognized_building[1], shot_num), []) tmp.append( (recognized_building[0], timestamp, recognized_building[2])) obj_cat_report[(recognized_building[1], shot_num)] = tmp log.info('-----------------------------------------------------') log.info('Number of distinct recognized buildings: {}'.format( len(object_ids))) log.info('Number of distinct concepts: {}'.format(len(concepts))) print_out = 'Report of detected concepts per shot:\n' for k in sorted(shot_list.keys()): v = '{ }' if len(shot_list[k]) == 0 else shot_list[k] print_out += ("{}:\t{}\n".format(k, v)) log.info(print_out) # get item "city" code crepo = CreationRepository(self.graph) city = crepo.get_belonging_city(item) log.debug('This item belong to city: {0}'.format(city)) repo = AnnotationRepository(self.graph) saved_counter = 0 # keep count of saved annotation wrong_city_counter = 0 for (key, timestamps) in obj_cat_report.items(): # detect the concept concept_name = key[0] building_resource = building_mapping.get(concept_name) if building_resource is None: log.warn('Cannot find building <{0}> in the mapping'.format( concept_name)) # DO NOT create this annotation continue # filter by belonging city if city != building_resource.get('city'): # log.debug('Building {0} does NOT belong to city of {1}' # .format(concept_name, city)) wrong_city_counter += 1 continue concept = { 'iri': building_resource.get('iri'), 'name': building_resource.get('name'), 'spatial': building_resource.get('coord') } log.debug(concept) # use first id as building recognition id br_id = timestamps[0][0] log.debug("Building Recognition ID: {}".format(br_id)) # detect target segment ONLY for videos start_frame = timestamps[0][1] end_frame = timestamps[-1][1] selector = { 'type': 'FragmentSelector', 'value': 't=' + str(start_frame) + ',' + str(end_frame) } log.debug(selector) # detection confidence confidence = [] for frame in list(range(start_frame, end_frame + 1)): found = False for value in timestamps: # value is a tuple like (id<str>, timestamp<int>, confidence<float>) if value[1] == frame: confidence.append(value[2]) found = True break if not found: confidence.append(None) od_confidences = [e for e in confidence if e is not None] avg_confidence = sum(od_confidences) / float(len(od_confidences)) log.debug('AVG confidence for building {0} in shot {1}: {2}'.format( key[0], key[1], avg_confidence)) if avg_confidence < 0.5: # discard detections with low confidence continue log.debug('~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~') # save annotation bodies = [] br_body = { 'type': 'BRBody', 'object_id': br_id, 'confidence': avg_confidence } br_body['concept'] = {'type': 'ResourceBody', 'source': concept} bodies.append(br_body) try: from neomodel import db as transaction transaction.begin() # log.debug("Automatic TAG. Body {0}, Selector {1}".format( # bodies, selector)) repo.create_tag_annotation(None, bodies, item, selector, False, None, True) transaction.commit() except Exception as e: log.verbose("Neomodel transaction ROLLBACK") try: transaction.rollback() except Exception as rollback_exp: log.warning("Exception raised during rollback: %s" % rollback_exp) raise e saved_counter += 1 log.info( 'Number of discarded recognized buildings with wrong city: {counter}'. format(counter=wrong_city_counter)) log.info('Number of saved automatic annotations: {counter}'.format( counter=saved_counter)) log.info('-----------------------------------------------------')
def extract_od_annotations(self, item, analyze_dir_path): ''' Extract object detection results given by automatic analysis tool and ingest valueable annotations as 'automatic' TAGs. ''' orf_results_filename = 'orf.xml' orf_results_path = os.path.join(os.path.dirname(analyze_dir_path), orf_results_filename) if not os.path.exists(orf_results_path): raise IOError("Analyze results does not exist in the path %s", orf_results_path) log.info('get automatic object detection from file [%s]' % orf_results_path) parser = ORF_XMLParser() frames = parser.parse(orf_results_path) # get the shot list of the item shots = item.shots.all() shot_list = {} for s in shots: shot_list[s.shot_num] = set() if item.item_type == 'Image': # consider a image like single frame shot shot_list[0] = set() object_ids = set() concepts = set() report = {} obj_cat_report = {} for timestamp, od_list in frames.items(): ''' A frame is a <dict> with timestamp<int> as key and a <list> as value. Each element in the list is a <tuple> that contains: (obj_id<str>, concept_label<str>, confidence<float>, region<list>). ''' log.debug("timestamp: {0}, element: {1}".format(timestamp, od_list)) for detected_object in od_list: concepts.add(detected_object[1]) if detected_object[0] not in object_ids: report[detected_object[1]] = report.get(detected_object[1], 0) + 1 object_ids.add(detected_object[0]) if item.item_type == 'Video': shot_uuid, shot_num = shot_lookup(self, shots, timestamp) elif item.item_type == 'Image': shot_num = 0 if shot_num is not None: shot_list[shot_num].add(detected_object[1]) # collect timestamp for keys: # (obj,cat) --> [(t1, confidence1, region1), (t2, confidence2, region2), ...] tmp = obj_cat_report.get((detected_object[0], detected_object[1]), []) tmp.append((timestamp, detected_object[2], detected_object[3])) obj_cat_report[(detected_object[0], detected_object[1])] = tmp log.info('-----------------------------------------------------') log.info('Number of distinct detected objects: {}'.format(len(object_ids))) log.info('Number of distinct concepts: {}'.format(len(concepts))) log.info('Report of detected concepts per shot: {}'.format(shot_list)) repo = AnnotationRepository(self.graph) counter = 0 # keep count of saved annotation for (key, timestamps) in obj_cat_report.items(): if item.item_type == 'Video' and len(timestamps) < 5: # discard detections for short times continue # detect the concept concept_name = key[1] concept_iri = concept_mapping.get(concept_name) if concept_iri is None: log.warn('Cannot find concept <{0}> in the mapping'.format( concept_name)) # DO NOT create this annotation continue concept = {'iri': concept_iri, 'name': concept_name} # detect target segment ONLY for videos start_frame = timestamps[0][0] end_frame = timestamps[-1][0] selector = { 'type': 'FragmentSelector', 'value': 't=' + str(start_frame) + ',' + str(end_frame) } # detection confidence confidence = [] region_sequence = [] for frame in list(range(start_frame, end_frame + 1)): found = False for value in timestamps: # value is a tuple like (timestamp<int>, confidence<float>, region<list>) if value[0] == frame: confidence.append(value[1]) region_sequence.append(value[2]) found = True break if not found: confidence.append(None) region_sequence.append(None) od_confidences = [e for e in confidence if e is not None] avg_confidence = sum(od_confidences) / float(len(od_confidences)) if avg_confidence < 0.5: # discard detections with low confidence continue # save annotation bodies = [] # warkaround for very huge area sequence! if len(region_sequence) > 1000: huge_size = len(region_sequence) region_sequence = [] log.warn( 'Detected Object [{objID}/{concept}]: area sequence too big! Number of regions: {size}' .format(objID=key[0], concept=concept['name'], size=huge_size)) od_body = { 'type': 'ODBody', 'object_id': key[0], 'confidence': avg_confidence, 'region_sequence': region_sequence } od_body['concept'] = {'type': 'ResourceBody', 'source': concept} bodies.append(od_body) try: from neomodel import db as transaction transaction.begin() # log.debug("Automatic TAG. Body {0}, Selector {1}".format( # bodies, selector)) repo.create_tag_annotation(None, bodies, item, selector, False, None, True) transaction.commit() except Exception as e: log.verbose("Neomodel transaction ROLLBACK") try: transaction.rollback() except Exception as rollback_exp: log.warning("Exception raised during rollback: %s" % rollback_exp) raise e counter += 1 log.info('Number of saved automatic annotations: {counter}'.format( counter=counter)) log.info('-----------------------------------------------------')
def write(request): """Tweet Writing API""" try: body_data = json.loads(request.body) except JSONDecodeError: return JsonResponse({ 'status': 'error', 'status_code': 'invalid_body', 'status_message': 'No json data exists.' }) text = body_data.get('text', None) if not text: return JsonResponse({ 'status': 'error', 'status_code': 'invalid_data', 'status_message': 'Data `text` is required.' }) graphdb.begin( ) # Set a save point to make a relationship between a user and a tweet. try: user_node = request.user.get_or_create_node() score = user_node.get_score() tweet = Tweet(text=text, score=score).save() tweet.user.connect(user_node) graphdb.commit() ScoringHistory.objects.create( user=request.user, tweet_id=tweet.pk, type=ScoringHistoryType.POST_COMMENT.value, cumulative_score=score, delta_score=score) try: # Parse tags tags = extract_hashtags(text) for tag in tags: try: tag_node = Tag(tag=tag).save() except UniqueProperty: tag_node = Tag.nodes.get(tag=tag) tweet.tags.connect(tag_node) except Exception as e: # to prevent to be input a weird string logger.error(f'Failed to parse tags : {text} {str(e)}') except Exception as e: logger.error(e) graphdb.rollback() return JsonResponse({ 'status': 'error', 'status_code': 'tweet_failed', 'status_message': 'Failed to tweet.' }) return JsonResponse({ 'status': 'success', 'status_code': '', 'status_message': 'The tweet has been written', 'contents': { 'tweet': { 'id': tweet.pk } } })
def createBuilding(self, building): db.begin() try: Building(name=building.get("name")).save() if building.get("floors") is not None: for floor in building.get("floors"): Floor(level=floor.get("level"), buildingName=building.get("name")).save() if floor.get("waypoints") is not None: for waypoint in floor.get("waypoints"): if ("type" in waypoint): if (waypoint.get("type") == WAYPOINT_TYPES["ClassRoom"]): classRoom = ClassRoom( name=waypoint.get("name"), markerId=waypoint.get("markerId"), buildingName=building.get("name"), floorLevel=floor.get("level"), shapeType=waypoint.get("shapeType"), color=waypoint.get("color"), width=waypoint.get("width"), length=waypoint.get("length"), x=waypoint.get("x"), y=waypoint.get("y")).save() for schedule in waypoint["schedule"]: group = Group.nodes.get_or_none( name=schedule["group"], buildingName=building.get("name")) if group is None: group = Group( name=schedule["group"], buildingName=building.get( "name")).save() group.classes.connect( classRoom, { 'course': schedule["course"], 'dayOfWeek': schedule["dayOfWeek"], 'startTime': schedule["startTime"], 'finishTime': schedule["finishTime"] }) elif (waypoint.get("type") == WAYPOINT_TYPES["Office"]): office = Office( name=waypoint.get("name"), markerId=waypoint.get("markerId"), buildingName=building.get("name"), floorLevel=floor.get("level"), shapeType=waypoint.get("shapeType"), color=waypoint.get("color"), width=waypoint.get("width"), length=waypoint.get("length"), x=waypoint.get("x"), y=waypoint.get("y")).save() for prof in waypoint["professors"]: teacher = Teacher.nodes.get_or_none( name=prof, buildingName=building.get("name")) if teacher is None: teacher = Teacher( name=prof, buildingName=building.get( "name")).save() teacher.office.connect(office) elif (waypoint.get("type") == WAYPOINT_TYPES["Connector"]): Connector( name=waypoint.get("name"), markerId=waypoint.get("markerId"), buildingName=building.get("name"), floorLevel=floor.get("level"), shapeType=waypoint.get("shapeType"), color=waypoint.get("color"), width=waypoint.get("width"), length=waypoint.get("length"), x=waypoint.get("x"), y=waypoint.get("y")).save() else: Waypoint( name=waypoint.get("name"), markerId=waypoint.get("markerId"), buildingName=building.get("name"), floorLevel=floor.get("level"), shapeType=waypoint.get("shapeType"), color=waypoint.get("color"), width=waypoint.get("width"), length=waypoint.get("length"), x=waypoint.get("x"), y=waypoint.get("y")).save() else: Waypoint(name=waypoint.get("name"), markerId=waypoint.get("markerId"), buildingName=building.get("name"), floorLevel=floor.get("level"), shapeType=waypoint.get("shapeType"), color=waypoint.get("color"), width=waypoint.get("width"), length=waypoint.get("length"), x=waypoint.get("x"), y=waypoint.get("y")).save() if floor.get("hallways") is not None: for hallway in floor.get("hallways"): Hallway(name=hallway["name"], markerId=hallway["markerId"], buildingName=building.get("name"), floorLevel=floor.get("level"), shapeType=hallway["shapeType"], color=hallway["color"], width=hallway["width"], length=hallway["length"], x=hallway["x"], y=hallway["y"]).save() if building.get("floors") is not None: for floor in building.get("floors"): if floor.get("waypoints") is not None: for waypoint in floor.get("waypoints"): base = Waypoint.nodes.get( name=waypoint.get("name"), floorLevel=floor.get("level"), buildingName=building.get("name")) for neighbor in waypoint["neighbors"]: base.neighbors.connect( Waypoint.nodes.get( name=neighbor.get("name"), floorLevel=floor.get("level"), buildingName=building.get("name")), { 'direction': neighbor.get("direction") }) self.checkBuilding(building.get("name")) db.commit() return self.get(building.get("name")), 200 except Exception as e: db.rollback() return str(e), 500