def start_loop(stream_name=None,
network='s3fd',
redis_host='localhost',
redis_port=6379):
"""
Starts the processing loop
"""
killer = GracefulKiller()
LOG.info('Initializing Redis cache...')
cache = redis.StrictRedis(host=redis_host, port=redis_port, db=0)
LOG.info('Initializing annotator...')
annotator = Annotator(network=network)
LOG.info('Starting processing loop...')
while True:
# Get next available frame from the cache
image_bytes = cache.get('{}_raw'.format(stream_name))
if image_bytes is None:
continue
# Annotate with bounding boxes
# TODO: Get size from feed configs
image = Image.frombytes('RGB', (1920, 1080), image_bytes)
image = annotator.annotate(image)
# Save frame to cache as raw bytes
cache.set('{}_annotated'.format(stream_name), image.tobytes())
# Capture kill signals and terminate loop
if killer.kill_now:
LOG.info('Shutting down gracefully...')
break
def executeAnnotator(logger, startTime, datetime, confpath, pattern, paths):
print("execute Annotator")
momentum = datetime.now()
now = momentum - startTime
l = list(paths)
for path in l:
mags = results = None
mags, results = readKataMagazines(logger)
units = None
magazines = None
results, magazines, minl, maxl = readFile(logger, pattern, now, path,
mags)
annotator = Annotator(None, confpath)
units = annotator.doAnnotationWithConfig(results, mags, magazines)
momentum = logQueryData(logger, momentum, units, None)
print("execute Annotator")
writeResultsToRDF(units)
now = datetime.now() - momentum
end = datetime.now() - startTime
print("Finished queries in " + str(now))
print("REACHED THE END in " + str(end))
logger.info("Application execution ended, and it lasted for " + str(end))
def __init__(self, dock_area, params_tree, config):
self.dock_area = dock_area
self.db = ParameterDB()
self.curves = {}
self.qtree_widget_items = {}
self.plot_widget_items = []
# self.params = {} # [Parameter, Plot, Curve, QTreeWidgetItem]
self.params_tree = params_tree
self.last_dock_added = None
self.xlink = None
self.annotators = {
a['param']: Annotator(a['param'], a)
for a in config.get('annotators', [])
}
self.auto_pens = [
pg.mkPen('#f00'),
pg.mkPen('#0f0'),
pg.mkPen('#00f'),
pg.mkPen('#AA0'),
pg.mkPen('#0AA'),
pg.mkPen('#A0A'),
]
self.initialize_plots(config)
def annotate_data(self, nr_docs=-1):
self.logger.println("data annotator called")
start_time = timeit.default_timer()
annotator = Annotator()
annotator.prepare_dataset(nr_docs)
elapsed_seconds = timeit.default_timer() - start_time
self.logger.print_time_taken("data annotation operation took",
elapsed_seconds)
def generate_annotator(self, event):
annotation = Annotator(self.channel_fnames, self.probability_table,
self.output_dir)
new_table = annotation.annotate([self.x0, self.y0], [self.x1, self.y1])
# add the new table to the groundtruth table
self.groundtruth_table.loc[
new_table.index,
self.column_names] = new_table.loc[:, self.column_names]
def run_annotator():
parser = argparse.ArgumentParser(description='')
parser.add_argument('--source-files',
dest='source_files',
type=str,
nargs='+',
help='')
args = parser.parse_args()
anno = Annotator()
anno.annotate(args.source_files)
def default_annotator(mix_path, src_track_paths):
from nodes import Unmixer, GenericProvider, SegmentFinder, XFadeFinder, Fingerprinter
from annotator import Annotator
return Annotator([
GenericProvider(src_track_paths=src_track_paths, mix_path=mix_path),
Fingerprinter(lazy=False),
Unmixer(),
XFadeFinder(),
SegmentFinder()
])
def handle_resume_post(self):
# Get the name of the uploaded file
file = request.files['file']
if file and self.__allowed_file(file.filename):
# Save file to upload folder
file.save(os.path.join(self.__app.config['UPLOAD_FOLDER'], file.filename))
# use crf here
annotator = Annotator()
annotated_resume = annotator.annotate_using_trained_model(self.__app.config['UPLOAD_FOLDER'] + self.__seperator + file.filename)
tagged_resume = self.__crfsuite.tag_doc(annotated_resume)
template = render_template('%s.xml' % self.__path_output_xml, entities=tagged_resume)
response = make_response(template)
response.headers['Content-Type'] = 'application/xml'
return response
else:
return "Invalid file type, use PDF, DOC or DOCX", 406
#clips forlder for this video
clips_folder = os.path.join(clips_folder,each)
annotation_path = os.path.join("annotation",each)
annotation_path_for_vid = os.path.exists(os.path.join(annotation_path,"labels.json"))
if not os.path.exists(annotation_path):
os.makedirs(annotation_path)
# Initialise the annotator
annotator = Annotator([
{'name': 'goal', 'color': (0, 255, 0)},
{'name': 'others', 'color': (0, 0, 255)},
{'name': 'startgame', 'color': (0, 255, 255)},
{'name': 'endgame', 'color': (255, 255, 255)},
{'name': 'replay_goal', 'color': (0, 0, 0)},
{'name': 'resume','color':(64,244,226)},
{'name':'SOT','color':(66, 86, 244)},
{'name':'play','color':(244, 155, 65)},
{'name':'replay','color':(193, 52, 156)}],
clips_folder, sort_files_list=True, N_show_approx=20, screen_ratio=16/9,
image_resize=1, loop_duration=None, annotation_file=os.path.join(annotation_path,'labels.json'))
if not os.path.exists(clips_folder):
# Split the video into clips
os.makedirs(os.path.join(clips_folder))
print('Generating clips from the video...')
annotator.video_to_clips(os.path.join(videos_folder,each), clips_folder, clip_length=60, overlap=0, resize=0.5)
if annotation_path_for_vid:
if loadflag is None:
resp = input("the anotations will be overwritten. continue? (y/n)")
# Create the clips folder
clips_folder = 'test_overlap_clips'
if os.path.exists(clips_folder):
shutil.rmtree(clips_folder)
os.makedirs(clips_folder)
# Test the annotator
from annotator import Annotator
# Initialise the annotator
annotator = Annotator([{
'name': 'test_label_1',
'color': (0, 1, 0)
}, {
'name': 'test_label_2',
'color': (0, 0, 1)
}, {
'name': 'test_label_3',
'color': (0, 1, 1)
}],
clips_folder,
loop_duration=2,
annotation_file='overlap_annotation.json',
status_file='overlap_status.json')
# Create the overlapping clips
annotator.video_to_clips('dummy_digits.mp4',
clips_folder,
resize=0.5,
overlap=0.5,
clip_length=6)
# Run!
annotator.main()
def setUp(self):
self.annotator = Annotator(TestFiles.make_io_helper())
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
'--model', help='File path of Tflite model.', required=True)
parser.add_argument('--label', help='File path of label file.')
args = parser.parse_args()
labels = dataset_utils.read_label_file(args.label) if args.label else None
engine = DetectionEngine(args.model)
with picamera.PiCamera() as camera:
preview_size = (640, 480)
camera.resolution = preview_size
camera.framerate = 30
# camera.hflip = True
# camera.vflip = True
# camera.rotation = 90
_, input_height, input_width, _ = engine.get_input_tensor_shape()
input_size = (input_width, input_height)
# Width is rounded up to the nearest multiple of 32,
# height to the nearest multiple of 16.
capture_size = (math.ceil(input_width / 32) * 32,
math.ceil(input_height / 16) * 16)
# Actual detection area on preview.
detect_size = (preview_size[0] * input_size[0] / capture_size[0],
preview_size[1] * input_size[1] / capture_size[1])
# Make annotator smaller for efficiency.
annotator_factor = 0.5
annotator_size = (int(preview_size[0] * annotator_factor),
int(preview_size[1] * annotator_factor))
# Font for drawing detection candidates
font = ImageFont.truetype(
'/usr/share/fonts/truetype/freefont/FreeMonoBold.ttf',
size=12)
camera.start_preview()
annotator = Annotator(camera,
dimensions=annotator_size,
default_color=(255, 255, 255, 64))
def annotate(candidates):
annotator.clear()
# Get actual coordinates to draw
def translate(relative_coord):
return (detect_size[0] * relative_coord[0] * annotator_factor,
detect_size[1] * relative_coord[1] * annotator_factor)
for c in candidates:
top_left = translate(c.bounding_box[0])
bottom_right = translate(c.bounding_box[1])
annotator.bounding_box(top_left + bottom_right)
text = '{} {:.2f}'.format(labels[c.label_id], c.score) \
if labels else '{:.2f}'.format(c.score)
annotator.text(top_left, text, font=font)
annotator.update()
try:
stream = io.BytesIO()
for _ in camera.capture_continuous(
stream, format='rgb', use_video_port=True, resize=capture_size):
stream.truncate()
stream.seek(0)
input_tensor = np.frombuffer(stream.getvalue(), dtype=np.uint8)
if input_size != capture_size:
# Crop to input size. Note dimension order (height, width, channels)
input_tensor = input_tensor.reshape(
(capture_size[1], capture_size[0], 3))[
0:input_height, 0:input_width, :].ravel()
start_ms = time.time()
results = engine.detect_with_input_tensor(input_tensor, top_k=3)
elapsed_ms = time.time() - start_ms
annotate(results)
camera.annotate_text = '{:.2f}ms'.format(elapsed_ms * 1000.0)
finally:
# Maybe should make this an annotator method
camera.remove_overlay(annotator._overlay)
camera.stop_preview()
def __init__(self):
self.annotator = Annotator()
# make Qt logs go to logger
QtCore.qInstallMessageHandler(qt_message_handler)
logger = logging.getLogger(__name__)
logger.info('Annotator %s starting on %s (%s)', '0.0.0.0', platform.system(), sys.platform)
if platform.system() == 'Windows':
logger.info('Applying Windows-specific setup')
# enable automatic scaling for high-DPI screens
os.environ['QT_AUTO_SCREEN_SCALE_FACTOR'] = '1'
# set the App ID for Windows 7 to properly display the icon in the
# taskbar.
import ctypes
myappid = '??????' # arbitrary string
try:
ctypes.windll.shell32.SetCurrentProcessExplicitAppUserModelID(myappid)
except:
logger.error('Could not set the app model ID. If the plaftorm is older than Windows 7, this is normal.')
elif platform.system() == 'Darwin':
pass
# logger.info('Applying Mac OS-specific setup')
app = QApplication(sys.argv)
window = Annotator()
window.show()
return_code = app.exec_()
del window # prevent mac errors
sys.exit(return_code)
def main():
images_path = get_image_path_from_user()
annotator = Annotator(categories=["dog", "cat"], images_path=images_path)
annotator.begin_annotation()
parser.add_argument('-cl', type=float, default=5)
parser.add_argument('-l', choices=['semantic', 'segment'], default='segment')
args = parser.parse_args()
video_file = args.video
video_basename = os.path.basename(video_file)
video_name, ext = os.path.split(video_basename)
out_file = args.out_file
if os.path.isdir(video_file):
clip_dir = video_file
else:
clip_dir = 'clips-' + video_basename
annotator = Annotator(segment_labels if args.l == 'shot' else semantic_labels,
clip_dir,
annotation_file=out_file,
N_show_approx=100)
if not os.path.exists(clip_dir):
vc = cv2.VideoCapture(video_file)
fps = vc.get(cv2.CAP_PROP_FPS)
vc.release()
clip_frames = round(fps * args.cl)
print('Clips contain {} frames'.format(clip_frames))
os.makedirs(clip_dir)
with open(os.path.join(clip_dir, 'frame_count.txt'), 'w') as fp:
fp.write(str(clip_frames))
annotator.video_to_clips(video_file,
clip_dir,
clip_length=clip_frames,
overlap=0,
# Save the clip
print('\rClip %d complete' % clip_counter, end=' ')
clip.release()
clip_time = 0
clip_counter += 1
clip = cv2.VideoWriter(
os.path.join(clips_folder, 'clip_%04d.mp4' % clip_counter),
fourcc, fps, (fdim[1], fdim[0]))
if video_time < n_frames - 1:
video_time += 1
else:
cap.release()
break
# Run the annotator
annotator = Annotator([{
'name': 'result_table',
'color': (0, 1, 0)
}, {
'name': 'olympics_logo',
'color': (0, 0, 1)
}, {
'name': 'stretching',
'color': (0, 1, 1)
}],
clips_folder,
N_show_approx=100,
annotation_file='demo_labels.json')
annotator.main()
def initialize():
global annotator, answer_catalog, analysis
annotator = Annotator()
answer_catalog = Answers()
analysis = Analysis()
if not os.path.exists(clips_folder):
os.mkdir(clips_folder)
# Initialise the annotator
annotator = Annotator(
# [
# {'name': 'clarity 100', 'color': (0, 255, 0)},
# {'name': 'clarity 80', 'color': (0, 0, 255)},
# {'name': 'clarity 60', 'color': (0, 255, 255)},
# {'name': 'clarity 40', 'color': (255, 100, 0)},
# {'name': 'clarity 20', 'color': (0, 100, 255)}],
[
{'name': '1', 'color': (0, 255, 0)},
{'name': '2', 'color': (0, 0, 255)},
{'name': '3', 'color': (0, 255, 255)},
{'name': '4', 'color': (255, 100, 0)},
{'name': '5', 'color': (0, 100, 255)},
{'name': '6', 'color': (0, 100, 50)},
{'name': '7', 'color': (0, 150, 100)},
{'name': '8', 'color': (50, 100, 255)},
{'name': '9', 'color': (100, 50, 50)},
{'name': '10', 'color': (50, 100, 150)},
{'name': '11', 'color': (100, 100, 200)}
],
clips_folder, sort_files_list=True, N_show_approx=20, screen_ratio=16 / 9,
image_resize=1, loop_duration=None, annotation_file='ourdata_section.json')
# Split the video into clips
print('Generating clips from the video...')
annotator.video_to_clips(ourdata_filename, clips_folder, clip_length=150, overlap=0, resize=1)
'name': 'sluty',
'color': (0, 0, 1)
}, {
'name': 'sexy',
'color': (0, 1, 1)
}, {
'name': 'normal',
'color': (0, 1, 0)
}]
# Initialise MuViLab
name = 'H.mp4'
fn = 'E:\\download\\593310496_saturdays85\\test\\%s' % name
clips_folder = './%s' % name.rsplit('.', 1)[0]
# Split the main video into clips
annotator = Annotator(labels,
clips_folder,
annotation_file='%s.json' % name,
N_show_approx=10)
import os
if not os.path.exists(clips_folder):
os.mkdir(clips_folder)
annotator.video_to_clips(fn,
clips_folder,
clip_length=1200,
overlap=0,
resize=0.5)
# Run the GUI
annotator.main()
def get_ies_scores(self):
extractor = Extractor()
ies_filenames = extractor.populate_file_names(self.__ies_accuracy_test)
ies_filenames = extractor.filter_by_valid_exts(ies_filenames)
filenames, resume_content = extractor.read_resume_content_tika_api(
ies_filenames, self.__ies_accuracy_test)
filenames, resume_content = extractor.remove_empty_resumes(
filenames, resume_content)
resume_labels = extractor.read_resume_labels(self.__ies_accuracy_test,
filenames)
true_edu_insts = [
extractor.get_edu_institutions(xml_tree)
for xml_tree in resume_labels
]
true_edu_majors = [
extractor.get_edu_majors(xml_tree) for xml_tree in resume_labels
]
true_emp_names = [
extractor.get_company_names(xml_tree) for xml_tree in resume_labels
]
true_emp_jtitles = [
extractor.get_job_titles(xml_tree) for xml_tree in resume_labels
]
cs = CrfSuite()
cs.load_tagger()
annotator = Annotator()
annotated_resumes = [
annotator.annotate_using_trained_model(self.__ies_accuracy_test +
self.__seperator +
filename[0] + filename[1])
for filename in filenames
]
predicted_entity_list = [
cs.tag_doc(resume) for resume in annotated_resumes
]
ies_edu_insts = [
extractor.get_edu_institutions_from_list(entity_list)
for entity_list in predicted_entity_list
]
ies_edu_majors = [
extractor.get_edu_major_from_list(entity_list)
for entity_list in predicted_entity_list
]
ies_emp_names = [
extractor.get_company_names_from_list(entity_list)
for entity_list in predicted_entity_list
]
ies_emp_jtitles = [
extractor.get_company_position_from_list(entity_list)
for entity_list in predicted_entity_list
]
tokeniser = Tokeniser()
true_edu_insts = tokeniser.docs_tolower(
tokeniser.tokenise_doclines_to_words(true_edu_insts))
true_edu_majors = tokeniser.docs_tolower(
tokeniser.tokenise_doclines_to_words(true_edu_majors))
true_emp_names = tokeniser.docs_tolower(
tokeniser.tokenise_doclines_to_words(true_emp_names))
true_emp_jtitles = tokeniser.docs_tolower(
tokeniser.tokenise_doclines_to_words(true_emp_jtitles))
ies_edu_insts = tokeniser.docs_tolower(
tokeniser.tokenise_doclines_to_words(ies_edu_insts))
ies_edu_majors = tokeniser.docs_tolower(
tokeniser.tokenise_doclines_to_words(ies_edu_majors))
ies_emp_names = tokeniser.docs_tolower(
tokeniser.tokenise_doclines_to_words(ies_emp_names))
ies_emp_jtitles = tokeniser.docs_tolower(
tokeniser.tokenise_doclines_to_words(ies_emp_jtitles))
edu_insts_match_score = self.score_matches(ies_edu_insts,
true_edu_insts)
edu_majors_match_score = self.score_matches(ies_edu_majors,
true_edu_majors)
emp_names_match_score = self.score_matches(ies_emp_names,
true_emp_names)
emp_jtitles_match_score = self.score_matches(ies_emp_jtitles,
true_emp_jtitles)
print(edu_insts_match_score)
print(edu_majors_match_score)
print(emp_names_match_score)
print(emp_jtitles_match_score)
ap.add_argument("-v",
"--version",
dest="version",
default=False,
action="store_true",
help="displays the current version of the application")
return ap.parse_args()
if __name__ == "__main__":
arguments = parse_args()
if arguments.test:
doctest.testmod() # unit testing
sys.exit()
# displays version of the program
if arguments.version:
sys.exit("{} {}".format(APP_TITLE, VERSION_NUMBER))
# creates the annotation engine
annotator = Annotator()
if arguments.fullscreen:
annotator.toggle_fullscreen() # toggles fullscreen
annotator.parent.protocol("WM_DELETE_WINDOW",
annotator.exit_prompt) # quit event handler
annotator.mainloop() # runs the main tkinter loop
yt = YouTube('https://www.youtube.com/watch?v=VZvoufQy8qc')
stream = yt.streams.filter(res='144p', mime_type='video/mp4').first()
print('Downloading youtube file. This may take a while.\n' +
'Let\'s be honest, this _will_ take a while...')
stream.download(demo_folder, filename='youtube')
# Initialise the annotator
annotator = Annotator([{
'name': 'result_table',
'color': (0, 1, 0)
}, {
'name': 'olympics_logo',
'color': (0, 0, 1)
}, {
'name': 'stretching',
'color': (0, 1, 1)
}],
clips_folder,
sort_files_list=True,
N_show_approx=100,
screen_ratio=16 / 9,
image_resize=1,
loop_duration=None,
annotation_file='demo_labels.json')
# Split the video into clips
print('Generating clips from the video...')
annotator.video_to_clips(youtube_filename,
clips_folder,
clip_length=90,
overlap=0,
def run_tool(video_file, labels):
videoFileName = video_file.split("/")[-1].split(".")[0]
out_folder = './output/'
if not os.path.exists(out_folder):
os.mkdir(out_folder)
output_folder = f'{out_folder}{videoFileName}/'
if not os.path.exists(output_folder):
os.mkdir(output_folder)
clips_folder = output_folder + 'clips/'
json_file = f'{output_folder}{videoFileName}.json'
ref_labels = f'{output_folder}{videoFileName}_ref_labels.json'
#Copy the reference json file in to the folder
copyfile(labels, ref_labels)
# Create the folders
if not os.path.exists(clips_folder):
os.mkdir(clips_folder)
print(json_file)
colors = [(255, 0, 0), (0, 255, 0), (0, 0, 255), (255, 255, 0),
(0, 255, 255), (255, 0, 255), (192, 192, 192), (128, 128, 128),
(128, 0, 0), (128, 128, 0), (0, 128, 0), (128, 0, 128),
(0, 128, 128), (0, 0, 128)]
# Initialise the annotator
with open(labels) as f:
lines = f.readlines()
labels = [l.strip() for l in lines]
assert len(labels) > 0, "Labels file doesn't have any labels"
if len(labels) <= len(colors):
colors_need = colors[:len(labels)]
else:
t = len(labels) // len(colors)
r = len(labels) % len(colors)
colors_need = t * colors + colors[:r]
assert len(labels) == len(colors_need), "Check the code for error"
annotation_labels = []
for idx, label in enumerate(labels):
annotation_labels.append({'name': label, 'color': colors_need[idx]})
annotator = Annotator(annotation_labels,
clips_folder,
sort_files_list=True,
N_show_approx=16,
screen_ratio=16 / 9,
image_resize=1,
loop_duration=None,
annotation_file=json_file)
bClippingRequired = True
try:
filesArr = os.listdir(clips_folder)
if len(filesArr) > 0:
bClippingRequired = False
except:
print("no files")
if bClippingRequired:
# Split the video into clips
print('Generating clips from the video...')
annotator.video_to_clips(video_file,
clips_folder,
clip_length=60,
overlap=0,
resize=0.5)
# Run the annotator
annotator.main()
def bccwj_pipeline(input_path, output_path):
a = Annotator()
file_names = glob.glob(os.path.join(input_path, '**/*.xml'),
recursive=True)
docs = load_text(file_names, encoding='utf-8')
save_file(docs, a, output_path)
def main(argv):
#logging detup
logger = logging.getLogger('myapp')
hdlr = logging.FileHandler('/tmp/myapp.log')
formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
hdlr.setFormatter(formatter)
logger.addHandler(hdlr)
logger.setLevel(logging.DEBUG)
startTime = datetime.now()
logger.info("Application execution started at " + str(startTime))
#retrieve options for this job
configfile, filepattern, inputfilepath, csvlogging, target_format, source_file, source_format, special = extractConfigurations(
argv)
momentum = datetime.now()
now = momentum - startTime
units = None
counter = 1
annotator = Annotator(None, configfile)
full_list_magazines = None
minm = 0
maxm = 0
print(filepattern)
print(inputfilepath)
print(configfile)
#kata=True
for path in inputfilepath:
try:
mags = results = None
#print("Before anything")
#print(full_list_magazines)
if special == "kata":
mags, results = readKataMagazines(logger)
magazines = None
results, magazines, min, max = readFile(logger, filepattern, now,
path, mags)
minm, maxm = checkLength(min, minm, maxm, logger)
minm, maxm = checkLength(max, minm, maxm, logger)
#print("After reading files")
#print(minm)
#print(maxm)
#print(results)
#print(magazines)
#print(full_list_magazines)
u = annotator.doAnnotationWithConfig(results, mags, magazines,
csvlogging, units)
#print("After annotating them")
#print(magazines)
if full_list_magazines != None:
for m in magazines:
if m not in full_list_magazines:
full_list_magazines.append(m)
#print(magazines)
#print(full_list_magazines)
else:
full_list_magazines = magazines
#for m in full_list_magazines:
# print(m)
if units != None:
#units.extend(u)
for node in u:
if node not in units:
units.append(node)
else:
units = u
logger.info("Processed " + str(counter) + "/" +
str(len(inputfilepath)))
counter = counter + 1
#print("After everything")
#print(full_list_magazines)
#if counter == 10:
#remove this later
except Exception as e:
print("Error happened during execution: " + str(path))
print("Error happened during execution: " + str(e))
logger.warn(
"Unexpected error while processing data " + str(path) + ":", e)
error = traceback.format_exc()
print(error.upper())
if full_list_magazines != None:
writeTextOutput(full_list_magazines)
else:
logger.error("magaine list is empty!")
rank = annotator.doRanking()
tfidf, limit = logCandicates(logger, full_list_magazines, rank)
#logger.debug(tfidf)
if units != None:
print("Check ranking" + str(limit))
rank, rank_range = annotator.doRanking()
print(len(rank))
if len(rank) > 0:
limits = (minm, maxm)
logger.info("Execute candidate ranking for " + str(rank) + " " +
str(limit))
apply_weights(units, tfidf, rank, logger, limits, rank_range,
limit)
#use when using ranges
#apply_weights(units, tfidf, rank, logger, limits, rank_range)
print("convert to rdf")
writeResultsToRDF(units, annotator, counter, target_format,
source_file, source_format)
writeXmlOutput(full_list_magazines)
writeCSVOutputOfResults(full_list_magazines)
annotator.writeToCSV(full_list_magazines)
annotator.logConseptsByIndex(full_list_magazines)
#writeResultsToRDF(u,annotator,counter, target_format, source_file, source_format)
annotator.print_filtered_terms(full_list_magazines)
annotator.print_included_terms(full_list_magazines)
annotator.print_stats(full_list_magazines)
now = datetime.now() - momentum
end = datetime.now() - startTime
print("Finished queries in " + str(now))
print("REACHED THE END in " + str(end))
logger.info("Application execution ended, and it lasted for " + str(end))
def mainichi_pipeline(input_path, output_path):
a = Annotator()
file_names = glob.glob(os.path.join(input_path, '*.sgml'))
docs = load_text(file_names, encoding='shift_jis')
save_file(docs, a, output_path)
# hyper parameters. May modify to change the behaviour of the model
# l determines the importance of the foreground and background probability terms
# bins determines how many bins to use in the histograms
l = 1
num_bins = 2
# Get image
if len(sys.argv) == 2:
imgURL = sys.argv[1]
else:
imgURL = "http://www.python.org/static/community_logos/python-logo.png"
img = getImage(imgURL)
#annotate it with foreground and background points
annotator = Annotator(img)
# extract foreground and background points
fg = annotator.fg
bg = annotator.bg
if (len(fg) == 0) or (len(bg) == 0):
print("No Foreground or Background points selected, exiting")
sys.exit()
#time Imitalisation
start = time.clock()
print("Converting to grayscale and computing statistics")
#compute intensities (grayscale) and collect statistics
gray = rgb2gray(img)
def test(nlp,
src,
gen,
bert=False,
print_annotations=False,
print_latex=False,
verbose=False):
if print_annotations:
print("source:", src_line[:50])
print("summary:", gen_line[:50])
src = nlp(src)
gen = nlp(gen)
if verbose:
print("clusters:", src._.coref_clusters, gen._.coref_clusters)
ce = CompoundEquivalency()
spe = SpeakerPronounEquivalency()
spe.register(src)
spe.register(gen)
kg = KnowledgeGraph(nlp,
use_bert=bert,
equivalencies=[ce, spe],
verbose=verbose)
if print_annotations:
annotator = Annotator(src, gen, latex=print_latex)
kg.add_document(src)
contained = 0
contained_bert = 0
missing = 0
missing_verb = 0
missing_actors = 0
missing_acteds = 0
contradiction = 0
contradiction_bert = 0
invalid_simplification = 0
total = 0
for token in gen:
if token.pos_ == "VERB":
total += 1
relation = kg.get_relation(token)
r = kg.query_relation(relation)
if r[0] == KnowledgeGraph.entailment:
if print_annotations:
print(util.format("contained", "blue", latex=print_latex),
"|", relation, "|", r[1])
contained += 1
if r[0] == KnowledgeGraph.entailment_bert:
if print_annotations:
print(
util.format("contained (BERT)",
"blue",
latex=print_latex), "|", relation, "|",
r[1])
contained_bert += 1
if r[0] == KnowledgeGraph.contradiction_bert:
if print_annotations:
print(
util.format("contradiction (BERT)",
"red",
latex=print_latex), "|", relation, "|",
r[1])
contradiction_bert += 1
elif r[0] == KnowledgeGraph.missing_dependencies:
missing += 1
if print_annotations:
print(
util.format("generic missing dependency",
"yellow",
latex=print_latex), "|", relation, "|",
r[1])
elif r[0] == KnowledgeGraph.missing_actors:
missing_actors += 1
if print_annotations:
print(
util.format("missing actors",
"magenta",
latex=print_latex), "|", relation, "|",
r[1])
elif r[0] == KnowledgeGraph.missing_acteds:
missing_acteds += 1
if print_annotations:
print(
util.format("missing acteds",
"magenta",
latex=print_latex), "|", relation, "|",
r[1])
elif r[0] == KnowledgeGraph.missing_verb:
missing_verb += 1
if print_annotations:
print(
util.format("missing verb",
"magenta",
latex=print_latex), "|", relation, "|",
r[1])
elif r[0] == KnowledgeGraph.invalid_simplification:
invalid_simplification += 1
if print_annotations:
print(
util.format("invalid simplification",
"magenta",
latex=print_latex), "|", relation, "|",
r[1])
elif r[0] == KnowledgeGraph.contradiction:
contradiction += 1
if print_annotations:
print(
util.format("contradiction", "red", latex=print_latex),
"|", relation, "|", r[1])
if print_annotations:
annotator.annotate(relation, r)
if print_annotations:
annotated_document, annotated_summary = annotator.annotated()
print("Document:", " ".join(annotated_document))
print("Summary:", " ".join(annotated_summary))
if total == 0:
return 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0
return 100.0 * contained / total, \
100.0 * contained_bert / total, \
100.0 * missing / total, \
100.0 * missing_verb / total, \
100.0 * missing_actors / total, \
100.0 * missing_acteds / total, \
100.0 * contradiction / total, \
100.0 * contradiction_bert / total, \
100.0 * invalid_simplification / total
